mirror of
https://github.com/garrytan/gbrain.git
synced 2026-07-27 22:15:33 +00:00
feat(calibration): domain_scorecards widening + 4 aggregators (v0.41 T10)
Replaces the v0.36.1.0 placeholder `JSON.stringify({})` in
calibration-profile.ts:336 with a real aggregator pass over the active
pack's calibration_domains declarations. domain_scorecards JSONB now
populates per declared domain with {n, brier, accuracy, aggregator,
page_types, extras}.
New module: src/core/calibration/domain-aggregators.ts
- aggregateDomainScorecards(engine, holder, domains, sourceId) → JSONB-shape
- 4 aggregator implementations matching the AggregatorKind closed enum:
- scalar_brier: AVG(POWER(weight - outcome::int, 2)). The default for
most predictive domains. Filters by holder + page_types +
resolved_outcome IS NOT NULL + active=TRUE + source_id.
- weighted_brier: Brier weighted by ABS(weight - 0.5) * 2 (conviction
proxy since takes table has no separate confidence column). A
0.95-conviction miss weights 9x more than a 0.55-conviction one.
Matches the investor pack's market_call semantics.
- count_based: simple SUM(hit)/COUNT(*) accuracy without Brier.
For domains where probability isn't natural.
- cluster_summary: page count + tier histogram via
frontmatter->>'tier' JSONB read. For concept_themes where there's
no binary outcome to score. Returns {n, tier_counts: {T1, T2,
T3, T4}}.
Wiring in src/core/cycle/calibration-profile.ts:
Try/catch wraps the loadActivePack → aggregator chain. Empty {}
scorecard on any pack-resolution error (R1 IRON RULE: byte-identical
v0.36.1.0 baseline when no active pack declares domains). Warning
appended to result.warnings so doctor surfaces silent failures
instead of crashing the phase.
Per-domain fail-soft: aggregateOneDomain's try/catch returns
{n: 0, brier: null, accuracy: null, extras: {error}} for any single
malformed domain. The other domains still aggregate. Phase keeps
running.
Tests (test/domain-aggregators.test.ts, 13 cases):
- R1 IRON RULE: empty domain list returns {} (byte-identical)
- scalar_brier: empty no-takes returns n:0/null/null; 2-take
Brier computed correctly (0.5 over (0, 1) sq_errs); accuracy
matches weight>=0.5 hit/miss; filters by holder; filters by
page_types; ignores unresolved takes
- weighted_brier: high-conviction miss weighted 9x more; accuracy
independent of conviction weighting
- count_based: accuracy without Brier
- cluster_summary: tier histogram from frontmatter; zero-concepts
returns n:0 + all-zero tiers
- Multi-domain: aggregates all declared in one call
- Fail-soft per domain: nonexistent page_type produces n:0 without
blocking other domains
89/89 across T1+T2+T3+T4+T9+T10 tests; typecheck clean.
Plan: ~/.claude/plans/system-instruction-you-are-working-toasty-milner.md
Task T10 of 13. The propose_takes-side wiring (populate
take_domain_assignments at write time from active pack's page_type→
domain mapping) is deferred to T5/T6 phase implementations, since
they are the natural producers of takes. Manual propose_takes via
fence write covers the operator path. v0.42+ adds a takes-fence
parser extension to read domain[] from fence rows.
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 4.7
parent
1850613e53
commit
c6f33491c2
@@ -0,0 +1,279 @@
|
||||
// v0.41 T10 — calibration domain aggregators.
|
||||
//
|
||||
// Closed registry of algorithms that aggregate resolved takes within a
|
||||
// calibration domain. Each aggregator runs against the active pack's
|
||||
// `calibration_domains: [{name, aggregator, page_types}]` declarations
|
||||
// (T3 schema) and produces a per-domain scorecard for the JSONB stored
|
||||
// in calibration_profiles.domain_scorecards.
|
||||
//
|
||||
// The aggregator algorithms are CLOSED (this enum + their SQL stays in
|
||||
// code; pack manifests reference them by name). Domain NAMES are open —
|
||||
// any pack can declare new domains pointing at known aggregators. See
|
||||
// T3 (codex refinement of D6).
|
||||
//
|
||||
// JOIN shape: takes → take_domain_assignments WHERE domain = $name
|
||||
// AND assignments.take_id = takes.id. Pages filtering happens via
|
||||
// page_types: [...] from the domain declaration — JOIN pages too and
|
||||
// filter by p.type = ANY($page_types::text[]).
|
||||
|
||||
import type { BrainEngine } from '../engine.ts';
|
||||
import type { AggregatorKind, CalibrationDomain } from '../schema-pack/manifest-v1.ts';
|
||||
|
||||
export interface DomainScorecard {
|
||||
/** Number of resolved takes contributing to this scorecard. */
|
||||
n: number;
|
||||
/** Brier score (lower = better). null when n === 0 or aggregator doesn't compute one. */
|
||||
brier: number | null;
|
||||
/** Accuracy fraction in [0, 1]. null when n === 0 or aggregator doesn't compute one. */
|
||||
accuracy: number | null;
|
||||
/** Aggregator algorithm used. Stamped for downstream debugging. */
|
||||
aggregator: AggregatorKind;
|
||||
/** Page types whose takes feed this domain (per pack manifest). */
|
||||
page_types: string[];
|
||||
/** Aggregator-specific extras. cluster_summary fills tier_counts here. */
|
||||
extras?: Record<string, unknown>;
|
||||
}
|
||||
|
||||
export type DomainScorecards = Record<string, DomainScorecard>;
|
||||
|
||||
/**
|
||||
* Aggregate every declared calibration_domain for the given holder.
|
||||
*
|
||||
* Returns a Record<domain_name, DomainScorecard>. Empty domains (n=0)
|
||||
* are STILL included in the result so consumers can distinguish
|
||||
* "domain is declared but has no resolved takes yet" from "domain
|
||||
* isn't declared by the active pack" (the latter case never appears
|
||||
* in this return shape).
|
||||
*
|
||||
* Fail-soft per domain: if one domain's SQL throws (e.g. the pack
|
||||
* declares a page_type that doesn't exist in the brain yet), the
|
||||
* domain gets {n: 0, brier: null, accuracy: null, extras: {error: msg}}
|
||||
* and the rest of the domains still aggregate. The phase keeps running.
|
||||
*/
|
||||
export async function aggregateDomainScorecards(
|
||||
engine: BrainEngine,
|
||||
holder: string,
|
||||
domains: CalibrationDomain[],
|
||||
sourceId: string,
|
||||
): Promise<DomainScorecards> {
|
||||
const out: DomainScorecards = {};
|
||||
for (const domain of domains) {
|
||||
try {
|
||||
const scorecard = await aggregateOneDomain(engine, holder, domain, sourceId);
|
||||
out[domain.name] = scorecard;
|
||||
} catch (err) {
|
||||
out[domain.name] = {
|
||||
n: 0,
|
||||
brier: null,
|
||||
accuracy: null,
|
||||
aggregator: domain.aggregator,
|
||||
page_types: [...domain.page_types],
|
||||
extras: { error: err instanceof Error ? err.message : String(err) },
|
||||
};
|
||||
}
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
async function aggregateOneDomain(
|
||||
engine: BrainEngine,
|
||||
holder: string,
|
||||
domain: CalibrationDomain,
|
||||
sourceId: string,
|
||||
): Promise<DomainScorecard> {
|
||||
switch (domain.aggregator) {
|
||||
case 'scalar_brier':
|
||||
return aggregateScalarBrier(engine, holder, domain, sourceId);
|
||||
case 'weighted_brier':
|
||||
return aggregateWeightedBrier(engine, holder, domain, sourceId);
|
||||
case 'count_based':
|
||||
return aggregateCountBased(engine, holder, domain, sourceId);
|
||||
case 'cluster_summary':
|
||||
return aggregateClusterSummary(engine, holder, domain, sourceId);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Standard Brier score over resolved binary takes.
|
||||
* Brier = mean((p - outcome)^2) where p = take.weight (probability),
|
||||
* outcome = resolved_outcome::int (0 or 1).
|
||||
*/
|
||||
async function aggregateScalarBrier(
|
||||
engine: BrainEngine,
|
||||
holder: string,
|
||||
domain: CalibrationDomain,
|
||||
sourceId: string,
|
||||
): Promise<DomainScorecard> {
|
||||
const rows = await engine.executeRaw<{
|
||||
n: number;
|
||||
brier: number | null;
|
||||
accuracy: number | null;
|
||||
}>(
|
||||
`SELECT
|
||||
COUNT(*)::int AS n,
|
||||
AVG(POWER(t.weight - (t.resolved_outcome::int)::real, 2))::real AS brier,
|
||||
(SUM(CASE WHEN (t.weight >= 0.5) = t.resolved_outcome THEN 1 ELSE 0 END)::real
|
||||
/ NULLIF(COUNT(*), 0))::real AS accuracy
|
||||
FROM takes t
|
||||
JOIN take_domain_assignments a ON a.take_id = t.id
|
||||
JOIN pages p ON p.id = t.page_id
|
||||
WHERE a.domain = $1
|
||||
AND t.holder = $2
|
||||
AND t.active = TRUE
|
||||
AND t.resolved_outcome IS NOT NULL
|
||||
AND p.type = ANY($3::text[])
|
||||
AND p.source_id = $4`,
|
||||
[domain.name, holder, domain.page_types, sourceId],
|
||||
);
|
||||
const row = rows[0] ?? { n: 0, brier: null, accuracy: null };
|
||||
return {
|
||||
n: row.n,
|
||||
brier: row.n > 0 ? row.brier : null,
|
||||
accuracy: row.n > 0 ? row.accuracy : null,
|
||||
aggregator: 'scalar_brier',
|
||||
page_types: [...domain.page_types],
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Weighted Brier — weight each prediction by its CONVICTION (ABS(weight - 0.5) * 2).
|
||||
* Low-conviction (weight ≈ 0.5) hunches count less than high-conviction calls.
|
||||
* Mirrors the "market_call cares more about strong-conviction misses" semantics
|
||||
* from the investor pack.
|
||||
*/
|
||||
async function aggregateWeightedBrier(
|
||||
engine: BrainEngine,
|
||||
holder: string,
|
||||
domain: CalibrationDomain,
|
||||
sourceId: string,
|
||||
): Promise<DomainScorecard> {
|
||||
const rows = await engine.executeRaw<{
|
||||
n: number;
|
||||
brier: number | null;
|
||||
accuracy: number | null;
|
||||
}>(
|
||||
`WITH scored AS (
|
||||
SELECT
|
||||
POWER(t.weight - (t.resolved_outcome::int)::real, 2) AS sq_err,
|
||||
ABS(t.weight - 0.5) * 2.0 AS conviction,
|
||||
(t.weight >= 0.5) = t.resolved_outcome AS hit
|
||||
FROM takes t
|
||||
JOIN take_domain_assignments a ON a.take_id = t.id
|
||||
JOIN pages p ON p.id = t.page_id
|
||||
WHERE a.domain = $1
|
||||
AND t.holder = $2
|
||||
AND t.active = TRUE
|
||||
AND t.resolved_outcome IS NOT NULL
|
||||
AND p.type = ANY($3::text[])
|
||||
AND p.source_id = $4
|
||||
)
|
||||
SELECT
|
||||
COUNT(*)::int AS n,
|
||||
(SUM(sq_err * conviction) / NULLIF(SUM(conviction), 0))::real AS brier,
|
||||
(SUM(CASE WHEN hit THEN 1 ELSE 0 END)::real / NULLIF(COUNT(*), 0))::real AS accuracy
|
||||
FROM scored`,
|
||||
[domain.name, holder, domain.page_types, sourceId],
|
||||
);
|
||||
const row = rows[0] ?? { n: 0, brier: null, accuracy: null };
|
||||
return {
|
||||
n: row.n,
|
||||
brier: row.n > 0 ? row.brier : null,
|
||||
accuracy: row.n > 0 ? row.accuracy : null,
|
||||
aggregator: 'weighted_brier',
|
||||
page_types: [...domain.page_types],
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Simple accuracy ratio (correct / resolved). Use when binary outcomes
|
||||
* don't have natural probability semantics — e.g., "did the event happen
|
||||
* at all" rather than "what's the probability it happens."
|
||||
*/
|
||||
async function aggregateCountBased(
|
||||
engine: BrainEngine,
|
||||
holder: string,
|
||||
domain: CalibrationDomain,
|
||||
sourceId: string,
|
||||
): Promise<DomainScorecard> {
|
||||
const rows = await engine.executeRaw<{ n: number; accuracy: number | null }>(
|
||||
`SELECT
|
||||
COUNT(*)::int AS n,
|
||||
(SUM(CASE WHEN (t.weight >= 0.5) = t.resolved_outcome THEN 1 ELSE 0 END)::real
|
||||
/ NULLIF(COUNT(*), 0))::real AS accuracy
|
||||
FROM takes t
|
||||
JOIN take_domain_assignments a ON a.take_id = t.id
|
||||
JOIN pages p ON p.id = t.page_id
|
||||
WHERE a.domain = $1
|
||||
AND t.holder = $2
|
||||
AND t.active = TRUE
|
||||
AND t.resolved_outcome IS NOT NULL
|
||||
AND p.type = ANY($3::text[])
|
||||
AND p.source_id = $4`,
|
||||
[domain.name, holder, domain.page_types, sourceId],
|
||||
);
|
||||
const row = rows[0] ?? { n: 0, accuracy: null };
|
||||
return {
|
||||
n: row.n,
|
||||
brier: null,
|
||||
accuracy: row.n > 0 ? row.accuracy : null,
|
||||
aggregator: 'count_based',
|
||||
page_types: [...domain.page_types],
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Descriptive rollup for domains where Brier doesn't apply (concept_themes
|
||||
* — concepts don't have binary outcomes to score). Returns count of pages
|
||||
* matching the domain's page_types plus tier histogram when frontmatter
|
||||
* carries `tier: T1|T2|T3|T4`.
|
||||
*
|
||||
* v0.41 minimal: returns n=page count, brier=null, accuracy=null, extras
|
||||
* carries tier_counts when available. v0.42+ can enrich with dominant
|
||||
* topics + recency + cross-source breadth.
|
||||
*/
|
||||
async function aggregateClusterSummary(
|
||||
engine: BrainEngine,
|
||||
holder: string,
|
||||
domain: CalibrationDomain,
|
||||
sourceId: string,
|
||||
): Promise<DomainScorecard> {
|
||||
// For cluster_summary, "holder" is informational only — concepts aren't
|
||||
// owned by a holder the way takes are. We still scope by source.
|
||||
const rows = await engine.executeRaw<{
|
||||
n: number;
|
||||
t1: number;
|
||||
t2: number;
|
||||
t3: number;
|
||||
t4: number;
|
||||
}>(
|
||||
`SELECT
|
||||
COUNT(*)::int AS n,
|
||||
SUM(CASE WHEN frontmatter->>'tier' = 'T1' OR frontmatter->>'tier' = '1' THEN 1 ELSE 0 END)::int AS t1,
|
||||
SUM(CASE WHEN frontmatter->>'tier' = 'T2' OR frontmatter->>'tier' = '2' THEN 1 ELSE 0 END)::int AS t2,
|
||||
SUM(CASE WHEN frontmatter->>'tier' = 'T3' OR frontmatter->>'tier' = '3' THEN 1 ELSE 0 END)::int AS t3,
|
||||
SUM(CASE WHEN frontmatter->>'tier' = 'T4' OR frontmatter->>'tier' = '4' THEN 1 ELSE 0 END)::int AS t4
|
||||
FROM pages
|
||||
WHERE type = ANY($1::text[])
|
||||
AND source_id = $2
|
||||
AND deleted_at IS NULL`,
|
||||
[domain.page_types, sourceId],
|
||||
);
|
||||
const row = rows[0] ?? { n: 0, t1: 0, t2: 0, t3: 0, t4: 0 };
|
||||
// holder is unused for cluster_summary but documented for symmetry
|
||||
void holder;
|
||||
return {
|
||||
n: row.n,
|
||||
brier: null,
|
||||
accuracy: null,
|
||||
aggregator: 'cluster_summary',
|
||||
page_types: [...domain.page_types],
|
||||
extras: {
|
||||
tier_counts: {
|
||||
T1: row.t1 ?? 0,
|
||||
T2: row.t2 ?? 0,
|
||||
T3: row.t3 ?? 0,
|
||||
T4: row.t4 ?? 0,
|
||||
},
|
||||
},
|
||||
};
|
||||
}
|
||||
@@ -29,6 +29,10 @@ import { BaseCyclePhase, type ScopedReadOpts, type BasePhaseOpts } from './base-
|
||||
import { chat as gatewayChat } from '../ai/gateway.ts';
|
||||
import { gateVoice, type VoiceGateGenerator, type VoiceGateJudge } from '../calibration/voice-gate.ts';
|
||||
import { patternStatementTemplate, type PatternStatementSlots } from '../calibration/templates.ts';
|
||||
// v0.41 T10 — domain widening. The aggregator module resolves the active
|
||||
// pack's calibration_domains declarations into per-domain Brier+accuracy+
|
||||
// extras scorecards stored in calibration_profiles.domain_scorecards JSONB.
|
||||
import { aggregateDomainScorecards, type DomainScorecards } from '../calibration/domain-aggregators.ts';
|
||||
import { GBrainError } from '../types.ts';
|
||||
import type { OperationContext } from '../operations.ts';
|
||||
import type { BrainEngine, TakesScorecard } from '../engine.ts';
|
||||
@@ -310,6 +314,39 @@ class CalibrationProfilePhase extends BaseCyclePhase {
|
||||
|
||||
// Write the profile row.
|
||||
const sourceId = scope.sourceId ?? 'default';
|
||||
|
||||
// v0.41 T10 — domain_scorecards widening (replaces v0.36.1.0 `{}`
|
||||
// placeholder). Resolve the active pack's calibration_domains
|
||||
// declarations and run each one's aggregator. Per-domain fail-soft:
|
||||
// a malformed domain or missing page_type produces a {n:0, error}
|
||||
// entry rather than crashing the whole phase. When no pack is
|
||||
// active or the active pack declares no calibration_domains, the
|
||||
// JSONB stays {} (byte-identical to v0.36.1.0 — R1 IRON RULE).
|
||||
let domainScorecards: DomainScorecards = {};
|
||||
try {
|
||||
const { loadActivePack } = await import('../schema-pack/load-active.ts');
|
||||
const { loadConfig } = await import('../config.ts');
|
||||
const cfg = loadConfig();
|
||||
const resolved = await loadActivePack({ cfg, remote: false });
|
||||
const domains = resolved.manifest.calibration_domains ?? [];
|
||||
if (domains.length > 0) {
|
||||
domainScorecards = await aggregateDomainScorecards(
|
||||
engine,
|
||||
holder,
|
||||
domains,
|
||||
sourceId,
|
||||
);
|
||||
}
|
||||
} catch (err) {
|
||||
// Pack resolution failed (e.g. registry not initialized, manifest
|
||||
// malformed). Don't crash calibration — log a warning and write the
|
||||
// empty {} scorecard. Matches the v0.36.1.0 baseline behavior so
|
||||
// R1 byte-identical regression survives the widening.
|
||||
result.warnings.push(
|
||||
`domain_scorecards_aggregation_failed: ${err instanceof Error ? err.message : String(err)}`,
|
||||
);
|
||||
}
|
||||
|
||||
await engine.executeRaw(
|
||||
`INSERT INTO calibration_profiles (
|
||||
source_id, holder, generated_at, published,
|
||||
@@ -330,10 +367,10 @@ class CalibrationProfilePhase extends BaseCyclePhase {
|
||||
scorecard.accuracy,
|
||||
scorecard.partial_rate,
|
||||
gradeCompletion,
|
||||
// domain_scorecards: per-domain breakdown placeholder — v0.36.1.0
|
||||
// ships with the overall scorecard only; per-domain comes when
|
||||
// batchGetTakesScorecards (F12) lands in Lane C.
|
||||
JSON.stringify({}),
|
||||
// v0.41 T10 — domain_scorecards JSONB populated by the
|
||||
// domain-aggregators pass above. Empty {} when no active pack
|
||||
// declares calibration_domains (R1 byte-identical regression).
|
||||
JSON.stringify(domainScorecards),
|
||||
result.pattern_statements,
|
||||
result.voice_gate_passed,
|
||||
result.voice_gate_attempts,
|
||||
|
||||
@@ -0,0 +1,377 @@
|
||||
// v0.41 T10 — calibration domain aggregators.
|
||||
//
|
||||
// Tests the per-aggregator SQL shape + the R1 IRON-RULE byte-identical
|
||||
// regression (empty {} JSONB when no active pack declares domains).
|
||||
//
|
||||
// Covers all 4 aggregator kinds (scalar_brier, weighted_brier,
|
||||
// count_based, cluster_summary) against real PGLite. Seeds takes +
|
||||
// take_domain_assignments + pages and asserts the expected scorecard
|
||||
// shape comes back.
|
||||
|
||||
import { describe, test, expect, beforeAll, afterAll, beforeEach } from 'bun:test';
|
||||
import { PGLiteEngine } from '../src/core/pglite-engine.ts';
|
||||
import { aggregateDomainScorecards } from '../src/core/calibration/domain-aggregators.ts';
|
||||
import { resetPgliteState } from './helpers/reset-pglite.ts';
|
||||
import type { CalibrationDomain } from '../src/core/schema-pack/manifest-v1.ts';
|
||||
|
||||
let engine: PGLiteEngine;
|
||||
|
||||
beforeAll(async () => {
|
||||
engine = new PGLiteEngine();
|
||||
await engine.connect({});
|
||||
await engine.initSchema();
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
await engine.disconnect();
|
||||
});
|
||||
|
||||
beforeEach(async () => {
|
||||
await resetPgliteState(engine);
|
||||
});
|
||||
|
||||
async function seedTakeWithAssignment(
|
||||
slug: string,
|
||||
pageType: string,
|
||||
takeOpts: {
|
||||
weight: number;
|
||||
resolved_outcome: boolean;
|
||||
holder?: string;
|
||||
sourceId?: string;
|
||||
rowNum?: number;
|
||||
},
|
||||
assignmentOpts: {
|
||||
domain: string;
|
||||
pack: string;
|
||||
confidence?: number;
|
||||
},
|
||||
): Promise<void> {
|
||||
const holder = takeOpts.holder ?? 'garry';
|
||||
const sourceId = takeOpts.sourceId ?? 'default';
|
||||
const rowNum = takeOpts.rowNum ?? 1;
|
||||
|
||||
await engine.putPage(slug, {
|
||||
title: slug,
|
||||
type: pageType,
|
||||
compiled_truth: '',
|
||||
frontmatter: {},
|
||||
timeline: '',
|
||||
});
|
||||
const pageRow = await engine.executeRaw<{ id: number }>(
|
||||
`SELECT id FROM pages WHERE slug = $1 AND source_id = $2 LIMIT 1`,
|
||||
[slug, sourceId],
|
||||
);
|
||||
const pageId = pageRow[0].id;
|
||||
await engine.executeRaw(
|
||||
`INSERT INTO takes (page_id, row_num, claim, kind, holder, weight, resolved_outcome, resolved_at, active)
|
||||
VALUES ($1, $2, $3, 'take', $4, $5, $6, now(), TRUE)`,
|
||||
[pageId, rowNum, `claim for ${slug}`, holder, takeOpts.weight, takeOpts.resolved_outcome],
|
||||
);
|
||||
const takeRow = await engine.executeRaw<{ id: number }>(
|
||||
`SELECT id FROM takes WHERE page_id = $1 AND row_num = $2 LIMIT 1`,
|
||||
[pageId, rowNum],
|
||||
);
|
||||
const takeId = takeRow[0].id;
|
||||
await engine.executeRaw(
|
||||
`INSERT INTO take_domain_assignments (take_id, domain, pack, confidence)
|
||||
VALUES ($1, $2, $3, $4)`,
|
||||
[takeId, assignmentOpts.domain, assignmentOpts.pack, assignmentOpts.confidence ?? 1.0],
|
||||
);
|
||||
}
|
||||
|
||||
describe('v0.41 T10 R1: empty domain list returns {} (byte-identical regression)', () => {
|
||||
test('aggregateDomainScorecards with [] domains returns {}', async () => {
|
||||
const result = await aggregateDomainScorecards(engine, 'garry', [], 'default');
|
||||
expect(result).toEqual({});
|
||||
});
|
||||
});
|
||||
|
||||
describe('v0.41 T10: scalar_brier aggregator', () => {
|
||||
const domain: CalibrationDomain = {
|
||||
name: 'deal_success',
|
||||
aggregator: 'scalar_brier',
|
||||
page_types: ['deal'],
|
||||
};
|
||||
|
||||
test('returns n:0 + null brier when no takes match', async () => {
|
||||
const result = await aggregateDomainScorecards(engine, 'garry', [domain], 'default');
|
||||
expect(result.deal_success.n).toBe(0);
|
||||
expect(result.deal_success.brier).toBeNull();
|
||||
expect(result.deal_success.accuracy).toBeNull();
|
||||
expect(result.deal_success.aggregator).toBe('scalar_brier');
|
||||
expect(result.deal_success.page_types).toEqual(['deal']);
|
||||
});
|
||||
|
||||
test('computes Brier over resolved deal-attached takes', async () => {
|
||||
// Two takes: one perfect (p=1, outcome=true → sq_err=0), one wrong (p=1, outcome=false → sq_err=1)
|
||||
await seedTakeWithAssignment(
|
||||
'deals/perfect',
|
||||
'deal',
|
||||
{ weight: 1.0, resolved_outcome: true, rowNum: 1 },
|
||||
{ domain: 'deal_success', pack: 'gbrain-investor' },
|
||||
);
|
||||
await seedTakeWithAssignment(
|
||||
'deals/wrong',
|
||||
'deal',
|
||||
{ weight: 1.0, resolved_outcome: false, rowNum: 1 },
|
||||
{ domain: 'deal_success', pack: 'gbrain-investor' },
|
||||
);
|
||||
|
||||
const result = await aggregateDomainScorecards(engine, 'garry', [domain], 'default');
|
||||
expect(result.deal_success.n).toBe(2);
|
||||
// Mean of (0, 1) = 0.5
|
||||
expect(result.deal_success.brier).toBeCloseTo(0.5, 2);
|
||||
// Accuracy: weight>=0.5 (true) === outcome → 1 hit, 1 miss → 0.5
|
||||
expect(result.deal_success.accuracy).toBeCloseTo(0.5, 2);
|
||||
});
|
||||
|
||||
test('filters by holder', async () => {
|
||||
await seedTakeWithAssignment(
|
||||
'deals/mine',
|
||||
'deal',
|
||||
{ weight: 0.9, resolved_outcome: true, holder: 'garry', rowNum: 1 },
|
||||
{ domain: 'deal_success', pack: 'gbrain-investor' },
|
||||
);
|
||||
await seedTakeWithAssignment(
|
||||
'deals/theirs',
|
||||
'deal',
|
||||
{ weight: 0.1, resolved_outcome: true, holder: 'alice-example', rowNum: 1 },
|
||||
{ domain: 'deal_success', pack: 'gbrain-investor' },
|
||||
);
|
||||
|
||||
const garryResult = await aggregateDomainScorecards(engine, 'garry', [domain], 'default');
|
||||
expect(garryResult.deal_success.n).toBe(1);
|
||||
const aliceResult = await aggregateDomainScorecards(engine, 'alice-example', [domain], 'default');
|
||||
expect(aliceResult.deal_success.n).toBe(1);
|
||||
});
|
||||
|
||||
test('filters by page_types (only matching types counted)', async () => {
|
||||
await seedTakeWithAssignment(
|
||||
'deals/match',
|
||||
'deal',
|
||||
{ weight: 0.8, resolved_outcome: true, rowNum: 1 },
|
||||
{ domain: 'deal_success', pack: 'gbrain-investor' },
|
||||
);
|
||||
await seedTakeWithAssignment(
|
||||
'people/no-match',
|
||||
'person',
|
||||
{ weight: 0.8, resolved_outcome: true, rowNum: 1 },
|
||||
{ domain: 'deal_success', pack: 'gbrain-investor' },
|
||||
);
|
||||
|
||||
const result = await aggregateDomainScorecards(engine, 'garry', [domain], 'default');
|
||||
expect(result.deal_success.n).toBe(1);
|
||||
});
|
||||
|
||||
test('ignores unresolved takes', async () => {
|
||||
await engine.putPage('deals/unresolved', {
|
||||
title: 'unresolved',
|
||||
type: 'deal',
|
||||
compiled_truth: '',
|
||||
frontmatter: {},
|
||||
timeline: '',
|
||||
});
|
||||
const pageRow = await engine.executeRaw<{ id: number }>(
|
||||
`SELECT id FROM pages WHERE slug = 'deals/unresolved' LIMIT 1`,
|
||||
);
|
||||
await engine.executeRaw(
|
||||
`INSERT INTO takes (page_id, row_num, claim, kind, holder, weight, active)
|
||||
VALUES ($1, 1, 'unresolved', 'take', 'garry', 0.7, TRUE)`,
|
||||
[pageRow[0].id],
|
||||
);
|
||||
const takeRow = await engine.executeRaw<{ id: number }>(
|
||||
`SELECT id FROM takes WHERE page_id = $1 LIMIT 1`,
|
||||
[pageRow[0].id],
|
||||
);
|
||||
await engine.executeRaw(
|
||||
`INSERT INTO take_domain_assignments (take_id, domain, pack)
|
||||
VALUES ($1, 'deal_success', 'gbrain-investor')`,
|
||||
[takeRow[0].id],
|
||||
);
|
||||
|
||||
const result = await aggregateDomainScorecards(engine, 'garry', [domain], 'default');
|
||||
expect(result.deal_success.n).toBe(0);
|
||||
});
|
||||
});
|
||||
|
||||
describe('v0.41 T10: weighted_brier aggregator', () => {
|
||||
const domain: CalibrationDomain = {
|
||||
name: 'market_call',
|
||||
aggregator: 'weighted_brier',
|
||||
page_types: ['thesis'],
|
||||
};
|
||||
|
||||
test('high-conviction miss weighted more than low-conviction miss', async () => {
|
||||
// High-conviction miss: weight=0.95 (conviction = ABS(0.95-0.5)*2 = 0.9), outcome=false → sq_err=0.9025
|
||||
await seedTakeWithAssignment(
|
||||
'theses/high-conv-miss',
|
||||
'thesis',
|
||||
{ weight: 0.95, resolved_outcome: false, rowNum: 1 },
|
||||
{ domain: 'market_call', pack: 'gbrain-investor' },
|
||||
);
|
||||
// Low-conviction hit: weight=0.55 (conviction = 0.1), outcome=true → sq_err=0.2025
|
||||
await seedTakeWithAssignment(
|
||||
'theses/low-conv-hit',
|
||||
'thesis',
|
||||
{ weight: 0.55, resolved_outcome: true, rowNum: 1 },
|
||||
{ domain: 'market_call', pack: 'gbrain-investor' },
|
||||
);
|
||||
|
||||
const result = await aggregateDomainScorecards(engine, 'garry', [domain], 'default');
|
||||
expect(result.market_call.n).toBe(2);
|
||||
// Weighted mean: (0.9025 * 0.9 + 0.2025 * 0.1) / (0.9 + 0.1) ≈ 0.8325
|
||||
expect(result.market_call.brier).toBeCloseTo(0.8325, 2);
|
||||
});
|
||||
|
||||
test('accuracy independent of conviction weighting', async () => {
|
||||
await seedTakeWithAssignment(
|
||||
'theses/a',
|
||||
'thesis',
|
||||
{ weight: 0.9, resolved_outcome: true, rowNum: 1 },
|
||||
{ domain: 'market_call', pack: 'gbrain-investor' },
|
||||
);
|
||||
await seedTakeWithAssignment(
|
||||
'theses/b',
|
||||
'thesis',
|
||||
{ weight: 0.6, resolved_outcome: true, rowNum: 1 },
|
||||
{ domain: 'market_call', pack: 'gbrain-investor' },
|
||||
);
|
||||
const result = await aggregateDomainScorecards(engine, 'garry', [domain], 'default');
|
||||
expect(result.market_call.accuracy).toBeCloseTo(1.0, 2);
|
||||
});
|
||||
});
|
||||
|
||||
describe('v0.41 T10: count_based aggregator', () => {
|
||||
const domain: CalibrationDomain = {
|
||||
name: 'simple_acc',
|
||||
aggregator: 'count_based',
|
||||
page_types: ['deal'],
|
||||
};
|
||||
|
||||
test('computes accuracy without brier', async () => {
|
||||
await seedTakeWithAssignment(
|
||||
'deals/right',
|
||||
'deal',
|
||||
{ weight: 0.8, resolved_outcome: true, rowNum: 1 },
|
||||
{ domain: 'simple_acc', pack: 'test' },
|
||||
);
|
||||
await seedTakeWithAssignment(
|
||||
'deals/wrong',
|
||||
'deal',
|
||||
{ weight: 0.8, resolved_outcome: false, rowNum: 1 },
|
||||
{ domain: 'simple_acc', pack: 'test' },
|
||||
);
|
||||
const result = await aggregateDomainScorecards(engine, 'garry', [domain], 'default');
|
||||
expect(result.simple_acc.n).toBe(2);
|
||||
expect(result.simple_acc.brier).toBeNull();
|
||||
expect(result.simple_acc.accuracy).toBeCloseTo(0.5, 2);
|
||||
expect(result.simple_acc.aggregator).toBe('count_based');
|
||||
});
|
||||
});
|
||||
|
||||
describe('v0.41 T10: cluster_summary aggregator', () => {
|
||||
const domain: CalibrationDomain = {
|
||||
name: 'concept_themes',
|
||||
aggregator: 'cluster_summary',
|
||||
page_types: ['concept'],
|
||||
};
|
||||
|
||||
test('returns page count + tier histogram', async () => {
|
||||
await engine.putPage('concepts/canon-a', {
|
||||
title: 'a',
|
||||
type: 'concept',
|
||||
compiled_truth: '',
|
||||
frontmatter: { tier: 'T1' },
|
||||
timeline: '',
|
||||
});
|
||||
await engine.putPage('concepts/canon-b', {
|
||||
title: 'b',
|
||||
type: 'concept',
|
||||
compiled_truth: '',
|
||||
frontmatter: { tier: 'T1' },
|
||||
timeline: '',
|
||||
});
|
||||
await engine.putPage('concepts/dev', {
|
||||
title: 'dev',
|
||||
type: 'concept',
|
||||
compiled_truth: '',
|
||||
frontmatter: { tier: 'T2' },
|
||||
timeline: '',
|
||||
});
|
||||
await engine.putPage('concepts/no-tier', {
|
||||
title: 'no-tier',
|
||||
type: 'concept',
|
||||
compiled_truth: '',
|
||||
frontmatter: {},
|
||||
timeline: '',
|
||||
});
|
||||
|
||||
const result = await aggregateDomainScorecards(engine, 'garry', [domain], 'default');
|
||||
expect(result.concept_themes.n).toBe(4);
|
||||
expect(result.concept_themes.brier).toBeNull();
|
||||
expect(result.concept_themes.accuracy).toBeNull();
|
||||
expect(result.concept_themes.aggregator).toBe('cluster_summary');
|
||||
const tiers = (result.concept_themes.extras as { tier_counts: Record<string, number> }).tier_counts;
|
||||
expect(tiers.T1).toBe(2);
|
||||
expect(tiers.T2).toBe(1);
|
||||
expect(tiers.T3).toBe(0);
|
||||
expect(tiers.T4).toBe(0);
|
||||
});
|
||||
|
||||
test('returns n:0 + all-zero tiers when no concepts exist', async () => {
|
||||
const result = await aggregateDomainScorecards(engine, 'garry', [domain], 'default');
|
||||
expect(result.concept_themes.n).toBe(0);
|
||||
});
|
||||
});
|
||||
|
||||
describe('v0.41 T10: multi-domain aggregation', () => {
|
||||
test('aggregates all declared domains in one call', async () => {
|
||||
await seedTakeWithAssignment(
|
||||
'deals/d1',
|
||||
'deal',
|
||||
{ weight: 0.9, resolved_outcome: true, rowNum: 1 },
|
||||
{ domain: 'deal_success', pack: 'gbrain-investor' },
|
||||
);
|
||||
await seedTakeWithAssignment(
|
||||
'people/p1',
|
||||
'person',
|
||||
{ weight: 0.7, resolved_outcome: true, rowNum: 1 },
|
||||
{ domain: 'founder_evaluation', pack: 'gbrain-investor' },
|
||||
);
|
||||
|
||||
const domains: CalibrationDomain[] = [
|
||||
{ name: 'deal_success', aggregator: 'scalar_brier', page_types: ['deal'] },
|
||||
{ name: 'founder_evaluation', aggregator: 'scalar_brier', page_types: ['person'] },
|
||||
{ name: 'empty_domain', aggregator: 'scalar_brier', page_types: ['deal'] },
|
||||
];
|
||||
const result = await aggregateDomainScorecards(engine, 'garry', domains, 'default');
|
||||
|
||||
expect(Object.keys(result).sort()).toEqual([
|
||||
'deal_success',
|
||||
'empty_domain',
|
||||
'founder_evaluation',
|
||||
]);
|
||||
expect(result.deal_success.n).toBe(1);
|
||||
expect(result.founder_evaluation.n).toBe(1);
|
||||
expect(result.empty_domain.n).toBe(0);
|
||||
});
|
||||
});
|
||||
|
||||
describe('v0.41 T10: fail-soft per domain', () => {
|
||||
test('one domain SQL error does NOT block other domains', async () => {
|
||||
// Inject a malformed-but-shape-valid domain. The aggregator JOINs
|
||||
// pages by source_id='default'; a domain pointing at a non-existent
|
||||
// page_type still completes (returns n=0). Errors come from things
|
||||
// like SQL syntax mistakes — harder to trigger via the public API.
|
||||
// For now, assert that an empty page_types-mismatch domain produces
|
||||
// a clean n=0 result without throwing.
|
||||
const domains: CalibrationDomain[] = [
|
||||
{ name: 'good', aggregator: 'scalar_brier', page_types: ['deal'] },
|
||||
{ name: 'nonexistent', aggregator: 'scalar_brier', page_types: ['__fake_type__'] },
|
||||
];
|
||||
const result = await aggregateDomainScorecards(engine, 'garry', domains, 'default');
|
||||
expect(result.good).toBeDefined();
|
||||
expect(result.nonexistent).toBeDefined();
|
||||
expect(result.nonexistent.n).toBe(0);
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user