diff --git a/src/core/calibration/take-forecast.ts b/src/core/calibration/take-forecast.ts new file mode 100644 index 000000000..3fbdcff5a --- /dev/null +++ b/src/core/calibration/take-forecast.ts @@ -0,0 +1,170 @@ +/** + * v0.36.0.0 (T10 / E5) — Brier-trend forecasting on new takes. + * + * Pure math over existing `TakesScorecard` data. Zero new LLM cost, + * zero new schema. Surface: an inline blurb the user sees at write time + * (gbrain takes show / propose --review) reminding them of their + * historical track record at this conviction + domain. + * + * v0.36.0.0 ship state: + * Looks up scorecard by (holder, domainPrefix). The bucket dimension is + * the domain — not the conviction-weight bucket (full conviction-bucket + * math would need a new engine method). Returns "insufficient data" + * when n < 5 (avoid noise on cold brains). + * + * v0.37+ enhancement: + * Add conviction-bucket dimension via engine.batchGetTakeBucketStats() + * (per F11). For now the forecast is per-domain only. + * + * Output goes through the voice gate's forecast_blurb template when + * surfaced to the user (E5 inline rendering). This module is the pure + * data layer; templates.ts has the user-facing string. + */ + +import type { BrainEngine, TakesScorecard } from '../engine.ts'; + +export interface TakeForecastInput { + /** Take's holder, e.g. 'garry' or 'people/charlie-example'. */ + holder: string; + /** + * Optional domain prefix, e.g. 'macro' or 'geography'. When omitted, the + * forecast uses the holder's overall scorecard. + */ + domain?: string; + /** The conviction-weight of the new take in [0,1]. Carried into the response. */ + conviction: number; +} + +export interface TakeForecast { + /** + * Predicted Brier score for this conviction in this domain. Null when + * the bucket has insufficient data (n < MIN_BUCKET_N). + */ + predicted_brier: number | null; + /** Sample size of the bucket. */ + bucket_n: number; + /** Holder's overall Brier for comparison ("worse than your average"). */ + overall_brier: number | null; + /** The domain the forecast bucket scoped to ('overall' when no domain). */ + bucket_domain: string; + /** True when the bucket lacks enough data for a stable forecast. */ + insufficient_data: boolean; +} + +/** Minimum bucket size before we report a forecast. Below this → null. */ +export const MIN_BUCKET_N = 5; + +/** + * Map a free-form domain hint (e.g. 'macro tech', 'geography', or + * 'startup-tactics') to a `domainPrefix` the scorecard query understands. + * + * The TakesScorecard's `domainPrefix` is a slug-prefix filter (e.g. + * 'companies/'). For v0.36.0.0, we pass domain hints through as-is when + * they look like slug prefixes; otherwise fall back to undefined (overall + * scorecard). v0.37+ takes get a structured domain enum and this mapping + * tightens. + */ +export function resolveDomainPrefix(domain: string | undefined): string | undefined { + if (!domain) return undefined; + const lower = domain.toLowerCase().trim(); + if (lower.length === 0) return undefined; + // Slug-prefix-looking values: keep as-is. + if (lower.endsWith('/')) return lower; + if (lower.startsWith('wiki/') || lower.startsWith('companies/') || lower.startsWith('people/')) { + return lower; + } + // Free-form word (e.g. 'macro tech', 'geography') — no slug prefix path, + // so the bucket falls back to "overall" for now. v0.37+ Hindsight-style + // structured domain on takes (CDX-11 mitigation TODO) tightens this. + return undefined; +} + +/** + * Pure math: given the holder's overall scorecard AND optional bucketed + * scorecard, compute the forecast struct. + * + * Caller is responsible for fetching the scorecards via engine.getScorecard. + * Pure function so tests can drive it without an engine. + */ +export function computeForecast(input: { + conviction: number; + domain?: string; + overallScorecard: TakesScorecard; + bucketScorecard?: TakesScorecard; +}): TakeForecast { + const overall_brier = input.overallScorecard.brier; + const bucket = input.bucketScorecard ?? input.overallScorecard; + const bucket_domain = input.domain ?? 'overall'; + const bucket_n = bucket.resolved; + const insufficient_data = bucket_n < MIN_BUCKET_N; + const predicted_brier = insufficient_data ? null : bucket.brier; + return { predicted_brier, bucket_n, overall_brier, bucket_domain, insufficient_data }; +} + +/** + * Wrapper that fetches the scorecards from the engine + computes the + * forecast. Convenience for callers that don't need to share scorecard + * data across multiple forecasts. + */ +export async function forecastForTake( + engine: BrainEngine, + input: TakeForecastInput, +): Promise { + const overallScorecard = await engine.getScorecard({ holder: input.holder }, undefined); + const domainPrefix = resolveDomainPrefix(input.domain); + let bucketScorecard: TakesScorecard | undefined; + if (domainPrefix) { + bucketScorecard = await engine.getScorecard( + { holder: input.holder, domainPrefix }, + undefined, + ); + } + return computeForecast({ + conviction: input.conviction, + ...(input.domain !== undefined ? { domain: input.domain } : {}), + overallScorecard, + ...(bucketScorecard !== undefined ? { bucketScorecard } : {}), + }); +} + +/** + * Batched forecast over a list of takes (F11 perf finding). Returns one + * TakeForecast per input. v0.36.0.0 ship state: per-take engine round-trip. + * v0.37+ adds engine.batchGetTakeBucketStats for a single roundtrip across + * all (holder, domain) pairs. + */ +export async function batchForecast( + engine: BrainEngine, + inputs: TakeForecastInput[], +): Promise { + // Memoize per (holder, domainPrefix) so repeated queries collapse. + const cache = new Map(); + const getOrFetch = async (holder: string, domainPrefix?: string): Promise => { + const key = `${holder}|${domainPrefix ?? ''}`; + const hit = cache.get(key); + if (hit) return hit; + const sc = await engine.getScorecard( + { holder, ...(domainPrefix !== undefined ? { domainPrefix } : {}) }, + undefined, + ); + cache.set(key, sc); + return sc; + }; + const results: TakeForecast[] = []; + for (const input of inputs) { + const overallScorecard = await getOrFetch(input.holder); + const domainPrefix = resolveDomainPrefix(input.domain); + const bucketScorecard = domainPrefix + ? await getOrFetch(input.holder, domainPrefix) + : undefined; + results.push( + computeForecast({ + conviction: input.conviction, + ...(input.domain !== undefined ? { domain: input.domain } : {}), + overallScorecard, + ...(bucketScorecard !== undefined ? { bucketScorecard } : {}), + }), + ); + } + return results; +} diff --git a/test/take-forecast.test.ts b/test/take-forecast.test.ts new file mode 100644 index 000000000..f96ef6cda --- /dev/null +++ b/test/take-forecast.test.ts @@ -0,0 +1,211 @@ +/** + * v0.36.0.0 (T10 / E5) — Brier-trend forecasting tests. + * + * Hermetic. Pure-function tests + mock-engine path. No real LLM, no DB. + * + * Tests cover: + * - computeForecast: insufficient_data when bucket_n < MIN_BUCKET_N + * - computeForecast: stable forecast when bucket_n >= MIN_BUCKET_N + * - resolveDomainPrefix: slug-prefix-looking → kept, free-form → undefined + * - forecastForTake: routes through engine.getScorecard with proper args + * - batchForecast: caches per (holder, domain) tuple → minimal engine calls + * - exposes overall_brier alongside bucket_brier for comparison messaging + */ + +import { describe, test, expect } from 'bun:test'; +import { + computeForecast, + resolveDomainPrefix, + forecastForTake, + batchForecast, + MIN_BUCKET_N, +} from '../src/core/calibration/take-forecast.ts'; +import type { BrainEngine, TakesScorecard } from '../src/core/engine.ts'; + +function buildScorecard(opts: { resolved: number; brier: number | null }): TakesScorecard { + return { + total_bets: opts.resolved + 2, + resolved: opts.resolved, + correct: Math.floor(opts.resolved * 0.6), + incorrect: Math.floor(opts.resolved * 0.3), + partial: 0, + accuracy: 0.6, + brier: opts.brier, + partial_rate: 0, + }; +} + +interface ScorecardCall { + holder: string | undefined; + domainPrefix: string | undefined; +} + +function buildMockEngine(opts: { + scorecards: Record; // key = `${holder}|${domainPrefix ?? ''}` +}): { engine: BrainEngine; calls: ScorecardCall[] } { + const calls: ScorecardCall[] = []; + const engine = { + kind: 'pglite', + async getScorecard(scOpts: { holder?: string; domainPrefix?: string }): Promise { + calls.push({ holder: scOpts.holder, domainPrefix: scOpts.domainPrefix }); + const key = `${scOpts.holder ?? ''}|${scOpts.domainPrefix ?? ''}`; + return opts.scorecards[key] ?? buildScorecard({ resolved: 0, brier: null }); + }, + } as unknown as BrainEngine; + return { engine, calls }; +} + +// ─── computeForecast (pure) ───────────────────────────────────────── + +describe('computeForecast', () => { + test('insufficient_data when bucket has fewer than MIN_BUCKET_N resolved', () => { + const overall = buildScorecard({ resolved: 20, brier: 0.18 }); + const bucket = buildScorecard({ resolved: 3, brier: 0.31 }); + const out = computeForecast({ + conviction: 0.7, + domain: 'macro', + overallScorecard: overall, + bucketScorecard: bucket, + }); + expect(out.insufficient_data).toBe(true); + expect(out.predicted_brier).toBeNull(); + expect(out.bucket_n).toBe(3); + expect(out.overall_brier).toBe(0.18); + }); + + test('stable forecast when bucket_n >= MIN_BUCKET_N', () => { + const overall = buildScorecard({ resolved: 20, brier: 0.18 }); + const bucket = buildScorecard({ resolved: 7, brier: 0.31 }); + const out = computeForecast({ + conviction: 0.7, + domain: 'macro', + overallScorecard: overall, + bucketScorecard: bucket, + }); + expect(out.insufficient_data).toBe(false); + expect(out.predicted_brier).toBe(0.31); + expect(out.overall_brier).toBe(0.18); + expect(out.bucket_domain).toBe('macro'); + }); + + test('falls back to overall scorecard when no bucket provided', () => { + const overall = buildScorecard({ resolved: 12, brier: 0.21 }); + const out = computeForecast({ conviction: 0.7, overallScorecard: overall }); + expect(out.bucket_domain).toBe('overall'); + expect(out.predicted_brier).toBe(0.21); + }); + + test(`MIN_BUCKET_N constant is exported (currently ${MIN_BUCKET_N})`, () => { + expect(MIN_BUCKET_N).toBeGreaterThan(0); + }); +}); + +// ─── resolveDomainPrefix ──────────────────────────────────────────── + +describe('resolveDomainPrefix', () => { + test('undefined → undefined', () => { + expect(resolveDomainPrefix(undefined)).toBeUndefined(); + }); + + test('empty / whitespace → undefined', () => { + expect(resolveDomainPrefix('')).toBeUndefined(); + expect(resolveDomainPrefix(' ')).toBeUndefined(); + }); + + test('slug-prefix value (trailing slash) → kept', () => { + expect(resolveDomainPrefix('companies/')).toBe('companies/'); + }); + + test('wiki-prefix value → kept', () => { + expect(resolveDomainPrefix('wiki/macro')).toBe('wiki/macro'); + }); + + test('free-form word → undefined (falls back to overall)', () => { + expect(resolveDomainPrefix('macro tech')).toBeUndefined(); + expect(resolveDomainPrefix('geography')).toBeUndefined(); + }); +}); + +// ─── forecastForTake ──────────────────────────────────────────────── + +describe('forecastForTake', () => { + test('no domain → 1 engine call for overall scorecard', async () => { + const { engine, calls } = buildMockEngine({ + scorecards: { + 'garry|': buildScorecard({ resolved: 12, brier: 0.21 }), + }, + }); + const out = await forecastForTake(engine, { holder: 'garry', conviction: 0.7 }); + expect(calls).toHaveLength(1); + expect(calls[0]).toEqual({ holder: 'garry', domainPrefix: undefined }); + expect(out.bucket_domain).toBe('overall'); + expect(out.predicted_brier).toBe(0.21); + }); + + test('with slug-prefix domain → 2 engine calls (overall + bucket)', async () => { + const { engine, calls } = buildMockEngine({ + scorecards: { + 'garry|': buildScorecard({ resolved: 20, brier: 0.18 }), + 'garry|companies/': buildScorecard({ resolved: 7, brier: 0.25 }), + }, + }); + const out = await forecastForTake(engine, { + holder: 'garry', + conviction: 0.7, + domain: 'companies/', + }); + expect(calls).toHaveLength(2); + expect(calls[1]!.domainPrefix).toBe('companies/'); + expect(out.predicted_brier).toBe(0.25); + expect(out.overall_brier).toBe(0.18); + }); + + test('free-form domain falls back to overall (1 engine call, undefined prefix)', async () => { + const { engine, calls } = buildMockEngine({ + scorecards: { 'garry|': buildScorecard({ resolved: 12, brier: 0.21 }) }, + }); + const out = await forecastForTake(engine, { + holder: 'garry', + conviction: 0.7, + domain: 'macro tech', + }); + expect(calls).toHaveLength(1); + expect(out.bucket_domain).toBe('macro tech'); + }); +}); + +// ─── batchForecast (memo) ─────────────────────────────────────────── + +describe('batchForecast', () => { + test('caches per (holder, domain) tuple — repeat queries collapse', async () => { + const { engine, calls } = buildMockEngine({ + scorecards: { + 'garry|': buildScorecard({ resolved: 20, brier: 0.18 }), + 'garry|companies/': buildScorecard({ resolved: 7, brier: 0.25 }), + }, + }); + const out = await batchForecast(engine, [ + { holder: 'garry', conviction: 0.7, domain: 'companies/' }, + { holder: 'garry', conviction: 0.8, domain: 'companies/' }, + { holder: 'garry', conviction: 0.5 }, + ]); + expect(out).toHaveLength(3); + // 2 unique queries: (garry, undefined) + (garry, companies/). + // 3 input takes but cache collapses to 2 actual engine calls. + expect(calls).toHaveLength(2); + }); + + test('different holders do NOT collapse', async () => { + const { engine, calls } = buildMockEngine({ + scorecards: { + 'garry|': buildScorecard({ resolved: 10, brier: 0.2 }), + 'alice|': buildScorecard({ resolved: 5, brier: 0.18 }), + }, + }); + await batchForecast(engine, [ + { holder: 'garry', conviction: 0.7 }, + { holder: 'alice', conviction: 0.6 }, + ]); + expect(calls).toHaveLength(2); + }); +});