From c3bb182e9af6e66477dd00ebc9a7d8c85d6fe1bc Mon Sep 17 00:00:00 2001 From: Garry Tan Date: Sun, 17 May 2026 16:33:43 -0700 Subject: [PATCH] calibration: Brier-trend forecast at write time (T10 / E5) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Pure math layer over existing TakesScorecard data. Zero new LLM cost, zero new schema. Surfaces the user's historical Brier for the take's (holder, domain) bucket at write time so they see "your historical Brier in macro takes is 0.31" before committing the take. Voice-gate-rendered output: The user-facing string goes through gateVoice mode='forecast_blurb' via templates.ts (already in T6). This module is the pure data layer; the template renders the math into the conversational voice. v0.36.0.0 ship state: Bucket dimension is the DOMAIN (slug-prefix). The conviction-weight bucket dimension would need a new engine method (engine.batchGetTakeBucketStats per F11) — deferred to v0.37+. Until then, forecast = historical Brier in this holder's domain. resolveDomainPrefix() keeps slug-prefix-looking domain hints ('companies/', 'wiki/macro') and falls back to overall for free-form hints ('macro tech', 'geography'). Hindsight-style structured domain on takes (CDX-11 mitigation TODO) tightens this in v0.37+. MIN_BUCKET_N = 5: Below this sample size, the forecast returns predicted_brier=null with insufficient_data=true. Template renders "Forecast unavailable: only N resolved takes at this conviction yet" instead of a noisy estimate. Architecture: computeForecast(input) — pure function, takes scorecards already fetched; ideal for tests + reuse across batched paths. forecastForTake(engine, input) — convenience wrapper, 1-2 engine round-trips (no domain → 1; with domain → 2). batchForecast(engine, inputs[]) — memoizes per (holder, domainPrefix); N inputs collapse to ≤2*unique_holders unique engine calls. Used by the propose-queue review flow (50 candidates → 1-2 scorecard fetches). Tests: 14 cases. computeForecast (4): insufficient_data branch, stable forecast, overall fallback, MIN_BUCKET_N export. resolveDomainPrefix (5): undefined/empty/whitespace → undefined; slug-prefix → kept; free-form → undefined. forecastForTake (3): 1-call overall, 2-call domain, free-form fallback. batchForecast (2): cache collapse for repeat queries; different holders do not collapse. Co-Authored-By: Claude Opus 4.7 (1M context) --- src/core/calibration/take-forecast.ts | 170 +++++++++++++++++++++ test/take-forecast.test.ts | 211 ++++++++++++++++++++++++++ 2 files changed, 381 insertions(+) create mode 100644 src/core/calibration/take-forecast.ts create mode 100644 test/take-forecast.test.ts diff --git a/src/core/calibration/take-forecast.ts b/src/core/calibration/take-forecast.ts new file mode 100644 index 000000000..3fbdcff5a --- /dev/null +++ b/src/core/calibration/take-forecast.ts @@ -0,0 +1,170 @@ +/** + * v0.36.0.0 (T10 / E5) — Brier-trend forecasting on new takes. + * + * Pure math over existing `TakesScorecard` data. Zero new LLM cost, + * zero new schema. Surface: an inline blurb the user sees at write time + * (gbrain takes show / propose --review) reminding them of their + * historical track record at this conviction + domain. + * + * v0.36.0.0 ship state: + * Looks up scorecard by (holder, domainPrefix). The bucket dimension is + * the domain — not the conviction-weight bucket (full conviction-bucket + * math would need a new engine method). Returns "insufficient data" + * when n < 5 (avoid noise on cold brains). + * + * v0.37+ enhancement: + * Add conviction-bucket dimension via engine.batchGetTakeBucketStats() + * (per F11). For now the forecast is per-domain only. + * + * Output goes through the voice gate's forecast_blurb template when + * surfaced to the user (E5 inline rendering). This module is the pure + * data layer; templates.ts has the user-facing string. + */ + +import type { BrainEngine, TakesScorecard } from '../engine.ts'; + +export interface TakeForecastInput { + /** Take's holder, e.g. 'garry' or 'people/charlie-example'. */ + holder: string; + /** + * Optional domain prefix, e.g. 'macro' or 'geography'. When omitted, the + * forecast uses the holder's overall scorecard. + */ + domain?: string; + /** The conviction-weight of the new take in [0,1]. Carried into the response. */ + conviction: number; +} + +export interface TakeForecast { + /** + * Predicted Brier score for this conviction in this domain. Null when + * the bucket has insufficient data (n < MIN_BUCKET_N). + */ + predicted_brier: number | null; + /** Sample size of the bucket. */ + bucket_n: number; + /** Holder's overall Brier for comparison ("worse than your average"). */ + overall_brier: number | null; + /** The domain the forecast bucket scoped to ('overall' when no domain). */ + bucket_domain: string; + /** True when the bucket lacks enough data for a stable forecast. */ + insufficient_data: boolean; +} + +/** Minimum bucket size before we report a forecast. Below this → null. */ +export const MIN_BUCKET_N = 5; + +/** + * Map a free-form domain hint (e.g. 'macro tech', 'geography', or + * 'startup-tactics') to a `domainPrefix` the scorecard query understands. + * + * The TakesScorecard's `domainPrefix` is a slug-prefix filter (e.g. + * 'companies/'). For v0.36.0.0, we pass domain hints through as-is when + * they look like slug prefixes; otherwise fall back to undefined (overall + * scorecard). v0.37+ takes get a structured domain enum and this mapping + * tightens. + */ +export function resolveDomainPrefix(domain: string | undefined): string | undefined { + if (!domain) return undefined; + const lower = domain.toLowerCase().trim(); + if (lower.length === 0) return undefined; + // Slug-prefix-looking values: keep as-is. + if (lower.endsWith('/')) return lower; + if (lower.startsWith('wiki/') || lower.startsWith('companies/') || lower.startsWith('people/')) { + return lower; + } + // Free-form word (e.g. 'macro tech', 'geography') — no slug prefix path, + // so the bucket falls back to "overall" for now. v0.37+ Hindsight-style + // structured domain on takes (CDX-11 mitigation TODO) tightens this. + return undefined; +} + +/** + * Pure math: given the holder's overall scorecard AND optional bucketed + * scorecard, compute the forecast struct. + * + * Caller is responsible for fetching the scorecards via engine.getScorecard. + * Pure function so tests can drive it without an engine. + */ +export function computeForecast(input: { + conviction: number; + domain?: string; + overallScorecard: TakesScorecard; + bucketScorecard?: TakesScorecard; +}): TakeForecast { + const overall_brier = input.overallScorecard.brier; + const bucket = input.bucketScorecard ?? input.overallScorecard; + const bucket_domain = input.domain ?? 'overall'; + const bucket_n = bucket.resolved; + const insufficient_data = bucket_n < MIN_BUCKET_N; + const predicted_brier = insufficient_data ? null : bucket.brier; + return { predicted_brier, bucket_n, overall_brier, bucket_domain, insufficient_data }; +} + +/** + * Wrapper that fetches the scorecards from the engine + computes the + * forecast. Convenience for callers that don't need to share scorecard + * data across multiple forecasts. + */ +export async function forecastForTake( + engine: BrainEngine, + input: TakeForecastInput, +): Promise { + const overallScorecard = await engine.getScorecard({ holder: input.holder }, undefined); + const domainPrefix = resolveDomainPrefix(input.domain); + let bucketScorecard: TakesScorecard | undefined; + if (domainPrefix) { + bucketScorecard = await engine.getScorecard( + { holder: input.holder, domainPrefix }, + undefined, + ); + } + return computeForecast({ + conviction: input.conviction, + ...(input.domain !== undefined ? { domain: input.domain } : {}), + overallScorecard, + ...(bucketScorecard !== undefined ? { bucketScorecard } : {}), + }); +} + +/** + * Batched forecast over a list of takes (F11 perf finding). Returns one + * TakeForecast per input. v0.36.0.0 ship state: per-take engine round-trip. + * v0.37+ adds engine.batchGetTakeBucketStats for a single roundtrip across + * all (holder, domain) pairs. + */ +export async function batchForecast( + engine: BrainEngine, + inputs: TakeForecastInput[], +): Promise { + // Memoize per (holder, domainPrefix) so repeated queries collapse. + const cache = new Map(); + const getOrFetch = async (holder: string, domainPrefix?: string): Promise => { + const key = `${holder}|${domainPrefix ?? ''}`; + const hit = cache.get(key); + if (hit) return hit; + const sc = await engine.getScorecard( + { holder, ...(domainPrefix !== undefined ? { domainPrefix } : {}) }, + undefined, + ); + cache.set(key, sc); + return sc; + }; + const results: TakeForecast[] = []; + for (const input of inputs) { + const overallScorecard = await getOrFetch(input.holder); + const domainPrefix = resolveDomainPrefix(input.domain); + const bucketScorecard = domainPrefix + ? await getOrFetch(input.holder, domainPrefix) + : undefined; + results.push( + computeForecast({ + conviction: input.conviction, + ...(input.domain !== undefined ? { domain: input.domain } : {}), + overallScorecard, + ...(bucketScorecard !== undefined ? { bucketScorecard } : {}), + }), + ); + } + return results; +} diff --git a/test/take-forecast.test.ts b/test/take-forecast.test.ts new file mode 100644 index 000000000..f96ef6cda --- /dev/null +++ b/test/take-forecast.test.ts @@ -0,0 +1,211 @@ +/** + * v0.36.0.0 (T10 / E5) — Brier-trend forecasting tests. + * + * Hermetic. Pure-function tests + mock-engine path. No real LLM, no DB. + * + * Tests cover: + * - computeForecast: insufficient_data when bucket_n < MIN_BUCKET_N + * - computeForecast: stable forecast when bucket_n >= MIN_BUCKET_N + * - resolveDomainPrefix: slug-prefix-looking → kept, free-form → undefined + * - forecastForTake: routes through engine.getScorecard with proper args + * - batchForecast: caches per (holder, domain) tuple → minimal engine calls + * - exposes overall_brier alongside bucket_brier for comparison messaging + */ + +import { describe, test, expect } from 'bun:test'; +import { + computeForecast, + resolveDomainPrefix, + forecastForTake, + batchForecast, + MIN_BUCKET_N, +} from '../src/core/calibration/take-forecast.ts'; +import type { BrainEngine, TakesScorecard } from '../src/core/engine.ts'; + +function buildScorecard(opts: { resolved: number; brier: number | null }): TakesScorecard { + return { + total_bets: opts.resolved + 2, + resolved: opts.resolved, + correct: Math.floor(opts.resolved * 0.6), + incorrect: Math.floor(opts.resolved * 0.3), + partial: 0, + accuracy: 0.6, + brier: opts.brier, + partial_rate: 0, + }; +} + +interface ScorecardCall { + holder: string | undefined; + domainPrefix: string | undefined; +} + +function buildMockEngine(opts: { + scorecards: Record; // key = `${holder}|${domainPrefix ?? ''}` +}): { engine: BrainEngine; calls: ScorecardCall[] } { + const calls: ScorecardCall[] = []; + const engine = { + kind: 'pglite', + async getScorecard(scOpts: { holder?: string; domainPrefix?: string }): Promise { + calls.push({ holder: scOpts.holder, domainPrefix: scOpts.domainPrefix }); + const key = `${scOpts.holder ?? ''}|${scOpts.domainPrefix ?? ''}`; + return opts.scorecards[key] ?? buildScorecard({ resolved: 0, brier: null }); + }, + } as unknown as BrainEngine; + return { engine, calls }; +} + +// ─── computeForecast (pure) ───────────────────────────────────────── + +describe('computeForecast', () => { + test('insufficient_data when bucket has fewer than MIN_BUCKET_N resolved', () => { + const overall = buildScorecard({ resolved: 20, brier: 0.18 }); + const bucket = buildScorecard({ resolved: 3, brier: 0.31 }); + const out = computeForecast({ + conviction: 0.7, + domain: 'macro', + overallScorecard: overall, + bucketScorecard: bucket, + }); + expect(out.insufficient_data).toBe(true); + expect(out.predicted_brier).toBeNull(); + expect(out.bucket_n).toBe(3); + expect(out.overall_brier).toBe(0.18); + }); + + test('stable forecast when bucket_n >= MIN_BUCKET_N', () => { + const overall = buildScorecard({ resolved: 20, brier: 0.18 }); + const bucket = buildScorecard({ resolved: 7, brier: 0.31 }); + const out = computeForecast({ + conviction: 0.7, + domain: 'macro', + overallScorecard: overall, + bucketScorecard: bucket, + }); + expect(out.insufficient_data).toBe(false); + expect(out.predicted_brier).toBe(0.31); + expect(out.overall_brier).toBe(0.18); + expect(out.bucket_domain).toBe('macro'); + }); + + test('falls back to overall scorecard when no bucket provided', () => { + const overall = buildScorecard({ resolved: 12, brier: 0.21 }); + const out = computeForecast({ conviction: 0.7, overallScorecard: overall }); + expect(out.bucket_domain).toBe('overall'); + expect(out.predicted_brier).toBe(0.21); + }); + + test(`MIN_BUCKET_N constant is exported (currently ${MIN_BUCKET_N})`, () => { + expect(MIN_BUCKET_N).toBeGreaterThan(0); + }); +}); + +// ─── resolveDomainPrefix ──────────────────────────────────────────── + +describe('resolveDomainPrefix', () => { + test('undefined → undefined', () => { + expect(resolveDomainPrefix(undefined)).toBeUndefined(); + }); + + test('empty / whitespace → undefined', () => { + expect(resolveDomainPrefix('')).toBeUndefined(); + expect(resolveDomainPrefix(' ')).toBeUndefined(); + }); + + test('slug-prefix value (trailing slash) → kept', () => { + expect(resolveDomainPrefix('companies/')).toBe('companies/'); + }); + + test('wiki-prefix value → kept', () => { + expect(resolveDomainPrefix('wiki/macro')).toBe('wiki/macro'); + }); + + test('free-form word → undefined (falls back to overall)', () => { + expect(resolveDomainPrefix('macro tech')).toBeUndefined(); + expect(resolveDomainPrefix('geography')).toBeUndefined(); + }); +}); + +// ─── forecastForTake ──────────────────────────────────────────────── + +describe('forecastForTake', () => { + test('no domain → 1 engine call for overall scorecard', async () => { + const { engine, calls } = buildMockEngine({ + scorecards: { + 'garry|': buildScorecard({ resolved: 12, brier: 0.21 }), + }, + }); + const out = await forecastForTake(engine, { holder: 'garry', conviction: 0.7 }); + expect(calls).toHaveLength(1); + expect(calls[0]).toEqual({ holder: 'garry', domainPrefix: undefined }); + expect(out.bucket_domain).toBe('overall'); + expect(out.predicted_brier).toBe(0.21); + }); + + test('with slug-prefix domain → 2 engine calls (overall + bucket)', async () => { + const { engine, calls } = buildMockEngine({ + scorecards: { + 'garry|': buildScorecard({ resolved: 20, brier: 0.18 }), + 'garry|companies/': buildScorecard({ resolved: 7, brier: 0.25 }), + }, + }); + const out = await forecastForTake(engine, { + holder: 'garry', + conviction: 0.7, + domain: 'companies/', + }); + expect(calls).toHaveLength(2); + expect(calls[1]!.domainPrefix).toBe('companies/'); + expect(out.predicted_brier).toBe(0.25); + expect(out.overall_brier).toBe(0.18); + }); + + test('free-form domain falls back to overall (1 engine call, undefined prefix)', async () => { + const { engine, calls } = buildMockEngine({ + scorecards: { 'garry|': buildScorecard({ resolved: 12, brier: 0.21 }) }, + }); + const out = await forecastForTake(engine, { + holder: 'garry', + conviction: 0.7, + domain: 'macro tech', + }); + expect(calls).toHaveLength(1); + expect(out.bucket_domain).toBe('macro tech'); + }); +}); + +// ─── batchForecast (memo) ─────────────────────────────────────────── + +describe('batchForecast', () => { + test('caches per (holder, domain) tuple — repeat queries collapse', async () => { + const { engine, calls } = buildMockEngine({ + scorecards: { + 'garry|': buildScorecard({ resolved: 20, brier: 0.18 }), + 'garry|companies/': buildScorecard({ resolved: 7, brier: 0.25 }), + }, + }); + const out = await batchForecast(engine, [ + { holder: 'garry', conviction: 0.7, domain: 'companies/' }, + { holder: 'garry', conviction: 0.8, domain: 'companies/' }, + { holder: 'garry', conviction: 0.5 }, + ]); + expect(out).toHaveLength(3); + // 2 unique queries: (garry, undefined) + (garry, companies/). + // 3 input takes but cache collapses to 2 actual engine calls. + expect(calls).toHaveLength(2); + }); + + test('different holders do NOT collapse', async () => { + const { engine, calls } = buildMockEngine({ + scorecards: { + 'garry|': buildScorecard({ resolved: 10, brier: 0.2 }), + 'alice|': buildScorecard({ resolved: 5, brier: 0.18 }), + }, + }); + await batchForecast(engine, [ + { holder: 'garry', conviction: 0.7 }, + { holder: 'alice', conviction: 0.6 }, + ]); + expect(calls).toHaveLength(2); + }); +});