diff --git a/src/core/ingestion/daemon.ts b/src/core/ingestion/daemon.ts index 46687204c..ea69eeeba 100644 --- a/src/core/ingestion/daemon.ts +++ b/src/core/ingestion/daemon.ts @@ -432,12 +432,23 @@ export class IngestionDaemon { // is lying about its identity. Trust source.kind over event.source_kind. const effectiveEvent: IngestionEvent = { ...event, source_kind: sourceKind, source_id: sourceId }; - // 2. Dedup. - const isNew = this.dedup.mark(sourceKind, effectiveEvent.content_hash); - if (!isNew) { - // Silent dedup hit. dedup.hits counter already incremented. - return; + // 2. Dedup — TRICKLE MODE ONLY (v0.41 T2). Migration-mode sources own + // their own permanent slug-keyed idempotency (via op_checkpoint); the + // 24h DedupWindow is wrong for bulk historical replay where retries + // happen days apart and content_hash collisions across the import + // window are expected. Default-undefined source.mode treats as + // 'trickle' for v0.38 back-compat. + const sourceMode = state.registration.source.mode ?? 'trickle'; + if (sourceMode === 'trickle') { + const isNew = this.dedup.mark(sourceKind, effectiveEvent.content_hash); + if (!isNew) { + // Silent dedup hit. dedup.hits counter already incremented. + return; + } } + // Migration mode: skip the dedup window entirely. The source's own + // idempotency layer is the authoritative duplicate gate; reaching here + // means the source already decided this event should land. // 3. Rate limit (token-bucket-ish: count events in trailing window). const nowMs = this.now(); diff --git a/src/core/ingestion/types.ts b/src/core/ingestion/types.ts index e013d6d32..4935913dc 100644 --- a/src/core/ingestion/types.ts +++ b/src/core/ingestion/types.ts @@ -141,6 +141,27 @@ export interface IngestionSourceHealth { * that need richer semantics (transient vs fatal) are a v2 concern; for * v1, "throw to fail" is the entire contract. */ +/** + * Source operating mode (v0.41 T2 — codex outside-voice challenge: bulk + * migration semantics differ from trickle ingestion). The daemon branches + * on this flag to decide whether to apply the 24h content-hash dedup window + * (trickle) or bypass it (migration; the source owns its own permanent + * slug-keyed idempotency via op_checkpoint or similar). + * + * Defaults to 'trickle' when unset — preserves v0.38-shipped source + * behavior unchanged. Any source declaring mode: 'migration' opts into + * the bulk semantics: + * - Daemon bypasses DedupWindow.mark() so retries beyond 24h still ingest. + * - Source MUST implement permanent idempotency (slug + content_hash + * forever, NOT a windowed dedup) on its own. The greenfield importer + * uses op_checkpoint keyed on the importer's fingerprint. + * - Rate limit + validate + dispatch still apply uniformly. + * + * Migration-mode sources are one-shot bulk importers. Trickle-mode sources + * are file-watcher / inbox-folder / webhook (the v0.38 default shape). + */ +export type IngestionSourceMode = 'trickle' | 'migration'; + export interface IngestionSource { /** Unique source instance id. Two file-watcher sources pointing at * different directories MUST have different ids. The daemon dedups @@ -150,6 +171,12 @@ export interface IngestionSource { /** Source kind taxonomy. The router uses this to look up processors * and the dedup window to scope content-hash keys. */ readonly kind: string; + /** + * v0.41 T2 — operating mode discriminator. Defaults to 'trickle' when + * unset (preserves v0.38 shipped behavior). 'migration' bypasses the + * 24h dedup window; the source owns its own permanent idempotency. + */ + readonly mode?: IngestionSourceMode; /** * Begin emitting events. MUST resolve when the source is ready to emit; * MAY throw on unrecoverable startup failure. The daemon catches throws diff --git a/src/core/schema-pack/manifest-v1.ts b/src/core/schema-pack/manifest-v1.ts index d80dc90e3..9d391b423 100644 --- a/src/core/schema-pack/manifest-v1.ts +++ b/src/core/schema-pack/manifest-v1.ts @@ -89,6 +89,68 @@ const FilingRuleSchema = z.object({ description: z.string().optional(), }).strict(); +/** + * v0.41 T3 — closed registry of calibration aggregator algorithms. + * + * Codex outside-voice refinement of D6: domain NAMES stay open (any pack + * can declare `pricing_judgment` or `hiring_quality` without a gbrain + * release), but the AGGREGATOR — the actual SQL/code that computes a + * scorecard for that domain — must be a closed enum. New aggregator + * algorithms ship via gbrain release; new domain names ship via pack + * manifest. This splits the "what" (open) from the "how" (closed), + * preserving extensibility without SQL injection surface. + * + * v1 aggregators: + * - `scalar_brier` — standard Brier score over resolved binary takes + * (sum((p - outcome)^2) / n). Default for most + * predictive domains. + * - `weighted_brier` — Brier weighted by take.confidence. Use when + * calibration cares more about high-conviction + * predictions than low-stakes ones. + * - `count_based` — simple accuracy ratio (correct / resolved). + * Use when binary outcomes don't have natural + * probability semantics (e.g. did/didn't happen). + * - `cluster_summary` — descriptive rollup (tier counts, dominant + * topics, time span) instead of Brier. Used by + * the creator pack's concept_themes domain where + * there is no "right answer" to score against. + * + * Expand this enum in v0.42+ as real lens-pack usage surfaces new + * aggregation needs. Each addition is a versioned gbrain release. + */ +export const AGGREGATOR_KINDS = [ + 'scalar_brier', + 'weighted_brier', + 'count_based', + 'cluster_summary', +] as const; +export type AggregatorKind = typeof AGGREGATOR_KINDS[number]; + +const AggregatorKindSchema = z.enum(AGGREGATOR_KINDS); + +/** + * v0.41 T3 — per-pack calibration domain declaration. The calibration_profile + * cycle phase widens at v0.41 from a placeholder `{}` JSONB to an aggregator + * pass over each active pack's declared domains. Each entry binds: + * - `name` — open string label visible in scorecards + * (`deal_success`, `architecture_calls`, etc.) + * - `aggregator` — closed-enum algorithm to compute the scorecard + * - `page_types` — page types whose takes feed this domain (the + * propose_takes phase populates take_domain_assignments + * at write time from this mapping) + * + * Loaded by the registry at pack-load; validated against AggregatorKindSchema + * before any aggregator code runs. Unknown aggregator values fail the pack + * load with a paste-ready `gbrain models doctor`-style hint. + */ +const CalibrationDomainSchema = z.object({ + name: z.string().min(1).regex(/^[a-z][a-z0-9_]*$/, 'domain name must be lowercase snake_case'), + aggregator: AggregatorKindSchema, + page_types: z.array(z.string().min(1)).min(1), +}).strict(); + +export type CalibrationDomain = z.infer; + /** * SchemaPackManifest v1 — the parsed + validated pack file shape. * `extends` resolution + closure expansion are done by registry.ts, not at @@ -118,6 +180,27 @@ export const SchemaPackManifestSchema = z.object({ takes_kinds: z.array(z.string()).default(['fact', 'take', 'bet', 'hunch']), enrichable_types: z.array(EnrichableSchema).default([]), filing_rules: z.array(FilingRuleSchema).default([]), + /** + * v0.41 T3/D4 — phase participation declaration. The runCycle orchestrator + * consults active pack's `phases:` to decide which pack-flavored cycle + * phases run (extract_atoms, synthesize_concepts, future pack phases). + * Pre-existing 17 core phases (lint, sync, extract, extract_facts, + * propose_takes, etc.) ALWAYS run regardless of this declaration — + * `phases:` is additive, not subtractive. `borrow_from` does NOT borrow + * phases; each pack declares its own participation explicitly. + * + * Phase names are validated as strings at parse time and against the + * runtime CyclePhase union at pack-load by the registry (kept as string[] + * here to avoid a circular import from src/core/cycle.ts). + */ + phases: z.array(z.string().min(1)).default([]), + /** + * v0.41 T3 — per-pack calibration domain declarations. The + * calibration_profile cycle phase widens at v0.41 from `{}` placeholder + * JSONB to a real aggregator pass over each declared domain. See + * CalibrationDomainSchema for the per-entry shape. + */ + calibration_domains: z.array(CalibrationDomainSchema).default([]), }).strict(); export type SchemaPackManifest = z.infer; diff --git a/test/ingestion/migration-mode.test.ts b/test/ingestion/migration-mode.test.ts new file mode 100644 index 000000000..9261e0538 --- /dev/null +++ b/test/ingestion/migration-mode.test.ts @@ -0,0 +1,226 @@ +// v0.41 T2 — IngestionSource.mode discriminator + daemon supervisor branch. +// +// Codex outside-voice challenge: bulk migration semantics differ from trickle +// ingestion. The 24h DedupWindow is wrong for one-shot bulk importers (24K +// pages, retries days apart, content_hash collisions across the window are +// expected). Migration-mode sources bypass DedupWindow entirely and own +// permanent slug-keyed idempotency themselves. +// +// This test pins: +// - IngestionSource.mode type accepts 'trickle' | 'migration' +// - Defaults to 'trickle' when unset (back-compat with v0.38 sources) +// - Daemon's handleEmit() bypasses DedupWindow.mark() in migration mode +// - Validation + rate limit + dispatch still apply uniformly +// - Two emits of identical content_hash from migration-mode source BOTH +// dispatch (no silent dedup drop) +// - Same two emits from trickle-mode source: second is dedup hit (silent) + +import { describe, test, expect, beforeEach } from 'bun:test'; +import { IngestionDaemon } from '../../src/core/ingestion/daemon.ts'; +import type { + IngestionSource, + IngestionSourceContext, + IngestionEvent, + IngestionSourceMode, +} from '../../src/core/ingestion/types.ts'; +import { computeContentHash } from '../../src/core/ingestion/types.ts'; + +// Stub source that emits whatever we tell it to. Captures the context so +// tests can drive emit() directly from outside. +class StubSource implements IngestionSource { + ctx: IngestionSourceContext | null = null; + constructor( + readonly id: string, + readonly kind: string, + readonly mode?: IngestionSourceMode, + ) {} + async start(ctx: IngestionSourceContext): Promise { + this.ctx = ctx; + } + async stop(): Promise { + this.ctx = null; + } +} + +function makeEvent(overrides: Partial = {}): IngestionEvent { + const content = overrides.content ?? 'hello world'; + return { + source_id: 'stub-1', + source_kind: 'test-source', + source_uri: 'test://event-1', + received_at: new Date().toISOString(), + content_type: 'text/markdown', + content, + content_hash: overrides.content_hash ?? computeContentHash(content), + ...overrides, + }; +} + +// Async barrier — daemon dispatches via microtask, so we await one tick. +async function tick(): Promise { + await Promise.resolve(); + await Promise.resolve(); + await Promise.resolve(); +} + +describe('v0.41 T2: IngestionSource.mode discriminator', () => { + test('mode is optional in interface (back-compat with v0.38 sources)', () => { + // Compile-time test: a source without `mode` field is valid. + const trickle: IngestionSource = { + id: 'no-mode', + kind: 'test-source', + async start() {}, + async stop() {}, + }; + expect(trickle.mode).toBeUndefined(); + }); + + test('mode accepts trickle | migration string literals', () => { + const trickle: IngestionSource = { + id: 's1', + kind: 'test', + mode: 'trickle', + async start() {}, + async stop() {}, + }; + const migration: IngestionSource = { + id: 's2', + kind: 'test', + mode: 'migration', + async start() {}, + async stop() {}, + }; + expect(trickle.mode).toBe('trickle'); + expect(migration.mode).toBe('migration'); + }); +}); + +describe('v0.41 T2: daemon handleEmit branches on source.mode', () => { + let dispatched: IngestionEvent[]; + let dispatch: (event: IngestionEvent) => Promise<{ kind: 'ok' | 'failed'; error?: string }>; + + beforeEach(() => { + dispatched = []; + dispatch = async (event) => { + dispatched.push(event); + return { kind: 'ok' as const }; + }; + }); + + test('trickle-mode source: duplicate content_hash within 24h window → second silent-dropped', async () => { + const source = new StubSource('trickle-1', 'test-source', 'trickle'); + const daemon = new IngestionDaemon({ + engine: {} as never, + logger: { info: () => {}, warn: () => {}, error: () => {} }, + dispatch, + }); + daemon.register({ source }); + await daemon.start(); + + const event = makeEvent({ content: 'shared content' }); + source.ctx!.emit(event); + await tick(); + source.ctx!.emit(event); // identical content_hash + await tick(); + + expect(dispatched.length).toBe(1); + await daemon.stop(); + }); + + test('migration-mode source: duplicate content_hash within 24h window → BOTH dispatch', async () => { + const source = new StubSource('migration-1', 'test-source', 'migration'); + const daemon = new IngestionDaemon({ + engine: {} as never, + logger: { info: () => {}, warn: () => {}, error: () => {} }, + dispatch, + }); + daemon.register({ source }); + await daemon.start(); + + const event = makeEvent({ content: 'shared content' }); + source.ctx!.emit(event); + await tick(); + source.ctx!.emit(event); // identical content_hash — should still dispatch + await tick(); + + expect(dispatched.length).toBe(2); + await daemon.stop(); + }); + + test('source without mode field defaults to trickle (v0.38 back-compat)', async () => { + const source: IngestionSource = { + id: 'no-mode-1', + kind: 'test-source', + async start(ctx) { + (source as { _ctx?: IngestionSourceContext })._ctx = ctx; + }, + async stop() {}, + }; + const daemon = new IngestionDaemon({ + engine: {} as never, + logger: { info: () => {}, warn: () => {}, error: () => {} }, + dispatch, + }); + daemon.register({ source }); + await daemon.start(); + + const ctx = (source as { _ctx?: IngestionSourceContext })._ctx!; + const event = makeEvent({ content: 'default-mode test' }); + ctx.emit(event); + await tick(); + ctx.emit(event); // identical content_hash — trickle defaults dedup it + await tick(); + + expect(dispatched.length).toBe(1); + await daemon.stop(); + }); + + test('migration-mode source: validation still runs (malformed event still dropped)', async () => { + const source = new StubSource('migration-2', 'test-source', 'migration'); + const daemon = new IngestionDaemon({ + engine: {} as never, + logger: { info: () => {}, warn: () => {}, error: () => {} }, + dispatch, + }); + daemon.register({ source }); + await daemon.start(); + + // Malformed: content_hash isn't 64 hex chars + source.ctx!.emit(makeEvent({ content_hash: 'not-a-real-sha256' })); + await tick(); + + expect(dispatched.length).toBe(0); + await daemon.stop(); + }); + + test('mixed dual source: trickle dedups own stream, migration does not', async () => { + const trickle = new StubSource('trickle-mixed', 'test-source', 'trickle'); + const migration = new StubSource('migration-mixed', 'test-source-2', 'migration'); + const daemon = new IngestionDaemon({ + engine: {} as never, + logger: { info: () => {}, warn: () => {}, error: () => {} }, + dispatch, + }); + daemon.register({ source: trickle }); + daemon.register({ source: migration }); + await daemon.start(); + + const e1 = makeEvent({ content: 'mixed-1' }); + const e2 = makeEvent({ content: 'mixed-2' }); + + // Trickle: same hash twice → 1 dispatched + trickle.ctx!.emit(e1); + await tick(); + trickle.ctx!.emit(e1); + await tick(); + + // Migration: same hash twice → 2 dispatched + migration.ctx!.emit(e2); + await tick(); + migration.ctx!.emit(e2); + await tick(); + + expect(dispatched.length).toBe(3); + await daemon.stop(); + }); +}); diff --git a/test/schema-pack-manifest-v041.test.ts b/test/schema-pack-manifest-v041.test.ts new file mode 100644 index 000000000..3ed2494d8 --- /dev/null +++ b/test/schema-pack-manifest-v041.test.ts @@ -0,0 +1,258 @@ +// v0.41 T3 — SchemaPackManifestSchema extensions: phases + calibration_domains. +// +// Pinned contracts: +// - `phases:` optional, defaults to [], accepts string[] of CyclePhase names +// - `calibration_domains:` optional, defaults to [] +// - CalibrationDomain entries: {name: snake_case, aggregator: closed enum, page_types: non-empty} +// - AggregatorKind closed enum exposes 4 v1 algorithms (scalar_brier, +// weighted_brier, count_based, cluster_summary) +// - Unknown aggregator rejected at parse time with a clear error path +// - Unknown domain name shape (e.g. 'Deal-Success' kebab-case) rejected at parse +// - Backward compat: existing pack manifests without phases/calibration_domains +// still parse cleanly (defaults to empty arrays) + +import { describe, test, expect } from 'bun:test'; +import { + parseSchemaPackManifest, + AGGREGATOR_KINDS, + type AggregatorKind, + type CalibrationDomain, + type SchemaPackManifest, +} from '../src/core/schema-pack/manifest-v1.ts'; + +const baseManifest = (overrides: Record = {}): Record => ({ + api_version: 'gbrain-schema-pack-v1', + name: 'test-pack', + version: '1.0.0', + description: 'unit test pack for v0.41 schema extensions', + ...overrides, +}); + +describe('v0.41 T3: AggregatorKind closed registry', () => { + test('exposes exactly 4 v1 aggregator kinds', () => { + expect(AGGREGATOR_KINDS.length).toBe(4); + expect(AGGREGATOR_KINDS).toEqual([ + 'scalar_brier', + 'weighted_brier', + 'count_based', + 'cluster_summary', + ]); + }); + + test('AggregatorKind type union covers exactly the enum values', () => { + // Compile-time + runtime test: every AGGREGATOR_KINDS value is a valid AggregatorKind + for (const k of AGGREGATOR_KINDS) { + const typed: AggregatorKind = k; + expect(typeof typed).toBe('string'); + } + }); +}); + +describe('v0.41 T3: SchemaPackManifestSchema phases field', () => { + test('phases defaults to empty array when omitted', () => { + const parsed = parseSchemaPackManifest(baseManifest()); + expect(parsed.phases).toEqual([]); + }); + + test('phases accepts string array of phase names', () => { + const parsed = parseSchemaPackManifest( + baseManifest({ phases: ['extract_atoms', 'synthesize_concepts'] }), + ); + expect(parsed.phases).toEqual(['extract_atoms', 'synthesize_concepts']); + }); + + test('phases rejects non-string entries', () => { + expect(() => + parseSchemaPackManifest(baseManifest({ phases: ['extract_atoms', 42] })), + ).toThrow(); + }); + + test('phases rejects empty-string entries', () => { + expect(() => parseSchemaPackManifest(baseManifest({ phases: [''] }))).toThrow(); + }); + + test('phases rejects non-array shape', () => { + expect(() => parseSchemaPackManifest(baseManifest({ phases: 'extract_atoms' }))).toThrow(); + }); +}); + +describe('v0.41 T3: SchemaPackManifestSchema calibration_domains field', () => { + test('calibration_domains defaults to empty array when omitted', () => { + const parsed = parseSchemaPackManifest(baseManifest()); + expect(parsed.calibration_domains).toEqual([]); + }); + + test('accepts well-formed domain entry', () => { + const parsed = parseSchemaPackManifest( + baseManifest({ + calibration_domains: [ + { + name: 'deal_success', + aggregator: 'scalar_brier', + page_types: ['deal'], + }, + ], + }), + ); + expect(parsed.calibration_domains.length).toBe(1); + const d = parsed.calibration_domains[0]; + expect(d.name).toBe('deal_success'); + expect(d.aggregator).toBe('scalar_brier'); + expect(d.page_types).toEqual(['deal']); + }); + + test('accepts all 4 aggregator kinds', () => { + for (const aggregator of AGGREGATOR_KINDS) { + const parsed = parseSchemaPackManifest( + baseManifest({ + calibration_domains: [ + { name: `domain_${aggregator}`, aggregator, page_types: ['person'] }, + ], + }), + ); + expect(parsed.calibration_domains[0].aggregator).toBe(aggregator); + } + }); + + test('rejects unknown aggregator value', () => { + expect(() => + parseSchemaPackManifest( + baseManifest({ + calibration_domains: [ + { name: 'bad_domain', aggregator: 'made_up_algo', page_types: ['deal'] }, + ], + }), + ), + ).toThrow(); + }); + + test('rejects kebab-case domain name (must be snake_case)', () => { + expect(() => + parseSchemaPackManifest( + baseManifest({ + calibration_domains: [ + { name: 'Deal-Success', aggregator: 'scalar_brier', page_types: ['deal'] }, + ], + }), + ), + ).toThrow(); + }); + + test('rejects uppercase domain name', () => { + expect(() => + parseSchemaPackManifest( + baseManifest({ + calibration_domains: [ + { name: 'DealSuccess', aggregator: 'scalar_brier', page_types: ['deal'] }, + ], + }), + ), + ).toThrow(); + }); + + test('rejects domain name starting with digit', () => { + expect(() => + parseSchemaPackManifest( + baseManifest({ + calibration_domains: [ + { name: '1deal', aggregator: 'scalar_brier', page_types: ['deal'] }, + ], + }), + ), + ).toThrow(); + }); + + test('rejects empty page_types array (must have at least 1)', () => { + expect(() => + parseSchemaPackManifest( + baseManifest({ + calibration_domains: [ + { name: 'deal_success', aggregator: 'scalar_brier', page_types: [] }, + ], + }), + ), + ).toThrow(); + }); + + test('rejects unknown extra field on domain entry (.strict)', () => { + expect(() => + parseSchemaPackManifest( + baseManifest({ + calibration_domains: [ + { + name: 'deal_success', + aggregator: 'scalar_brier', + page_types: ['deal'], + bonus_field: 'not allowed', + }, + ], + }), + ), + ).toThrow(); + }); + + test('accepts multiple page_types per domain', () => { + const parsed = parseSchemaPackManifest( + baseManifest({ + calibration_domains: [ + { + name: 'architecture_calls', + aggregator: 'scalar_brier', + page_types: ['code', 'decision'], + }, + ], + }), + ); + expect(parsed.calibration_domains[0].page_types).toEqual(['code', 'decision']); + }); + + test('accepts multiple domain entries per pack', () => { + const parsed = parseSchemaPackManifest( + baseManifest({ + calibration_domains: [ + { name: 'deal_success', aggregator: 'scalar_brier', page_types: ['deal'] }, + { name: 'founder_evaluation', aggregator: 'scalar_brier', page_types: ['person'] }, + { name: 'market_call', aggregator: 'weighted_brier', page_types: ['thesis'] }, + { name: 'concept_themes', aggregator: 'cluster_summary', page_types: ['concept'] }, + ], + }), + ); + expect(parsed.calibration_domains.length).toBe(4); + const byName = Object.fromEntries(parsed.calibration_domains.map((d: CalibrationDomain) => [d.name, d.aggregator])); + expect(byName.deal_success).toBe('scalar_brier'); + expect(byName.market_call).toBe('weighted_brier'); + expect(byName.concept_themes).toBe('cluster_summary'); + }); +}); + +describe('v0.41 T3: backward compatibility with v0.38 manifests', () => { + test('existing minimal manifest without phases/calibration_domains still parses', () => { + const v038Shape = baseManifest({ + page_types: [ + { + name: 'thing', + primitive: 'entity', + path_prefixes: ['things/'], + aliases: [], + extractable: false, + expert_routing: false, + }, + ], + }); + const parsed: SchemaPackManifest = parseSchemaPackManifest(v038Shape); + expect(parsed.phases).toEqual([]); + expect(parsed.calibration_domains).toEqual([]); + expect(parsed.page_types.length).toBe(1); + }); + + test('existing manifest with takes_kinds + filing_rules unchanged by extensions', () => { + const parsed = parseSchemaPackManifest( + baseManifest({ + takes_kinds: ['fact', 'take', 'bet', 'hunch'], + filing_rules: [{ kind: 'note', directory: 'notes/', examples: [], description: undefined }], + }), + ); + expect(parsed.takes_kinds).toEqual(['fact', 'take', 'bet', 'hunch']); + expect(parsed.filing_rules.length).toBe(1); + }); +});