From 07e4ca4dace30a10dcd2e57579ce694c56b9b004 Mon Sep 17 00:00:00 2001 From: Garry Tan Date: Thu, 11 Jun 2026 21:29:05 -0700 Subject: [PATCH] =?UTF-8?q?feat(schema):=20context=5Fvolunteer=5Fevents=20?= =?UTF-8?q?table=20(v116)=20=E2=80=94=20push-context=20feedback=20log=20(#?= =?UTF-8?q?2095)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit One row per page the brain volunteers (op / reflex / watch channels). "Used" is DERIVED, never written: pages.last_retrieved_at > volunteered_at (the existing bumpLastRetrievedAt write-back is the open/cite signal), so there is no second tracking path. session_id/turn are nullable caller-supplied attribution; rationale is a deterministic template string, never raw conversation text. - Migration v116 (idempotent) + mirrors in src/schema.sql + src/core/pglite-schema.ts + regenerated schema-embedded.ts (regen also folds in pre-existing comment-only drift from the v114 links edits). - src/core/context/volunteer-events.ts: insertVolunteerEvents (ONE multi-row parameterized INSERT — never per-row awaited round-trips) + purgeStaleVolunteerEvents (90-day GC, returns 0 on pre-v116 brains). - Dream cycle purge phase prunes stale events alongside op_checkpoints / brainstorm checkpoints / batch-retry audit files. - RLS on Postgres comes from the v35 auto_rls_on_create_table event trigger (the same mechanism that covered v110 page_aliases and v115 op_checkpoint_paths); the volunteer Postgres e2e pins it. - No ::jsonb anywhere; no bootstrap probe needed (nothing references the table pre-creation; writers guard with try/catch). Tests: v116 shape + columns + indexes + live insert/purge round-trip on PGLite (test/migrate.test.ts, 161 pass); schema-bootstrap-coverage green. Co-Authored-By: Claude Fable 5 --- src/core/context/volunteer-events.ts | 96 ++++++++++++++++++++++++++++ src/core/cycle.ts | 14 +++- src/core/migrate.ts | 36 +++++++++++ src/core/pglite-schema.ts | 21 ++++++ src/core/schema-embedded.ts | 71 +++++++++++++++----- src/schema.sql | 22 +++++++ test/migrate.test.ts | 64 +++++++++++++++++++ 7 files changed, 306 insertions(+), 18 deletions(-) create mode 100644 src/core/context/volunteer-events.ts diff --git a/src/core/context/volunteer-events.ts b/src/core/context/volunteer-events.ts new file mode 100644 index 000000000..09f1f4e8e --- /dev/null +++ b/src/core/context/volunteer-events.ts @@ -0,0 +1,96 @@ +/** + * v0.43 (#2095) — context_volunteer_events: the feedback-loop log behind + * push-based context. One row per page the brain VOLUNTEERED, written + * fire-and-forget by the volunteer_context op, the retrieval-reflex pointer + * path (channel 'reflex'), and `gbrain watch` (channel 'watch'). + * + * "Used" is DERIVED, never written: a volunteered page counts as used when + * pages.last_retrieved_at > volunteered_at (the existing bumpLastRetrievedAt + * write-back on get_page/search/query is the open/cite signal). The join is + * approximate by design — last-retrieved is 5-min throttled (false negatives) + * and unrelated reads match too (false positives); stats output carries the + * caveat. + * + * Retention: rows older than VOLUNTEER_EVENTS_TTL_DAYS are pruned by the + * dream cycle's purge phase so conversation-adjacent telemetry never grows + * unbounded. rationale is a deterministic template string, never raw + * conversation text. + */ + +import type { BrainEngine } from './../engine.ts'; + +export const VOLUNTEER_EVENTS_TTL_DAYS = 90; + +export type VolunteerChannel = 'op' | 'reflex' | 'watch'; + +export interface VolunteerEventRow { + source_id: string; + slug: string; + confidence: number; + match_arm: string; + rationale: string; + channel: VolunteerChannel; + session_id?: string | null; + turn?: number | null; +} + +/** + * ONE multi-row parameterized INSERT for a batch of volunteered pages (max 5 + * per call by the volunteer cap) — never per-row awaited INSERTs (up to 5 + * RTTs ≈ 355ms on a cross-region deployment; eng-review D4). Throws on + * failure; callers run it through the volunteer-events background-work sink + * with try/catch so logging can never fail the op. + */ +export async function insertVolunteerEvents( + engine: BrainEngine, + rows: VolunteerEventRow[], +): Promise { + if (!rows.length) return; + const params: unknown[] = []; + const tuples = rows.map((r) => { + const base = params.length; + params.push( + r.source_id, + r.slug, + r.confidence, + r.match_arm, + r.rationale, + r.channel, + r.session_id ?? null, + r.turn ?? null, + ); + const ph = Array.from({ length: 8 }, (_, i) => `$${base + i + 1}`); + return `(${ph.join(', ')})`; + }); + await engine.executeRaw( + `INSERT INTO context_volunteer_events + (source_id, slug, confidence, match_arm, rationale, channel, session_id, turn) + VALUES ${tuples.join(', ')}`, + params, + ); +} + +/** + * 90-day GC, called from the dream cycle's purge phase (mirrors + * purgeStaleCheckpoints). Best-effort: returns 0 on any failure (pre-v116 + * brains have no table yet). + */ +export async function purgeStaleVolunteerEvents( + engine: BrainEngine, + ttlDays = VOLUNTEER_EVENTS_TTL_DAYS, +): Promise { + try { + const rows = await engine.executeRaw<{ count: string | number }>( + `WITH deleted AS ( + DELETE FROM context_volunteer_events + WHERE volunteered_at < now() - ($1 || ' days')::interval + RETURNING 1 + ) + SELECT count(*)::text AS count FROM deleted`, + [String(ttlDays)], + ); + return Number(rows[0]?.count ?? 0); + } catch { + return 0; + } +} diff --git a/src/core/cycle.ts b/src/core/cycle.ts index a4dce8ffe..ddc697bd3 100644 --- a/src/core/cycle.ts +++ b/src/core/cycle.ts @@ -1270,6 +1270,16 @@ async function runPhasePurge(engine: BrainEngine, dryRun: boolean): Promise volunteered_at — no second write path. + // session_id/turn are caller-supplied attribution (nullable). rationale is + // a deterministic template string, NEVER raw conversation text. Rows are + // pruned past 90 days by the dream cycle's purge phase + // (purgeStaleVolunteerEvents). No ::jsonb anywhere. RLS: covered by the + // v35 auto_rls_on_create_table event trigger on Postgres (same as + // v110/v115 tables); pinned by the volunteer-context Postgres e2e. + // Created empty; plain CREATE INDEX is instant — no CONCURRENTLY needed. + // Keep in sync with src/schema.sql, src/core/pglite-schema.ts, + // src/core/schema-embedded.ts. + idempotent: true, + sql: ` + CREATE TABLE IF NOT EXISTS context_volunteer_events ( + id BIGSERIAL PRIMARY KEY, + source_id TEXT NOT NULL, + slug TEXT NOT NULL, + confidence DOUBLE PRECISION NOT NULL, + match_arm TEXT NOT NULL, + rationale TEXT NOT NULL DEFAULT '', + channel TEXT NOT NULL DEFAULT 'op', + session_id TEXT, + turn INTEGER, + volunteered_at TIMESTAMPTZ NOT NULL DEFAULT now() + ); + CREATE INDEX IF NOT EXISTS context_volunteer_events_src_time_idx + ON context_volunteer_events (source_id, volunteered_at DESC); + CREATE INDEX IF NOT EXISTS context_volunteer_events_src_slug_idx + ON context_volunteer_events (source_id, slug); + `, + }, ]; export const LATEST_VERSION = MIGRATIONS.length > 0 diff --git a/src/core/pglite-schema.ts b/src/core/pglite-schema.ts index 44c3cc418..6b41ad54d 100644 --- a/src/core/pglite-schema.ts +++ b/src/core/pglite-schema.ts @@ -945,6 +945,27 @@ CREATE TABLE IF NOT EXISTS op_checkpoint_paths ( FOREIGN KEY (op, fingerprint) REFERENCES op_checkpoints (op, fingerprint) ON DELETE CASCADE ); +-- #2095 push-based context: feedback-loop log of volunteered pages. +-- Mirrors migration v116 + src/schema.sql. "Used" derives from +-- pages.last_retrieved_at > volunteered_at; 90-day prune in the dream +-- cycle's purge phase. +CREATE TABLE IF NOT EXISTS context_volunteer_events ( + id BIGSERIAL PRIMARY KEY, + source_id TEXT NOT NULL, + slug TEXT NOT NULL, + confidence DOUBLE PRECISION NOT NULL, + match_arm TEXT NOT NULL, + rationale TEXT NOT NULL DEFAULT '', + channel TEXT NOT NULL DEFAULT 'op', + session_id TEXT, + turn INTEGER, + volunteered_at TIMESTAMPTZ NOT NULL DEFAULT now() +); +CREATE INDEX IF NOT EXISTS context_volunteer_events_src_time_idx + ON context_volunteer_events (source_id, volunteered_at DESC); +CREATE INDEX IF NOT EXISTS context_volunteer_events_src_slug_idx + ON context_volunteer_events (source_id, slug); + -- ============================================================ -- migration_impact_log (v0.41.18.0 — gbrain onboard wave) -- ============================================================ diff --git a/src/core/schema-embedded.ts b/src/core/schema-embedded.ts index dc288eabd..e9b2f620b 100644 --- a/src/core/schema-embedded.ts +++ b/src/core/schema-embedded.ts @@ -411,12 +411,24 @@ CREATE INDEX IF NOT EXISTS idx_code_edges_symbol_to -- ============================================================ -- links: cross-references between pages -- ============================================================ --- Provenance model (v0.13, extended issue #972): --- link_source — 'markdown' | 'frontmatter' | 'manual' | 'wikilink-resolved' | NULL --- 'wikilink-resolved' is the opt-in --- (link_resolution.global_basename) basename-match --- provenance — see issue #972 / migration v113. --- (NULL = legacy row written before v0.13; unknown source) +-- Provenance model (v0.13; opened to kebab provenance in v114 / issue #1941): +-- link_source — open kebab-case provenance tag, NOT a closed allowlist. +-- Format gate (CHECK): ^[a-z][a-z0-9]*(-[a-z0-9]+)*\$ and +-- char_length <= 64. NULL = legacy row (pre-v0.13). +-- Reconciliation-managed built-ins written internally: +-- 'markdown' — body markdown links +-- 'frontmatter' — YAML frontmatter edges (see origin_*) +-- 'mentions' — auto-linked body-text mentions +-- 'wikilink-resolved'— opt-in global-basename [[name]] (#972) +-- User/tool-facing: +-- 'manual' — hand- or tool-created edges (the +-- add_link op default + CLI link-add) +-- '' — external derivers, e.g. 'citation-graph', +-- stamp their own kebab tag (no migration). +-- The add_link OP forbids callers from passing the four +-- managed built-ins (they imply reconciliation semantics a +-- hand-created row can't honor); the DB CHECK still admits +-- them because internal writers use them. See operations.ts. -- origin_page_id — for link_source='frontmatter', the page whose YAML -- frontmatter created this edge; scopes reconciliation -- origin_field — the frontmatter field name (e.g. 'key_people') @@ -424,21 +436,21 @@ CREATE INDEX IF NOT EXISTS idx_code_edges_symbol_to -- The unique constraint includes link_source + origin_page_id so a manual edge -- and a frontmatter-derived edge with the same (from, to, type) tuple coexist. -- Reconciliation on put_page filters by (link_source='frontmatter' AND --- origin_page_id = written_page) — never touches other pages' edges. +-- origin_page_id = written_page) — never touches other pages' edges. (This is +-- exactly why a CLI-forged 'frontmatter' row with NULL origin would be a phantom +-- edge reconciliation never cleans — hence the op-layer guard.) CREATE TABLE IF NOT EXISTS links ( id SERIAL PRIMARY KEY, from_page_id INTEGER NOT NULL REFERENCES pages(id) ON DELETE CASCADE, to_page_id INTEGER NOT NULL REFERENCES pages(id) ON DELETE CASCADE, link_type TEXT NOT NULL DEFAULT '', context TEXT NOT NULL DEFAULT '', - -- v114 (#1941): link_source is an open kebab-case provenance, not a closed - -- allowlist — external derivers (e.g. 'citation-graph') stamp their own tag - -- without a gbrain migration. Format gate only: lowercase kebab, <=64 chars. - -- The reconciliation-managed built-ins ('markdown','frontmatter','mentions', - -- 'wikilink-resolved') still satisfy this and are written internally; the - -- add_link op forbids CALLERS from forging them (see operations.ts). 'manual' - -- is the user-facing default for hand/tool-created edges. - link_source TEXT CHECK (link_source IS NULL OR (link_source ~ '^[a-z][a-z0-9]*(-[a-z0-9]+)*$' AND char_length(link_source) <= 64)), + -- v0.41.18.0: 'mentions' added for auto-linked body-text mentions + -- (gbrain extract links --by-mention). Filtered OUT of backlink-count + -- for search ranking; only counts toward orphan-ratio + graph traversal. + -- v0.40.8.2 (#972): 'wikilink-resolved' added for opt-in global-basename + -- wikilink resolution (bare [[name]] resolved by slug tail). + link_source TEXT CHECK (link_source IS NULL OR (link_source ~ '^[a-z][a-z0-9]*(-[a-z0-9]+)*\$' AND char_length(link_source) <= 64)), -- v0.41.18.0: nullable link_kind distinguishes "plain body mention" from -- "verb-pattern-derived typed link" within link_source='mentions'. -- Codex finding #12 design: keep link_source stable; add link_kind @@ -675,8 +687,11 @@ CREATE TABLE IF NOT EXISTS op_checkpoints ( CREATE INDEX IF NOT EXISTS op_checkpoints_updated_at_idx ON op_checkpoints (updated_at); --- #1794: append-only delta storage (one row per completed path). FK cascade --- drops children with the parent. Mirrors migration v115 + src/schema.sql. +-- #1794: append-only delta storage. One row per completed path; sync's +-- appendCompleted INSERTs only the delta instead of rewriting the whole +-- completed_keys JSONB array each flush (O(N^2) -> O(delta)). FK cascade drops +-- children with the parent (clearOpCheckpoint + 7-day purge). PK prefix +-- (op,fingerprint) serves all reads; no separate index. Mirrors migration v115. CREATE TABLE IF NOT EXISTS op_checkpoint_paths ( op TEXT NOT NULL, fingerprint TEXT NOT NULL, @@ -687,6 +702,28 @@ CREATE TABLE IF NOT EXISTS op_checkpoint_paths ( FOREIGN KEY (op, fingerprint) REFERENCES op_checkpoints (op, fingerprint) ON DELETE CASCADE ); +-- #2095 push-based context: feedback-loop log of volunteered pages +-- (volunteer_context op / retrieval-reflex pointers / gbrain watch). +-- "Used" derives from pages.last_retrieved_at > volunteered_at; rationale is +-- a deterministic template string, never raw conversation text. 90-day prune +-- in the dream cycle's purge phase. Mirrors migration v116. +CREATE TABLE IF NOT EXISTS context_volunteer_events ( + id BIGSERIAL PRIMARY KEY, + source_id TEXT NOT NULL, + slug TEXT NOT NULL, + confidence DOUBLE PRECISION NOT NULL, + match_arm TEXT NOT NULL, + rationale TEXT NOT NULL DEFAULT '', + channel TEXT NOT NULL DEFAULT 'op', + session_id TEXT, + turn INTEGER, + volunteered_at TIMESTAMPTZ NOT NULL DEFAULT now() +); +CREATE INDEX IF NOT EXISTS context_volunteer_events_src_time_idx + ON context_volunteer_events (source_id, volunteered_at DESC); +CREATE INDEX IF NOT EXISTS context_volunteer_events_src_slug_idx + ON context_volunteer_events (source_id, slug); + -- migration_impact_log moved BELOW minion_jobs (was here, lines 645-676) -- because its \`job_id BIGINT REFERENCES minion_jobs(id)\` FK requires -- minion_jobs to exist FIRST during SCHEMA_SQL replay. v0.41.25.0 fix. diff --git a/src/schema.sql b/src/schema.sql index a900c6d1a..8d67adffa 100644 --- a/src/schema.sql +++ b/src/schema.sql @@ -698,6 +698,28 @@ CREATE TABLE IF NOT EXISTS op_checkpoint_paths ( FOREIGN KEY (op, fingerprint) REFERENCES op_checkpoints (op, fingerprint) ON DELETE CASCADE ); +-- #2095 push-based context: feedback-loop log of volunteered pages +-- (volunteer_context op / retrieval-reflex pointers / gbrain watch). +-- "Used" derives from pages.last_retrieved_at > volunteered_at; rationale is +-- a deterministic template string, never raw conversation text. 90-day prune +-- in the dream cycle's purge phase. Mirrors migration v116. +CREATE TABLE IF NOT EXISTS context_volunteer_events ( + id BIGSERIAL PRIMARY KEY, + source_id TEXT NOT NULL, + slug TEXT NOT NULL, + confidence DOUBLE PRECISION NOT NULL, + match_arm TEXT NOT NULL, + rationale TEXT NOT NULL DEFAULT '', + channel TEXT NOT NULL DEFAULT 'op', + session_id TEXT, + turn INTEGER, + volunteered_at TIMESTAMPTZ NOT NULL DEFAULT now() +); +CREATE INDEX IF NOT EXISTS context_volunteer_events_src_time_idx + ON context_volunteer_events (source_id, volunteered_at DESC); +CREATE INDEX IF NOT EXISTS context_volunteer_events_src_slug_idx + ON context_volunteer_events (source_id, slug); + -- migration_impact_log moved BELOW minion_jobs (was here, lines 645-676) -- because its `job_id BIGINT REFERENCES minion_jobs(id)` FK requires -- minion_jobs to exist FIRST during SCHEMA_SQL replay. v0.41.25.0 fix. diff --git a/test/migrate.test.ts b/test/migrate.test.ts index 67fd83248..58dd5b935 100644 --- a/test/migrate.test.ts +++ b/test/migrate.test.ts @@ -2179,3 +2179,67 @@ describe('v112 — pages_links_extracted_at', () => { }); }); + +// v0.43 (#2095): context_volunteer_events — push-based-context feedback log. +describe('v116 — context_volunteer_events_table', () => { + let engine: PGLiteEngine; + beforeAll(async () => { + engine = new PGLiteEngine(); + await engine.connect({}); + await engine.initSchema(); + }, 60_000); + afterAll(async () => { if (engine) await engine.disconnect(); }, 60_000); + + test('v116 entry exists, named + idempotent', () => { + const m = MIGRATIONS.find(x => x.version === 116); + expect(m).toBeDefined(); + expect(m!.name).toBe('context_volunteer_events_table'); + expect(m!.idempotent).toBe(true); + }); + + test('LATEST_VERSION is at or above 116', () => { + expect(LATEST_VERSION).toBeGreaterThanOrEqual(116); + }); + + test('table exists after initSchema with the documented columns', async () => { + const rows = await engine.executeRaw<{ column_name: string; is_nullable: string }>( + `SELECT column_name, is_nullable FROM information_schema.columns + WHERE table_name = 'context_volunteer_events' ORDER BY ordinal_position`, [], + ); + const byName = new Map(rows.map(r => [r.column_name, r.is_nullable])); + for (const required of ['source_id', 'slug', 'confidence', 'match_arm', 'rationale', 'channel', 'volunteered_at']) { + expect(byName.get(required)).toBe('NO'); + } + // Caller-supplied attribution columns are nullable by contract (codex D9). + expect(byName.get('session_id')).toBe('YES'); + expect(byName.get('turn')).toBe('YES'); + }); + + test('both source-scoped indexes exist after initSchema', async () => { + const rows = await engine.executeRaw<{ indexname: string }>( + `SELECT indexname FROM pg_indexes WHERE tablename = 'context_volunteer_events'`, [], + ); + const names = rows.map(r => r.indexname); + expect(names).toContain('context_volunteer_events_src_time_idx'); + expect(names).toContain('context_volunteer_events_src_slug_idx'); + }); + + test('insert + 90-day purge round-trip (purgeStaleVolunteerEvents)', async () => { + const { insertVolunteerEvents, purgeStaleVolunteerEvents } = await import('../src/core/context/volunteer-events.ts'); + await insertVolunteerEvents(engine, [ + { source_id: 'default', slug: 'people/alice-example', confidence: 0.9, match_arm: 'alias', rationale: 'alias match', channel: 'op' }, + { source_id: 'default', slug: 'projects/acme-example', confidence: 0.8, match_arm: 'title', rationale: 'exact title', channel: 'reflex', session_id: 's1', turn: 3 }, + ]); + // Age one row past the TTL, leave the other fresh. + await engine.executeRaw( + `UPDATE context_volunteer_events SET volunteered_at = now() - interval '120 days' + WHERE slug = 'projects/acme-example'`, [], + ); + const purged = await purgeStaleVolunteerEvents(engine, 90); + expect(purged).toBe(1); + const left = await engine.executeRaw<{ slug: string }>( + `SELECT slug FROM context_volunteer_events`, [], + ); + expect(left.map(r => r.slug)).toEqual(['people/alice-example']); + }); +});