fix: codex adversarial round — CI aggregate gap + first-landing verification + scoring honesty

(1) test-status never read needs.brainbench.result: a failing gate job left
the branch-protection aggregate green — merge-bypassable. Now in the loop.
(2) The first-landing path verifies the run against the COMMITTED baseline
(an unverified initial merge could seed a doctored baseline for every future
PR). (3) One stored fact row may satisfy at most one gold probe (merged/
broad extractions inflated fidelity via unconstrained find()). (4) The
slug-granularity limit of source-isolation detection is disclosed in the
methodology doc with a pointer to the engine-layer fuzz that covers the
content-leak case. Plus 30s timeouts on engine-touching tests (observed 5s
default flaking at load avg 20).

.github/workflows/test.yml

@@ -284,15 +284,16 @@ jobs:
           SERIAL="${{ needs.serial-tests.result }}"
           SLOW_EVAL="${{ needs.slow-eval-longmemeval.result }}"
           SLOW_PERF="${{ needs.slow-entity-resolve-perf.result }}"
+          BRAINBENCH="${{ needs.brainbench.result }}"
           TEST="${{ needs.test.result }}"
           echo "cache-check.hit=$HIT"
-          echo "gitleaks=$GITLEAKS  verify=$VERIFY  serial-tests=$SERIAL  slow-eval-longmemeval=$SLOW_EVAL  slow-entity-resolve-perf=$SLOW_PERF  test=$TEST"
+          echo "gitleaks=$GITLEAKS  verify=$VERIFY  serial-tests=$SERIAL  slow-eval-longmemeval=$SLOW_EVAL  slow-entity-resolve-perf=$SLOW_PERF  brainbench=$BRAINBENCH  test=$TEST"
           if [ "$HIT" = "true" ]; then
             echo "✓ cache HIT for hash ${{ needs.cache-check.outputs.hash }} — CI green"
             exit 0
           fi
           # Cache miss: every gated job must have succeeded.
-          for r in "$GITLEAKS" "$VERIFY" "$SERIAL" "$SLOW_EVAL" "$SLOW_PERF" "$TEST"; do
+          for r in "$GITLEAKS" "$VERIFY" "$SERIAL" "$SLOW_EVAL" "$SLOW_PERF" "$BRAINBENCH" "$TEST"; do
             if [ "$r" != "success" ]; then
               echo "✗ gated job did not succeed (got $r) — CI fail"
               exit 1

docs/eval/BRAINBENCH.md

@@ -49,7 +49,7 @@ All micro-averaged per (harness × suite) cell; registered in
 - `write_back_fidelity` = |gold facts that survive the PRODUCTION conversation→memory pipeline and are keyword-findable with correct entity attribution| / |gold facts|. The deterministic mode injects a gold extractor at the pipeline's extractor seam so segmentation, batching, dedup, and provenance stamping execute shipped code with zero LLM calls.
 - `provenance_accuracy` = |surviving facts with correct {source, source_session, source_markdown_slug}| / |surviving facts|.
 - `continuity_rate` = |decision probes recalled by the reader| / |probes|, per READER harness. The writer fixture's decisions persist through the production write-back pipeline — which is harness-INDEPENDENT in v1 — so each pair preps once and every harness replays the read-only reader against the same persisted state (an ordered writer×reader sweep would rebuild byte-identical brains for identical scores). A probe succeeds via pointer injection or stored-fact keyword lookup. The per-writer axis activates when harness-specific write paths land.
-- `source_isolation_violations` = count of injected slugs from a non-active source. **Gates at zero**, every run, regardless of baseline — cross-source leakage is the data-leak invariant.
+- `source_isolation_violations` = count of injected slugs from a non-active source. **Gates at zero**, every run, regardless of baseline — cross-source leakage is the data-leak invariant. Granularity disclosure: detection is slug-keyed, so it catches injection of slugs seeded ONLY in a foreign source; a same-slug cross-source CONTENT leak would require the engine's source-scoped SQL itself to fail, which the engine-layer source-isolation fuzz (gbrain-evals Cat 22) covers directly.
 - `avg_injected_tokens` = mean estimated tokens (chars/4) of injected context per replayed turn. Intrusion-budget diagnostic; reported, NOT gated (gating awaits calibration data — filed TODO).
 - `extraction_recall` / `extraction_precision` — `--llm` runs only: the real extractor's output vs gold keyword probes.
 

scripts/ci-brainbench-gate.sh

@@ -52,8 +52,16 @@ if git show "${MAIN_REF}:${BASELINE_PATH}" > "$MAIN_BASELINE" 2>/dev/null; then
   echo "[brainbench-gate] comparing against ${MAIN_REF}:${BASELINE_PATH}"
   bun src/cli.ts eval brainbench --compare "$MAIN_BASELINE" --out "$OUT"
 else
-  # First landing: the ref exists but carries no baseline yet. Run without a
-  # gate so the PR that introduces BrainBench can commit the initial baseline.
-  echo "[brainbench-gate] no baseline on ${MAIN_REF} yet — running ungated (initial-landing path)"
-  bun src/cli.ts eval brainbench --out "$OUT"
+  # First landing: the ref exists but carries no baseline yet. The COMMITTED
+  # baseline still gets verified against the actual run (codex adversarial
+  # finding: an unverified first landing could seed a doctored baseline for
+  # every future PR to compare against). Same-hash + committed==run logic
+  # inside --compare does the verification.
+  if [ -f "$BASELINE_PATH" ]; then
+    echo "[brainbench-gate] no baseline on ${MAIN_REF} yet — verifying the run against the COMMITTED baseline (first-landing path)"
+    bun src/cli.ts eval brainbench --compare "$BASELINE_PATH" --out "$OUT"
+  else
+    echo "[brainbench-gate] no baseline on ${MAIN_REF} and none committed — running ungated (pre-baseline tree)"
+    bun src/cli.ts eval brainbench --out "$OUT"
+  fi
 fi

src/eval/brainbench/metrics/write-back.ts

@@ -198,16 +198,23 @@ export async function runWriteBack(
   let survived = 0;
   let provenanceOk = 0;
   const failed: string[] = [];
+  // Each stored row may satisfy AT MOST one gold fact (codex adversarial
+  // finding: unconstrained find() let one merged/overly-broad extracted row
+  // inflate fidelity by matching several gold probes).
+  const consumed = new Set<number>();
   for (const { spec } of goldItems) {
-    const match = stored.find(
-      (row) =>
+    const matchIdx = stored.findIndex(
+      (row, idx) =>
+        !consumed.has(idx) &&
         spec.match_keywords.every((kw) => row.fact.toLowerCase().includes(kw.toLowerCase())) &&
         (spec.entity_slug === null || row.entity_slug === spec.entity_slug),
     );
-    if (!match) {
+    if (matchIdx === -1) {
       failed.push(`${fixture.fixture_id} (gold fact lost: ${spec.gist})`);
       continue;
     }
+    consumed.add(matchIdx);
+    const match = stored[matchIdx];
     survived++;
     if (
       match.source === PER_SEGMENT_SOURCE_PREFIX &&

test/brainbench-adapters.test.ts

@@ -60,7 +60,7 @@ describe('runReflexPipeline (the ONE pipeline, decision 13)', () => {
       suppression: 'prior-context',
     });
     expect(block?.pointers.map((p) => p.slug)).toContain('people/alice-example');
-  });
+  }, 30_000);
 
   test('budget cap: maxPointers=1 truncates a multi-entity turn', async () => {
     const block = await runReflexPipeline(
@@ -71,7 +71,7 @@ describe('runReflexPipeline (the ONE pipeline, decision 13)', () => {
       { maxPointers: 1, suppression: 'prior-context' },
     );
     expect(block?.pointers.length).toBe(1);
-  });
+  }, 30_000);
 
   test('suppression prior-context: an already-seen slug is not re-injected; suppression none re-injects', async () => {
     const prior = 'earlier we discussed people/alice-example in detail';
@@ -85,7 +85,7 @@ describe('runReflexPipeline (the ONE pipeline, decision 13)', () => {
       suppression: 'none',
     });
     expect(reinjected?.pointers.map((p) => p.slug)).toContain('people/alice-example');
-  });
+  }, 30_000);
 
   test('no candidates → null (silence, not an empty block)', async () => {
     const block = await runReflexPipeline(engine, 'default', turn('ok thanks, sounds good'), '', {
@@ -93,7 +93,7 @@ describe('runReflexPipeline (the ONE pipeline, decision 13)', () => {
       suppression: 'prior-context',
     });
     expect(block).toBeNull();
-  });
+  }, 30_000);
 });
 
 describe('OpenClawAdapter (production seam)', () => {
@@ -109,7 +109,7 @@ describe('OpenClawAdapter (production seam)', () => {
     const r2 = await a.replayTurn(turn('Does Alice Example have consent lined up?', 2), prior);
     expect(r2.injectedSlugs).not.toContain('people/alice-example');
     await a.endConversation();
-  });
+  }, 30_000);
 });
 
 describe('ClaudeCodeAdapter (contract seam: hook wire shape, no conversation memory)', () => {
@@ -121,7 +121,7 @@ describe('ClaudeCodeAdapter (contract seam: hook wire shape, no conversation mem
     // No memory: prior context does NOT suppress (the measured contract delta).
     expect(r.injectedSlugs).toContain('people/alice-example');
     await a.endConversation();
-  });
+  }, 30_000);
 
   test('respects the 2-pointer hook budget', async () => {
     const a = new ClaudeCodeAdapter();
@@ -129,7 +129,7 @@ describe('ClaudeCodeAdapter (contract seam: hook wire shape, no conversation mem
     const r = await a.replayTurn(turn('Memo: Alice Example, Charlie Example, and Widget Co all in one.'), '');
     expect(r.injectedSlugs.length).toBeLessThanOrEqual(2);
     await a.endConversation();
-  });
+  }, 30_000);
 });
 
 describe('CodexAdapter (contract seam: static preamble + ≤1 fragment)', () => {
@@ -144,7 +144,7 @@ describe('CodexAdapter (contract seam: static preamble + ≤1 fragment)', () =>
     // First turn carried the preamble cost; later turns don't re-pay it.
     expect(r1.injectedTokens).toBeGreaterThan(r2.injectedTokens);
     await a.endConversation();
-  });
+  }, 30_000);
 });
 
 describe('sentinel isolation (the engine-sharing guarantee, eng-review D9)', () => {
@@ -164,5 +164,5 @@ describe('sentinel isolation (the engine-sharing guarantee, eng-review D9)', ()
     expect(facts[0].n).toBe('0');
     // re-seed for any later test in this file (none currently, but keep the brain valid)
     await seedBrain(engine, FIXTURE);
-  });
+  }, 30_000);
 });

test/brainbench-continuity.test.ts

@@ -43,7 +43,7 @@ describe('writer openclaw → reader codex (and reverse) on a shared brain', ()
     const readerRows = out.turn_rows.filter((r) => r.suite === 'continuity');
     expect(readerRows.length).toBeGreaterThan(0);
     expect(readerRows.every((r) => r.fixture_id === 'cont-001-widget-pass-reader')).toBe(true);
-  });
+  }, 30_000);
 
   test('single-harness run falls back to the diagonal (writer == reader) instead of vanishing', async () => {
     const out = await runBrainBench(pairCorpus, {
@@ -55,7 +55,7 @@ describe('writer openclaw → reader codex (and reverse) on a shared brain', ()
     const cell = out.cells.find((c) => c.suite === 'continuity' && c.harness === 'openclaw');
     expect(cell).toBeDefined();
     expect(cell!.metrics.continuity_rate).toBe(1);
-  });
+  }, 30_000);
 });
 
 describe('factKeywordProbe + the miss path', () => {
@@ -81,11 +81,11 @@ describe('factKeywordProbe + the miss path', () => {
   test('AND semantics: every keyword must match; partial sets fail', async () => {
     expect(await factKeywordProbe(engine, 'default', ['pass', 'widget-co'])).toBe(true);
     expect(await factKeywordProbe(engine, 'default', ['pass', 'no-such-keyword'])).toBe(false);
-  });
+  }, 30_000);
 
   test('expired facts are excluded — a superseded decision is not "recalled"', async () => {
     expect(await factKeywordProbe(engine, 'default', ['lead', 'acme-example'])).toBe(false);
-  });
+  }, 30_000);
 
   test('a decision neither injected nor stored counts into gold_failed with a named item', async () => {
     const score = await scoreContinuityPair(engine, 'default', 'pair-x', [], [
@@ -97,5 +97,5 @@ describe('factKeywordProbe + the miss path', () => {
     expect(score.hits['d-missing']).toBe(false);
     expect(score.hits['d-hit']).toBe(true);
     expect(score.failed_items[0]).toContain('pair-x/d-missing');
-  });
+  }, 30_000);
 });

test/brainbench-writeback.test.ts

@@ -56,7 +56,7 @@ describe('makeGoldExtractor', () => {
     expect(out[0].entity_slug).toBe('people/alice-example');
     const none = await extractor({ turnText: 'unrelated segment text', source: 'cli:x' });
     expect(none.length).toBe(0);
-  });
+  }, 30_000);
 });
 
 describe('runWriteBack (deterministic, production pipeline)', () => {
@@ -77,7 +77,7 @@ describe('runWriteBack (deterministic, production pipeline)', () => {
     );
     expect(rows.length).toBeGreaterThanOrEqual(2);
     expect(rows[0].source_session).toBe(`${PER_SEGMENT_SOURCE_PREFIX}:${slug}`);
-  });
+  }, 30_000);
 
   test('a gold fact the pipeline drops is counted as failed, named in failed_items', async () => {
     await resetTables(engine);
@@ -91,7 +91,7 @@ describe('runWriteBack (deterministic, production pipeline)', () => {
     expect(score.gold_failed).toBe(1);
     expect(score.metrics.write_back_fidelity).toBeCloseTo(2 / 3);
     expect(score.failed_items[0]).toContain('gold fact lost');
-  });
+  }, 30_000);
 
   test('--llm branch: real-extractor lane emits extraction metrics (stubbed transport)', async () => {
     await resetTables(engine);
@@ -130,7 +130,7 @@ describe('runWriteBack (deterministic, production pipeline)', () => {
       __setChatTransportForTests(null);
       __setEmbedTransportForTests(null);
     }
-  });
+  }, 30_000);
 
   test('--llm extraction metrics reach the harness CELLS (review finding: they were dropped in aggregation)', async () => {
     __setChatTransportForTests(async (): Promise<ChatResult> => ({
@@ -168,7 +168,7 @@ describe('runWriteBack (deterministic, production pipeline)', () => {
       __setChatTransportForTests(null);
       __setEmbedTransportForTests(null);
     }
-  });
+  }, 30_000);
 
   test('multi-segment conversations extract per segment (the 45-min gap splits)', async () => {
     await resetTables(engine);
@@ -179,5 +179,5 @@ describe('runWriteBack (deterministic, production pipeline)', () => {
     const score = await runWriteBack(engine, genWb.fixture, genWb.gold, { llm: false });
     expect(score.gold_failed).toBe(0);
     expect(score.metrics.write_back_fidelity).toBe(1);
-  });
+  }, 30_000);
 });