admin/gbrain
1
0
Fork 0
mirror of https://github.com/garrytan/gbrain.git synced 2026-07-30 11:22:34 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix: codex adversarial round — CI aggregate gap + first-landing verification + scoring honesty

Browse Source 
(1) test-status never read needs.brainbench.result: a failing gate job left
the branch-protection aggregate green — merge-bypassable. Now in the loop.
(2) The first-landing path verifies the run against the COMMITTED baseline
(an unverified initial merge could seed a doctored baseline for every future
PR). (3) One stored fact row may satisfy at most one gold probe (merged/
broad extractions inflated fidelity via unconstrained find()). (4) The
slug-granularity limit of source-isolation detection is disclosed in the
methodology doc with a pointer to the engine-layer fuzz that covers the
content-leak case. Plus 30s timeouts on engine-touching tests (observed 5s
default flaking at load avg 20).
This commit is contained in:
Garry Tan
2026-06-12 13:24:34 -07:00
parent 1077868e36
commit b3fd66bfc6
7 changed files with 46 additions and 30 deletions
Download Patch File Download Diff File Expand all files Collapse all files
test/brainbench-adapters.test.ts
+9 -9
View File
@@ -60,7 +60,7 @@ describe('runReflexPipeline (the ONE pipeline, decision 13)', () => {
suppression: 'prior-context',
});
expect(block?.pointers.map((p) => p.slug)).toContain('people/alice-example');
});
}, 30_000);
test('budget cap: maxPointers=1 truncates a multi-entity turn', async () => {
const block = await runReflexPipeline(
@@ -71,7 +71,7 @@ describe('runReflexPipeline (the ONE pipeline, decision 13)', () => {
{ maxPointers: 1, suppression: 'prior-context' },
);
expect(block?.pointers.length).toBe(1);
});
}, 30_000);
test('suppression prior-context: an already-seen slug is not re-injected; suppression none re-injects', async () => {
const prior = 'earlier we discussed people/alice-example in detail';
@@ -85,7 +85,7 @@ describe('runReflexPipeline (the ONE pipeline, decision 13)', () => {
suppression: 'none',
});
expect(reinjected?.pointers.map((p) => p.slug)).toContain('people/alice-example');
});
}, 30_000);
test('no candidates → null (silence, not an empty block)', async () => {
const block = await runReflexPipeline(engine, 'default', turn('ok thanks, sounds good'), '', {
@@ -93,7 +93,7 @@ describe('runReflexPipeline (the ONE pipeline, decision 13)', () => {
suppression: 'prior-context',
});
expect(block).toBeNull();
});
}, 30_000);
});
describe('OpenClawAdapter (production seam)', () => {
@@ -109,7 +109,7 @@ describe('OpenClawAdapter (production seam)', () => {
const r2 = await a.replayTurn(turn('Does Alice Example have consent lined up?', 2), prior);
expect(r2.injectedSlugs).not.toContain('people/alice-example');
await a.endConversation();
});
}, 30_000);
});
describe('ClaudeCodeAdapter (contract seam: hook wire shape, no conversation memory)', () => {
@@ -121,7 +121,7 @@ describe('ClaudeCodeAdapter (contract seam: hook wire shape, no conversation mem
// No memory: prior context does NOT suppress (the measured contract delta).
expect(r.injectedSlugs).toContain('people/alice-example');
await a.endConversation();
});
}, 30_000);
test('respects the 2-pointer hook budget', async () => {
const a = new ClaudeCodeAdapter();
@@ -129,7 +129,7 @@ describe('ClaudeCodeAdapter (contract seam: hook wire shape, no conversation mem
const r = await a.replayTurn(turn('Memo: Alice Example, Charlie Example, and Widget Co all in one.'), '');
expect(r.injectedSlugs.length).toBeLessThanOrEqual(2);
await a.endConversation();
});
}, 30_000);
});
describe('CodexAdapter (contract seam: static preamble + ≤1 fragment)', () => {
@@ -144,7 +144,7 @@ describe('CodexAdapter (contract seam: static preamble + ≤1 fragment)', () =>
// First turn carried the preamble cost; later turns don't re-pay it.
expect(r1.injectedTokens).toBeGreaterThan(r2.injectedTokens);
await a.endConversation();
});
}, 30_000);
});
describe('sentinel isolation (the engine-sharing guarantee, eng-review D9)', () => {
@@ -164,5 +164,5 @@ describe('sentinel isolation (the engine-sharing guarantee, eng-review D9)', ()
expect(facts[0].n).toBe('0');
// re-seed for any later test in this file (none currently, but keep the brain valid)
await seedBrain(engine, FIXTURE);
});
}, 30_000);
});
test/brainbench-continuity.test.ts
+5 -5
View File
@@ -43,7 +43,7 @@ describe('writer openclaw → reader codex (and reverse) on a shared brain', ()
const readerRows = out.turn_rows.filter((r) => r.suite === 'continuity');
expect(readerRows.length).toBeGreaterThan(0);
expect(readerRows.every((r) => r.fixture_id === 'cont-001-widget-pass-reader')).toBe(true);
});
}, 30_000);
test('single-harness run falls back to the diagonal (writer == reader) instead of vanishing', async () => {
const out = await runBrainBench(pairCorpus, {
@@ -55,7 +55,7 @@ describe('writer openclaw → reader codex (and reverse) on a shared brain', ()
const cell = out.cells.find((c) => c.suite === 'continuity' && c.harness === 'openclaw');
expect(cell).toBeDefined();
expect(cell!.metrics.continuity_rate).toBe(1);
});
}, 30_000);
});
describe('factKeywordProbe + the miss path', () => {
@@ -81,11 +81,11 @@ describe('factKeywordProbe + the miss path', () => {
test('AND semantics: every keyword must match; partial sets fail', async () => {
expect(await factKeywordProbe(engine, 'default', ['pass', 'widget-co'])).toBe(true);
expect(await factKeywordProbe(engine, 'default', ['pass', 'no-such-keyword'])).toBe(false);
});
}, 30_000);
test('expired facts are excluded — a superseded decision is not "recalled"', async () => {
expect(await factKeywordProbe(engine, 'default', ['lead', 'acme-example'])).toBe(false);
});
}, 30_000);
test('a decision neither injected nor stored counts into gold_failed with a named item', async () => {
const score = await scoreContinuityPair(engine, 'default', 'pair-x', [], [
@@ -97,5 +97,5 @@ describe('factKeywordProbe + the miss path', () => {
expect(score.hits['d-missing']).toBe(false);
expect(score.hits['d-hit']).toBe(true);
expect(score.failed_items[0]).toContain('pair-x/d-missing');
});
}, 30_000);
});
test/brainbench-writeback.test.ts
+6 -6
View File
@@ -56,7 +56,7 @@ describe('makeGoldExtractor', () => {
expect(out[0].entity_slug).toBe('people/alice-example');
const none = await extractor({ turnText: 'unrelated segment text', source: 'cli:x' });
expect(none.length).toBe(0);
});
}, 30_000);
});
describe('runWriteBack (deterministic, production pipeline)', () => {
@@ -77,7 +77,7 @@ describe('runWriteBack (deterministic, production pipeline)', () => {
);
expect(rows.length).toBeGreaterThanOrEqual(2);
expect(rows[0].source_session).toBe(`${PER_SEGMENT_SOURCE_PREFIX}:${slug}`);
});
}, 30_000);
test('a gold fact the pipeline drops is counted as failed, named in failed_items', async () => {
await resetTables(engine);
@@ -91,7 +91,7 @@ describe('runWriteBack (deterministic, production pipeline)', () => {
expect(score.gold_failed).toBe(1);
expect(score.metrics.write_back_fidelity).toBeCloseTo(2 / 3);
expect(score.failed_items[0]).toContain('gold fact lost');
});
}, 30_000);
test('--llm branch: real-extractor lane emits extraction metrics (stubbed transport)', async () => {
await resetTables(engine);
@@ -130,7 +130,7 @@ describe('runWriteBack (deterministic, production pipeline)', () => {
__setChatTransportForTests(null);
__setEmbedTransportForTests(null);
}
});
}, 30_000);
test('--llm extraction metrics reach the harness CELLS (review finding: they were dropped in aggregation)', async () => {
__setChatTransportForTests(async (): Promise<ChatResult> => ({
@@ -168,7 +168,7 @@ describe('runWriteBack (deterministic, production pipeline)', () => {
__setChatTransportForTests(null);
__setEmbedTransportForTests(null);
}
});
}, 30_000);
test('multi-segment conversations extract per segment (the 45-min gap splits)', async () => {
await resetTables(engine);
@@ -179,5 +179,5 @@ describe('runWriteBack (deterministic, production pipeline)', () => {
const score = await runWriteBack(engine, genWb.fixture, genWb.gold, { llm: false });
expect(score.gold_failed).toBe(0);
expect(score.metrics.write_back_fidelity).toBe(1);
});
}, 30_000);
});