mirror of
https://github.com/garrytan/gbrain.git
synced 2026-07-27 22:15:33 +00:00
* fix(extract,ingest): well-form lone UTF-16 surrogates before jsonb (#2011) excerpt() in link-extraction.ts sliced the link-context window by raw UTF-16 index, so a boundary landing inside a non-BMP char (emoji, math, CJK) left an unpaired surrogate half in `context`. Serialized to JSONB for the jsonb_to_recordset batch insert, Postgres rejects it at the ::jsonb cast and aborts the whole batch — wedging `extract --stale` because the staleness bookmark only advances on a clean finish. - text-safe.ts: new ensureWellFormed() (Bun isWellFormed/toWellFormed) — one shared surrogate-cleaning primitive. - link-extraction.ts: excerpt() well-forms the slice (root-cause fix). - batch-rows.ts: new sanitizeForJsonb() = ensureWellFormed(stripNul(s)) applied to every free-text body field (link context; timeline summary/detail/source; take claim/source). Identity/security fields stay un-sanitized and fail closed. - postgres-engine.ts + pglite-engine.ts: scalar addLink + addTimelineEntry use sanitizeForJsonb too, matching the batch path on both engines. - brainstorm/orchestrator.ts: consolidate hand-rolled sanitizeUnicode onto ensureWellFormed (also fixes consecutive-lone-surrogate mishandling). Tests: ensureWellFormed unit cases (incl. consecutive lone surrogates), an excerpt window-split regression, PGLite + Postgres-e2e surrogate cases across all free-text fields and scalar paths, and fail-closed identity-field tests proving sanitization was NOT extended to slugs/holders. * v0.42.39.0 chore: bump version and changelog (#2011) Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com> * docs: update project documentation for v0.42.39.0 Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com> * v0.42.40.0 chore: re-slot release version (was 0.42.39.0) Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com> * test: fix env-mutation isolation violations in retrieval-reflex tests check:test-isolation (rule R1) flags direct process.env mutation in non-serial test files — bun's parallel runner loads multiple files into one process, so a leaked GBRAIN_RETRIEVAL_REFLEX flips reflex behavior in unrelated tests. Both files landed via the #2019 merge; convert the beforeEach/afterEach env juggling to the canonical withEnv() wrapper, which restores the prior value via try/finally even on throw. Fixes the failing `verify` CI check on #2031 (and the `test-status` aggregate that inherits it). All 30 verify checks green locally. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com> --------- Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
185 lines
8.0 KiB
TypeScript
185 lines
8.0 KiB
TypeScript
/**
|
|
* Retrieval Reflex — resolver + assemble() regression tests (#1981, T5).
|
|
*
|
|
* Encodes the motivating failure: a turn naming an entity with an existing brain
|
|
* page must surface a pointer BEFORE the agent answers. Runs against a hermetic
|
|
* in-memory PGLite engine (no file lock). The PGLite-in-production path is
|
|
* covered by exercising the resolver through an injected resolver (the same
|
|
* shape the serve IPC / host ctx.brainQuery supply).
|
|
*/
|
|
import { describe, test, expect, beforeAll, afterAll, beforeEach } from 'bun:test';
|
|
import { PGLiteEngine } from '../src/core/pglite-engine.ts';
|
|
import { withEnv } from './helpers/with-env.ts';
|
|
import { normalizeAlias } from '../src/core/search/alias-normalize.ts';
|
|
import { resolveEntitiesToPointers } from '../src/core/context/retrieval-reflex.ts';
|
|
import { extractCandidates } from '../src/core/context/entity-salience.ts';
|
|
import { createGBrainContextEngine } from '../src/core/context-engine.ts';
|
|
import { disposeReflex } from '../src/core/context/reflex.ts';
|
|
import { TAKES_FENCE_BEGIN, TAKES_FENCE_END } from '../src/core/takes-fence.ts';
|
|
|
|
let engine: PGLiteEngine;
|
|
|
|
async function seed(slug: string, title: string, body: string, source = 'default') {
|
|
await engine.executeRaw(
|
|
`INSERT INTO pages (slug, source_id, type, title, compiled_truth, timeline)
|
|
VALUES ($1, $2, 'person', $3, $4, '')`,
|
|
[slug, source, title, body],
|
|
);
|
|
}
|
|
|
|
beforeAll(async () => {
|
|
engine = new PGLiteEngine();
|
|
await engine.connect({});
|
|
await engine.initSchema();
|
|
}, 60_000);
|
|
|
|
afterAll(async () => {
|
|
await engine.disconnect();
|
|
await disposeReflex();
|
|
});
|
|
|
|
beforeEach(async () => {
|
|
await engine.executeRaw('DELETE FROM page_aliases').catch(() => {});
|
|
await engine.executeRaw('DELETE FROM pages');
|
|
});
|
|
|
|
describe('resolveEntitiesToPointers', () => {
|
|
test('namespaced slug resolves from a bare title (the recall fix, D6)', async () => {
|
|
await seed('people/alice-example', 'Alice Example', 'Alice is an early founder.');
|
|
const candidates = extractCandidates('what do you think about Alice Example?');
|
|
const block = await resolveEntitiesToPointers(engine, 'default', candidates, {});
|
|
expect(block).not.toBeNull();
|
|
expect(block!.pointers[0].slug).toBe('people/alice-example');
|
|
expect(block!.text).toContain('people/alice-example');
|
|
expect(block!.text).toContain('use get_page');
|
|
});
|
|
|
|
test('alias arm resolves an unambiguous single-slug hit', async () => {
|
|
await seed('people/swami-x', 'Swami X', 'A close friend.');
|
|
await engine.setPageAliases('people/swami-x', 'default', [normalizeAlias('Swami')]);
|
|
const block = await resolveEntitiesToPointers(engine, 'default', extractCandidates('Spoke with Swami today'), {});
|
|
expect(block).not.toBeNull();
|
|
expect(block!.pointers.some((p) => p.slug === 'people/swami-x')).toBe(true);
|
|
});
|
|
|
|
test('privacy (D5): takes-fence content never leaks into the synopsis', async () => {
|
|
const body = `${TAKES_FENCE_BEGIN}\nSECRET_HUNCH_DO_NOT_LEAK\n${TAKES_FENCE_END}\nAlice is a founder.`;
|
|
await seed('people/alice-example', 'Alice Example', body);
|
|
const block = await resolveEntitiesToPointers(engine, 'default', extractCandidates('about Alice Example'), {});
|
|
expect(block).not.toBeNull();
|
|
expect(block!.text).not.toContain('SECRET_HUNCH_DO_NOT_LEAK');
|
|
});
|
|
|
|
test('suppression: a slug already in PRIOR context is dropped', async () => {
|
|
await seed('people/alice-example', 'Alice Example', 'A founder.');
|
|
const candidates = extractCandidates('tell me about Alice Example');
|
|
const block = await resolveEntitiesToPointers(engine, 'default', candidates, {
|
|
priorContextText: 'earlier we already opened people/alice-example and read it',
|
|
});
|
|
expect(block).toBeNull();
|
|
});
|
|
|
|
test('empty candidates → null', async () => {
|
|
expect(await resolveEntitiesToPointers(engine, 'default', [], {})).toBeNull();
|
|
});
|
|
|
|
test('cap to maxPointers', async () => {
|
|
await seed('people/aa', 'Aa Bb', 'x');
|
|
await seed('people/cc', 'Cc Dd', 'y');
|
|
await seed('people/ee', 'Ee Ff', 'z');
|
|
const block = await resolveEntitiesToPointers(
|
|
engine,
|
|
'default',
|
|
extractCandidates('met Aa Bb, Cc Dd, and Ee Ff'),
|
|
{ maxPointers: 2 },
|
|
);
|
|
expect(block!.pointers.length).toBe(2);
|
|
});
|
|
|
|
test('pre-v110 brains: alias-table absence does not break the slug arm', async () => {
|
|
await seed('people/alice-example', 'Alice Example', 'A founder.');
|
|
// Simulate no page_aliases table.
|
|
await engine.executeRaw('DROP TABLE IF EXISTS page_aliases');
|
|
const block = await resolveEntitiesToPointers(engine, 'default', extractCandidates('about Alice Example'), {});
|
|
expect(block).not.toBeNull();
|
|
expect(block!.pointers[0].slug).toBe('people/alice-example');
|
|
// restore for other tests
|
|
await engine.initSchema();
|
|
});
|
|
});
|
|
|
|
describe('context-engine assemble() — Retrieval Reflex integration', () => {
|
|
// Each test wraps its body in withEnv (NOT a beforeEach env mutation) so the
|
|
// flag is restored even on throw — required by check-test-isolation rule R1.
|
|
const REFLEX_ON = { GBRAIN_RETRIEVAL_REFLEX: 'true' };
|
|
|
|
test('regression: a named entity with a page surfaces a pointer (host resolver path)', async () => {
|
|
await withEnv(REFLEX_ON, async () => {
|
|
await seed('people/alice-example', 'Alice Example', 'Alice is a founder.');
|
|
// Inject a resolver the way the OpenClaw host (ctx.brainQuery) or serve IPC would.
|
|
const ce = createGBrainContextEngine({
|
|
workspaceDir: '/tmp/rr-test-ws',
|
|
resolveEntities: (candidates, opts) =>
|
|
resolveEntitiesToPointers(engine, 'default', candidates, opts),
|
|
});
|
|
const res = await ce.assemble({
|
|
sessionId: 's1',
|
|
messages: [{ role: 'user', content: 'what do you think about Alice Example?' }],
|
|
});
|
|
expect(res.systemPromptAddition).toContain('Brain pages mentioned this turn');
|
|
expect(res.systemPromptAddition).toContain('people/alice-example');
|
|
expect(res.systemPromptAddition).toContain('use get_page');
|
|
});
|
|
});
|
|
|
|
test('no resolver available (PGLite, no serve/host) → no throw, live context still present', async () => {
|
|
await withEnv(REFLEX_ON, async () => {
|
|
const ce = createGBrainContextEngine({ workspaceDir: '/tmp/rr-test-ws-2' });
|
|
const res = await ce.assemble({
|
|
sessionId: 's2',
|
|
messages: [{ role: 'user', content: 'what about Alice Example?' }],
|
|
});
|
|
// Live Context block always ships; no pointer block (nothing resolved).
|
|
expect(res.systemPromptAddition).toContain('Live Context');
|
|
expect(res.systemPromptAddition).not.toContain('Brain pages mentioned this turn');
|
|
});
|
|
});
|
|
|
|
test('zero salient candidates → no brain touch, no pointer block', async () => {
|
|
await withEnv(REFLEX_ON, async () => {
|
|
let called = false;
|
|
const ce = createGBrainContextEngine({
|
|
workspaceDir: '/tmp/rr-test-ws-3',
|
|
resolveEntities: async () => { called = true; return null; },
|
|
});
|
|
const res = await ce.assemble({
|
|
sessionId: 's3',
|
|
messages: [{ role: 'user', content: 'can you help me with this?' }],
|
|
});
|
|
expect(called).toBe(false);
|
|
expect(res.systemPromptAddition).not.toContain('Brain pages mentioned this turn');
|
|
});
|
|
});
|
|
|
|
test('suppression uses PRIOR turns only, not the current message', async () => {
|
|
await withEnv(REFLEX_ON, async () => {
|
|
await seed('people/alice-example', 'Alice Example', 'A founder.');
|
|
const ce = createGBrainContextEngine({
|
|
workspaceDir: '/tmp/rr-test-ws-4',
|
|
resolveEntities: (candidates, opts) =>
|
|
resolveEntitiesToPointers(engine, 'default', candidates, opts),
|
|
});
|
|
// The current message names Alice Example; prior context does NOT. Must fire.
|
|
const res = await ce.assemble({
|
|
sessionId: 's4',
|
|
messages: [
|
|
{ role: 'user', content: 'hello' },
|
|
{ role: 'assistant', content: 'hi there' },
|
|
{ role: 'user', content: 'what do you think about Alice Example?' },
|
|
],
|
|
});
|
|
expect(res.systemPromptAddition).toContain('people/alice-example');
|
|
});
|
|
});
|
|
});
|