mirror of
https://github.com/garrytan/gbrain.git
synced 2026-07-27 22:15:33 +00:00
* fix(extract,ingest): well-form lone UTF-16 surrogates before jsonb (#2011) excerpt() in link-extraction.ts sliced the link-context window by raw UTF-16 index, so a boundary landing inside a non-BMP char (emoji, math, CJK) left an unpaired surrogate half in `context`. Serialized to JSONB for the jsonb_to_recordset batch insert, Postgres rejects it at the ::jsonb cast and aborts the whole batch — wedging `extract --stale` because the staleness bookmark only advances on a clean finish. - text-safe.ts: new ensureWellFormed() (Bun isWellFormed/toWellFormed) — one shared surrogate-cleaning primitive. - link-extraction.ts: excerpt() well-forms the slice (root-cause fix). - batch-rows.ts: new sanitizeForJsonb() = ensureWellFormed(stripNul(s)) applied to every free-text body field (link context; timeline summary/detail/source; take claim/source). Identity/security fields stay un-sanitized and fail closed. - postgres-engine.ts + pglite-engine.ts: scalar addLink + addTimelineEntry use sanitizeForJsonb too, matching the batch path on both engines. - brainstorm/orchestrator.ts: consolidate hand-rolled sanitizeUnicode onto ensureWellFormed (also fixes consecutive-lone-surrogate mishandling). Tests: ensureWellFormed unit cases (incl. consecutive lone surrogates), an excerpt window-split regression, PGLite + Postgres-e2e surrogate cases across all free-text fields and scalar paths, and fail-closed identity-field tests proving sanitization was NOT extended to slugs/holders. * v0.42.39.0 chore: bump version and changelog (#2011) Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com> * docs: update project documentation for v0.42.39.0 Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com> * v0.42.40.0 chore: re-slot release version (was 0.42.39.0) Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com> * test: fix env-mutation isolation violations in retrieval-reflex tests check:test-isolation (rule R1) flags direct process.env mutation in non-serial test files — bun's parallel runner loads multiple files into one process, so a leaked GBRAIN_RETRIEVAL_REFLEX flips reflex behavior in unrelated tests. Both files landed via the #2019 merge; convert the beforeEach/afterEach env juggling to the canonical withEnv() wrapper, which restores the prior value via try/finally even on throw. Fixes the failing `verify` CI check on #2031 (and the `test-status` aggregate that inherits it). All 30 verify checks green locally. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com> --------- Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
Garry Tan
co-authored by
Claude Opus 4.8
Claude Opus 4.8
parent
8f45624e55
commit
ecd6ae8772
@@ -1,46 +1,44 @@
|
||||
/**
|
||||
* Doctor retrieval_reflex_health check (#1981, T8).
|
||||
*/
|
||||
import { describe, test, expect, afterEach } from 'bun:test';
|
||||
import { describe, test, expect } from 'bun:test';
|
||||
import { mkdtempSync, mkdirSync, writeFileSync, rmSync } from 'node:fs';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { join } from 'node:path';
|
||||
import { buildRetrievalReflexCheck } from '../src/commands/doctor.ts';
|
||||
|
||||
const origEnv = process.env.GBRAIN_RETRIEVAL_REFLEX;
|
||||
afterEach(() => {
|
||||
if (origEnv === undefined) delete process.env.GBRAIN_RETRIEVAL_REFLEX;
|
||||
else process.env.GBRAIN_RETRIEVAL_REFLEX = origEnv;
|
||||
});
|
||||
import { withEnv } from './helpers/with-env.ts';
|
||||
|
||||
describe('buildRetrievalReflexCheck', () => {
|
||||
test('disabled via env → warn, names the right check', () => {
|
||||
process.env.GBRAIN_RETRIEVAL_REFLEX = 'false';
|
||||
const c = buildRetrievalReflexCheck(null);
|
||||
expect(c.name).toBe('retrieval_reflex_health');
|
||||
expect(c.status).toBe('warn');
|
||||
expect(c.message).toContain('disabled');
|
||||
expect((c.details as any)?.enabled).toBe(false);
|
||||
test('disabled via env → warn, names the right check', async () => {
|
||||
await withEnv({ GBRAIN_RETRIEVAL_REFLEX: 'false' }, async () => {
|
||||
const c = buildRetrievalReflexCheck(null);
|
||||
expect(c.name).toBe('retrieval_reflex_health');
|
||||
expect(c.status).toBe('warn');
|
||||
expect(c.message).toContain('disabled');
|
||||
expect((c.details as any)?.enabled).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
test('enabled → reports policy-skill install state in details', () => {
|
||||
process.env.GBRAIN_RETRIEVAL_REFLEX = 'true';
|
||||
const dir = mkdtempSync(join(tmpdir(), 'rr-doctor-'));
|
||||
mkdirSync(join(dir, 'retrieval-reflex'), { recursive: true });
|
||||
writeFileSync(join(dir, 'retrieval-reflex', 'SKILL.md'), '# stub\n');
|
||||
const c = buildRetrievalReflexCheck(dir);
|
||||
expect(c.name).toBe('retrieval_reflex_health');
|
||||
expect((c.details as any)?.enabled).toBe(true);
|
||||
expect((c.details as any)?.policy_skill_installed).toBe(true);
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
test('enabled → reports policy-skill install state in details', async () => {
|
||||
await withEnv({ GBRAIN_RETRIEVAL_REFLEX: 'true' }, async () => {
|
||||
const dir = mkdtempSync(join(tmpdir(), 'rr-doctor-'));
|
||||
mkdirSync(join(dir, 'retrieval-reflex'), { recursive: true });
|
||||
writeFileSync(join(dir, 'retrieval-reflex', 'SKILL.md'), '# stub\n');
|
||||
const c = buildRetrievalReflexCheck(dir);
|
||||
expect(c.name).toBe('retrieval_reflex_health');
|
||||
expect((c.details as any)?.enabled).toBe(true);
|
||||
expect((c.details as any)?.policy_skill_installed).toBe(true);
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
});
|
||||
});
|
||||
|
||||
test('enabled, policy skill absent → message includes the install hint', () => {
|
||||
process.env.GBRAIN_RETRIEVAL_REFLEX = 'true';
|
||||
const dir = mkdtempSync(join(tmpdir(), 'rr-doctor-2-'));
|
||||
const c = buildRetrievalReflexCheck(dir);
|
||||
expect((c.details as any)?.policy_skill_installed).toBe(false);
|
||||
expect(c.message).toContain('gbrain integrations install retrieval-reflex');
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
test('enabled, policy skill absent → message includes the install hint', async () => {
|
||||
await withEnv({ GBRAIN_RETRIEVAL_REFLEX: 'true' }, async () => {
|
||||
const dir = mkdtempSync(join(tmpdir(), 'rr-doctor-2-'));
|
||||
const c = buildRetrievalReflexCheck(dir);
|
||||
expect((c.details as any)?.policy_skill_installed).toBe(false);
|
||||
expect(c.message).toContain('gbrain integrations install retrieval-reflex');
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
@@ -20,6 +20,15 @@ const POISON =
|
||||
'"Q2 sync" — notes {a,b}, path C:\\Users\\x, em–dash } and { trailing';
|
||||
const NUL = String.fromCharCode(0);
|
||||
|
||||
// #2011 — lone UTF-16 high surrogate. THIS is the value that crashed Postgres at
|
||||
// the ::jsonb cast (22P02) and aborted `extract --stale` on a managed Supabase
|
||||
// brain. The PGLite sibling rejects it too on current builds, but this lane is
|
||||
// the production engine the bug was reported against. Built via fromCharCode so
|
||||
// no lone surrogate is a literal in this file.
|
||||
const LONE_HI = String.fromCharCode(0xd83c);
|
||||
const SURROGATE = `before${LONE_HI}after`;
|
||||
const SURROGATE_CLEAN = 'before�after';
|
||||
|
||||
let engine: PostgresEngine;
|
||||
|
||||
async function seed(slug: string) {
|
||||
@@ -100,4 +109,78 @@ d('JSONB batch poison — Postgres (#1861)', () => {
|
||||
expect(rows[0].active).toBe(false);
|
||||
expect(rows[0].since_date).toBeNull();
|
||||
});
|
||||
|
||||
// ── #2011: lone-surrogate ::jsonb crash lock (the abort this PR fixes) ──
|
||||
|
||||
it('addLinksBatch survives a lone surrogate in context (the #2011 crash)', async () => {
|
||||
await seed('from-page'); await seed('to-page');
|
||||
const n = await engine.addLinksBatch([
|
||||
{ from_slug: 'from-page', to_slug: 'to-page', link_type: 'mentions',
|
||||
context: SURROGATE, link_source: 'manual' },
|
||||
]);
|
||||
expect(n).toBe(1); // pre-fix: threw "invalid input syntax for type json"
|
||||
const links = await engine.getLinks('from-page', { sourceId: 'default' });
|
||||
expect(links[0].context).toBe(SURROGATE_CLEAN);
|
||||
expect(links[0].context.isWellFormed()).toBe(true);
|
||||
});
|
||||
|
||||
it('addTimelineEntriesBatch survives a lone surrogate in source (D2)', async () => {
|
||||
await seed('meeting-page');
|
||||
const n = await engine.addTimelineEntriesBatch([
|
||||
{ slug: 'meeting-page', date: '2026-06-04', source: SURROGATE,
|
||||
summary: SURROGATE, detail: SURROGATE },
|
||||
]);
|
||||
expect(n).toBe(1);
|
||||
const rows = await engine.executeRaw<{ summary: string; source: string }>(
|
||||
`SELECT te.summary, te.source FROM timeline_entries te
|
||||
JOIN pages p ON p.id = te.page_id WHERE p.slug = 'meeting-page'`,
|
||||
);
|
||||
expect(rows[0].summary).toBe(SURROGATE_CLEAN);
|
||||
expect(rows[0].source).toBe(SURROGATE_CLEAN);
|
||||
});
|
||||
|
||||
it('addTakesBatch survives a lone surrogate in claim and source', async () => {
|
||||
await seed('take-page');
|
||||
const pid = await pageId('take-page');
|
||||
const n = await engine.addTakesBatch([
|
||||
{ page_id: pid, row_num: 1, claim: SURROGATE, kind: 'fact', holder: 'h', source: SURROGATE },
|
||||
]);
|
||||
expect(n).toBe(1);
|
||||
const rows = await engine.executeRaw<{ claim: string; source: string }>(
|
||||
`SELECT claim, source FROM takes t JOIN pages p ON p.id = t.page_id
|
||||
WHERE p.slug = 'take-page' AND t.row_num = 1`,
|
||||
);
|
||||
expect(rows[0].claim).toBe(SURROGATE_CLEAN);
|
||||
expect(rows[0].source).toBe(SURROGATE_CLEAN); // free-text provenance, sanitized too
|
||||
});
|
||||
|
||||
it('scalar addLink + addTimelineEntry survive a lone surrogate', async () => {
|
||||
await seed('sa'); await seed('sb');
|
||||
await engine.addLink('sa', 'sb', SURROGATE, 'mentions', 'manual');
|
||||
const links = await engine.getLinks('sa', { sourceId: 'default' });
|
||||
expect(links[0].context).toBe(SURROGATE_CLEAN);
|
||||
|
||||
await engine.addTimelineEntry('sa', {
|
||||
date: '2026-06-05', source: SURROGATE, summary: SURROGATE, detail: SURROGATE,
|
||||
});
|
||||
const rows = await engine.executeRaw<{ source: string }>(
|
||||
`SELECT te.source FROM timeline_entries te JOIN pages p ON p.id = te.page_id
|
||||
WHERE p.slug = 'sa'`,
|
||||
);
|
||||
expect(rows[0].source).toBe(SURROGATE_CLEAN);
|
||||
});
|
||||
|
||||
it('fail-closed: a lone surrogate in an IDENTITY field still rejects the batch', async () => {
|
||||
// The NUL/surrogate policy deliberately leaves identity fields raw so a
|
||||
// malformed slug/holder can never silently retarget a row. On Postgres the
|
||||
// raw lone surrogate hits the ::jsonb cast and rejects the whole batch —
|
||||
// loud failure, not silent normalization. This locks that we did NOT extend
|
||||
// sanitization to identity fields.
|
||||
await seed('idfrom'); await seed('idto');
|
||||
await expect(
|
||||
engine.addLinksBatch([
|
||||
{ from_slug: `idfrom${LONE_HI}`, to_slug: 'idto', link_type: 'mentions', link_source: 'manual' },
|
||||
]),
|
||||
).rejects.toThrow();
|
||||
});
|
||||
});
|
||||
|
||||
@@ -228,6 +228,32 @@ describe('extractPageLinks', () => {
|
||||
expect(aliceLink!.linkType).toBe('works_at');
|
||||
});
|
||||
|
||||
test('#2011: excerpt window slicing a non-BMP char yields well-formed context', async () => {
|
||||
// Reproduce the abort trigger: a markdown ref whose 240-char context window
|
||||
// boundary lands inside an emoji's surrogate pair. Pre-fix, the slice kept a
|
||||
// lone high surrogate in `context`, which Postgres rejected at the ::jsonb
|
||||
// cast and aborted the whole `extract --stale` run.
|
||||
const ROCKET = '🚀'; // U+1F680 = [0xD83D, 0xDE80]
|
||||
const head = '[Alice](people/alice)';
|
||||
const idx = head.indexOf('Alice'); // excerpt centers on ref.name
|
||||
const half = 120; // width 240 / 2
|
||||
// Place the emoji so its HIGH half sits at index (idx+half-1) and its LOW
|
||||
// half at (idx+half) — exactly the excerpt `end` boundary, splitting it.
|
||||
const padLen = idx + half - 1 - head.length;
|
||||
const content = head + 'x'.repeat(padLen) + ROCKET + ' trailing context';
|
||||
|
||||
// Sanity: confirm the fixture actually splits a pair (the raw window is
|
||||
// malformed). If this ever stops being malformed, the regression is moot.
|
||||
const rawWindow = content.slice(Math.max(0, idx - half), idx + half);
|
||||
expect(rawWindow.isWellFormed()).toBe(false);
|
||||
|
||||
const { candidates } = await extractPageLinks('docs/x', content, {}, 'concept', allowAllResolver);
|
||||
const alice = candidates.find(c => c.targetSlug === 'people/alice');
|
||||
expect(alice).toBeDefined();
|
||||
expect(alice!.context.isWellFormed()).toBe(true);
|
||||
expect(JSON.parse(JSON.stringify(alice!.context))).toBe(alice!.context);
|
||||
});
|
||||
|
||||
test('dedups multiple mentions of same entity (within-page dedup)', async () => {
|
||||
const content = '[Alice](people/alice) said this. Later, [Alice](people/alice) said that.';
|
||||
const { candidates } = await extractPageLinks('docs/x', content, {}, 'concept', allowAllResolver);
|
||||
|
||||
@@ -23,6 +23,17 @@ const POISON =
|
||||
// fromCharCode so no literal NUL byte ever lands in this source file.
|
||||
const NUL = String.fromCharCode(0);
|
||||
|
||||
// #2011 — a lone UTF-16 high surrogate (no low partner). This is what excerpt()
|
||||
// left behind when a context window boundary sliced an emoji; Postgres rejects
|
||||
// it inside the ::jsonb cast and aborts the whole batch. The row builders now
|
||||
// well-form it to U+FFFD. Built via fromCharCode so no lone surrogate is ever a
|
||||
// literal in this source file. PGLite serializes jsonb differently and may not
|
||||
// reproduce the original Postgres crash, but it still locks the JS-side
|
||||
// sanitization (buildLinkRows/...): the stored value must be well-formed.
|
||||
const LONE_HI = String.fromCharCode(0xd83c);
|
||||
const SURROGATE = `before${LONE_HI}after`;
|
||||
const SURROGATE_CLEAN = 'before�after';
|
||||
|
||||
let engine: PGLiteEngine;
|
||||
|
||||
beforeAll(async () => {
|
||||
@@ -244,3 +255,98 @@ describe('scalar addLink — NUL strip on free-text context (#1861 codex #3)', (
|
||||
expect(links[0].context).toBe('beforeafter');
|
||||
});
|
||||
});
|
||||
|
||||
describe('#2011 — lone-surrogate well-forming across all prose write paths', () => {
|
||||
it('batch link context: lone surrogate well-formed, insert succeeds', async () => {
|
||||
await seed('a'); await seed('b');
|
||||
const n = await engine.addLinksBatch([
|
||||
{ from_slug: 'a', to_slug: 'b', link_type: 'mentions', context: SURROGATE, link_source: 'manual' },
|
||||
]);
|
||||
expect(n).toBe(1);
|
||||
const links = await engine.getLinks('a', { sourceId: 'default' });
|
||||
expect(links[0].context).toBe(SURROGATE_CLEAN);
|
||||
expect(links[0].context.isWellFormed()).toBe(true);
|
||||
});
|
||||
|
||||
it('batch timeline summary/detail/source: all lone surrogates well-formed', async () => {
|
||||
await seed('m');
|
||||
const n = await engine.addTimelineEntriesBatch([
|
||||
{ slug: 'm', date: '2026-06-04', source: SURROGATE, summary: SURROGATE, detail: SURROGATE },
|
||||
]);
|
||||
expect(n).toBe(1);
|
||||
const rows = await engine.executeRaw<{ summary: string; detail: string; source: string }>(
|
||||
`SELECT te.summary, te.detail, te.source FROM timeline_entries te
|
||||
JOIN pages p ON p.id = te.page_id WHERE p.slug = 'm'`,
|
||||
);
|
||||
expect(rows[0].summary).toBe(SURROGATE_CLEAN);
|
||||
expect(rows[0].detail).toBe(SURROGATE_CLEAN);
|
||||
expect(rows[0].source).toBe(SURROGATE_CLEAN); // D2: timeline source now sanitized
|
||||
expect(rows[0].source.isWellFormed()).toBe(true);
|
||||
});
|
||||
|
||||
it('batch take claim: lone surrogate well-formed', async () => {
|
||||
await seed('tk');
|
||||
const pid = await pageId('tk');
|
||||
await engine.addTakesBatch([
|
||||
{ page_id: pid, row_num: 1, claim: SURROGATE, kind: 'fact', holder: 'h' },
|
||||
]);
|
||||
const rows = await engine.executeRaw<{ claim: string }>(
|
||||
`SELECT claim FROM takes t JOIN pages p ON p.id = t.page_id
|
||||
WHERE p.slug = 'tk' AND t.row_num = 1`,
|
||||
);
|
||||
expect(rows[0].claim).toBe(SURROGATE_CLEAN);
|
||||
expect(rows[0].claim.isWellFormed()).toBe(true);
|
||||
});
|
||||
|
||||
it('batch take source (free-text provenance): lone surrogate well-formed', async () => {
|
||||
await seed('tks');
|
||||
const pid = await pageId('tks');
|
||||
await engine.addTakesBatch([
|
||||
{ page_id: pid, row_num: 1, claim: 'c', kind: 'fact', holder: 'h', source: SURROGATE },
|
||||
]);
|
||||
const rows = await engine.executeRaw<{ source: string }>(
|
||||
`SELECT source FROM takes t JOIN pages p ON p.id = t.page_id
|
||||
WHERE p.slug = 'tks' AND t.row_num = 1`,
|
||||
);
|
||||
expect(rows[0].source).toBe(SURROGATE_CLEAN);
|
||||
expect(rows[0].source.isWellFormed()).toBe(true);
|
||||
});
|
||||
|
||||
it('scalar addLink: lone surrogate in context well-formed', async () => {
|
||||
await seed('sa'); await seed('sb');
|
||||
await engine.addLink('sa', 'sb', SURROGATE, 'mentions', 'manual');
|
||||
const links = await engine.getLinks('sa', { sourceId: 'default' });
|
||||
expect(links[0].context).toBe(SURROGATE_CLEAN);
|
||||
expect(links[0].context.isWellFormed()).toBe(true);
|
||||
});
|
||||
|
||||
it('scalar addTimelineEntry: lone surrogate in summary/detail/source well-formed', async () => {
|
||||
await seed('sm');
|
||||
await engine.addTimelineEntry('sm', {
|
||||
date: '2026-06-05', source: SURROGATE, summary: SURROGATE, detail: SURROGATE,
|
||||
});
|
||||
const rows = await engine.executeRaw<{ summary: string; detail: string; source: string }>(
|
||||
`SELECT te.summary, te.detail, te.source FROM timeline_entries te
|
||||
JOIN pages p ON p.id = te.page_id WHERE p.slug = 'sm'`,
|
||||
);
|
||||
expect(rows[0].summary).toBe(SURROGATE_CLEAN);
|
||||
expect(rows[0].detail).toBe(SURROGATE_CLEAN);
|
||||
expect(rows[0].source).toBe(SURROGATE_CLEAN);
|
||||
});
|
||||
|
||||
it('fail-closed: identity fields are NOT sanitized — a lone surrogate in to_slug rejects the batch', async () => {
|
||||
// The NUL/surrogate policy deliberately leaves identity fields raw so a
|
||||
// malformed slug can never be silently normalized into one that matches a
|
||||
// different page. This PGLite build rejects the lone surrogate at the
|
||||
// ::jsonb cast (22P02) — a loud failure, exactly the intended boundary
|
||||
// (and proof we did NOT extend sanitizeForJsonb to identity columns).
|
||||
await seed('idfrom'); await seed('idto');
|
||||
await expect(
|
||||
engine.addLinksBatch([
|
||||
{ from_slug: 'idfrom', to_slug: `idto${LONE_HI}`, link_type: 'mentions', link_source: 'manual' },
|
||||
]),
|
||||
).rejects.toThrow();
|
||||
const links = await engine.getLinks('idfrom', { sourceId: 'default' });
|
||||
expect(links).toHaveLength(0);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -9,6 +9,7 @@
|
||||
*/
|
||||
import { describe, test, expect, beforeAll, afterAll, beforeEach } from 'bun:test';
|
||||
import { PGLiteEngine } from '../src/core/pglite-engine.ts';
|
||||
import { withEnv } from './helpers/with-env.ts';
|
||||
import { normalizeAlias } from '../src/core/search/alias-normalize.ts';
|
||||
import { resolveEntitiesToPointers } from '../src/core/context/retrieval-reflex.ts';
|
||||
import { extractCandidates } from '../src/core/context/entity-salience.ts';
|
||||
@@ -108,68 +109,76 @@ describe('resolveEntitiesToPointers', () => {
|
||||
});
|
||||
|
||||
describe('context-engine assemble() — Retrieval Reflex integration', () => {
|
||||
beforeEach(() => {
|
||||
process.env.GBRAIN_RETRIEVAL_REFLEX = 'true';
|
||||
});
|
||||
// Each test wraps its body in withEnv (NOT a beforeEach env mutation) so the
|
||||
// flag is restored even on throw — required by check-test-isolation rule R1.
|
||||
const REFLEX_ON = { GBRAIN_RETRIEVAL_REFLEX: 'true' };
|
||||
|
||||
test('regression: a named entity with a page surfaces a pointer (host resolver path)', async () => {
|
||||
await seed('people/alice-example', 'Alice Example', 'Alice is a founder.');
|
||||
// Inject a resolver the way the OpenClaw host (ctx.brainQuery) or serve IPC would.
|
||||
const ce = createGBrainContextEngine({
|
||||
workspaceDir: '/tmp/rr-test-ws',
|
||||
resolveEntities: (candidates, opts) =>
|
||||
resolveEntitiesToPointers(engine, 'default', candidates, opts),
|
||||
await withEnv(REFLEX_ON, async () => {
|
||||
await seed('people/alice-example', 'Alice Example', 'Alice is a founder.');
|
||||
// Inject a resolver the way the OpenClaw host (ctx.brainQuery) or serve IPC would.
|
||||
const ce = createGBrainContextEngine({
|
||||
workspaceDir: '/tmp/rr-test-ws',
|
||||
resolveEntities: (candidates, opts) =>
|
||||
resolveEntitiesToPointers(engine, 'default', candidates, opts),
|
||||
});
|
||||
const res = await ce.assemble({
|
||||
sessionId: 's1',
|
||||
messages: [{ role: 'user', content: 'what do you think about Alice Example?' }],
|
||||
});
|
||||
expect(res.systemPromptAddition).toContain('Brain pages mentioned this turn');
|
||||
expect(res.systemPromptAddition).toContain('people/alice-example');
|
||||
expect(res.systemPromptAddition).toContain('use get_page');
|
||||
});
|
||||
const res = await ce.assemble({
|
||||
sessionId: 's1',
|
||||
messages: [{ role: 'user', content: 'what do you think about Alice Example?' }],
|
||||
});
|
||||
expect(res.systemPromptAddition).toContain('Brain pages mentioned this turn');
|
||||
expect(res.systemPromptAddition).toContain('people/alice-example');
|
||||
expect(res.systemPromptAddition).toContain('use get_page');
|
||||
});
|
||||
|
||||
test('no resolver available (PGLite, no serve/host) → no throw, live context still present', async () => {
|
||||
const ce = createGBrainContextEngine({ workspaceDir: '/tmp/rr-test-ws-2' });
|
||||
const res = await ce.assemble({
|
||||
sessionId: 's2',
|
||||
messages: [{ role: 'user', content: 'what about Alice Example?' }],
|
||||
await withEnv(REFLEX_ON, async () => {
|
||||
const ce = createGBrainContextEngine({ workspaceDir: '/tmp/rr-test-ws-2' });
|
||||
const res = await ce.assemble({
|
||||
sessionId: 's2',
|
||||
messages: [{ role: 'user', content: 'what about Alice Example?' }],
|
||||
});
|
||||
// Live Context block always ships; no pointer block (nothing resolved).
|
||||
expect(res.systemPromptAddition).toContain('Live Context');
|
||||
expect(res.systemPromptAddition).not.toContain('Brain pages mentioned this turn');
|
||||
});
|
||||
// Live Context block always ships; no pointer block (nothing resolved).
|
||||
expect(res.systemPromptAddition).toContain('Live Context');
|
||||
expect(res.systemPromptAddition).not.toContain('Brain pages mentioned this turn');
|
||||
});
|
||||
|
||||
test('zero salient candidates → no brain touch, no pointer block', async () => {
|
||||
let called = false;
|
||||
const ce = createGBrainContextEngine({
|
||||
workspaceDir: '/tmp/rr-test-ws-3',
|
||||
resolveEntities: async () => { called = true; return null; },
|
||||
await withEnv(REFLEX_ON, async () => {
|
||||
let called = false;
|
||||
const ce = createGBrainContextEngine({
|
||||
workspaceDir: '/tmp/rr-test-ws-3',
|
||||
resolveEntities: async () => { called = true; return null; },
|
||||
});
|
||||
const res = await ce.assemble({
|
||||
sessionId: 's3',
|
||||
messages: [{ role: 'user', content: 'can you help me with this?' }],
|
||||
});
|
||||
expect(called).toBe(false);
|
||||
expect(res.systemPromptAddition).not.toContain('Brain pages mentioned this turn');
|
||||
});
|
||||
const res = await ce.assemble({
|
||||
sessionId: 's3',
|
||||
messages: [{ role: 'user', content: 'can you help me with this?' }],
|
||||
});
|
||||
expect(called).toBe(false);
|
||||
expect(res.systemPromptAddition).not.toContain('Brain pages mentioned this turn');
|
||||
});
|
||||
|
||||
test('suppression uses PRIOR turns only, not the current message', async () => {
|
||||
await seed('people/alice-example', 'Alice Example', 'A founder.');
|
||||
const ce = createGBrainContextEngine({
|
||||
workspaceDir: '/tmp/rr-test-ws-4',
|
||||
resolveEntities: (candidates, opts) =>
|
||||
resolveEntitiesToPointers(engine, 'default', candidates, opts),
|
||||
await withEnv(REFLEX_ON, async () => {
|
||||
await seed('people/alice-example', 'Alice Example', 'A founder.');
|
||||
const ce = createGBrainContextEngine({
|
||||
workspaceDir: '/tmp/rr-test-ws-4',
|
||||
resolveEntities: (candidates, opts) =>
|
||||
resolveEntitiesToPointers(engine, 'default', candidates, opts),
|
||||
});
|
||||
// The current message names Alice Example; prior context does NOT. Must fire.
|
||||
const res = await ce.assemble({
|
||||
sessionId: 's4',
|
||||
messages: [
|
||||
{ role: 'user', content: 'hello' },
|
||||
{ role: 'assistant', content: 'hi there' },
|
||||
{ role: 'user', content: 'what do you think about Alice Example?' },
|
||||
],
|
||||
});
|
||||
expect(res.systemPromptAddition).toContain('people/alice-example');
|
||||
});
|
||||
// The current message names Alice Example; prior context does NOT. Must fire.
|
||||
const res = await ce.assemble({
|
||||
sessionId: 's4',
|
||||
messages: [
|
||||
{ role: 'user', content: 'hello' },
|
||||
{ role: 'assistant', content: 'hi there' },
|
||||
{ role: 'user', content: 'what do you think about Alice Example?' },
|
||||
],
|
||||
});
|
||||
expect(res.systemPromptAddition).toContain('people/alice-example');
|
||||
});
|
||||
});
|
||||
|
||||
+58
-1
@@ -10,7 +10,7 @@
|
||||
*/
|
||||
|
||||
import { describe, test, expect } from 'bun:test';
|
||||
import { truncateUtf8, safeSplitIndex } from '../src/core/text-safe.ts';
|
||||
import { truncateUtf8, safeSplitIndex, ensureWellFormed } from '../src/core/text-safe.ts';
|
||||
|
||||
// 🚀 = U+1F680 (surrogate pair: 0xD83D 0xDE80; JS string length = 2).
|
||||
// 𝕏 = U+1D54F (surrogate pair: 0xD835 0xDD4F; JS string length = 2).
|
||||
@@ -19,6 +19,8 @@ const ROCKET = '🚀'; // 🚀
|
||||
const MATH_X = '𝕏'; // 𝕏
|
||||
const NBMP_HAN = '𠀀'; // 𠀀
|
||||
const STRAY_HIGH = '\uD83D'; // orphaned high surrogate (invalid alone)
|
||||
const STRAY_LOW = '\uDE80'; // orphaned low surrogate (invalid alone)
|
||||
const REPLACEMENT = '�'; // U+FFFD, what toWellFormed() substitutes
|
||||
|
||||
describe('truncateUtf8', () => {
|
||||
test('returns empty for empty input', () => {
|
||||
@@ -73,6 +75,61 @@ describe('truncateUtf8', () => {
|
||||
});
|
||||
});
|
||||
|
||||
describe('ensureWellFormed (#2011)', () => {
|
||||
test('lone high surrogate → U+FFFD', () => {
|
||||
const out = ensureWellFormed('before' + STRAY_HIGH + 'after');
|
||||
expect(out).toBe('before' + REPLACEMENT + 'after');
|
||||
expect(out.isWellFormed()).toBe(true);
|
||||
});
|
||||
|
||||
test('lone low surrogate → U+FFFD', () => {
|
||||
const out = ensureWellFormed('before' + STRAY_LOW + 'after');
|
||||
expect(out).toBe('before' + REPLACEMENT + 'after');
|
||||
expect(out.isWellFormed()).toBe(true);
|
||||
});
|
||||
|
||||
test('consecutive lone low surrogates → both replaced (the case the old regex got wrong)', () => {
|
||||
// The prior hand-rolled two-pass regex produced "�\uDE80" here (the second
|
||||
// lookbehind consumed the just-inserted boundary, leaving the 2nd low
|
||||
// surrogate orphaned). The built-in replaces both → "��".
|
||||
const out = ensureWellFormed(STRAY_LOW + STRAY_LOW);
|
||||
expect(out).toBe(REPLACEMENT + REPLACEMENT);
|
||||
expect(out.isWellFormed()).toBe(true);
|
||||
});
|
||||
|
||||
test('consecutive lone high surrogates → both replaced', () => {
|
||||
const out = ensureWellFormed(STRAY_HIGH + STRAY_HIGH);
|
||||
expect(out).toBe(REPLACEMENT + REPLACEMENT);
|
||||
expect(out.isWellFormed()).toBe(true);
|
||||
});
|
||||
|
||||
test('valid pairs (emoji / math / non-BMP CJK) preserved unchanged', () => {
|
||||
const text = 'a' + ROCKET + 'b' + MATH_X + 'c' + NBMP_HAN + 'd';
|
||||
expect(ensureWellFormed(text)).toBe(text);
|
||||
expect(ensureWellFormed(text).isWellFormed()).toBe(true);
|
||||
});
|
||||
|
||||
test('clean ASCII / empty input returns an equal well-formed value', () => {
|
||||
expect(ensureWellFormed('plain text')).toBe('plain text');
|
||||
expect(ensureWellFormed('')).toBe('');
|
||||
});
|
||||
|
||||
test('mixed valid pair + lone half: pair kept, orphan replaced', () => {
|
||||
// Valid 🚀 then a stray high then text.
|
||||
const out = ensureWellFormed(ROCKET + STRAY_HIGH + 'x');
|
||||
expect(out).toBe(ROCKET + REPLACEMENT + 'x');
|
||||
expect(out.isWellFormed()).toBe(true);
|
||||
});
|
||||
|
||||
test('output is JSON-serializable without orphaned escapes', () => {
|
||||
// The #2011 failure was Postgres ::jsonb rejecting a lone surrogate.
|
||||
// After ensureWellFormed, a JSON round-trip preserves the value.
|
||||
const out = ensureWellFormed('emoji ' + STRAY_HIGH + ' tail');
|
||||
expect(JSON.parse(JSON.stringify(out))).toBe(out);
|
||||
expect(out.isWellFormed()).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
describe('safeSplitIndex', () => {
|
||||
test('maxChars ≤ 0 returns 0', () => {
|
||||
expect(safeSplitIndex('any', 0)).toBe(0);
|
||||
|
||||
Reference in New Issue
Block a user