mirror of
https://github.com/garrytan/gbrain.git
synced 2026-07-27 21:19:18 +00:00
* fix(engine): batch inserts use jsonb_to_recordset, not text[] array literals (#1861) addLinksBatch/addTimelineEntriesBatch/addTakesBatch passed free text through unnest(${arr}::text[]); postgres.js serialized it to a Postgres text[] literal that array_in rejected ("malformed array literal") on calendar/Zoom context, aborting the whole `extract links --stale` sweep. Bind the batch as one JSONB doc via jsonb_to_recordset(($1::jsonb)->'rows') through the audited executeRawJsonb contract instead. Shared row builders (src/core/batch-rows.ts) keep both engines byte-identical; NUL is stripped only from free-text body fields (context/summary/detail/claim), while identity/security fields (slugs/source_ids/holder/kind/dates) still reject NUL. addTakesBatch is now batchRetry-wrapped ('addTakesBatch' audit site) and its BrainEngine signature takes BatchOpts. Scalar addLink context is NUL-stripped too. Regression tests on both engines: PGLite always-on poison/NUL/parity suite + DATABASE_URL-gated Postgres lane (the engine that actually crashed). * test: make "no Anthropic key" tests hermetic via withoutAnthropicKey hasAnthropicKey() reads both ANTHROPIC_API_KEY and ~/.gbrain config; tests that only deleted the env var fired a real LLM call on configured machines (warning flipped NO_ANTHROPIC_API_KEY -> LLM_OUTPUT_NOT_JSON). New test/helpers/no-anthropic-key.ts neutralizes both sources (env + GBRAIN_HOME temp dir) for the duration of the call. Refactors the five no-key tests in think-pipeline + takes-mcp-allowlist to use it, including two that previously passed only by luck of the live LLM output. * chore: docs + version bump (v0.42.28.0) KEY_FILES.md/RETRIEVAL.md describe the jsonb_to_recordset batch path; TODOS.md files the #1861 follow-ups (element-isolation, remaining ::text[] sites, shared SQL-string hoist, batch-insert edge-case tests). CHANGELOG + VERSION + package.json to 0.42.28.0. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com> * docs: sync TESTING.md batch-insert references for v0.42.28.0 The #1861 fix migrated links/timeline/takes batch inserts from unnest(::text[]) to jsonb_to_recordset. Update the stale "postgres-js unnest() binding" note and add the two new poison-regression test files (test/links-timeline-jsonb-poison.test.ts PGLite half, test/e2e/jsonb-batch-poison-postgres.test.ts Postgres lane) to the inventory. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com> * fix(sql-query): reject top-level array jsonb params in executeRawJsonb (#1861 P2a) The "no top-level array" rule was only a comment. A bare JS array bound to a $N::jsonb position can serialize as a Postgres array literal (not jsonb) through postgres.js, silently re-entering the "malformed array literal" class #1861 just escaped. executeRawJsonb now throws a clear error steering callers to the { rows: [...] } object wrapper. Verified breaks zero call sites (all pass objects or null). Codex adversarial P2a; batch-size enforcement (P2b) filed as a TODO. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com> --------- Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
192 lines
7.8 KiB
TypeScript
192 lines
7.8 KiB
TypeScript
import { afterAll, beforeAll, describe, expect, test } from 'bun:test';
|
|
import { PGLiteEngine } from '../src/core/pglite-engine.ts';
|
|
import { sqlQueryForEngine, executeRawJsonb } from '../src/core/sql-query.ts';
|
|
|
|
let engine: PGLiteEngine;
|
|
|
|
beforeAll(async () => {
|
|
engine = new PGLiteEngine();
|
|
await engine.connect({});
|
|
}, 30_000);
|
|
|
|
afterAll(async () => {
|
|
if (engine) await engine.disconnect();
|
|
});
|
|
|
|
describe('sqlQueryForEngine', () => {
|
|
test('runs parameterized tagged-template SQL against PGLite', async () => {
|
|
const sql = sqlQueryForEngine(engine);
|
|
const rows = await sql`SELECT ${'pglite'}::text AS engine, ${3}::int AS count`;
|
|
expect(rows).toEqual([{ engine: 'pglite', count: 3 }]);
|
|
});
|
|
|
|
test('rejects postgres.js-style fragment / object values explicitly', async () => {
|
|
const sql = sqlQueryForEngine(engine);
|
|
await expect(
|
|
sql`SELECT ${(Promise.resolve([]) as any)}::text AS bad`
|
|
).rejects.toThrow(/only supports scalar bind values/);
|
|
await expect(
|
|
sql`SELECT ${(['read', 'write'] as any)}::text[] AS bad`
|
|
).rejects.toThrow(/only supports scalar bind values/);
|
|
await expect(
|
|
sql`SELECT ${({ takes_holders: ['world'] } as any)}::jsonb AS bad`
|
|
).rejects.toThrow(/only supports scalar bind values/);
|
|
});
|
|
});
|
|
|
|
describe('executeRawJsonb (D1 wave / v0.31)', () => {
|
|
test('round-trips an object as JSONB on PGLite (jsonb_typeof = object, ->> reads value)', async () => {
|
|
// Verifies the cross-engine JSONB write helper produces a real Postgres
|
|
// JSONB object — not a quoted JSON string. Codex's plan-review #9 said
|
|
// "use the actual JSONB contract, not string-grep for backslash-quote",
|
|
// and that's what this asserts: jsonb_typeof + ->>.
|
|
const tableName = `t_jsonb_${Math.random().toString(36).slice(2, 10)}`;
|
|
await engine.executeRaw(`CREATE TEMP TABLE ${tableName} (j jsonb)`);
|
|
try {
|
|
await executeRawJsonb(
|
|
engine,
|
|
`INSERT INTO ${tableName} (j) VALUES ($1::jsonb)`,
|
|
[],
|
|
[{ k: 'v', n: 42 }],
|
|
);
|
|
const rows = await engine.executeRaw<{ kind: string; k: string; n: number }>(
|
|
`SELECT jsonb_typeof(j) AS kind, j->>'k' AS k, (j->>'n')::int AS n FROM ${tableName}`,
|
|
);
|
|
expect(rows).toHaveLength(1);
|
|
expect(rows[0].kind).toBe('object');
|
|
expect(rows[0].k).toBe('v');
|
|
expect(rows[0].n).toBe(42);
|
|
} finally {
|
|
await engine.executeRaw(`DROP TABLE ${tableName}`);
|
|
}
|
|
});
|
|
|
|
test('takes-holders shape: object preserved, ->> returns the encoded array, NOT a double-encoded string (v0.12.0 regression guard)', async () => {
|
|
// The v0.12.0 silent-data-loss bug stored `${JSON.stringify(perms)}::jsonb`
|
|
// as a JSON string-of-an-object, so `permissions->>'takes_holders'`
|
|
// would return a string with backslashes instead of the array.
|
|
// Post-fix: real JSONB object, ->> on the array key returns the
|
|
// pretty-printed JSON of the array (because jsonb -> array -> ->> is
|
|
// the array as a text), and jsonb_typeof on the parent stays 'object'.
|
|
const tableName = `t_perms_${Math.random().toString(36).slice(2, 10)}`;
|
|
await engine.executeRaw(
|
|
`CREATE TEMP TABLE ${tableName} (id serial PRIMARY KEY, permissions jsonb)`,
|
|
);
|
|
try {
|
|
const perms = { takes_holders: ['world', 'garry'] };
|
|
await executeRawJsonb(
|
|
engine,
|
|
`INSERT INTO ${tableName} (permissions) VALUES ($1::jsonb)`,
|
|
[],
|
|
[perms],
|
|
);
|
|
const rows = await engine.executeRaw<{
|
|
outer_kind: string;
|
|
holders_kind: string;
|
|
first_holder: string;
|
|
text_form: string;
|
|
}>(
|
|
`SELECT
|
|
jsonb_typeof(permissions) AS outer_kind,
|
|
jsonb_typeof(permissions->'takes_holders') AS holders_kind,
|
|
permissions->'takes_holders'->>0 AS first_holder,
|
|
permissions::text AS text_form
|
|
FROM ${tableName}`,
|
|
);
|
|
expect(rows).toHaveLength(1);
|
|
// The parent JSONB is an object; the takes_holders child is an array.
|
|
// Pre-fix this would be 'string' / 'string' (string-of-object).
|
|
expect(rows[0].outer_kind).toBe('object');
|
|
expect(rows[0].holders_kind).toBe('array');
|
|
expect(rows[0].first_holder).toBe('world');
|
|
// Defense in depth: the text representation must NOT contain
|
|
// backslash-quote sequences, which is what double-encoded JSONB
|
|
// looked like in the v0.12.0 incident (e.g. `"{\"takes_holders\":...}"`).
|
|
expect(rows[0].text_form).not.toContain('\\"');
|
|
// And the text representation should look like a normal JSON object
|
|
// — starts with `{`, not `"{`.
|
|
expect(rows[0].text_form.startsWith('{')).toBe(true);
|
|
} finally {
|
|
await engine.executeRaw(`DROP TABLE ${tableName}`);
|
|
}
|
|
});
|
|
|
|
test('null jsonb value stores NULL, not the string "null"', async () => {
|
|
// serve-http.ts sometimes inserts NULL params (e.g. tools/list, scope-
|
|
// rejected paths). The helper must accept null without trying to
|
|
// encode it as the string "null" or rejecting it.
|
|
const tableName = `t_jnull_${Math.random().toString(36).slice(2, 10)}`;
|
|
await engine.executeRaw(`CREATE TEMP TABLE ${tableName} (j jsonb)`);
|
|
try {
|
|
await executeRawJsonb(
|
|
engine,
|
|
`INSERT INTO ${tableName} (j) VALUES ($1::jsonb)`,
|
|
[],
|
|
[null],
|
|
);
|
|
const rows = await engine.executeRaw<{ kind: string | null; is_null: boolean }>(
|
|
`SELECT jsonb_typeof(j) AS kind, (j IS NULL) AS is_null FROM ${tableName}`,
|
|
);
|
|
expect(rows[0].is_null).toBe(true);
|
|
// jsonb_typeof on SQL NULL returns NULL.
|
|
expect(rows[0].kind).toBeNull();
|
|
} finally {
|
|
await engine.executeRaw(`DROP TABLE ${tableName}`);
|
|
}
|
|
});
|
|
|
|
test('mixes scalar params and jsonb params in positional order', async () => {
|
|
// Real call shape: scalars first ($1..$N), JSONB params next
|
|
// ($N+1..$N+M). Mirrors the auth.ts `INSERT INTO access_tokens
|
|
// (name, token_hash, permissions) VALUES ($1, $2, $3::jsonb)` pattern.
|
|
const tableName = `t_mix_${Math.random().toString(36).slice(2, 10)}`;
|
|
await engine.executeRaw(
|
|
`CREATE TEMP TABLE ${tableName} (name text, weight int, payload jsonb)`,
|
|
);
|
|
try {
|
|
await executeRawJsonb(
|
|
engine,
|
|
`INSERT INTO ${tableName} (name, weight, payload) VALUES ($1, $2, $3::jsonb)`,
|
|
['alice', 7],
|
|
[{ tags: ['a', 'b'] }],
|
|
);
|
|
const rows = await engine.executeRaw<{ name: string; weight: number; first_tag: string }>(
|
|
`SELECT name, weight, payload->'tags'->>0 AS first_tag FROM ${tableName}`,
|
|
);
|
|
expect(rows).toEqual([{ name: 'alice', weight: 7, first_tag: 'a' }]);
|
|
} finally {
|
|
await engine.executeRaw(`DROP TABLE ${tableName}`);
|
|
}
|
|
});
|
|
|
|
test('rejects non-scalar values in scalarParams (defense in depth)', async () => {
|
|
// The scalar position validator should fire even when a misuse passes
|
|
// an object via scalarParams instead of jsonbParams. Catches the
|
|
// cross-up-the-positions footgun loud at the helper boundary.
|
|
await expect(
|
|
executeRawJsonb(
|
|
engine,
|
|
`SELECT $1::text AS bad`,
|
|
[{ object: 'in scalar position' } as any],
|
|
[],
|
|
),
|
|
).rejects.toThrow(/only supports scalar bind values/);
|
|
});
|
|
|
|
test('rejects a top-level array jsonb param (gbrain#1861 P2a guard)', async () => {
|
|
// A bare JS array bound to a $N::jsonb position can serialize as a Postgres
|
|
// array literal (not jsonb) through postgres.js, re-entering the
|
|
// "malformed array literal" class #1861 escaped. The helper must reject it
|
|
// and steer callers to the { rows: [...] } wrapper. Objects + null are fine
|
|
// (covered by the tests above).
|
|
await expect(
|
|
executeRawJsonb(
|
|
engine,
|
|
`INSERT INTO nope (j) VALUES ($1::jsonb)`,
|
|
[],
|
|
[[{ a: 1 }, { a: 2 }] as any],
|
|
),
|
|
).rejects.toThrow(/top-level array jsonb param/);
|
|
});
|
|
});
|