v0.42.53.0 fix(sync,db): #2339 op_checkpoints jsonb double-encode + bug-class sweep + CI guard (#2375)

* fix(sync): op_checkpoints pin write double-encodes jsonb — every sync aborts (#2339) recordCompleted bound JSON.stringify(array) to a $3::jsonb param via postgres.js .unsafe(), double-encoding it into a jsonb string scalar that violates the v119 op_checkpoints_completed_keys_array CHECK — aborting every multi-source sync on real Postgres at the first checkpoint write. PGLite parses the string silently, which is why unit tests stayed green and it shipped. Cast through $3::text::jsonb so the text->jsonb cast parses a genuine array. Adds a DATABASE_URL-gated parity test + a dedicated Postgres CI job so the guard can never silently skip. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com> * fix(db): sweep positional jsonb double-encode sites + AST CI guard (#2324) Every executeRaw/.unsafe site that bound JSON.stringify(x) to a bare positional jsonb cast double-encodes on real Postgres (same class as #2339). Sweep them all to the text::jsonb form across query-cache, sources-ops, llm-base, calibration-profile, impact-capture, subagent, receipt-write, traversal-cache, symbol-resolver, and the agent/sources commands. Adds scripts/check-jsonb-params.mjs (AST-lite scanner for the positional form the legacy template grep misses, incl. generic-typed calls), wired into check-jsonb-pattern.sh, with a self-test. PGLite's native db.query is not scanned — it parses text to jsonb natively, so the bug can't occur there. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com> * fix(search,eval): alias-hop injected results carry page_id (contradiction-probe crash) applyAliasHop injected synthetic SearchResults without page_id (the `as SearchResult` cast hid the missing field), so listActiveTakesForPages bound undefined/NaN into ANY($1::int[]) and crashed the whole contradiction probe on real Postgres. Stamp page_id=page.id at the injection site and add a finite-id filter in generateIntraPagePairs as a defensive backstop (mirrors hybrid.ts:63). Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com> * docs(engines): positional jsonb binding rule (text::jsonb vs the double-encode trap) Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com> * v0.42.53.0 fix(sync,db): #2339 op_checkpoints jsonb double-encode + bug-class sweep + CI guard Bumps VERSION + package.json to 0.42.53.0, adds the CHANGELOG entry, and regenerates llms-full.txt. Ships the #2339 sync-abort hotfix, the repo-wide positional jsonb double-encode sweep, the alias-hop contradiction-probe crash fix, and the new positional-form CI guard. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com> * docs: post-ship sync — jsonb invariant now covers the positional form + new guard CLAUDE.md JSONB invariant + KEY_FILES (sql-query, check-jsonb-pattern, op-checkpoint) now describe the #2339 positional double-encode class, the $N::text::jsonb fix, and the new check-jsonb-params.mjs guard. Regenerates llms-full.txt. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com> --------- Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
2026-07-27 22:15:33 +00:00 · 2026-06-24 06:05:16 -07:00
parent bb2e88c42a
commit 814258dda6
27 changed files with 524 additions and 55 deletions
@@ -0,0 +1,83 @@
+/**
+ * Self-test for scripts/check-jsonb-params.mjs — the positional jsonb
+ * double-encode guard (#2339 / #2324 class). Verifies it catches the bug shape
+ * (including generic-typed calls and the `jsonStr` variable case is acknowledged
+ * as out of scope) and does NOT false-positive on the sanctioned forms.
+ *
+ * Fixtures are written to a temp dir and the scanner is pointed at it via argv,
+ * so this never touches the real src/ tree.
+ */
+import { describe, test, expect, beforeAll, afterAll } from 'bun:test';
+import { mkdtempSync, writeFileSync, mkdirSync, rmSync } from 'node:fs';
+import { join } from 'node:path';
+import { tmpdir } from 'node:os';
+
+const SCRIPT = join(import.meta.dir, '..', 'scripts', 'check-jsonb-params.mjs');
+
+let root: string;
+let badDir: string;
+let goodDir: string;
+
+function runGuard(dir: string): { code: number; err: string } {
+  const res = Bun.spawnSync([process.execPath, SCRIPT, dir]);
+  return { code: res.exitCode, err: res.stderr.toString() + res.stdout.toString() };
+}
+
+beforeAll(() => {
+  root = mkdtempSync(join(tmpdir(), 'jsonb-guard-'));
+  badDir = join(root, 'bad');
+  goodDir = join(root, 'good');
+  mkdirSync(badDir, { recursive: true });
+  mkdirSync(goodDir, { recursive: true });
+
+  // BAD: positional $1::jsonb bound to a JSON.stringify'd value.
+  writeFileSync(
+    join(badDir, 'bad.ts'),
+    "await engine.executeRaw(`INSERT INTO t (a) VALUES ($1::jsonb)`, [JSON.stringify(x)]);\n",
+  );
+  // BAD: generic-typed executeRaw<T>(...) must still be caught.
+  writeFileSync(
+    join(badDir, 'bad_generic.ts'),
+    "await engine.executeRaw<{ id: string }>(`UPDATE t SET a = $2::jsonb WHERE id = $1`, [id, JSON.stringify(x)]);\n",
+  );
+
+  // GOOD: the fix — $1::text::jsonb.
+  writeFileSync(
+    join(goodDir, 'good_text.ts'),
+    "await engine.executeRaw(`INSERT INTO t (a) VALUES ($1::text::jsonb)`, [JSON.stringify(x)]);\n",
+  );
+  // GOOD: text[] array path (the appendCompleted unnest shape).
+  writeFileSync(
+    join(goodDir, 'good_array.ts'),
+    "await engine.executeRaw(`INSERT INTO t (a) SELECT unnest($1::text[])`, [JSON.stringify(arr)]);\n",
+  );
+  // GOOD: executeRawJsonb passes a raw object, not a string — excluded.
+  writeFileSync(
+    join(goodDir, 'good_helper.ts'),
+    "await executeRawJsonb(engine, `INSERT INTO t (a) VALUES ($1::jsonb)`, [], [JSON.stringify(x)]);\n",
+  );
+  // GOOD: explicit opt-out for a rare legitimate object-binding case.
+  writeFileSync(
+    join(goodDir, 'good_optout.ts'),
+    "await engine.executeRaw(`INSERT INTO t (a) VALUES ($1::jsonb)` /* jsonb-guard-ok */, [JSON.stringify(x)]);\n",
+  );
+});
+
+afterAll(() => {
+  rmSync(root, { recursive: true, force: true });
+});
+
+describe('check-jsonb-params guard', () => {
+  test('flags positional $N::jsonb + JSON.stringify (incl. generic-typed calls)', () => {
+    const { code, err } = runGuard(badDir);
+    expect(code).toBe(1);
+    expect(err).toContain('bad.ts');
+    expect(err).toContain('bad_generic.ts');
+  });
+
+  test('passes the sanctioned forms (::text::jsonb, ::text[], executeRawJsonb, opt-out)', () => {
+    const { code, err } = runGuard(goodDir);
+    expect(code).toBe(0);
+    expect(err).toContain('clean');
+  });
+});
@@ -0,0 +1,80 @@
+/**
+ * E2E: op_checkpoints.completed_keys JSONB parity — #2339 regression guard.
+ *
+ * #2339: `recordCompleted` bound `JSON.stringify(array)` to a `$3::jsonb` param
+ * via postgres.js `.unsafe()` (executeRawDirect). That double-encodes the value
+ * into a jsonb *string scalar*, which violates the v119
+ * `op_checkpoints_completed_keys_array CHECK (jsonb_typeof = 'array')` and aborts
+ * EVERY sync on real Postgres at the first checkpoint write. PGLite's driver
+ * parses the string silently, which is exactly why the unit suite stayed green
+ * and the bug shipped — so this assertion can ONLY be made on real Postgres.
+ *
+ * This file uses the standard `hasDatabase()` skip gate (consistent with the
+ * other e2e tests). The X2-A guarantee that it actually RUNS lives in a dedicated
+ * CI job (.github/workflows) that provisions a Postgres service so DATABASE_URL
+ * is always present there — rather than a fail-on-skip hack inside this file,
+ * which would red-fail legitimate DB-less local runs.
+ *
+ * Fix under test: `$3::text::jsonb` binds the value as text, so the text→jsonb
+ * cast parses it into a genuine jsonb array.
+ */
+import { describe, test, expect, beforeAll, afterAll } from 'bun:test';
+import { hasDatabase, setupDB, teardownDB, getEngine, getConn } from './helpers.ts';
+import { recordCompleted, loadOpCheckpoint } from '../../src/core/op-checkpoint.ts';
+
+const describeE2E = hasDatabase() ? describe : describe.skip;
+
+describeE2E('E2E: op_checkpoints completed_keys jsonb parity (#2339)', () => {
+  beforeAll(async () => {
+    await setupDB();
+  });
+  afterAll(async () => {
+    await teardownDB();
+  });
+
+  const key = { op: 'sync-target', fingerprint: 'jsonb-parity-2339' };
+
+  test('recordCompleted stores completed_keys as a jsonb ARRAY, not a double-encoded scalar', async () => {
+    const engine = getEngine();
+
+    // Pre-fix this throws SQLSTATE 23514 (the CHECK), durableWrite exhausts its
+    // retries, and recordCompleted returns false. Post-fix it stores a real array.
+    const ok = await recordCompleted(engine, key, ['b/2.md', 'a/1.md', 'c/3.md']);
+    expect(ok).toBe(true);
+
+    const sql = getConn();
+    const [row] = await sql`
+      SELECT jsonb_typeof(completed_keys)   AS t,
+             jsonb_array_length(completed_keys) AS len,
+             completed_keys ->> 0           AS first
+      FROM op_checkpoints
+      WHERE op = ${key.op} AND fingerprint = ${key.fingerprint}
+    `;
+    expect(row.t).toBe('array'); // pre-fix: 'string' (scalar) — the bug
+    expect(Number(row.len)).toBe(3);
+    expect(row.first).toBe('a/1.md'); // recordCompleted sorts the set
+  }, 30_000);
+
+  test('loadOpCheckpoint round-trips the recorded set', async () => {
+    const engine = getEngine();
+    const got = await loadOpCheckpoint(engine, key);
+    expect(new Set(got)).toEqual(new Set(['a/1.md', 'b/2.md', 'c/3.md']));
+  }, 30_000);
+
+  test('REPLACE semantics: re-recording a smaller set drops stale keys (stays an array)', async () => {
+    const engine = getEngine();
+    const ok = await recordCompleted(engine, key, ['only/1.md']);
+    expect(ok).toBe(true);
+
+    const got = await loadOpCheckpoint(engine, key);
+    expect(new Set(got)).toEqual(new Set(['only/1.md']));
+
+    const sql = getConn();
+    const [row] = await sql`
+      SELECT jsonb_typeof(completed_keys) AS t
+      FROM op_checkpoints
+      WHERE op = ${key.op} AND fingerprint = ${key.fingerprint}
+    `;
+    expect(row.t).toBe('array');
+  }, 30_000);
+});