mirror of
https://github.com/garrytan/gbrain.git
synced 2026-07-28 14:59:47 +00:00
A single markdown page whose compiled_truth exceeds Postgres's hard 1,048,575-byte tsvector cap made update_page_search_vector() throw "string is too long for tsvector" INSIDE the pages UPSERT transaction. Not a per-file ledger entry — a transaction abort. The whole source's sync checkpoint stayed pinned (Sync BLOCKED) until the oversized file was fixed or manually skipped, even though every other file in the run imported fine. --retry-failed re-failed the same files every run; the 3-consecutive-failure auto-skip eventually moved past them, but for a scheduled collector that meant hours of blocked cycles per oversized file, per source. Root cause: pages.search_vector indexed compiled_truth — the unbounded whole-page body — even though it's write-only dead weight for actual search. searchKeyword() (postgres-engine.ts / pglite-engine.ts) ranks and queries content_chunks.search_vector exclusively (Cathedral II Layer 3, chunk-grain, already populated separately from compiled_truth via chunking at import time, and well under the tsvector cap since chunkText() targets embedding-sized pieces). Verified directly: no pages.search_vector / bare search_vector read appears anywhere outside this trigger's own definition and the reindex/backfill machinery that maintains it. Fix: v124 migration recreates update_page_search_vector() without compiled_truth — title + timeline stay (both naturally small), so the column keeps carrying some signal rather than going fully inert. Updated in lockstep (documented contract, see reindex-search-vector.ts's own comment): migrate.ts's new v124, reindex-search-vector.ts's recreatePagesFn, and the fresh-install baselines in pglite-schema.ts + src/schema.sql (regenerates schema-embedded.ts via `bun run build:schema`). No backfill: existing rows keep whatever search_vector they already computed until their next UPDATE — harmless, since nothing reads this column, and the brains that actually hit this bug never successfully wrote a value for the oversized page in the first place. Considered (from the issue) and rejected: truncating compiled_truth to fit under the cap. Silent, position-dependent recall loss, and the byte cap doesn't line up cleanly with any natural character/token boundary for UTF-8 content. content_chunks.search_vector already gives full, untruncated chunk-grain coverage for large pages — truncating a now-redundant whole-page vector would trade a real bug for a subtler one. ## Test plan - New test/page-search-vector-overflow.test.ts: a >1MB page (genuinely diverse tokens — a repetitive lorem-ipsum-style fixture does NOT reproduce this bug, since to_tsvector's cap is on its DEDUPLICATED output size, not raw input length) now imports successfully instead of throwing; remains keyword-searchable via the chunk-grain path; a normal page's search_vector still carries title signal (not fully inert). Verified the test is meaningful both directions: fails with the exact reported error on the pre-fix trigger (git-stashed the fix, reran, confirmed byte-for-byte match: "string is too long for tsvector (2684620 bytes, max 1048575 bytes)"), passes with the fix restored. - Updated fts-language-migration.serial.test.ts: removed an assertion that configurable_fts_language (v123) is LATEST_VERSION — that was only ever true until the next migration landed; the codebase's own pattern elsewhere for this (migrate.test.ts) uses toBeGreaterThanOrEqual, not exact-match, for exactly this reason. - bun run typecheck clean, bun run verify 31/31 green. - test/page-search-vector-overflow.test.ts (3/3), test/reindex-search-vector.serial.test.ts, test/fts-language-migration.serial.test.ts, test/migration-v120.test.ts, test/sync.test.ts, test/bootstrap.test.ts, test/migrate.test.ts — 254 total, 0 fail, no regressions. ## Design consultation Investigated jointly with masa-codex (async design review) before implementing — their read of the codebase (content_chunks.search_vector already covers keyword search; pages.search_vector's compiled_truth feed is the only overflow-prone, effectively-dead write) matched independent verification and shaped the "remove from trigger" fix over the issue's alternative options (chunk-grain rebuild — largely already exists; input truncation — rejected above; ledger-entry-only — insufficient alone, since the page upsert failing in the same transaction also loses the chunk write, so checkpoint advancing without this fix would make the content permanently unsearchable, not just delayed). Claude-Session: https://claude.ai/code/session_01NCVEvUVm15bFuKqskWgfNS Co-authored-by: Claude Sonnet 5 <noreply@anthropic.com>
122 lines
4.4 KiB
TypeScript
122 lines
4.4 KiB
TypeScript
import { describe, test, expect, beforeEach, afterEach } from 'bun:test';
|
|
import type { BrainEngine } from '../src/core/engine.ts';
|
|
import { MIGRATIONS } from '../src/core/migrate.ts';
|
|
import { resetFtsLanguageCache } from '../src/core/fts-language.ts';
|
|
|
|
const ENV_KEY = 'GBRAIN_FTS_LANGUAGE';
|
|
const originalLang = process.env[ENV_KEY];
|
|
|
|
beforeEach(() => {
|
|
delete process.env[ENV_KEY];
|
|
resetFtsLanguageCache();
|
|
});
|
|
|
|
afterEach(() => {
|
|
delete process.env[ENV_KEY];
|
|
if (originalLang !== undefined) process.env[ENV_KEY] = originalLang;
|
|
resetFtsLanguageCache();
|
|
});
|
|
|
|
describe('configurable_fts_language migration', () => {
|
|
test('migration is registered', () => {
|
|
const ftsMig = MIGRATIONS.find(m => m.name === 'configurable_fts_language');
|
|
expect(ftsMig).toBeDefined();
|
|
expect(ftsMig?.version).toBeGreaterThan(115);
|
|
});
|
|
|
|
// #2704 (v124, page_search_vector_drop_compiled_truth) landed after this
|
|
// migration — "is the latest migration" was only ever true at the
|
|
// moment v123 was added and would break on every subsequent migration,
|
|
// so it's removed rather than bumped to a hardcoded v124. The
|
|
// registration + shape assertions below don't depend on migration order.
|
|
|
|
test('ftsMig uses handler (not static SQL) because language interpolation is dynamic', () => {
|
|
const ftsMig = MIGRATIONS.find(m => m.name === 'configurable_fts_language');
|
|
expect(ftsMig?.sql).toBe('');
|
|
expect(ftsMig?.handler).toBeTypeOf('function');
|
|
});
|
|
|
|
test('ftsMig handler is async', () => {
|
|
const ftsMig = MIGRATIONS.find(m => m.name === 'configurable_fts_language');
|
|
// Async function check: the constructor name is 'AsyncFunction'
|
|
expect(ftsMig?.handler?.constructor.name).toBe('AsyncFunction');
|
|
});
|
|
|
|
test('migration handler issues recreate-function calls (smoke check via mock engine)', async () => {
|
|
const ftsMig = MIGRATIONS.find(m => m.name === 'configurable_fts_language');
|
|
const calls: string[] = [];
|
|
|
|
const mockEngine = {
|
|
executeRaw: async (sql: string) => {
|
|
calls.push(sql);
|
|
return [];
|
|
},
|
|
} as unknown as BrainEngine;
|
|
|
|
process.env[ENV_KEY] = 'english';
|
|
resetFtsLanguageCache();
|
|
|
|
await ftsMig?.handler?.(mockEngine);
|
|
|
|
// Default 'english' \u2014 no backfill, only 2 CREATE OR REPLACE calls.
|
|
expect(calls.length).toBe(2);
|
|
expect(calls[0]).toContain('CREATE OR REPLACE FUNCTION update_page_search_vector');
|
|
expect(calls[0]).toContain("to_tsvector('english'");
|
|
expect(calls[1]).toContain('CREATE OR REPLACE FUNCTION update_chunk_search_vector');
|
|
expect(calls[1]).toContain("to_tsvector('english'");
|
|
// v120/#1647 hardening must survive the CREATE OR REPLACE (which resets
|
|
// proconfig): both recreated bodies pin search_path.
|
|
expect(calls[0]).toContain('SET search_path = pg_catalog, public');
|
|
expect(calls[1]).toContain('SET search_path = pg_catalog, public');
|
|
});
|
|
|
|
test('non-english language triggers backfill', async () => {
|
|
const ftsMig = MIGRATIONS.find(m => m.name === 'configurable_fts_language');
|
|
const calls: string[] = [];
|
|
|
|
const mockEngine = {
|
|
executeRaw: async (sql: string) => {
|
|
calls.push(sql);
|
|
return [];
|
|
},
|
|
} as unknown as BrainEngine;
|
|
|
|
process.env[ENV_KEY] = 'pt_br';
|
|
resetFtsLanguageCache();
|
|
|
|
await ftsMig?.handler?.(mockEngine);
|
|
|
|
// pt_br \u2014 2 CREATE + 2 backfill UPDATEs = 4 calls
|
|
expect(calls.length).toBe(4);
|
|
expect(calls[0]).toContain("to_tsvector('pt_br'");
|
|
expect(calls[1]).toContain("to_tsvector('pt_br'");
|
|
expect(calls[2]).toMatch(/UPDATE pages/);
|
|
expect(calls[3]).toContain("to_tsvector('pt_br'");
|
|
expect(calls[3]).toMatch(/UPDATE content_chunks/);
|
|
});
|
|
|
|
test('invalid language falls back to english (no SQL injection)', async () => {
|
|
const ftsMig = MIGRATIONS.find(m => m.name === 'configurable_fts_language');
|
|
const calls: string[] = [];
|
|
|
|
const mockEngine = {
|
|
executeRaw: async (sql: string) => {
|
|
calls.push(sql);
|
|
return [];
|
|
},
|
|
} as unknown as BrainEngine;
|
|
|
|
process.env[ENV_KEY] = "english'; DROP TABLE pages; --";
|
|
resetFtsLanguageCache();
|
|
|
|
await ftsMig?.handler?.(mockEngine);
|
|
|
|
// Falls back to english: 2 CREATE OR REPLACE only, no DROP TABLE in any SQL.
|
|
expect(calls.length).toBe(2);
|
|
for (const sql of calls) {
|
|
expect(sql).not.toContain('DROP TABLE');
|
|
expect(sql).toContain("to_tsvector('english'");
|
|
}
|
|
});
|
|
});
|