/** * Engine Parity E2E * * Codex flagged that searchKeyword behavior differs structurally between * the two engines (Postgres uses a CTE that ranks pages then picks best * chunk; PGLite returns chunks directly). Without verification, source-aware * ranking could pass on PGLite and silently fail on Postgres. * * Strategy: seed identical corpora into both engines, run identical queries, * assert top-5 slug ordering matches. * * Gated by DATABASE_URL — skips gracefully if no real Postgres. Always runs * the PGLite half so the seed/query path is at least exercised. */ import { describe, test, expect, beforeAll, afterAll } from 'bun:test'; import { PGLiteEngine } from '../../src/core/pglite-engine.ts'; import type { ChunkInput, SearchResult } from '../../src/core/types.ts'; import type { BrainEngine } from '../../src/core/engine.ts'; import { hasDatabase, setupDB, teardownDB, getEngine } from './helpers.ts'; const SKIP_PG = !hasDatabase(); const describeBoth = SKIP_PG ? describe.skip : describe; function basisEmbedding(idx: number, dim = 1536): Float32Array { const emb = new Float32Array(dim); emb[idx % dim] = 1.0; return emb; } interface SeedPage { slug: string; type: 'writing' | 'concept' | 'note' | 'person' | 'company'; title: string; body: string; embeddingDim: number; } const SEED_PAGES: SeedPage[] = [ { slug: 'originals/talks/article-outline-fat-code', type: 'writing', title: 'Fat Code Thin Harness — Part 3', body: 'fat code thin harness pattern part 3 production case studies', embeddingDim: 7, }, { slug: 'concepts/fat-code-thin-harness', type: 'concept', title: 'Fat Code Thin Harness', body: 'reusable concept fat code thin harness architecture', embeddingDim: 14, }, { slug: 'openclaw/chat/2026-04-15', type: 'note', title: '2026-04-15 chat', body: 'fat code thin harness fat code thin harness discussion went on at length, ' + 'fat code thin harness came up again and again, fat code thin harness fat code thin harness.', embeddingDim: 8, }, { slug: 'openclaw/chat/2026-04-16', type: 'note', title: '2026-04-16 chat', body: 'fat code thin harness once more, fat code thin harness fat code thin harness, ' + 'still talking about fat code thin harness fat code thin harness.', embeddingDim: 9, }, { slug: 'people/example-founder', type: 'person', title: 'Example Founder', body: 'example founder unrelated content for distraction', embeddingDim: 50, }, ]; async function seedEngine(eng: BrainEngine) { for (const p of SEED_PAGES) { await eng.putPage(p.slug, { type: p.type, title: p.title, compiled_truth: p.body, timeline: '', }); const chunks: ChunkInput[] = [ { chunk_index: 0, chunk_text: p.body, chunk_source: 'compiled_truth', embedding: basisEmbedding(p.embeddingDim), token_count: p.body.split(/\s+/).length, }, ]; await eng.upsertChunks(p.slug, chunks); } } const QUERIES = [ 'fat code thin harness', 'fat code thin harness part 3', 'fat code production', ]; describeBoth('Engine parity — Postgres vs PGLite', () => { let pgEngine: BrainEngine; let pgliteEngine: PGLiteEngine; beforeAll(async () => { pgEngine = await setupDB(); await seedEngine(pgEngine); pgliteEngine = new PGLiteEngine(); await pgliteEngine.connect({}); await pgliteEngine.initSchema(); await seedEngine(pgliteEngine); }, 90_000); afterAll(async () => { await pgliteEngine.disconnect(); await teardownDB(); }); for (const q of QUERIES) { test(`searchKeyword: top-5 slugs match for "${q}"`, async () => { const pgResults = await pgEngine.searchKeyword(q, { limit: 5 }); const pgliteResults = await pgliteEngine.searchKeyword(q, { limit: 5 }); const pgSlugs = pgResults.map((r: SearchResult) => r.slug); const pgliteSlugs = pgliteResults.map((r: SearchResult) => r.slug); // Top result MUST match (the swamp-resistance guarantee). expect(pgSlugs[0]).toBe(pgliteSlugs[0]); // Sets should match (allowing some ordering drift on lower-ranked // results since FTS rank function differences between engines are // out of scope for this fix). expect(new Set(pgSlugs)).toEqual(new Set(pgliteSlugs)); }); } test('searchVector: top result matches between engines', async () => { const queryVec = basisEmbedding(7); // article direction const pgResults = await pgEngine.searchVector(queryVec, { limit: 5 }); const pgliteResults = await pgliteEngine.searchVector(queryVec, { limit: 5 }); expect(pgResults[0]?.slug).toBe(pgliteResults[0]?.slug); }); test('hard-exclude is consistent across engines', async () => { // Both engines should hide test/ pages by default; both should opt // them back in via include_slug_prefixes. await pgEngine.putPage('test/parity-fixture', { type: 'note', title: 'parity test fixture', compiled_truth: 'parity test fixture content', timeline: '', }); await pgEngine.upsertChunks('test/parity-fixture', [{ chunk_index: 0, chunk_text: 'parity test fixture content', chunk_source: 'compiled_truth', embedding: basisEmbedding(20), token_count: 5, }] satisfies ChunkInput[]); await pgliteEngine.putPage('test/parity-fixture', { type: 'note', title: 'parity test fixture', compiled_truth: 'parity test fixture content', timeline: '', }); await pgliteEngine.upsertChunks('test/parity-fixture', [{ chunk_index: 0, chunk_text: 'parity test fixture content', chunk_source: 'compiled_truth', embedding: basisEmbedding(20), token_count: 5, }] satisfies ChunkInput[]); const pgDefault = await pgEngine.searchKeyword('parity test fixture'); const pgliteDefault = await pgliteEngine.searchKeyword('parity test fixture'); expect(pgDefault.map((r: SearchResult) => r.slug)).not.toContain('test/parity-fixture'); expect(pgliteDefault.map((r: SearchResult) => r.slug)).not.toContain('test/parity-fixture'); const pgOptIn = await pgEngine.searchKeyword('parity test fixture', { include_slug_prefixes: ['test/'], }); const pgliteOptIn = await pgliteEngine.searchKeyword('parity test fixture', { include_slug_prefixes: ['test/'], }); expect(pgOptIn.map((r: SearchResult) => r.slug)).toContain('test/parity-fixture'); expect(pgliteOptIn.map((r: SearchResult) => r.slug)).toContain('test/parity-fixture'); }); test('detail=high produces a different ranking than default on at least one engine', async () => { // Source-boost gates on `detail !== 'high'`. If the gate works on both // engines, the ordering for `detail=high` should differ from default in // any case where the swamp / curated pages have different raw scores. // // Postgres's CTE ranks pages then picks best chunk; ts_rank normalizes // by doc length so chat pages don't always swamp at the page level. // PGLite scores chunks directly — chat chunks beat article chunks on // raw ts_rank. The two engines need different parity contracts here. // // Common assertion that holds on both: detail=high must include the // chat pages in its result set (they're not filtered by detail), and // the result set should not be identical to default-detail (the boost // must be doing _something_ visible). const pgDefault = await pgEngine.searchKeyword('fat code thin harness', { limit: 5 }); const pgHigh = await pgEngine.searchKeyword('fat code thin harness', { detail: 'high', limit: 5 }); const pgliteDefault = await pgliteEngine.searchKeyword('fat code thin harness', { limit: 5 }); const pgliteHigh = await pgliteEngine.searchKeyword('fat code thin harness', { detail: 'high', limit: 5 }); // Chat pages must be present in detail=high results on both engines. expect(pgHigh.some((r: SearchResult) => r.slug.startsWith('openclaw/chat/'))).toBe(true); expect(pgliteHigh.some((r: SearchResult) => r.slug.startsWith('openclaw/chat/'))).toBe(true); // The boost must be doing something — at least one engine's ordering // should change between default and detail=high. const pgChanged = pgDefault.map((r: SearchResult) => r.slug).join(',') !== pgHigh.map((r: SearchResult) => r.slug).join(','); const pgliteChanged = pgliteDefault.map((r: SearchResult) => r.slug).join(',') !== pgliteHigh.map((r: SearchResult) => r.slug).join(','); expect(pgChanged || pgliteChanged).toBe(true); }); });