diff --git a/docs/architecture/KEY_FILES.md b/docs/architecture/KEY_FILES.md index c102a7ef5..bd4c80623 100644 --- a/docs/architecture/KEY_FILES.md +++ b/docs/architecture/KEY_FILES.md @@ -42,7 +42,7 @@ per-release `**vX.Y.Z:**` narration — CI enforces this - `src/commands/reindex-search-vector.ts` — `gbrain reindex-search-vector [--dry-run] [--yes] [--json]`. Escape hatch for changing `GBRAIN_FTS_LANGUAGE` after the `configurable_fts_language` migration has run (the migration shows applied and is skipped): recreates `update_page_search_vector` + `update_chunk_search_vector` with the configured language — bodies mirror the migration's and KEEP the `SET search_path = pg_catalog, public` hardening (CREATE OR REPLACE resets proconfig) — then backfills `pages` (UPDATE-to-self re-fires the trigger) and `content_chunks` (direct vector recompute) in id-keyset batches of `BACKFILL_BATCH_SIZE` (5000) via `UPDATE … WHERE id IN (SELECT … LIMIT n) RETURNING id`, streaming phases `reindex_search_vector.pages`/`.chunks` through the shared progress reporter (stderr). Confirmation gate: `--yes`, or an interactive TTY [y/N]; `--json` does NOT bypass the gate (non-TTY without `--yes` refuses with a ConfirmationRequired envelope, exit 2). Idempotent. Pinned by `test/reindex-search-vector.serial.test.ts`. - `src/commands/sync.ts:resolveSlugByPathOrSourcePath` — Resolves a slug by `pages.source_path` first (returns the stored slug for frontmatter-fallback pages whose path doesn't derive a slug), then falls back to `resolveSlugForPath(path)`. Threaded into all 4 delete/rename call sites (`performSync`'s un-syncable cleanup at ~:531, deletes at ~:603, rename oldSlug at ~:622). Without this, emoji-only / Thai / Arabic filenames whose slug came from frontmatter would orphan on delete/rename (the delete path would compute the wrong path-derived slug). Best-effort query — pre-migration brains fall through to the legacy path. - `src/core/sources-ops.ts` — Multi-source registration + clone-lifecycle ops (`addSource`, `recloneIfMissing`, `defaultCloneDir`, `isOwnedClone`, `unownedHint`). **Reclone-ownership invariant (must-never-violate): gbrain may only delete/re-clone a clone it created, NEVER a user working tree.** `recloneIfMissing` deletes `local_path`, so it gates on `isOwnedClone(src)` and throws a `SourceOpError('unmanaged_path', ...)` BEFORE any filesystem op when ownership is unprovable — fail-closed. Ownership is proven by `config.managed_clone === true` (written by `addSource`'s `--url` path, covering default-location and `--clone-dir` clones) OR `local_path === defaultCloneDir(id)` (back-compat for pre-marker clones, via exact normalized-path equality, symlink-free). A row with `remote_url` + an unowned `local_path` (a user-registered working tree, e.g. `sources add --path`) is refused untouched; re-add with `--url` to regain auto-reclone. The reclone is EXDEV-safe: clone into a SIBLING temp of `local_path` (not the shared `clones/.tmp`, which may sit on a different mount than a `--clone-dir` target), then swap (move old aside → move new in → drop old) so `local_path` is never left missing-and-unrecoverable; on swap failure the original is restored, and if restore fails the error names the `aside` path so it's never reflexively deleted. A TOCTOU re-check re-confirms ownership immediately before the destructive move and rejects a symlink leaf swapped in after the entry check (`symlink_escape`). `unownedHint(src, state)` is the shared recovery message used by both the core error and the `gbrain sync --source` CLI error; `gbrain sources restore` special-cases `unmanaged_path` to print "DB row restored; gbrain syncs this path read-only" instead of the misleading "try sync to recover" guidance. `SourceOpErrorCode` includes `unmanaged_path`. Pinned by `test/sources-ops.test.ts`, `test/sources-resync-recovery.test.ts`. -- `src/core/utils.ts` — Shared SQL utilities extracted from postgres-engine.ts. Exports `parseEmbedding(value)` (throws on unknown input, used by migration + ingest paths where data integrity matters) and `tryParseEmbedding(value)` (returns `null` + warns once per process, used by search/rescore paths where availability matters more than strictness). `isUndefinedColumnError(err)` predicate — pattern-matches Postgres SQLSTATE 42703 / "column ... does not exist" with engine-driver shape variation tolerated; replaces bare `catch {}` blocks in `oauth-provider.ts` so genuine errors (lock timeout, network blip, permission denied) propagate while column-missing falls through to the legacy fallback. `validateSourceId(id)` throws on anything outside `^[a-z0-9_-]+$`, used by the per-source disk-layout fix in patterns.ts/synthesize.ts before any `join(brainDir, '.sources', source_id, slug+'.md')` so source_id can't traverse out of brainDir. `rowToPage` populates the required `Page.source_id` from the SELECT projection (`scripts/check-source-id-projection.sh` enforces every projection feeding `rowToPage` includes the column). +- `src/core/utils.ts` — Shared SQL utilities extracted from postgres-engine.ts. Exports `parseEmbedding(value)` (throws on unknown input, used by migration + ingest paths where data integrity matters) and `tryParseEmbedding(value)` (returns `null` + warns once per process, used by search/rescore paths where availability matters more than strictness). `isUndefinedColumnError(err)` predicate — pattern-matches Postgres SQLSTATE 42703 / "column ... does not exist" with engine-driver shape variation tolerated; replaces bare `catch {}` blocks in `oauth-provider.ts` so genuine errors (lock timeout, network blip, permission denied) propagate while column-missing falls through to the legacy fallback. `validateSourceId(id)` throws on anything outside `^[a-z0-9_-]+$`, used by the per-source disk-layout fix in patterns.ts/synthesize.ts before any `join(brainDir, '.sources', source_id, slug+'.md')` so source_id can't traverse out of brainDir. `rowToSearchResult` projects email `message_id` / `thread_id` metadata and exposes `source_subject` only when a non-empty Message-ID proves the page is an email, so generated page titles never become authoritative email subjects. `rowToPage` populates the required `Page.source_id` from the SELECT projection (`scripts/check-source-id-projection.sh` enforces every projection feeding `rowToPage` includes the column). - `src/core/db.ts` — Connection management, schema initialization. `resolveSessionTimeouts()` returns `statement_timeout` + `idle_in_transaction_session_timeout` (defaults 5min each, env-overridable via `GBRAIN_STATEMENT_TIMEOUT`/`GBRAIN_IDLE_TX_TIMEOUT`/`GBRAIN_CLIENT_CHECK_INTERVAL`). Both `connect()` (module singleton) and `PostgresEngine.connect()` (worker pool) consume the result via postgres.js's `connection` option, sending GUCs as startup parameters that survive PgBouncer transaction mode (`setSessionDefaults` kept as a back-compat no-op shim). `connect()` returns `Promise` — `true` iff THIS call created the module singleton, `false` if it joined an existing one; the decision is atomic (no `await` between the `if (sql)` null-check and the synchronous `sql = postgres(...)` assignment), so two concurrent module connects can't both claim creation. `PostgresEngine` stores the return as its `_ownsModuleSingleton` token and only the creating engine may `db.disconnect()` the singleton — a borrower probe engine (lint/doctor config-lift) no-ops its disconnect, so its teardown can't null the connection the long-lived cycle owner is still using (the dream-cycle "connect() has not been called" failure). The module `sql` is only ever nulled by `db.disconnect()` (postgres.js auto-reconnects its own internal pool and never touches our reference). `disconnect()` snapshots + nulls `sql` before awaiting the pool end so a concurrent connect can't join a pool that's already closing. The end routes through `endPoolBounded(pool)` (#1972) — a gbrain-owned `Promise.race` of `pool.end({ timeout: POOL_END_TIMEOUT_SECONDS })` against a hard timer — so a PgBouncer transaction-mode drain that never settles can't hang teardown — the #2084 contract (finishCliTeardown's computed-deadline backstop + flushThenExit's fence-and-grace exit in cli-force-exit.ts) bounds it and delivers pending stdout before exit. `connection-manager.ts` ends its direct + read pools concurrently through the same helper so the per-pool bounds don't stack. - `src/commands/migrate-engine.ts` — Bidirectional engine migration (`gbrain migrate --to supabase/pglite`). Copies the complete source catalog FIRST (`copyMigrationSources` — every `sources` row incl. archived rows and sync/routing metadata, `ON CONFLICT (id) DO UPDATE`, `default` ordered first) so every page write has a valid `pages.source_id` FK parent and the target preserves per-source behavior; pages copy afterward, tracked in the resume manifest by composite `(source_id, slug)` key. The resume manifest is target-aware: `migrationTargetId(config)` hashes `(engine, locator)` (`database_url` for Postgres, resolved `database_path` for PGLite) and `manifestMatchesTarget` requires `schema_version === 2` plus a matching `target_id` — a legacy engine-only manifest, or one from a DIFFERENT target of the same engine kind, starts fresh instead of skipping "completed" pages the new target never received. Pinned by `test/migrate-engine-resume.test.ts` (manifest identity) + `test/e2e/migrate-engine-sources-postgres.test.ts` (source catalog lands before overlapping-slug pages, PGLite → real Postgres). - `src/core/import-file.ts` — importFromFile + importFromContent (chunk + embed + tags). `importFromContent` and `importCodeFile` stamp `pages.embedding_signature` via `setPageEmbeddingSignature(slug, {sourceId, signature: currentEmbeddingSignature()})` when the import actually embedded (not `--no-embed`) so a model/dims swap is detectable as stale; `importCodeFile` only stamps when every chunk was freshly embedded this call (`needsEmbedIndexes.length === chunks.length`), mixed reuse-by-hash pages stay unstamped (`reindex --code --force` / `embed --stale` handle those). `importFromContent`'s tag reconciliation is ADD-ONLY: it only `addTag` (idempotent, ON CONFLICT DO NOTHING). The `tags` table has no provenance column and frontmatter tags are stripped from stored `pages.frontmatter` (markdown.ts:118), so a frontmatter-origin tag can't be distinguished from a DB-enrichment tag (auto-tag / dream synthesize / signal-detector) at re-import — deletion is unsafe (would wipe enrichment under `gbrain reindex --markdown`). Accepted trade-off: removing a tag from frontmatter no longer removes it from the DB on next sync (needs a `tag_source` provenance column). Pinned by `test/reindex-preserve-tags.test.ts` + `test/import-file.test.ts`. diff --git a/src/core/pglite-engine.ts b/src/core/pglite-engine.ts index b8b23aa0e..8d2e211f9 100644 --- a/src/core/pglite-engine.ts +++ b/src/core/pglite-engine.ts @@ -1649,6 +1649,10 @@ export class PGLiteEngine implements BrainEngine { SELECT p.slug, p.id as page_id, p.title, p.type, p.source_id, p.effective_date, p.effective_date_source, + CASE WHEN NULLIF(regexp_replace(p.frontmatter->>'message_id', '^[[:space:]]+|[[:space:]]+$', '', 'g'), '') IS NOT NULL + THEN p.frontmatter->>'message_id' END AS message_id, p.frontmatter->>'thread_id' AS thread_id, + CASE WHEN NULLIF(regexp_replace(p.frontmatter->>'message_id', '^[[:space:]]+|[[:space:]]+$', '', 'g'), '') IS NOT NULL + THEN NULLIF(p.frontmatter->>'subject', '') END AS source_subject, cc.id as chunk_id, cc.chunk_index, cc.chunk_text, cc.chunk_source, ts_rank(cc.search_vector, websearch_to_tsquery('${ftsLang}', $1)) * ${sourceFactorCase} AS score, CASE WHEN p.updated_at < ( @@ -1893,6 +1897,10 @@ export class PGLiteEngine implements BrainEngine { SELECT p.slug, p.id as page_id, p.title, p.type, p.source_id, p.effective_date, p.effective_date_source, + CASE WHEN NULLIF(regexp_replace(p.frontmatter->>'message_id', '^[[:space:]]+|[[:space:]]+$', '', 'g'), '') IS NOT NULL + THEN p.frontmatter->>'message_id' END AS message_id, p.frontmatter->>'thread_id' AS thread_id, + CASE WHEN NULLIF(regexp_replace(p.frontmatter->>'message_id', '^[[:space:]]+|[[:space:]]+$', '', 'g'), '') IS NOT NULL + THEN NULLIF(p.frontmatter->>'subject', '') END AS source_subject, cc.id as chunk_id, cc.chunk_index, cc.chunk_text, cc.chunk_source, ${scoreExpr} AS score, CASE WHEN p.updated_at < ( @@ -1918,6 +1926,10 @@ export class PGLiteEngine implements BrainEngine { `SELECT p.slug, p.id as page_id, p.title, p.type, p.source_id, p.effective_date, p.effective_date_source, + CASE WHEN NULLIF(regexp_replace(p.frontmatter->>'message_id', '^[[:space:]]+|[[:space:]]+$', '', 'g'), '') IS NOT NULL + THEN p.frontmatter->>'message_id' END AS message_id, p.frontmatter->>'thread_id' AS thread_id, + CASE WHEN NULLIF(regexp_replace(p.frontmatter->>'message_id', '^[[:space:]]+|[[:space:]]+$', '', 'g'), '') IS NOT NULL + THEN NULLIF(p.frontmatter->>'subject', '') END AS source_subject, cc.id as chunk_id, cc.chunk_index, cc.chunk_text, cc.chunk_source, ${scoreExpr} AS score, CASE WHEN p.updated_at < ( @@ -2014,6 +2026,10 @@ export class PGLiteEngine implements BrainEngine { `SELECT p.slug, p.id as page_id, p.title, p.type, p.source_id, p.effective_date, p.effective_date_source, + CASE WHEN NULLIF(regexp_replace(p.frontmatter->>'message_id', '^[[:space:]]+|[[:space:]]+$', '', 'g'), '') IS NOT NULL + THEN p.frontmatter->>'message_id' END AS message_id, p.frontmatter->>'thread_id' AS thread_id, + CASE WHEN NULLIF(regexp_replace(p.frontmatter->>'message_id', '^[[:space:]]+|[[:space:]]+$', '', 'g'), '') IS NOT NULL + THEN NULLIF(p.frontmatter->>'subject', '') END AS source_subject, cc.id as chunk_id, cc.chunk_index, cc.chunk_text, cc.chunk_source, ts_rank(cc.search_vector, websearch_to_tsquery('${ftsLang}', $1)) * ${sourceFactorCase} AS score, CASE WHEN p.updated_at < ( @@ -2126,6 +2142,10 @@ export class PGLiteEngine implements BrainEngine { SELECT p.slug, p.id as page_id, p.title, p.type, p.source_id, p.updated_at, p.effective_date, p.effective_date_source, + CASE WHEN NULLIF(regexp_replace(p.frontmatter->>'message_id', '^[[:space:]]+|[[:space:]]+$', '', 'g'), '') IS NOT NULL + THEN p.frontmatter->>'message_id' END AS message_id, p.frontmatter->>'thread_id' AS thread_id, + CASE WHEN NULLIF(regexp_replace(p.frontmatter->>'message_id', '^[[:space:]]+|[[:space:]]+$', '', 'g'), '') IS NOT NULL + THEN NULLIF(p.frontmatter->>'subject', '') END AS source_subject, cc.id as chunk_id, cc.chunk_index, cc.chunk_text, cc.chunk_source, 1 - (cc.${col} <=> ${castSql}) AS raw_score FROM content_chunks cc @@ -2148,6 +2168,7 @@ export class PGLiteEngine implements BrainEngine { SELECT bpp.slug, bpp.page_id, bpp.title, bpp.type, bpp.source_id, bpp.effective_date, bpp.effective_date_source, + bpp.message_id, bpp.thread_id, bpp.source_subject, bpp.chunk_id, bpp.chunk_index, bpp.chunk_text, bpp.chunk_source, bpp.score, CASE WHEN bpp.updated_at < ( diff --git a/src/core/postgres-engine.ts b/src/core/postgres-engine.ts index b58d748f6..f63481ed0 100644 --- a/src/core/postgres-engine.ts +++ b/src/core/postgres-engine.ts @@ -1770,6 +1770,10 @@ export class PostgresEngine implements BrainEngine { SELECT p.slug, p.id as page_id, p.title, p.type, p.source_id, p.effective_date, p.effective_date_source, + CASE WHEN NULLIF(regexp_replace(p.frontmatter->>'message_id', '^[[:space:]]+|[[:space:]]+$', '', 'g'), '') IS NOT NULL + THEN p.frontmatter->>'message_id' END AS message_id, p.frontmatter->>'thread_id' AS thread_id, + CASE WHEN NULLIF(regexp_replace(p.frontmatter->>'message_id', '^[[:space:]]+|[[:space:]]+$', '', 'g'), '') IS NOT NULL + THEN NULLIF(p.frontmatter->>'subject', '') END AS source_subject, cc.id as chunk_id, cc.chunk_index, cc.chunk_text, cc.chunk_source, ts_rank(cc.search_vector, websearch_to_tsquery('${ftsLang}', $1)) * ${sourceFactorCase} AS score FROM content_chunks cc @@ -1797,6 +1801,7 @@ export class PostgresEngine implements BrainEngine { ${buildBestPerPagePoolCte('ranked_chunks')} SELECT slug, page_id, title, type, source_id, effective_date, effective_date_source, + message_id, thread_id, source_subject, chunk_id, chunk_index, chunk_text, chunk_source, score, false AS stale FROM best_per_page @@ -2068,6 +2073,10 @@ export class PostgresEngine implements BrainEngine { SELECT p.slug, p.id as page_id, p.title, p.type, p.source_id, p.effective_date, p.effective_date_source, + CASE WHEN NULLIF(regexp_replace(p.frontmatter->>'message_id', '^[[:space:]]+|[[:space:]]+$', '', 'g'), '') IS NOT NULL + THEN p.frontmatter->>'message_id' END AS message_id, p.frontmatter->>'thread_id' AS thread_id, + CASE WHEN NULLIF(regexp_replace(p.frontmatter->>'message_id', '^[[:space:]]+|[[:space:]]+$', '', 'g'), '') IS NOT NULL + THEN NULLIF(p.frontmatter->>'subject', '') END AS source_subject, cc.id as chunk_id, cc.chunk_index, cc.chunk_text, cc.chunk_source, ts_rank(cc.search_vector, websearch_to_tsquery('${ftsLang}', $1)) * ${sourceFactorCase} AS score, false AS stale @@ -2220,6 +2229,10 @@ export class PostgresEngine implements BrainEngine { SELECT p.slug, p.id as page_id, p.title, p.type, p.source_id, p.effective_date, p.effective_date_source, + CASE WHEN NULLIF(regexp_replace(p.frontmatter->>'message_id', '^[[:space:]]+|[[:space:]]+$', '', 'g'), '') IS NOT NULL + THEN p.frontmatter->>'message_id' END AS message_id, p.frontmatter->>'thread_id' AS thread_id, + CASE WHEN NULLIF(regexp_replace(p.frontmatter->>'message_id', '^[[:space:]]+|[[:space:]]+$', '', 'g'), '') IS NOT NULL + THEN NULLIF(p.frontmatter->>'subject', '') END AS source_subject, cc.id as chunk_id, cc.chunk_index, cc.chunk_text, cc.chunk_source, 1 - (cc.${col} <=> ${castSql}) AS raw_score FROM content_chunks cc @@ -2254,6 +2267,7 @@ export class PostgresEngine implements BrainEngine { SELECT slug, page_id, title, type, source_id, effective_date, effective_date_source, + message_id, thread_id, source_subject, chunk_id, chunk_index, chunk_text, chunk_source, score, false AS stale diff --git a/src/core/types.ts b/src/core/types.ts index 9464d16de..8fc339ed2 100644 --- a/src/core/types.ts +++ b/src/core/types.ts @@ -723,6 +723,12 @@ export interface SearchResult { */ effective_date?: string | null; effective_date_source?: string | null; + /** RFC 5322 Message-ID projected from allowlisted email frontmatter. */ + message_id?: string; + /** Gmail thread id projected from allowlisted email frontmatter. */ + thread_id?: string; + /** Exact email subject, projected only when the page has a Message-ID. */ + source_subject?: string; /** * v0.40.4 graph signals — populated by applyGraphSignals when the * graph_signals mode-bundle knob is on. Surfaced in JSON envelope diff --git a/src/core/utils.ts b/src/core/utils.ts index 65ecf84ea..5f79d8cf7 100644 --- a/src/core/utils.ts +++ b/src/core/utils.ts @@ -381,6 +381,19 @@ export function rowToSearchResult(row: Record): SearchResult { result.effective_date_source = raw; } } + if (typeof row.message_id === 'string' && row.message_id.trim().length > 0) { + result.message_id = row.message_id; + } + if (typeof row.thread_id === 'string' && row.thread_id.length > 0) { + result.thread_id = row.thread_id; + } + if ( + result.message_id && + typeof row.source_subject === 'string' && + row.source_subject.length > 0 + ) { + result.source_subject = row.source_subject; + } return result; } diff --git a/test/e2e/engine-parity.test.ts b/test/e2e/engine-parity.test.ts index 85b7a0144..bad2e4e6c 100644 --- a/test/e2e/engine-parity.test.ts +++ b/test/e2e/engine-parity.test.ts @@ -149,6 +149,113 @@ describeBoth('Engine parity — Postgres vs PGLite', () => { expect(pgResults[0]?.slug).toBe(pgliteResults[0]?.slug); }); + test('email citation metadata projects identically across engines', async () => { + const slug = 'mail/example-citation'; + const page = { + type: 'note' as const, + title: 'Generated page title', + compiled_truth: 'unique citation projection evidence', + timeline: '', + frontmatter: { + message_id: '', + thread_id: 'thread-example', + subject: 'Example exact email subject', + }, + }; + const chunks = [{ + chunk_index: 0, + chunk_text: page.compiled_truth, + chunk_source: 'compiled_truth' as const, + embedding: basisEmbedding(77), + }]; + + await pgEngine.putPage(slug, page); + await pgEngine.upsertChunks(slug, chunks); + await pgliteEngine.putPage(slug, page); + await pgliteEngine.upsertChunks(slug, chunks); + + const results = [ + (await pgEngine.searchKeyword('unique citation projection evidence'))[0], + (await pgliteEngine.searchKeyword('unique citation projection evidence'))[0], + (await pgEngine.searchKeywordChunks('unique citation projection evidence'))[0], + (await pgliteEngine.searchKeywordChunks('unique citation projection evidence'))[0], + (await pgEngine.searchVector(basisEmbedding(77)))[0], + (await pgliteEngine.searchVector(basisEmbedding(77)))[0], + ]; + + for (const result of results) { + expect(result?.message_id).toBe(''); + expect(result?.thread_id).toBe('thread-example'); + expect(result?.source_subject).toBe('Example exact email subject'); + } + + const nonEmailSlug = 'notes/generated-title-subject-gate'; + const nonEmailPage = { + type: 'note' as const, + title: 'Generated page title must stay a title', + compiled_truth: 'unique non-email subject gate evidence', + timeline: '', + frontmatter: { + subject: 'Frontmatter subject without an email identity', + thread_id: 'standalone-thread-id', + }, + }; + const nonEmailChunks = [{ + chunk_index: 0, + chunk_text: nonEmailPage.compiled_truth, + chunk_source: 'compiled_truth' as const, + }]; + await pgEngine.putPage(nonEmailSlug, nonEmailPage); + await pgEngine.upsertChunks(nonEmailSlug, nonEmailChunks); + await pgliteEngine.putPage(nonEmailSlug, nonEmailPage); + await pgliteEngine.upsertChunks(nonEmailSlug, nonEmailChunks); + + for (const result of [ + (await pgEngine.searchKeyword('unique non-email subject gate evidence'))[0], + (await pgliteEngine.searchKeyword('unique non-email subject gate evidence'))[0], + ]) { + expect(result?.message_id).toBeUndefined(); + expect(result?.thread_id).toBe('standalone-thread-id'); + expect(result?.source_subject).toBeUndefined(); + } + + const whitespaceSlug = 'mail/whitespace-message-id'; + const whitespacePage = { + type: 'note' as const, + title: 'Whitespace Message-ID', + compiled_truth: 'unique whitespace message id evidence', + timeline: '', + frontmatter: { + message_id: ' \t\n ', + thread_id: 'thread-whitespace', + subject: 'Subject must remain gated', + }, + }; + const whitespaceChunks = [{ + chunk_index: 0, + chunk_text: whitespacePage.compiled_truth, + chunk_source: 'compiled_truth' as const, + embedding: basisEmbedding(78), + }]; + await pgEngine.putPage(whitespaceSlug, whitespacePage); + await pgEngine.upsertChunks(whitespaceSlug, whitespaceChunks); + await pgliteEngine.putPage(whitespaceSlug, whitespacePage); + await pgliteEngine.upsertChunks(whitespaceSlug, whitespaceChunks); + + for (const result of [ + (await pgEngine.searchKeyword('unique whitespace message id evidence'))[0], + (await pgliteEngine.searchKeyword('unique whitespace message id evidence'))[0], + (await pgEngine.searchKeywordChunks('unique whitespace message id evidence'))[0], + (await pgliteEngine.searchKeywordChunks('unique whitespace message id evidence'))[0], + (await pgEngine.searchVector(basisEmbedding(78)))[0], + (await pgliteEngine.searchVector(basisEmbedding(78)))[0], + ]) { + expect(result?.message_id).toBeUndefined(); + expect(result?.thread_id).toBe('thread-whitespace'); + expect(result?.source_subject).toBeUndefined(); + } + }); + test('hard-exclude is consistent across engines', async () => { // Both engines should hide test/ pages by default; both should opt // them back in via include_slug_prefixes. diff --git a/test/e2e/source-isolation-pglite.test.ts b/test/e2e/source-isolation-pglite.test.ts index 71567d87a..5eb4650fb 100644 --- a/test/e2e/source-isolation-pglite.test.ts +++ b/test/e2e/source-isolation-pglite.test.ts @@ -20,11 +20,17 @@ import { PGLiteEngine } from '../../src/core/pglite-engine.ts'; import { resetPgliteState } from '../helpers/reset-pglite.ts'; let engine: PGLiteEngine; +let chunkEmbedDim = 0; beforeAll(async () => { engine = new PGLiteEngine(); await engine.connect({}); await engine.initSchema(); + const dim = await (engine as any).db.query( + `SELECT atttypmod FROM pg_attribute + WHERE attrelid = 'content_chunks'::regclass AND attname = 'embedding'`, + ); + chunkEmbedDim = (dim.rows[0] as { atttypmod: number }).atttypmod; }); afterAll(async () => { @@ -48,13 +54,18 @@ beforeEach(async () => { title: 'Alice Source-A', compiled_truth: 'Alice works on widgets in source A. Important context here.', timeline: '', - frontmatter: {}, + frontmatter: { + message_id: '', + thread_id: 'thread-source-a', + subject: 'Source A exact subject', + }, }, { sourceId: 'default' }); await engine.upsertChunks('people/alice', [{ chunk_index: 0, chunk_text: 'Alice works on widgets in source A. Important context here.', chunk_source: 'compiled_truth', token_count: 12, + embedding: Float32Array.from({ length: chunkEmbedDim }, (_, i) => i === 0 ? 1 : 0), }], { sourceId: 'default' }); await engine.putPage('people/alice', { @@ -62,13 +73,18 @@ beforeEach(async () => { title: 'Alice Source-B', compiled_truth: 'Alice works on gadgets in source B. Important context here.', timeline: '', - frontmatter: {}, + frontmatter: { + message_id: '', + thread_id: 'thread-source-b', + subject: 'Source B exact subject', + }, }, { sourceId: 'src-b' }); await engine.upsertChunks('people/alice', [{ chunk_index: 0, chunk_text: 'Alice works on gadgets in source B. Important context here.', chunk_source: 'compiled_truth', token_count: 12, + embedding: Float32Array.from({ length: chunkEmbedDim }, (_, i) => i === 1 ? 1 : 0), }], { sourceId: 'src-b' }); await engine.putPage('people/bob', { @@ -95,6 +111,9 @@ describe('v0.34.1 source-isolation regression (#861)', () => { expect(results.length).toBeGreaterThan(0); for (const r of results) { expect(r.source_id).toBe('default'); + expect(r.message_id).toBe(''); + expect(r.thread_id).toBe('thread-source-a'); + expect(r.source_subject).toBe('Source A exact subject'); } }); @@ -103,6 +122,9 @@ describe('v0.34.1 source-isolation regression (#861)', () => { expect(results.length).toBeGreaterThan(0); for (const r of results) { expect(r.source_id).toBe('src-b'); + expect(r.message_id).toBe(''); + expect(r.thread_id).toBe('thread-source-b'); + expect(r.source_subject).toBe('Source B exact subject'); } }); @@ -170,16 +192,32 @@ describe('v0.34.1 source-isolation regression (#861)', () => { }); test('searchVector with sourceId filters HNSW candidate pool', async () => { - // No real embeddings on the test pages; the WHERE cc.embedding IS NOT NULL - // gate filters them out. We assert the contract via an empty result - // rather than a positive match: with sourceId set, the SQL still runs - // (no type or undefined-column errors). - const synth = new Float32Array(1536).fill(0.01); - const results = await engine.searchVector(synth, { sourceId: 'src-b' }); - // Either empty (no embeddings) or all from src-b. Both prove the - // filter is wired without a runtime error. - for (const r of results) { - expect(r.source_id).toBe('src-b'); + const fixtures = [ + { + sourceId: 'default', embeddingIndex: 0, + message_id: '', thread_id: 'thread-source-a', + source_subject: 'Source A exact subject', + }, + { + sourceId: 'src-b', embeddingIndex: 1, + message_id: '', thread_id: 'thread-source-b', + source_subject: 'Source B exact subject', + }, + ]; + + for (const fixture of fixtures) { + const synth = Float32Array.from( + { length: chunkEmbedDim }, + (_, i) => i === fixture.embeddingIndex ? 1 : 0, + ); + const results = await engine.searchVector(synth, { sourceId: fixture.sourceId }); + expect(results.length).toBeGreaterThan(0); + for (const r of results) { + expect(r.source_id).toBe(fixture.sourceId); + expect(r.message_id).toBe(fixture.message_id); + expect(r.thread_id).toBe(fixture.thread_id); + expect(r.source_subject).toBe(fixture.source_subject); + } } }); diff --git a/test/pglite-engine.test.ts b/test/pglite-engine.test.ts index b41624267..620067a8d 100644 --- a/test/pglite-engine.test.ts +++ b/test/pglite-engine.test.ts @@ -227,6 +227,41 @@ describe('PGLiteEngine: Search', () => { await engine.upsertChunks('concepts/rag', [ { chunk_index: 0, chunk_text: 'RAG combines retrieval with generation', chunk_source: 'compiled_truth' }, ]); + await engine.putPage('mail/example', { + type: 'note', title: 'Launch message', + compiled_truth: 'Launch evidence for citation metadata.', + frontmatter: { + message_id: '', + thread_id: 'thread-123', + subject: 'Example launch subject', + }, + }); + await engine.upsertChunks('mail/example', [ + { chunk_index: 0, chunk_text: 'Launch evidence for citation metadata', chunk_source: 'compiled_truth' }, + ]); + await engine.putPage('notes/generated-title', { + type: 'note', title: 'Generated page title must stay a title', + compiled_truth: 'Non-email evidence for subject gating.', + frontmatter: { + subject: 'Frontmatter subject without an email identity', + thread_id: 'standalone-thread-id', + }, + }); + await engine.upsertChunks('notes/generated-title', [ + { chunk_index: 0, chunk_text: 'Non-email evidence for subject gating', chunk_source: 'compiled_truth' }, + ]); + await engine.putPage('mail/whitespace-message-id', { + type: 'note', title: 'Whitespace message id', + compiled_truth: 'Whitespace-only email identity evidence.', + frontmatter: { + message_id: ' \t\n ', + thread_id: 'thread-whitespace', + subject: 'Subject must remain gated', + }, + }); + await engine.upsertChunks('mail/whitespace-message-id', [ + { chunk_index: 0, chunk_text: 'Whitespace-only email identity evidence', chunk_source: 'compiled_truth' }, + ]); }); test('searchKeyword returns results for matching term', async () => { @@ -235,6 +270,27 @@ describe('PGLiteEngine: Search', () => { expect(results[0].slug).toBe('companies/novamind'); }); + test('searchKeyword projects email citation identifiers from frontmatter', async () => { + const results = await engine.searchKeyword('Launch evidence'); + expect(results[0].message_id).toBe(''); + expect(results[0].thread_id).toBe('thread-123'); + expect(results[0].source_subject).toBe('Example launch subject'); + }); + + test('searchKeyword never promotes a non-email title or subject to source_subject', async () => { + const results = await engine.searchKeyword('Non-email evidence'); + expect(results[0].message_id).toBeUndefined(); + expect(results[0].thread_id).toBe('standalone-thread-id'); + expect(results[0].source_subject).toBeUndefined(); + }); + + test('searchKeyword treats whitespace-only message_id as absent', async () => { + const results = await engine.searchKeyword('Whitespace-only email identity'); + expect(results[0].message_id).toBeUndefined(); + expect(results[0].thread_id).toBe('thread-whitespace'); + expect(results[0].source_subject).toBeUndefined(); + }); + test('searchKeyword returns empty for non-matching term', async () => { const results = await engine.searchKeyword('xyznonexistent'); expect(results.length).toBe(0); @@ -256,6 +312,42 @@ describe('PGLiteEngine: Search', () => { const results = await engine.searchVector(fakeEmbedding); expect(results.length).toBe(0); }); + + test('searchVector carries email citation metadata through the outer CTE', async () => { + const embedding = new Float32Array(CHUNK_EMBED_DIM); + embedding[0] = 1; + await engine.upsertChunks('mail/example', [ + { + chunk_index: 0, + chunk_text: 'Launch evidence for citation metadata', + chunk_source: 'compiled_truth', + embedding, + }, + ]); + + const results = await engine.searchVector(embedding); + expect(results[0].message_id).toBe(''); + expect(results[0].thread_id).toBe('thread-123'); + expect(results[0].source_subject).toBe('Example launch subject'); + }); + + test('searchVector treats whitespace-only message_id as absent', async () => { + const embedding = new Float32Array(CHUNK_EMBED_DIM); + embedding[1] = 1; + await engine.upsertChunks('mail/whitespace-message-id', [ + { + chunk_index: 0, + chunk_text: 'Whitespace-only email identity evidence', + chunk_source: 'compiled_truth', + embedding, + }, + ]); + + const results = await engine.searchVector(embedding); + expect(results[0].message_id).toBeUndefined(); + expect(results[0].thread_id).toBe('thread-whitespace'); + expect(results[0].source_subject).toBeUndefined(); + }); }); // ───────────────────────────────────────────────────────────────── @@ -299,6 +391,19 @@ describe('PGLiteEngine: CJK keyword fallback (v0.32.7)', () => { await engine.upsertChunks('originals/english-essay', [ { chunk_index: 0, chunk_text: 'NovaMind builds AI agents for enterprise', chunk_source: 'compiled_truth' }, ]); + + await engine.putPage('mail/cjk-example', { + type: 'note', title: 'Generated CJK page title', + compiled_truth: '郵件引用識別', + frontmatter: { + message_id: '', + thread_id: 'thread-cjk', + subject: 'Example CJK email subject', + }, + }); + await engine.upsertChunks('mail/cjk-example', [ + { chunk_index: 0, chunk_text: '郵件引用識別', chunk_source: 'compiled_truth' }, + ]); }); test('CJK query routes to LIKE branch and finds Chinese substring', async () => { @@ -319,6 +424,17 @@ describe('PGLiteEngine: CJK keyword fallback (v0.32.7)', () => { expect(results[0].slug).toBe('originals/korean-essay'); }); + test('CJK keyword page and chunk paths project email citation metadata', async () => { + for (const result of [ + (await engine.searchKeyword('郵件引用'))[0], + (await engine.searchKeywordChunks('郵件引用'))[0], + ]) { + expect(result.message_id).toBe(''); + expect(result.thread_id).toBe('thread-cjk'); + expect(result.source_subject).toBe('Example CJK email subject'); + } + }); + test('bigram ranking: 3-hit page outranks 1-hit page', async () => { // Add another Chinese page with only ONE occurrence of 测试. await engine.putPage('originals/chinese-one-hit', { diff --git a/test/search/hybrid-reranker-integration.serial.test.ts b/test/search/hybrid-reranker-integration.serial.test.ts index 350f7ec8d..3ba02c371 100644 --- a/test/search/hybrid-reranker-integration.serial.test.ts +++ b/test/search/hybrid-reranker-integration.serial.test.ts @@ -19,7 +19,11 @@ import { afterAll, beforeAll, describe, expect, test } from 'bun:test'; import { PGLiteEngine } from '../../src/core/pglite-engine.ts'; -import { hybridSearch } from '../../src/core/search/hybrid.ts'; +import { + awaitPendingSearchCacheWrites, + hybridSearch, + hybridSearchCached, +} from '../../src/core/search/hybrid.ts'; import { configureGateway, resetGateway, @@ -79,6 +83,23 @@ beforeAll(async () => { env: { OPENAI_API_KEY: 'sk-test' }, }); stubEmbeddings(); + + await engine.putPage('mail/vector-first', { + type: 'note', + title: 'Vector-first email', + compiled_truth: 'vector first duplicate metadata evidence', + frontmatter: { + message_id: '', + thread_id: 'thread-vector-first', + subject: 'Vector-first exact subject', + }, + }); + await engine.upsertChunks('mail/vector-first', [{ + chunk_index: 0, + chunk_text: 'vector first duplicate metadata evidence', + chunk_source: 'compiled_truth', + embedding: Float32Array.from(FAKE_EMB), + }]); }); afterAll(async () => { @@ -105,6 +126,28 @@ describe('hybridSearch — reranker disabled (pass-through)', () => { }); }); +describe('hybridSearchCached — email metadata through vector-first fusion', () => { + test('fresh cache miss preserves metadata through vector-first RRF duplicate handling', async () => { + await engine.executeRaw(`DELETE FROM query_cache`); + let cacheStatus: string | undefined; + const out = await hybridSearchCached(engine, 'vector first duplicate metadata evidence', { + limit: 10, + useCache: true, + autocut: false, + graph_signals: false, + onMeta: (meta) => { cacheStatus = meta.cache?.status; }, + }); + + expect(cacheStatus).toBe('miss'); + const matches = out.filter(r => r.slug === 'mail/vector-first'); + expect(matches).toHaveLength(1); + expect(matches[0].message_id).toBe(''); + expect(matches[0].thread_id).toBe('thread-vector-first'); + expect(matches[0].source_subject).toBe('Vector-first exact subject'); + await awaitPendingSearchCacheWrites(); + }); +}); + describe('hybridSearch — reranker enabled (reorder)', () => { test('rerankerFn receives a non-empty document list', async () => { let receivedDocs: string[] = []; diff --git a/test/utils.test.ts b/test/utils.test.ts index 6682b290c..568e9a207 100644 --- a/test/utils.test.ts +++ b/test/utils.test.ts @@ -183,4 +183,42 @@ describe('rowToSearchResult', () => { expect(typeof r.score).toBe('number'); expect(r.score).toBe(0.95); }); + + test('projects allowlisted email identifiers when present', () => { + const r = rowToSearchResult({ + slug: 'mail/example', page_id: 2, title: 'Example email', type: 'note', + chunk_text: 'text', chunk_source: 'compiled_truth', chunk_id: 3, chunk_index: 0, + score: 0.9, stale: false, + message_id: '', thread_id: 'abc123', + source_subject: 'Example launch subject', + }); + expect(r.message_id).toBe(''); + expect(r.thread_id).toBe('abc123'); + expect(r.source_subject).toBe('Example launch subject'); + }); + + test('does not invent email identifiers when projections are absent', () => { + const r = rowToSearchResult({ + slug: 'concept/example', page_id: 3, title: 'Example', type: 'concept', + chunk_text: 'text', chunk_source: 'compiled_truth', chunk_id: 4, chunk_index: 0, + score: 0.8, stale: false, + source_subject: 'Generated title must not become an email subject', + }); + expect(r.message_id).toBeUndefined(); + expect(r.thread_id).toBeUndefined(); + expect(r.source_subject).toBeUndefined(); + }); + + test('whitespace-only message_id does not project or authorize source_subject', () => { + const r = rowToSearchResult({ + slug: 'mail/whitespace-id', page_id: 4, title: 'Whitespace ID', type: 'note', + chunk_text: 'text', chunk_source: 'compiled_truth', chunk_id: 5, chunk_index: 0, + score: 0.7, stale: false, + message_id: ' \t\n ', thread_id: 'thread-whitespace', + source_subject: 'Must remain gated', + }); + expect(r.message_id).toBeUndefined(); + expect(r.thread_id).toBe('thread-whitespace'); + expect(r.source_subject).toBeUndefined(); + }); });