From a8a3b6df9f44270d88cb3b41bd21c9058cd89d58 Mon Sep 17 00:00:00 2001 From: Time Attakc <89218912+time-attack@users.noreply.github.com> Date: Tue, 28 Jul 2026 19:14:39 -0700 Subject: [PATCH] fix(engine): exclude soft-deleted pages from getHealth counts (#1305) (#3556) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit getStats() has excluded soft-deleted pages since v0.26.5, but getHealth() kept counting raw pages rows: page_count, the islanded/orphan scan, the entity_pages CTE (link/timeline coverage denominators), and most_connected all included deleted pages, so brain_score never moved when a user soft-deleted pages. Repro: 50 pages, soft-delete 40 -> getStats 10 vs getHealth 50, orphan_pages 50, brain_score byte-identical. Fix: every page-scoped count in getHealth now filters deleted_at IS NULL, identically in both engines (engine-parity SQL shapes match). Deliberate boundary: chunk/link storage counts (embed_coverage, missing_embeddings, link_count, dead_links) stay raw until the purge phase runs — matching getStats' documented posture — and destructive-removal counts (#2235) deliberately keep counting all rows. stale_pages already filtered via buildStalePagesWhere. Test: test/health-soft-delete.test.ts — 3 of 4 tests fail behaviorally on unmodified master (page_count 10 vs 4, orphan_pages 8 vs 0, link_coverage 0.5 vs 1), all pass with the fix. Co-authored-by: Garry Tan Co-authored-by: Claude Opus 5 --- src/core/pglite-engine.ts | 11 ++- src/core/postgres-engine.ts | 11 ++- test/health-soft-delete.test.ts | 123 ++++++++++++++++++++++++++++++++ 3 files changed, 139 insertions(+), 6 deletions(-) create mode 100644 test/health-soft-delete.test.ts diff --git a/src/core/pglite-engine.ts b/src/core/pglite-engine.ts index df2896102..94bed2de2 100644 --- a/src/core/pglite-engine.ts +++ b/src/core/pglite-engine.ts @@ -5332,12 +5332,16 @@ export class PGLiteEngine implements BrainEngine { // pages_with_timeline) and v0.10.3 graph layer (link_coverage, timeline_coverage, // most_connected). Both coexist: master's brain_score is the composite // dashboard, v0.10.3 metrics give entity-page-level granularity. + // #1305: every page-scoped count here excludes soft-deleted rows — same + // posture as getStats — so brain_score moves when the user deletes pages. + // Chunk/link counts stay raw (storage until the purge phase), matching + // getStats, and destructive-removal counts elsewhere deliberately stay raw. const { rows: [h] } = await this.db.query(` WITH entity_pages AS ( - SELECT id, slug FROM pages WHERE type IN ('entity', 'person', 'company') + SELECT id, slug FROM pages WHERE type IN ('entity', 'person', 'company') AND deleted_at IS NULL ) SELECT - (SELECT count(*) FROM pages) as page_count, + (SELECT count(*) FROM pages WHERE deleted_at IS NULL) as page_count, (SELECT count(*) FROM content_chunks WHERE embedded_at IS NOT NULL)::float / GREATEST((SELECT count(*) FROM content_chunks), 1)::float as embed_coverage, 0 as stale_pages, @@ -5362,7 +5366,7 @@ export class PGLiteEngine implements BrainEngine { SELECT p.slug, (SELECT count(*) FROM links l WHERE l.from_page_id = p.id OR l.to_page_id = p.id)::int as link_count FROM pages p - WHERE p.type IN ('entity', 'person', 'company') + WHERE p.type IN ('entity', 'person', 'company') AND p.deleted_at IS NULL ORDER BY link_count DESC LIMIT 5 `); @@ -5381,6 +5385,7 @@ export class PGLiteEngine implements BrainEngine { AND NOT EXISTS (SELECT 1 FROM links l WHERE l.from_page_id = p.id)) as islanded, EXISTS (SELECT 1 FROM timeline_entries te WHERE te.page_id = p.id) as has_timeline FROM pages p + WHERE p.deleted_at IS NULL `); const r = h as Record; diff --git a/src/core/postgres-engine.ts b/src/core/postgres-engine.ts index cb0244c49..f116fe405 100644 --- a/src/core/postgres-engine.ts +++ b/src/core/postgres-engine.ts @@ -5432,12 +5432,16 @@ export class PostgresEngine implements BrainEngine { // no outbound links). The raw islanded list is filtered through the same // policy as `gbrain orphans` so convention pages do not count against // dashboard health. + // #1305: every page-scoped count here excludes soft-deleted rows — same + // posture as getStats — so brain_score moves when the user deletes pages. + // Chunk/link counts stay raw (storage until the purge phase), matching + // getStats, and destructive-removal counts elsewhere deliberately stay raw. const [h] = await sql` WITH entity_pages AS ( - SELECT id, slug FROM pages WHERE type IN ('entity', 'person', 'company') + SELECT id, slug FROM pages WHERE type IN ('entity', 'person', 'company') AND deleted_at IS NULL ) SELECT - (SELECT count(*) FROM pages) as page_count, + (SELECT count(*) FROM pages WHERE deleted_at IS NULL) as page_count, (SELECT count(*) FROM content_chunks WHERE embedded_at IS NOT NULL)::float / GREATEST((SELECT count(*) FROM content_chunks), 1)::float as embed_coverage, 0 as stale_pages, @@ -5459,7 +5463,7 @@ export class PostgresEngine implements BrainEngine { SELECT p.slug, (SELECT count(*) FROM links l WHERE l.from_page_id = p.id OR l.to_page_id = p.id)::int as link_count FROM pages p - WHERE p.type IN ('entity', 'person', 'company') + WHERE p.type IN ('entity', 'person', 'company') AND p.deleted_at IS NULL ORDER BY link_count DESC LIMIT 5 `; @@ -5478,6 +5482,7 @@ export class PostgresEngine implements BrainEngine { AND NOT EXISTS (SELECT 1 FROM links l WHERE l.from_page_id = p.id)) as islanded, EXISTS (SELECT 1 FROM timeline_entries te WHERE te.page_id = p.id) as has_timeline FROM pages p + WHERE p.deleted_at IS NULL `; const pageCount = Number(h.page_count); diff --git a/test/health-soft-delete.test.ts b/test/health-soft-delete.test.ts new file mode 100644 index 000000000..b4991a6c3 --- /dev/null +++ b/test/health-soft-delete.test.ts @@ -0,0 +1,123 @@ +/** + * #1305 — getHealth() must exclude soft-deleted pages from every + * page-scoped count, the same posture getStats() has had since v0.26.5. + * + * Pre-fix, getHealth counted raw `pages` rows: page_count and orphan_pages + * included soft-deleted pages, the entity_pages CTE kept deleted entities in + * the link/timeline coverage denominators and in most_connected, and + * brain_score therefore never moved when a user soft-deleted pages. + * + * Boundary (deliberate): chunk- and link-scoped counts (embed_coverage, + * missing_embeddings, link_count, dead_links) stay RAW — they occupy storage + * until the autopilot purge phase, matching getStats. Destructive-removal + * counts (purge paths, #2235) also deliberately count all rows and are + * untouched here. + * + * Runs against PGLite — the fixed SQL shapes are identical in both engines. + */ + +import { describe, test, expect, beforeAll, afterAll, beforeEach } from 'bun:test'; +import { PGLiteEngine } from '../src/core/pglite-engine.ts'; + +let engine: PGLiteEngine; + +beforeAll(async () => { + engine = new PGLiteEngine(); + await engine.connect({}); + await engine.initSchema(); +}, 60_000); + +afterAll(async () => { + await engine.disconnect(); +}); + +beforeEach(async () => { + for (const t of ['links', 'content_chunks', 'timeline_entries', 'tags', 'page_versions', 'pages']) { + await (engine as any).db.exec(`DELETE FROM ${t}`); + } +}); + +async function seedNote(slug: string): Promise { + await engine.putPage(slug, { type: 'note', title: slug, compiled_truth: `content of ${slug}`, frontmatter: {} }); +} + +async function pageId(slug: string): Promise { + return (await (engine as any).db.query(`SELECT id FROM pages WHERE slug=$1`, [slug])).rows[0].id; +} + +describe('#1305 — getHealth excludes soft-deleted pages', () => { + test('page_count and orphan_pages match getStats after soft-delete (the issue repro)', async () => { + for (let i = 0; i < 10; i++) await seedNote(`wiki/note-${i}`); + for (let i = 0; i < 6; i++) await engine.softDeletePage(`wiki/note-${i}`); + + const stats = await engine.getStats(); + const health = await engine.getHealth(); + expect(stats.page_count).toBe(4); + // Pre-fix: 10 (raw rows). getHealth must agree with getStats. + expect(health.page_count).toBe(4); + // Pre-fix: 10 — deleted pages stayed in the islanded scan. + expect(health.orphan_pages).toBe(4); + }); + + test('brain_score moves when the user soft-deletes the islanded pages', async () => { + // 2 connected pages + 8 islanded ones. + await seedNote('wiki/hub'); + await seedNote('wiki/leaf'); + await (engine as any).db.query( + `INSERT INTO links (from_page_id, to_page_id, link_type) VALUES ($1, $2, 'mentions')`, + [await pageId('wiki/hub'), await pageId('wiki/leaf')], + ); + for (let i = 0; i < 8; i++) await seedNote(`wiki/clutter-${i}`); + + const before = await engine.getHealth(); + for (let i = 0; i < 8; i++) await engine.softDeletePage(`wiki/clutter-${i}`); + const after = await engine.getHealth(); + + // Pre-fix both assertions fail: orphan_pages stayed 8 and brain_score + // was byte-identical before/after the delete. + expect(after.orphan_pages).toBe(0); + expect(after.brain_score).toBeGreaterThan(before.brain_score); + }); + + test('entity coverage denominators and most_connected exclude deleted entities', async () => { + // Live entity: inbound link + timeline entry → full coverage. + await engine.putPage('people/alice-example', { type: 'person', title: 'Alice', compiled_truth: 'a person', frontmatter: {} }); + await engine.putPage('people/bob-example', { type: 'person', title: 'Bob', compiled_truth: 'another person', frontmatter: {} }); + await seedNote('wiki/mentions-alice'); + const aliceId = await pageId('people/alice-example'); + await (engine as any).db.query( + `INSERT INTO links (from_page_id, to_page_id, link_type) VALUES ($1, $2, 'mentions')`, + [await pageId('wiki/mentions-alice'), aliceId], + ); + await (engine as any).db.query( + `INSERT INTO timeline_entries (page_id, date, summary) VALUES ($1, '2026-01-01', 'met alice')`, + [aliceId], + ); + + await engine.softDeletePage('people/bob-example'); + const h = await engine.getHealth(); + + // Pre-fix: bob stayed in the entity_pages CTE → coverage 0.5 each, + // and bob appeared in most_connected. + expect(h.link_coverage).toBe(1); + expect(h.timeline_coverage).toBe(1); + expect(h.most_connected.map((c) => c.slug)).not.toContain('people/bob-example'); + }); + + test('chunk storage counts stay raw (the deliberate boundary)', async () => { + await seedNote('wiki/kept'); + await seedNote('wiki/gone'); + for (const slug of ['wiki/kept', 'wiki/gone']) { + await (engine as any).db.query( + `INSERT INTO content_chunks (page_id, chunk_index, chunk_text) VALUES ($1, 0, 'chunk')`, + [await pageId(slug)], + ); + } + await engine.softDeletePage('wiki/gone'); + + const h = await engine.getHealth(); + // Soft-deleted pages' chunks still occupy storage until purge; the + // missing_embeddings count keeps seeing them, same as getStats. + expect(h.missing_embeddings).toBe(2); + }); +});