v0.42.13.0 fix(search): archive/ content findable by default, demoted not hard-excluded (#1777) (#1797)

* fix(search): archive/ findable by default — demote not hard-exclude (#1777)

Move archive/ out of DEFAULT_HARD_EXCLUDES into a 0.5 source-boost demote so
archived historical content is findable by default, ranked below curated
content. Add a hidden_by_search_policy doctor check so the surviving exclude
policy (test/, attachments/, .raw/) is auditable. Bump KNOBS_HASH_VERSION 8->9
so the policy change invalidates archive-excluded query_cache rows.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

* chore: bump version and changelog (v0.42.13.0)

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

* docs: correct search-exclude.test.ts annotation for archive demote (#1777)

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

---------

Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

test/cross-modal-phase1.test.ts

@@ -136,13 +136,14 @@ describe('D2 — knobsHash differs across cross-modal knob values', () => {
     return resolveSearchMode({ mode: 'balanced' });
   }
 
-  test('KNOBS_HASH_VERSION is 6 (v=4 graph_signals + schema-pack; v=5 contextual_retrieval; v=6 alias_resolved; cross-modal still appended)', () => {
+  test('KNOBS_HASH_VERSION is 9 (cross-modal still appended; 8→9 archive-demote #1777)', () => {
     // v0.35 ladder: 1→2 reranker, 2→3 floor_ratio. v0.36 piggybacks on v=3
     // with 7 cross-modal knobs + column/provider context. v0.40.4 (salem) +
     // v0.39 T21 (master) bump to v=4 for graph_signals + schema-pack fields.
     // v0.40.3.0 D8 bumps to v=5 (sequenced behind salem's v=4 graph-signals).
     // v0.41.22.0 (type-unification): 5→6 for alias_resolved post-fusion boost.
-    expect(KNOBS_HASH_VERSION).toBe(8);
+    // T2: 6→7 title_boost. v0.42.3.0: 7→8 autocut. issue #1777: 8→9 archive/ demote.
+    expect(KNOBS_HASH_VERSION).toBe(9);
   });
 
   test('flipping unified_multimodal changes the hash', () => {

test/doctor-hidden-by-search-policy.test.ts

@@ -0,0 +1,153 @@
+/**
+ * hidden_by_search_policy doctor check (issue #1777)
+ *
+ * Counts CHUNKED pages withheld from default search by the hard-exclude prefix
+ * policy. Verifies:
+ *  - default-only excludes (test/) → status ok with a prescriptive message
+ *  - a NON-default env exclude hiding pages → status warn
+ *  - multi-chunk page counted once (DISTINCT)
+ *  - soft-deleted / quarantined pages NOT counted (visibility mirror)
+ *  - archive/ never appears (it's demoted, not excluded)
+ *  - SQL error → warn, not throw
+ *  - the check name categorizes as a brain check
+ */
+
+import { describe, test, expect, beforeAll, afterAll, beforeEach } from 'bun:test';
+import { PGLiteEngine } from '../src/core/pglite-engine.ts';
+import { resetPgliteState } from './helpers/reset-pglite.ts';
+import { withEnv } from './helpers/with-env.ts';
+import { checkHiddenBySearchPolicy } from '../src/commands/doctor.ts';
+import { categorizeCheck } from '../src/core/doctor-categories.ts';
+import { buildQuarantineMarker } from '../src/core/quarantine.ts';
+import type { BrainEngine } from '../src/core/engine.ts';
+import type { ChunkInput } from '../src/core/types.ts';
+
+let engine: PGLiteEngine;
+
+function basisEmbedding(idx: number, dim = 1536): Float32Array {
+  const emb = new Float32Array(dim);
+  emb[idx % dim] = 1.0;
+  return emb;
+}
+
+async function seed(
+  slug: string,
+  opts: { chunks?: number; frontmatter?: Record<string, unknown> } = {},
+): Promise<void> {
+  const chunks = opts.chunks ?? 1;
+  await engine.putPage(slug, {
+    type: 'note',
+    title: slug,
+    compiled_truth: `body for ${slug}`,
+    timeline: '',
+    ...(opts.frontmatter ? { frontmatter: opts.frontmatter } : {}),
+  });
+  if (chunks > 0) {
+    const rows: ChunkInput[] = [];
+    for (let i = 0; i < chunks; i++) {
+      rows.push({
+        chunk_index: i,
+        chunk_text: `chunk ${i} of ${slug}`,
+        chunk_source: 'compiled_truth',
+        embedding: basisEmbedding(100 + i),
+        token_count: 4,
+      });
+    }
+    await engine.upsertChunks(slug, rows);
+  }
+}
+
+beforeAll(async () => {
+  engine = new PGLiteEngine();
+  await engine.connect({});
+  await engine.initSchema();
+}, 60_000);
+
+afterAll(async () => {
+  await engine.disconnect();
+});
+
+beforeEach(async () => {
+  await resetPgliteState(engine);
+});
+
+describe('checkHiddenBySearchPolicy', () => {
+  test('default-only excludes → ok, prescriptive message, names test/ not archive/concepts', async () => {
+    await seed('test/fixtures/widget');
+    await seed('concepts/widget-pattern');
+    await seed('archive/old/widget-2020');
+
+    const r = await checkHiddenBySearchPolicy(engine);
+    expect(r.name).toBe('hidden_by_search_policy');
+    expect(r.status).toBe('ok'); // test/ is a DEFAULT exclude → not a warn
+    expect(r.message).toContain("under 'test/'");
+    expect(r.message).toContain('chunked');
+    expect(r.message).not.toContain('searchable'); // honest superset, not "searchable"
+    expect(r.message).toContain('include_slug_prefixes'); // prescriptive guidance
+    // archive/ is demoted, not excluded — must never surface here.
+    expect(r.message).not.toContain('archive/');
+    // concepts/ is not excluded at all.
+    expect(r.message).not.toContain('concepts/');
+    const counts = (r.details?.counts ?? {}) as Record<string, number>;
+    expect(counts['test/']).toBe(1);
+    expect(counts['archive/']).toBeUndefined();
+    expect(counts['concepts/']).toBeUndefined();
+  });
+
+  test('a NON-default env exclude hiding pages → warn', async () => {
+    await withEnv({ GBRAIN_SEARCH_EXCLUDE: 'scratch/' }, async () => {
+      await seed('scratch/notes');
+      await seed('concepts/keeper');
+      const r = await checkHiddenBySearchPolicy(engine);
+      expect(r.status).toBe('warn');
+      expect(r.message).toContain("under 'scratch/'");
+    });
+  });
+
+  test('multi-chunk page counted once (DISTINCT)', async () => {
+    await seed('test/multi', { chunks: 3 });
+    const r = await checkHiddenBySearchPolicy(engine);
+    const counts = (r.details?.counts ?? {}) as Record<string, number>;
+    expect(counts['test/']).toBe(1);
+  });
+
+  test('soft-deleted and quarantined pages are NOT counted', async () => {
+    await seed('test/soft-deleted');
+    await engine.softDeletePage('test/soft-deleted');
+    await seed('test/quarantined', {
+      frontmatter: { quarantine: buildQuarantineMarker('junk_pattern', 'test fixture') },
+    });
+    await seed('concepts/keeper');
+
+    const r = await checkHiddenBySearchPolicy(engine);
+    // Both test/ pages are hidden by the visibility clause, so 0 remain counted.
+    expect(r.status).toBe('ok');
+    expect(r.message).toBe('No pages hidden by search-exclude policy.');
+    const counts = (r.details?.counts ?? {}) as Record<string, number>;
+    expect(counts['test/']).toBeUndefined();
+  });
+
+  test('zero hidden (only curated content) → ok', async () => {
+    await seed('concepts/keeper');
+    const r = await checkHiddenBySearchPolicy(engine);
+    expect(r.status).toBe('ok');
+    expect(r.message).toBe('No pages hidden by search-exclude policy.');
+  });
+
+  test('SQL error → warn, not throw', async () => {
+    const broken = {
+      executeRaw: async () => {
+        throw new Error('boom');
+      },
+    } as unknown as BrainEngine;
+    const r = await checkHiddenBySearchPolicy(broken);
+    expect(r.status).toBe('warn');
+    expect(r.message).toContain('Could not check');
+  });
+});
+
+describe('hidden_by_search_policy categorization', () => {
+  test('categorizes as a brain check', () => {
+    expect(categorizeCheck('hidden_by_search_policy')).toBe('brain');
+  });
+});

test/e2e/search-exclude.test.ts

@@ -1,13 +1,16 @@
 /**
  * Hard-Exclude E2E
  *
- * Verifies the new exclude_slug_prefixes / include_slug_prefixes plumbing.
- * test/, archive/, attachments/, .raw/ are hard-excluded by default.
- * include_slug_prefixes opts back in.
+ * Verifies the exclude_slug_prefixes / include_slug_prefixes plumbing.
+ * test/, attachments/, .raw/ are hard-excluded by default.
+ * archive/ is NOT excluded (issue #1777) — it is demoted (0.5x) so it stays
+ * findable by default but ranks below curated content. include_slug_prefixes
+ * opts back into the genuinely-excluded prefixes.
  */
 
 import { describe, test, expect, beforeAll, afterAll } from 'bun:test';
 import { PGLiteEngine } from '../../src/core/pglite-engine.ts';
+import { hybridSearch } from '../../src/core/search/hybrid.ts';
 import type { ChunkInput } from '../../src/core/types.ts';
 
 let engine: PGLiteEngine;
@@ -70,6 +73,42 @@ beforeAll(async () => {
       token_count: 9,
     },
   ] satisfies ChunkInput[]);
+
+  // A genuinely-excluded prefix that stays hidden by default (locks "only
+  // archive moved" — test/, attachments/, .raw/ are unchanged).
+  await engine.putPage('.raw/widget-dump', {
+    type: 'note',
+    title: 'Widget raw dump',
+    compiled_truth: 'widget raw sidecar dump',
+    timeline: '',
+  });
+  await engine.upsertChunks('.raw/widget-dump', [
+    {
+      chunk_index: 0,
+      chunk_text: 'widget raw sidecar dump noise',
+      chunk_source: 'compiled_truth',
+      embedding: basisEmbedding(14),
+      token_count: 5,
+    },
+  ] satisfies ChunkInput[]);
+
+  // A term that appears in NO other page, only in an archive page. Proves
+  // archive is reachable even when it is the ONLY match (issue #1777).
+  await engine.putPage('archive/2019/quokkanaut-memo', {
+    type: 'note',
+    title: 'Quokkanaut memo',
+    compiled_truth: 'the quokkanaut project shipped in 2019',
+    timeline: '',
+  });
+  await engine.upsertChunks('archive/2019/quokkanaut-memo', [
+    {
+      chunk_index: 0,
+      chunk_text: 'the quokkanaut project shipped in 2019',
+      chunk_source: 'compiled_truth',
+      embedding: basisEmbedding(15),
+      token_count: 6,
+    },
+  ] satisfies ChunkInput[]);
 }, 60_000);
 
 afterAll(async () => {
@@ -83,10 +122,33 @@ describe('searchKeyword default hard-excludes', () => {
     expect(slugs).not.toContain('test/fixtures/widget');
   });
 
-  test('archive/ pages are hidden by default', async () => {
+  test('.raw/ pages stay hidden by default (only archive moved)', async () => {
     const results = await engine.searchKeyword('widget');
     const slugs = results.map(r => r.slug);
-    expect(slugs).not.toContain('archive/old-stuff/widget-2020');
+    expect(slugs).not.toContain('.raw/widget-dump');
+  });
+
+  test('archive/ pages ARE returned by default (issue #1777)', async () => {
+    const results = await engine.searchKeyword('widget');
+    const slugs = results.map(r => r.slug);
+    expect(slugs).toContain('archive/old-stuff/widget-2020');
+  });
+
+  test('archive/ is demoted below curated content (keyword ordering)', async () => {
+    const results = await engine.searchKeyword('widget');
+    const slugs = results.map(r => r.slug);
+    const archiveIdx = slugs.indexOf('archive/old-stuff/widget-2020');
+    const conceptsIdx = slugs.indexOf('concepts/widget-pattern');
+    expect(archiveIdx).toBeGreaterThanOrEqual(0); // present
+    expect(conceptsIdx).toBeGreaterThanOrEqual(0);
+    // concepts/ (boost 1.3) ranks ABOVE archive/ (boost 0.5).
+    expect(conceptsIdx).toBeLessThan(archiveIdx);
+  });
+
+  test('archive is reachable even when it is the ONLY match (unique phrase)', async () => {
+    const results = await engine.searchKeyword('quokkanaut');
+    const slugs = results.map(r => r.slug);
+    expect(slugs).toContain('archive/2019/quokkanaut-memo');
   });
 
   test('curated content is unaffected', async () => {
@@ -103,17 +165,17 @@ describe('searchKeyword include_slug_prefixes opt-back-in', () => {
     });
     const slugs = results.map(r => r.slug);
     expect(slugs).toContain('test/fixtures/widget');
-    // archive/ is still excluded.
-    expect(slugs).not.toContain('archive/old-stuff/widget-2020');
+    // .raw/ is still excluded.
+    expect(slugs).not.toContain('.raw/widget-dump');
   });
 
-  test('include_slug_prefixes lets caller opt back into both', async () => {
+  test('include_slug_prefixes opts back into multiple excluded prefixes', async () => {
     const results = await engine.searchKeyword('widget', {
-      include_slug_prefixes: ['test/', 'archive/'],
+      include_slug_prefixes: ['test/', '.raw/'],
     });
     const slugs = results.map(r => r.slug);
     expect(slugs).toContain('test/fixtures/widget');
-    expect(slugs).toContain('archive/old-stuff/widget-2020');
+    expect(slugs).toContain('.raw/widget-dump');
   });
 });
 
@@ -126,6 +188,13 @@ describe('searchVector hard-excludes', () => {
     expect(slugs).not.toContain('test/fixtures/widget');
   });
 
+  test('archive/ pages ARE returned by default in vector search (#1777)', async () => {
+    // basisEmbedding(12) points at archive/old-stuff/widget-2020.
+    const results = await engine.searchVector(basisEmbedding(12));
+    const slugs = results.map(r => r.slug);
+    expect(slugs).toContain('archive/old-stuff/widget-2020');
+  });
+
   test('include_slug_prefixes lets it back in', async () => {
     const results = await engine.searchVector(basisEmbedding(11), {
       include_slug_prefixes: ['test/'],
@@ -135,16 +204,32 @@ describe('searchVector hard-excludes', () => {
   });
 });
 
+describe('hybridSearch (the real user/MCP surface)', () => {
+  test('archive/ is returned by default through hybridSearch', async () => {
+    // No embedding provider in this hermetic run, so hybridSearch's vector path
+    // is skipped and it exercises keyword + dedup + post-fusion. archive/ must
+    // still come back by default (it is no longer hard-excluded).
+    const results = await hybridSearch(engine, 'widget', { limit: 20 });
+    const slugs = results.map(r => r.slug);
+    expect(slugs).toContain('archive/old-stuff/widget-2020');
+    // test/ and .raw/ stay hidden through the full pipeline.
+    expect(slugs).not.toContain('test/fixtures/widget');
+    expect(slugs).not.toContain('.raw/widget-dump');
+  });
+});
+
 describe('caller-supplied exclude_slug_prefixes (additive)', () => {
   test('caller can add a custom exclude prefix on top of defaults', async () => {
     const results = await engine.searchKeyword('widget', {
       exclude_slug_prefixes: ['concepts/'],
     });
     const slugs = results.map(r => r.slug);
-    // concepts/ now also excluded; with all three categories filtered, no
-    // hits remain.
+    // concepts/ now also excluded on top of the defaults (test/, .raw/).
     expect(slugs).not.toContain('concepts/widget-pattern');
     expect(slugs).not.toContain('test/fixtures/widget');
-    expect(slugs).not.toContain('archive/old-stuff/widget-2020');
+    expect(slugs).not.toContain('.raw/widget-dump');
+    // archive/ is NOT a default exclude and was not caller-excluded, so it
+    // remains visible (issue #1777).
+    expect(slugs).toContain('archive/old-stuff/widget-2020');
   });
 });

test/search-alias-resolved-boost.test.ts

@@ -89,7 +89,7 @@ describe('alias_resolved boost stage', () => {
 });
 
 describe('KNOBS_HASH_VERSION', () => {
-  it('bumped to 6 to invalidate caches across v0.42 boost stage addition', () => {
-    expect(KNOBS_HASH_VERSION).toBe(8);
+  it('is 9 (8→9 archive-demote invalidates archive-excluded cache rows, #1777)', () => {
+    expect(KNOBS_HASH_VERSION).toBe(9);
   });
 });

test/search-mode.test.ts

@@ -388,10 +388,11 @@ describe('knobsHash determinism + cross-mode separation (CDX-4)', () => {
     // embedding spaces. Sequenced behind salem's v=4 graph-signals work.
     // v0.41.22.0 (type-unification): bumped 5→6 for the new alias_resolved
     // post-fusion boost stage. T2: bumped 6→7 for title_boost. v0.42.3.0:
-    // bumped 7→8 for autocut (ac=/acj=). A query against a brain with
-    // slug_aliases populated must not be served from a cache row written
-    // before the boost stage existed.
-    expect(KNOBS_HASH_VERSION).toBe(8);
+    // bumped 7→8 for autocut (ac=/acj=). issue #1777: bumped 8→9 for the
+    // archive/ demote (search-exclude policy change isn't in the hash, so the
+    // version bump is what invalidates archive-excluded cache rows). A query
+    // must not be served from a cache row written before the policy change.
+    expect(KNOBS_HASH_VERSION).toBe(9);
   });
 
   test('T1 (codex): floor_ratio set vs unset produces DIFFERENT hashes (cache contamination prevention)', () => {
@@ -556,8 +557,8 @@ describe('v0.40.4 — graph_signals knob', () => {
 });
 
 describe('v0.42.3.0 — autocut knobs', () => {
-  test('KNOBS_HASH_VERSION bumped to 7', () => {
-    expect(KNOBS_HASH_VERSION).toBe(8);
+  test('KNOBS_HASH_VERSION is 9 (8→9 archive-demote, issue #1777)', () => {
+    expect(KNOBS_HASH_VERSION).toBe(9);
   });
 
   test('bundle defaults: conservative off, balanced/tokenmax on @0.20', () => {

test/search/knobs-hash-reranker.test.ts

@@ -43,7 +43,7 @@ function baseKnobs(): ResolvedSearchKnobs {
 }
 
 describe('KNOBS_HASH_VERSION + version invariants', () => {
-  test('version is 6 (1→2 reranker; 2→3 floor_ratio + cross-modal + column; 3→4 graph_signals + schema_pack; 4→5 contextual_retrieval; 5→6 v0.41.22 alias_resolved boost)', () => {
+  test('version is 9 (…; 6→7 title_boost; 7→8 autocut; 8→9 archive-demote #1777)', () => {
     // v0.35.0.0: 1→2 to fold reranker fields. v0.35.6.0: 2→3 to fold
     // floor_ratio. v0.36 wave: piggybacks on v=3 with 7 cross-modal knobs
     // (D2) PLUS column + provider context (D8/CDX-2 cross-column isolation).
@@ -54,8 +54,10 @@ describe('KNOBS_HASH_VERSION + version invariants', () => {
     // sequenced behind salem's v=4 graph-signals.
     // v0.41.22.0 (type-unification): 5→6 to fold the alias_resolved
     // post-fusion boost. Cache rows written before the boost stage
-    // cannot leak past the new stage.
-    expect(KNOBS_HASH_VERSION).toBe(8);
+    // cannot leak past the new stage. T2: 6→7 title_boost. v0.42.3.0: 7→8
+    // autocut. issue #1777: 8→9 archive/ demote (search-exclude policy change
+    // isn't in the hash, so the bump invalidates archive-excluded cache rows).
+    expect(KNOBS_HASH_VERSION).toBe(9);
   });
 
   test('hash is 16 hex chars regardless of reranker config', () => {

test/sql-ranking.test.ts

@@ -3,6 +3,7 @@ import {
   buildSourceFactorCase,
   buildHardExcludeClause,
   buildVisibilityClause,
+  escapeLikePattern as topLevelEscapeLikePattern,
   __test__,
 } from '../src/core/search/sql-ranking.ts';
 import {
@@ -241,7 +242,7 @@ describe('resolveHardExcludes', () => {
     const r = resolveHardExcludes(undefined, ['test/'], undefined);
     expect(r).not.toContain('test/');
     // Other defaults still present.
-    expect(r).toContain('archive/');
+    expect(r).toContain('attachments/');
   });
 
   test('env GBRAIN_SEARCH_EXCLUDE adds to the union', () => {
@@ -255,6 +256,40 @@ describe('resolveHardExcludes', () => {
   });
 });
 
+// issue #1777 — archive/ moved from hard-exclude to a 0.5 source-boost demote.
+describe('archive demote (issue #1777)', () => {
+  test('archive/ is NOT a default hard-exclude (regression guard)', () => {
+    expect(DEFAULT_HARD_EXCLUDES).not.toContain('archive/');
+    // The genuine-noise prefixes stay excluded.
+    expect(DEFAULT_HARD_EXCLUDES).toContain('test/');
+    expect(DEFAULT_HARD_EXCLUDES).toContain('attachments/');
+    expect(DEFAULT_HARD_EXCLUDES).toContain('.raw/');
+  });
+
+  test('resolveHardExcludes() never includes archive/ by default', () => {
+    expect(resolveHardExcludes()).not.toContain('archive/');
+  });
+
+  test('archive/ is demoted to 0.5 in the boost map', () => {
+    expect(DEFAULT_SOURCE_BOOSTS['archive/']).toBe(0.5);
+    expect(resolveBoostMap()['archive/']).toBe(0.5);
+  });
+
+  test('buildSourceFactorCase emits an archive/ demote branch', () => {
+    const sql = buildSourceFactorCase('p.slug', resolveBoostMap(), undefined);
+    expect(sql).toContain("WHEN p.slug LIKE 'archive/%' THEN 0.5");
+  });
+
+  test('detail=high bypasses the source factor (archive ranks normally)', () => {
+    expect(buildSourceFactorCase('p.slug', resolveBoostMap(), 'high')).toBe('1.0');
+  });
+
+  test('escapeLikePattern is exported at top level (CV-3a contract)', () => {
+    expect(typeof topLevelEscapeLikePattern).toBe('function');
+    expect(topLevelEscapeLikePattern('a_b%c\\d')).toBe('a\\_b\\%c\\\\d');
+  });
+});
+
 // v0.26.5 — visibility clause for soft-deleted pages and archived sources.
 describe('buildVisibilityClause (v0.26.5)', () => {
   test('emits both predicates joined by AND with a leading AND', () => {