Files
gbrain/test/quarantine.test.ts
T
0bfe0d0c7e v0.42.8.0 feat: content-quality gate on sync — quarantine junk + flag boilerplate (#1699) (#1756)
* feat: content-quality gate on sync — quarantine junk + flag boilerplate (#1699)

Three-tier disposition at the importFromContent narrow waist:
- High-confidence junk (Cloudflare/CAPTCHA interstitial patterns + operator
  literals) -> quarantine (hidden from search, zero chunks) or reject.
- Fuzzy markup-heavy (prose-vs-markup ratio, warn-tier window, code-exempt)
  -> content_flag marker, stays searchable, agent warned.
- Oversize -> existing embed_skip soft-block + content_flag:oversized warning.

Agent-warning channel: SearchResult.content_flag (stamped in hybridSearch +
the keyword-only search op) and a top-level content_flag on get_page.
New quarantine.ts markers, gbrain quarantine CLI (list/clear/scan), doctor
quarantined_pages + flagged_pages checks (engine.executeRaw, works on PGLite),
sources-audit disposition awareness, markup-heavy lint rule, config keys.

Security: gate-owned markers stripped from untrusted (remote MCP) frontmatter
so a write-scoped client can't hide pages or inject the warning channel.
Markers excluded from content_hash so flagged pages don't re-embed every sync.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

* chore: bump version and changelog (v0.42.8.0)

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

* docs: document content-quality gate (quarantine + content_flag) for v0.42.8.0

Add CLAUDE.md Key Files + Commands entries for the #1699 content-quality
gate: src/core/quarantine.ts, gbrain quarantine CLI (list/clear/scan),
the agent-warning channel (SearchResult.content_flag + get_page), doctor
quarantined_pages/flagged_pages checks, the markup-heavy lint rule,
sources-audit disposition awareness, and the three new content_sanity
config keys. Regenerate llms-full.txt from CLAUDE.md.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

---------

Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
2026-06-01 23:03:09 -07:00

81 lines
3.3 KiB
TypeScript

import { describe, test, expect } from 'bun:test';
import {
QUARANTINE_KEY,
QUARANTINE_FILTER_FRAGMENT,
quarantineFilterFragment,
buildQuarantineMarker,
isQuarantined,
filterOutQuarantined,
CONTENT_FLAG_KEY,
buildContentFlagMarker,
getContentFlag,
hasContentFlag,
} from '../src/core/quarantine.ts';
describe('quarantine marker (hides)', () => {
test('buildQuarantineMarker shape', () => {
const m = buildQuarantineMarker('junk_pattern', 'cloudflare_ray_id', { bytes: 1234, now: new Date('2026-06-01T00:00:00Z') });
expect(m.reason).toBe('junk_pattern');
expect(m.detail).toBe('cloudflare_ray_id');
expect(m.bytes).toBe(1234);
expect(m.assessed_at).toBe('2026-06-01T00:00:00.000Z');
});
test('isQuarantined: true only when key present + non-null', () => {
expect(isQuarantined({ [QUARANTINE_KEY]: { reason: 'junk_pattern' } })).toBe(true);
expect(isQuarantined({})).toBe(false);
expect(isQuarantined(null)).toBe(false);
expect(isQuarantined(undefined)).toBe(false);
expect(isQuarantined({ [QUARANTINE_KEY]: null })).toBe(false);
});
test('filterOutQuarantined drops quarantined pages', () => {
const pages = [
{ slug: 'a', frontmatter: {} },
{ slug: 'b', frontmatter: { [QUARANTINE_KEY]: { reason: 'junk_pattern' } } },
{ slug: 'c', frontmatter: null },
];
expect(filterOutQuarantined(pages).map((p) => p.slug)).toEqual(['a', 'c']);
});
test('QUARANTINE_FILTER_FRAGMENT is a negated JSONB existence check on p', () => {
expect(QUARANTINE_FILTER_FRAGMENT).toContain("p.frontmatter");
expect(QUARANTINE_FILTER_FRAGMENT).toContain("? 'quarantine'");
expect(QUARANTINE_FILTER_FRAGMENT.startsWith('NOT (')).toBe(true);
});
test('quarantineFilterFragment(alias) parameterizes the page alias; constant is the p-instance', () => {
expect(quarantineFilterFragment('p')).toBe(QUARANTINE_FILTER_FRAGMENT);
expect(quarantineFilterFragment('xx')).toContain('xx.frontmatter');
expect(quarantineFilterFragment('xx')).toContain("? 'quarantine'");
});
});
describe('content_flag marker (warns, does NOT hide)', () => {
test('buildContentFlagMarker shape (markup_heavy)', () => {
const m = buildContentFlagMarker('markup_heavy', 'ratio 0.91', { markup_ratio: 0.91, now: new Date('2026-06-01T00:00:00Z') });
expect(m.reason).toBe('markup_heavy');
expect(m.markup_ratio).toBe(0.91);
expect(m.bytes).toBeUndefined();
});
test('getContentFlag returns reason+detail or null', () => {
expect(getContentFlag({ [CONTENT_FLAG_KEY]: { reason: 'oversized', detail: 'big' } })).toEqual({ reason: 'oversized', detail: 'big' });
expect(getContentFlag({ [CONTENT_FLAG_KEY]: { detail: 'no reason' } })).toBeNull();
expect(getContentFlag({})).toBeNull();
expect(getContentFlag(null)).toBeNull();
});
test('hasContentFlag mirrors getContentFlag presence', () => {
expect(hasContentFlag({ [CONTENT_FLAG_KEY]: { reason: 'markup_heavy' } })).toBe(true);
expect(hasContentFlag({})).toBe(false);
});
test('there is deliberately NO content_flag SQL filter fragment exported', async () => {
// Flagged pages stay searchable by design — the module must not export a
// QUARANTINE_FILTER_FRAGMENT analog for content_flag.
const mod = await import('../src/core/quarantine.ts');
expect('CONTENT_FLAG_FILTER_FRAGMENT' in mod).toBe(false);
});
});