mirror of
https://github.com/garrytan/gbrain.git
synced 2026-07-27 22:15:33 +00:00
* feat: content-quality gate on sync — quarantine junk + flag boilerplate (#1699) Three-tier disposition at the importFromContent narrow waist: - High-confidence junk (Cloudflare/CAPTCHA interstitial patterns + operator literals) -> quarantine (hidden from search, zero chunks) or reject. - Fuzzy markup-heavy (prose-vs-markup ratio, warn-tier window, code-exempt) -> content_flag marker, stays searchable, agent warned. - Oversize -> existing embed_skip soft-block + content_flag:oversized warning. Agent-warning channel: SearchResult.content_flag (stamped in hybridSearch + the keyword-only search op) and a top-level content_flag on get_page. New quarantine.ts markers, gbrain quarantine CLI (list/clear/scan), doctor quarantined_pages + flagged_pages checks (engine.executeRaw, works on PGLite), sources-audit disposition awareness, markup-heavy lint rule, config keys. Security: gate-owned markers stripped from untrusted (remote MCP) frontmatter so a write-scoped client can't hide pages or inject the warning channel. Markers excluded from content_hash so flagged pages don't re-embed every sync. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com> * chore: bump version and changelog (v0.42.8.0) Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com> * docs: document content-quality gate (quarantine + content_flag) for v0.42.8.0 Add CLAUDE.md Key Files + Commands entries for the #1699 content-quality gate: src/core/quarantine.ts, gbrain quarantine CLI (list/clear/scan), the agent-warning channel (SearchResult.content_flag + get_page), doctor quarantined_pages/flagged_pages checks, the markup-heavy lint rule, sources-audit disposition awareness, and the three new content_sanity config keys. Regenerate llms-full.txt from CLAUDE.md. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com> --------- Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
81 lines
3.3 KiB
TypeScript
81 lines
3.3 KiB
TypeScript
import { describe, test, expect } from 'bun:test';
|
|
import {
|
|
QUARANTINE_KEY,
|
|
QUARANTINE_FILTER_FRAGMENT,
|
|
quarantineFilterFragment,
|
|
buildQuarantineMarker,
|
|
isQuarantined,
|
|
filterOutQuarantined,
|
|
CONTENT_FLAG_KEY,
|
|
buildContentFlagMarker,
|
|
getContentFlag,
|
|
hasContentFlag,
|
|
} from '../src/core/quarantine.ts';
|
|
|
|
describe('quarantine marker (hides)', () => {
|
|
test('buildQuarantineMarker shape', () => {
|
|
const m = buildQuarantineMarker('junk_pattern', 'cloudflare_ray_id', { bytes: 1234, now: new Date('2026-06-01T00:00:00Z') });
|
|
expect(m.reason).toBe('junk_pattern');
|
|
expect(m.detail).toBe('cloudflare_ray_id');
|
|
expect(m.bytes).toBe(1234);
|
|
expect(m.assessed_at).toBe('2026-06-01T00:00:00.000Z');
|
|
});
|
|
|
|
test('isQuarantined: true only when key present + non-null', () => {
|
|
expect(isQuarantined({ [QUARANTINE_KEY]: { reason: 'junk_pattern' } })).toBe(true);
|
|
expect(isQuarantined({})).toBe(false);
|
|
expect(isQuarantined(null)).toBe(false);
|
|
expect(isQuarantined(undefined)).toBe(false);
|
|
expect(isQuarantined({ [QUARANTINE_KEY]: null })).toBe(false);
|
|
});
|
|
|
|
test('filterOutQuarantined drops quarantined pages', () => {
|
|
const pages = [
|
|
{ slug: 'a', frontmatter: {} },
|
|
{ slug: 'b', frontmatter: { [QUARANTINE_KEY]: { reason: 'junk_pattern' } } },
|
|
{ slug: 'c', frontmatter: null },
|
|
];
|
|
expect(filterOutQuarantined(pages).map((p) => p.slug)).toEqual(['a', 'c']);
|
|
});
|
|
|
|
test('QUARANTINE_FILTER_FRAGMENT is a negated JSONB existence check on p', () => {
|
|
expect(QUARANTINE_FILTER_FRAGMENT).toContain("p.frontmatter");
|
|
expect(QUARANTINE_FILTER_FRAGMENT).toContain("? 'quarantine'");
|
|
expect(QUARANTINE_FILTER_FRAGMENT.startsWith('NOT (')).toBe(true);
|
|
});
|
|
|
|
test('quarantineFilterFragment(alias) parameterizes the page alias; constant is the p-instance', () => {
|
|
expect(quarantineFilterFragment('p')).toBe(QUARANTINE_FILTER_FRAGMENT);
|
|
expect(quarantineFilterFragment('xx')).toContain('xx.frontmatter');
|
|
expect(quarantineFilterFragment('xx')).toContain("? 'quarantine'");
|
|
});
|
|
});
|
|
|
|
describe('content_flag marker (warns, does NOT hide)', () => {
|
|
test('buildContentFlagMarker shape (markup_heavy)', () => {
|
|
const m = buildContentFlagMarker('markup_heavy', 'ratio 0.91', { markup_ratio: 0.91, now: new Date('2026-06-01T00:00:00Z') });
|
|
expect(m.reason).toBe('markup_heavy');
|
|
expect(m.markup_ratio).toBe(0.91);
|
|
expect(m.bytes).toBeUndefined();
|
|
});
|
|
|
|
test('getContentFlag returns reason+detail or null', () => {
|
|
expect(getContentFlag({ [CONTENT_FLAG_KEY]: { reason: 'oversized', detail: 'big' } })).toEqual({ reason: 'oversized', detail: 'big' });
|
|
expect(getContentFlag({ [CONTENT_FLAG_KEY]: { detail: 'no reason' } })).toBeNull();
|
|
expect(getContentFlag({})).toBeNull();
|
|
expect(getContentFlag(null)).toBeNull();
|
|
});
|
|
|
|
test('hasContentFlag mirrors getContentFlag presence', () => {
|
|
expect(hasContentFlag({ [CONTENT_FLAG_KEY]: { reason: 'markup_heavy' } })).toBe(true);
|
|
expect(hasContentFlag({})).toBe(false);
|
|
});
|
|
|
|
test('there is deliberately NO content_flag SQL filter fragment exported', async () => {
|
|
// Flagged pages stay searchable by design — the module must not export a
|
|
// QUARANTINE_FILTER_FRAGMENT analog for content_flag.
|
|
const mod = await import('../src/core/quarantine.ts');
|
|
expect('CONTENT_FLAG_FILTER_FRAGMENT' in mod).toBe(false);
|
|
});
|
|
});
|