/** * Quarantine + content-flag markers: the two frontmatter markers the * content-quality gate writes (issue #1699). * * Why two markers, not one (Q1=A confidence split): * - `quarantine` HIDES. Set ONLY for high-confidence junk (Cloudflare / * CAPTCHA interstitial patterns + operator literals). The page lands * (reviewable via get_page / quarantine list) but writes zero chunks * and is excluded from search via `QUARANTINE_FILTER_FRAGMENT`. * - `content_flag` WARNS, does NOT hide. Set for the fuzzy markup-ratio * signal and for oversize. The page stays fully searchable; the marker * rides along so search results / get_page can tell the agent "this * looks like boilerplate / is unusually large — examine it." There is * NO SQL filter fragment for content_flag, by design. * * Three distinct markers, three reasons (Codex #6 — never overload one to * mean another): * - `embed_skip` (src/core/embed-skip.ts) = "oversized-but-clean, not embedded" * - `quarantine` (here) = "junk, hidden from search" * - `content_flag` (here) = "odd, examine, still here" * A page can carry more than one (oversize → embed_skip + content_flag:oversized); * each is cleared independently. * * Sibling of `src/core/embed-skip.ts` — same marker-as-JSONB-object pattern, * same JSONB `?` existence check that works identically on Postgres (real) * and PGLite (PostgreSQL 17.5 in WASM). No schema migration (D4): both are * frontmatter JSONB keys. * * v0.42 follow-up: promote `quarantine` to a schema column + * partial index if the quarantined subset grows large. Single change * site (this module). */ // --------------------------------------------------------------------------- // quarantine marker (HIDES) // --------------------------------------------------------------------------- /** Frontmatter key for the HIDE marker. Stable contract. */ export const QUARANTINE_KEY = 'quarantine'; /** SQL fragment that excludes quarantined pages from search, parameterized on * the page-table alias. Single source of truth — `buildVisibilityClause` * (`src/core/search/sql-ranking.ts`) calls this so the search filter and the * marker key can never drift. Mirrors `EMBED_SKIP_FILTER_FRAGMENT` shape: * JSONB `?` existence, negated so we KEEP rows WITHOUT the marker. * `pageAlias` is engine-supplied (never user input), so no escaping needed. */ export function quarantineFilterFragment(pageAlias: string): string { return `NOT (COALESCE(${pageAlias}.frontmatter, '{}'::jsonb) ? '${QUARANTINE_KEY}')`; } /** The `p`-aliased instance — the common case (all 6 search call sites alias * pages as `p`). Kept as a constant for parity with `EMBED_SKIP_FILTER_FRAGMENT` * and for any future stale/orphan-chunk query that needs it. */ export const QUARANTINE_FILTER_FRAGMENT = quarantineFilterFragment('p'); export interface QuarantineMarker { /** Why the page was quarantined. The high-confidence junk reasons. */ reason: 'junk_pattern' | 'literal_substring'; /** Human-readable detail (which pattern/literal names fired). */ detail: string; /** ISO 8601 timestamp at assessment time. */ assessed_at: string; /** Body bytes at assessment, for operator visibility. */ bytes?: number; } /** Build the canonical quarantine marker. Spread onto frontmatter before * write: `frontmatter[QUARANTINE_KEY] = buildQuarantineMarker(...)`. */ export function buildQuarantineMarker( reason: QuarantineMarker['reason'], detail: string, extra: { bytes?: number; now?: Date } = {}, ): QuarantineMarker { return { reason, detail, assessed_at: (extra.now ?? new Date()).toISOString(), ...(extra.bytes !== undefined ? { bytes: extra.bytes } : {}), }; } /** JS-side predicate. True when the frontmatter has the quarantine key set * to any non-null value. Accepts null/undefined frontmatter. Key-existence * is the trigger; marker contents are diagnostic (mirrors the SQL fragment). */ export function isQuarantined(frontmatter: Record | null | undefined): boolean { if (!frontmatter) return false; const value = frontmatter[QUARANTINE_KEY]; return value !== undefined && value !== null; } /** JS-side filter: returns a new array with quarantined pages excluded. */ export function filterOutQuarantined | null }>( pages: ReadonlyArray, ): T[] { return pages.filter((p) => !isQuarantined(p.frontmatter ?? null)); } // --------------------------------------------------------------------------- // content_flag marker (WARNS, does NOT hide) // --------------------------------------------------------------------------- /** Frontmatter key for the WARN marker. Stable contract. */ export const CONTENT_FLAG_KEY = 'content_flag'; export interface ContentFlagMarker { /** Which fuzzy/oversize tier fired. */ reason: 'markup_heavy' | 'oversized'; /** Human-readable detail surfaced to the agent on retrieval. */ detail: string; /** ISO 8601 timestamp at assessment time. */ assessed_at: string; /** Markup ratio (when reason is markup_heavy). */ markup_ratio?: number; /** Body bytes (when reason is oversized). */ bytes?: number; } /** Build the canonical content-flag marker. NOTE: there is deliberately NO * SQL filter fragment for content_flag — flagged pages stay searchable; the * marker is READ INTO search/get_page output, never used to exclude. */ export function buildContentFlagMarker( reason: ContentFlagMarker['reason'], detail: string, extra: { markup_ratio?: number; bytes?: number; now?: Date } = {}, ): ContentFlagMarker { return { reason, detail, assessed_at: (extra.now ?? new Date()).toISOString(), ...(extra.markup_ratio !== undefined ? { markup_ratio: extra.markup_ratio } : {}), ...(extra.bytes !== undefined ? { bytes: extra.bytes } : {}), }; } /** Read the content-flag marker from frontmatter, or null. The shape is * validated loosely — a `reason` string is the minimum contract. Used by * the search projection + get_page to populate the agent-warning channel. */ export function getContentFlag( frontmatter: Record | null | undefined, ): { reason: string; detail: string } | null { if (!frontmatter) return null; const value = frontmatter[CONTENT_FLAG_KEY]; if (!value || typeof value !== 'object') return null; const obj = value as Record; const reason = typeof obj.reason === 'string' ? obj.reason : null; if (!reason) return null; return { reason, detail: typeof obj.detail === 'string' ? obj.detail : '' }; } /** True when the frontmatter carries a content-flag marker. */ export function hasContentFlag(frontmatter: Record | null | undefined): boolean { return getContentFlag(frontmatter) !== null; }