mirror of
https://github.com/garrytan/gbrain.git
synced 2026-07-27 22:15:33 +00:00
* feat: content-quality gate on sync — quarantine junk + flag boilerplate (#1699) Three-tier disposition at the importFromContent narrow waist: - High-confidence junk (Cloudflare/CAPTCHA interstitial patterns + operator literals) -> quarantine (hidden from search, zero chunks) or reject. - Fuzzy markup-heavy (prose-vs-markup ratio, warn-tier window, code-exempt) -> content_flag marker, stays searchable, agent warned. - Oversize -> existing embed_skip soft-block + content_flag:oversized warning. Agent-warning channel: SearchResult.content_flag (stamped in hybridSearch + the keyword-only search op) and a top-level content_flag on get_page. New quarantine.ts markers, gbrain quarantine CLI (list/clear/scan), doctor quarantined_pages + flagged_pages checks (engine.executeRaw, works on PGLite), sources-audit disposition awareness, markup-heavy lint rule, config keys. Security: gate-owned markers stripped from untrusted (remote MCP) frontmatter so a write-scoped client can't hide pages or inject the warning channel. Markers excluded from content_hash so flagged pages don't re-embed every sync. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com> * chore: bump version and changelog (v0.42.8.0) Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com> * docs: document content-quality gate (quarantine + content_flag) for v0.42.8.0 Add CLAUDE.md Key Files + Commands entries for the #1699 content-quality gate: src/core/quarantine.ts, gbrain quarantine CLI (list/clear/scan), the agent-warning channel (SearchResult.content_flag + get_page), doctor quarantined_pages/flagged_pages checks, the markup-heavy lint rule, sources-audit disposition awareness, and the three new content_sanity config keys. Regenerate llms-full.txt from CLAUDE.md. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com> --------- Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
256 lines
10 KiB
TypeScript
256 lines
10 KiB
TypeScript
import { describe, test, expect } from 'bun:test';
|
|
import { readFileSync } from 'fs';
|
|
import { isSensitiveConfigKey, redactConfigValue } from '../src/commands/config.ts';
|
|
|
|
// redactUrl is not exported, so we test it by reading the source and
|
|
// reimplementing the regex to verify the pattern, then test via CLI
|
|
|
|
// Extract the redactUrl regex pattern from source
|
|
const configSource = readFileSync(
|
|
new URL('../src/commands/config.ts', import.meta.url),
|
|
'utf-8',
|
|
);
|
|
|
|
// Reimplemented from source for unit testing
|
|
function redactUrl(url: string): string {
|
|
return url.replace(
|
|
/(postgresql:\/\/[^:]+:)([^@]+)(@)/,
|
|
'$1***$3',
|
|
);
|
|
}
|
|
|
|
describe('redactUrl', () => {
|
|
test('redacts password in postgresql:// URL', () => {
|
|
const url = 'postgresql://user:secretpass@host:5432/dbname';
|
|
expect(redactUrl(url)).toBe('postgresql://user:***@host:5432/dbname');
|
|
});
|
|
|
|
test('redacts complex passwords with special chars', () => {
|
|
const url = 'postgresql://postgres:p@ss!w0rd#123@db.supabase.co:5432/postgres';
|
|
// The regex is greedy on [^@]+ so it captures up to the LAST @
|
|
const result = redactUrl(url);
|
|
expect(result).not.toContain('p@ss');
|
|
expect(result).toContain('***');
|
|
});
|
|
|
|
test('returns non-postgresql URLs unchanged', () => {
|
|
const url = 'https://example.com/api';
|
|
expect(redactUrl(url)).toBe(url);
|
|
});
|
|
|
|
test('returns plain strings unchanged', () => {
|
|
expect(redactUrl('hello')).toBe('hello');
|
|
});
|
|
|
|
test('handles URL without password', () => {
|
|
const url = 'postgresql://user@host:5432/dbname';
|
|
// No colon after user means regex doesn't match
|
|
expect(redactUrl(url)).toBe(url);
|
|
});
|
|
|
|
test('handles empty string', () => {
|
|
expect(redactUrl('')).toBe('');
|
|
});
|
|
});
|
|
|
|
describe('config source correctness', () => {
|
|
test('redactUrl function exists in config.ts', () => {
|
|
expect(configSource).toContain('function redactUrl');
|
|
});
|
|
|
|
test('redactUrl uses the correct regex pattern', () => {
|
|
expect(configSource).toContain('postgresql:\\/\\/');
|
|
});
|
|
});
|
|
|
|
describe('isSensitiveConfigKey (v0.36.x #892 regression)', () => {
|
|
test('matches common sensitive key shapes', () => {
|
|
expect(isSensitiveConfigKey('openai_api_key')).toBe(true);
|
|
expect(isSensitiveConfigKey('anthropic_api_key')).toBe(true);
|
|
expect(isSensitiveConfigKey('voyage_api_key')).toBe(true);
|
|
expect(isSensitiveConfigKey('admin_token')).toBe(true);
|
|
expect(isSensitiveConfigKey('database.password')).toBe(true);
|
|
expect(isSensitiveConfigKey('CLIENT_SECRET')).toBe(true);
|
|
expect(isSensitiveConfigKey('auth')).toBe(true);
|
|
expect(isSensitiveConfigKey('passwd')).toBe(true);
|
|
});
|
|
|
|
test('does NOT false-positive on lookalike substrings', () => {
|
|
// Pre-fix `.includes('key')` would have matched 'monkey' and 'parsekey'.
|
|
expect(isSensitiveConfigKey('monkey_id')).toBe(false);
|
|
expect(isSensitiveConfigKey('parsekeyword')).toBe(false);
|
|
expect(isSensitiveConfigKey('tokenize')).toBe(false);
|
|
expect(isSensitiveConfigKey('autocomplete')).toBe(false);
|
|
});
|
|
|
|
test('non-sensitive keys pass through', () => {
|
|
expect(isSensitiveConfigKey('search.mode')).toBe(false);
|
|
expect(isSensitiveConfigKey('sync.repo_path')).toBe(false);
|
|
expect(isSensitiveConfigKey('embedding_model')).toBe(false);
|
|
});
|
|
});
|
|
|
|
describe('redactConfigValue (v0.36.x #892 — set output regression)', () => {
|
|
test('redacts sensitive keys to ***', () => {
|
|
expect(redactConfigValue('openai_api_key', 'sk-test-123')).toBe('***');
|
|
expect(redactConfigValue('admin_token', 'eyJhbGciOiJIUzI1NiJ9')).toBe('***');
|
|
});
|
|
|
|
test('redacts postgresql URL passwords regardless of key', () => {
|
|
expect(redactConfigValue('database_url', 'postgresql://u:secret@h:5432/d'))
|
|
.toBe('postgresql://u:***@h:5432/d');
|
|
});
|
|
|
|
test('non-sensitive values pass through unchanged', () => {
|
|
expect(redactConfigValue('search.mode', 'balanced')).toBe('balanced');
|
|
expect(redactConfigValue('embedding_model', 'voyage:voyage-3-large'))
|
|
.toBe('voyage:voyage-3-large');
|
|
});
|
|
});
|
|
|
|
// v0.36.1.x #1086: loadConfig translates legacy `provider` + `model` to the
|
|
// canonical `embedding_model`. Without this, Voyage/Cohere/Mistral configs
|
|
// silently fell through to OpenAI.
|
|
describe('loadConfig — legacy provider+model migration (v0.36.1.x #1086)', () => {
|
|
test('translates {provider, model} to embedding_model', async () => {
|
|
const { mkdtempSync, mkdirSync, writeFileSync, rmSync } = await import('fs');
|
|
const { join } = await import('path');
|
|
const { tmpdir } = await import('os');
|
|
const { withEnv } = await import('./helpers/with-env.ts');
|
|
const tmpHome = mkdtempSync(join(tmpdir(), 'gbrain-cfg-test-'));
|
|
try {
|
|
mkdirSync(join(tmpHome, '.gbrain'), { recursive: true });
|
|
writeFileSync(
|
|
join(tmpHome, '.gbrain', 'config.json'),
|
|
JSON.stringify({ engine: 'pglite', database_path: '/tmp/x', provider: 'voyage', model: 'voyage-4-large' }),
|
|
);
|
|
await withEnv({ GBRAIN_HOME: tmpHome }, async () => {
|
|
const { loadConfig } = await import('../src/core/config.ts');
|
|
const cfg = loadConfig();
|
|
expect(cfg).not.toBeNull();
|
|
expect(cfg!.embedding_model).toBe('voyage:voyage-4-large');
|
|
expect((cfg as unknown as Record<string, unknown>).provider).toBeUndefined();
|
|
expect((cfg as unknown as Record<string, unknown>).model).toBeUndefined();
|
|
});
|
|
} finally {
|
|
rmSync(tmpHome, { recursive: true, force: true });
|
|
}
|
|
});
|
|
|
|
test('canonical embedding_model wins over legacy provider+model when both present', async () => {
|
|
const { mkdtempSync, mkdirSync, writeFileSync, rmSync } = await import('fs');
|
|
const { join } = await import('path');
|
|
const { tmpdir } = await import('os');
|
|
const { withEnv } = await import('./helpers/with-env.ts');
|
|
const tmpHome = mkdtempSync(join(tmpdir(), 'gbrain-cfg-test-'));
|
|
try {
|
|
mkdirSync(join(tmpHome, '.gbrain'), { recursive: true });
|
|
writeFileSync(
|
|
join(tmpHome, '.gbrain', 'config.json'),
|
|
JSON.stringify({
|
|
engine: 'pglite',
|
|
database_path: '/tmp/x',
|
|
embedding_model: 'openai:text-embedding-3-large',
|
|
provider: 'voyage',
|
|
model: 'voyage-4-large',
|
|
}),
|
|
);
|
|
await withEnv({ GBRAIN_HOME: tmpHome }, async () => {
|
|
const { loadConfig } = await import('../src/core/config.ts');
|
|
const cfg = loadConfig();
|
|
expect(cfg!.embedding_model).toBe('openai:text-embedding-3-large');
|
|
});
|
|
} finally {
|
|
rmSync(tmpHome, { recursive: true, force: true });
|
|
}
|
|
});
|
|
|
|
test('config without provider+model is untouched', async () => {
|
|
const { mkdtempSync, mkdirSync, writeFileSync, rmSync } = await import('fs');
|
|
const { join } = await import('path');
|
|
const { tmpdir } = await import('os');
|
|
const { withEnv } = await import('./helpers/with-env.ts');
|
|
const tmpHome = mkdtempSync(join(tmpdir(), 'gbrain-cfg-test-'));
|
|
try {
|
|
mkdirSync(join(tmpHome, '.gbrain'), { recursive: true });
|
|
writeFileSync(
|
|
join(tmpHome, '.gbrain', 'config.json'),
|
|
JSON.stringify({ engine: 'pglite', database_path: '/tmp/x', embedding_model: 'voyage:voyage-3-large' }),
|
|
);
|
|
await withEnv({ GBRAIN_HOME: tmpHome }, async () => {
|
|
const { loadConfig } = await import('../src/core/config.ts');
|
|
const cfg = loadConfig();
|
|
expect(cfg!.embedding_model).toBe('voyage:voyage-3-large');
|
|
});
|
|
} finally {
|
|
rmSync(tmpHome, { recursive: true, force: true });
|
|
}
|
|
});
|
|
});
|
|
|
|
// v0.36.1.x #1019 (cherry-pick #1083): configDir uses path.isAbsolute and
|
|
// dual-separator '..' rejection so Windows paths are accepted.
|
|
describe('configDir — GBRAIN_HOME Windows path acceptance (v0.36.1.x #1019)', () => {
|
|
test('relative paths are rejected with an absolute-path error', async () => {
|
|
const { withEnv } = await import('./helpers/with-env.ts');
|
|
await withEnv({ GBRAIN_HOME: 'relative/path' }, async () => {
|
|
const { configDir } = await import('../src/core/config.ts');
|
|
expect(() => configDir()).toThrow(/absolute/);
|
|
});
|
|
});
|
|
|
|
test("'..' segments rejected on POSIX-style absolute paths", async () => {
|
|
const { withEnv } = await import('./helpers/with-env.ts');
|
|
await withEnv({ GBRAIN_HOME: '/tmp/foo/../bar' }, async () => {
|
|
const { configDir } = await import('../src/core/config.ts');
|
|
expect(() => configDir()).toThrow(/'..' segments/);
|
|
});
|
|
});
|
|
|
|
test("'..' segments rejected via backslash separator (Windows path shape)", async () => {
|
|
// The dual-separator split is the regression we lock in. On a POSIX
|
|
// host, `path.isAbsolute('C:\\foo')` returns false (no drive letters),
|
|
// so we use a forward-slash-prefixed path that also contains a
|
|
// backslash '..' segment — that's the case where the pre-fix
|
|
// single-separator split would have let it through.
|
|
const { withEnv } = await import('./helpers/with-env.ts');
|
|
await withEnv({ GBRAIN_HOME: '/tmp/foo\\..\\bar' }, async () => {
|
|
const { configDir } = await import('../src/core/config.ts');
|
|
expect(() => configDir()).toThrow(/'..' segments/);
|
|
});
|
|
});
|
|
});
|
|
|
|
// v0.42 (#1699): content_sanity.max_markup_ratio env parsing.
|
|
describe('loadConfig — GBRAIN_MAX_MARKUP_RATIO env (v0.42 #1699)', () => {
|
|
async function withHomeAndEnv(env: Record<string, string | undefined>, fn: (cfg: unknown) => void) {
|
|
const { mkdtempSync, mkdirSync, writeFileSync, rmSync } = await import('fs');
|
|
const { join } = await import('path');
|
|
const { tmpdir } = await import('os');
|
|
const { withEnv } = await import('./helpers/with-env.ts');
|
|
const tmpHome = mkdtempSync(join(tmpdir(), 'gbrain-cfg-mk-'));
|
|
try {
|
|
mkdirSync(join(tmpHome, '.gbrain'), { recursive: true });
|
|
writeFileSync(join(tmpHome, '.gbrain', 'config.json'), JSON.stringify({ engine: 'pglite', database_path: '/tmp/x' }));
|
|
await withEnv({ GBRAIN_HOME: tmpHome, ...env }, async () => {
|
|
const { loadConfig } = await import('../src/core/config.ts');
|
|
fn(loadConfig());
|
|
});
|
|
} finally {
|
|
rmSync(tmpHome, { recursive: true, force: true });
|
|
}
|
|
}
|
|
|
|
test('valid ratio in (0,1] is parsed onto content_sanity', async () => {
|
|
await withHomeAndEnv({ GBRAIN_MAX_MARKUP_RATIO: '0.7' }, (cfg) => {
|
|
expect((cfg as { content_sanity?: { max_markup_ratio?: number } }).content_sanity?.max_markup_ratio).toBe(0.7);
|
|
});
|
|
});
|
|
|
|
test('out-of-range value (>1) is ignored', async () => {
|
|
await withHomeAndEnv({ GBRAIN_MAX_MARKUP_RATIO: '1.5' }, (cfg) => {
|
|
expect((cfg as { content_sanity?: { max_markup_ratio?: number } }).content_sanity?.max_markup_ratio).toBeUndefined();
|
|
});
|
|
});
|
|
});
|