mirror of
https://github.com/garrytan/gbrain.git
synced 2026-07-27 22:15:33 +00:00
* fix(reindex): add-only tag reconciliation + DB-only re-chunk preserves frontmatter (#1621) reindex --markdown and re-import no longer wipe DB-side enrichment tags. Tag reconciliation is now ADD-ONLY (import-file.ts): re-import adds current frontmatter tags and never deletes, so auto/dream/signal-detector tags survive. The reindex DB-only fallback reconstructs full markdown via serializeMarkdown so re-chunking a page with no on-disk source preserves frontmatter/title/timeline. * fix(migrations): v0.13.1 grandfather chunked, source-safe, soft-delete-filtered (#1581) phaseCGrandfather rewritten from a per-page getPage+putPage loop (which hung 70+ min on an 82K-page PGLite brain) to a chunked bulk SQL pass keyed on pages.id (NOT slug — slug isn't globally unique), filtering deleted_at IS NULL, with a batched rollback log carrying source identity. * fix(migrations): run schema phases in-process to fix Windows getaddrinfo ENOTFOUND (#1605) The 9 'gbrain init --migrate-only' execSync spawns died on Windows+bun+Supabase (child DNS resolution). runMigrateOnlyCore (extracted from initMigrateOnly) runs the schema bring-up in-process for all engines, unblocking schema_version advancement. Includes async-call-site audit, a wall-clock guard, and a runGbrainSubprocess stderr-capture wrapper for the remaining backfill spawns. * fix(sync): ReDoS hardening + diagnostics for schema-pack regexes (#1569) Input-length cap in runRegexBounded + route the unbounded link-inference path through it (closes the only no-timeout ReDoS hole); star-height lint rule warns on nested-quantifier patterns; --no-schema-pack sync escape hatch; GBRAIN_SYNC_TRACE per-file begin heartbeat; PGLite serve/sync concurrency doc. Defensive hardening + diagnostics — the deterministic ~3100-file wedge root cause remains open (no repro). * docs(todos): file v0.41.37.0 fix-wave follow-ups (#1621/#1605/#1569) * chore: bump version and changelog (v0.41.37.0) Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com> * docs: sync README + CLAUDE.md for v0.41.37.0 critical fix wave Add reindex add-only tag reconciliation (#1621), v0.13.1 grandfather + Windows in-process migration (#1581/#1605), and schema-pack ReDoS hardening + sync --no-schema-pack / GBRAIN_SYNC_TRACE triage (#1569) to CLAUDE.md key-files annotations and README Troubleshooting. Regenerated llms-full.txt. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com> * fix(ci): bump llms-full.txt budget 700KB→750KB (CLAUDE.md crossed 700KB after master merge) The build-llms size-budget test failed: llms-full.txt is 703,244 bytes after the v0.41.37.0 key-files annotations merged on top of master's v0.41.34/35/36 CLAUDE.md additions. Matches the v0.41.9.0 precedent (600→700); the single-fetch bundle still fits comfortably in modern long-context models. --------- Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
97 lines
4.2 KiB
TypeScript
97 lines
4.2 KiB
TypeScript
// v0.41.37.0 #1569 — ReDoS hardening + diagnostics for schema-pack regexes.
|
|
//
|
|
// The real exposure was link-inference.ts:85 running `new RegExp(pattern)
|
|
// .test(context)` UNBOUNDED when no PageRegexBudget was passed. Fix: an input-
|
|
// length cap (the runtime safety net) + routing that path through the bounded
|
|
// executor + an advisory star-height lint rule.
|
|
import { describe, test, expect } from 'bun:test';
|
|
import { readFileSync } from 'fs';
|
|
import { join } from 'path';
|
|
import {
|
|
runRegexBounded,
|
|
PageRegexBudget,
|
|
RegexInputTooLargeError,
|
|
MAX_REGEX_INPUT_CHARS,
|
|
} from '../src/core/schema-pack/redos-guard.ts';
|
|
import { inferLinkTypeFromPack } from '../src/core/schema-pack/link-inference.ts';
|
|
import { linkRegexCatastrophicBacktrack } from '../src/core/schema-pack/lint-rules.ts';
|
|
import type { SchemaPackManifest } from '../src/core/schema-pack/manifest-v1.ts';
|
|
|
|
describe('#1569 input-length cap', () => {
|
|
test('runRegexBounded throws RegexInputTooLargeError over the cap', () => {
|
|
const big = 'a'.repeat(MAX_REGEX_INPUT_CHARS + 1);
|
|
expect(() => runRegexBounded('a', big)).toThrow(RegexInputTooLargeError);
|
|
});
|
|
|
|
test('runRegexBounded works normally under the cap', () => {
|
|
const m = runRegexBounded('wor', 'hello world');
|
|
expect(m).not.toBeNull();
|
|
expect(runRegexBounded('zzz', 'hello world')).toBeNull();
|
|
});
|
|
|
|
test('PageRegexBudget.runBounded degrades (null) on oversize input', () => {
|
|
const budget = new PageRegexBudget();
|
|
const big = 'a'.repeat(MAX_REGEX_INPUT_CHARS + 1);
|
|
expect(budget.runBounded('verb', 'a', big)).toBeNull();
|
|
});
|
|
|
|
test('inferLinkTypeFromPack (no budget) does NOT run regex unbounded on huge input', () => {
|
|
// Pre-#1569 this ran new RegExp().test() with no length cap → ReDoS risk.
|
|
// A catastrophic pattern + a long input must NOT hang; the cap skips it.
|
|
const pack = {
|
|
link_types: [{ name: 'founded', inference: { regex: '(a+)+$' } }],
|
|
} as unknown as Pick<SchemaPackManifest, 'link_types'>;
|
|
const huge = 'a'.repeat(MAX_REGEX_INPUT_CHARS + 100) + '!';
|
|
const t0 = Date.now();
|
|
const result = inferLinkTypeFromPack(pack, 'company', huge);
|
|
// Skipped via the cap → no match, and fast (no catastrophic backtrack).
|
|
expect(result).toBeNull();
|
|
expect(Date.now() - t0).toBeLessThan(2_000);
|
|
});
|
|
});
|
|
|
|
describe('#1569 star-height lint rule', () => {
|
|
const mk = (regex: string): SchemaPackManifest =>
|
|
({ name: 'testpack', page_types: [], link_types: [{ name: 'founded', inference: { regex } }] }) as unknown as SchemaPackManifest;
|
|
|
|
test('flags classic nested-quantifier shapes as warnings', () => {
|
|
for (const bad of ['(a+)+', '(a*)*', '(a+)*', '(\\w+)+$', '(.*)+']) {
|
|
const issues = linkRegexCatastrophicBacktrack(mk(bad)) as ReturnType<typeof linkRegexCatastrophicBacktrack> & any[];
|
|
expect(issues.length).toBe(1);
|
|
expect(issues[0].severity).toBe('warning');
|
|
expect(issues[0].rule).toBe('link_regex_catastrophic_backtrack');
|
|
expect(issues[0].link).toBe('founded');
|
|
}
|
|
});
|
|
|
|
test('does NOT flag benign patterns', () => {
|
|
for (const ok of ['a+', '(abc)+', '(a|b)+', 'founded\\s+\\w+', '[a-z]+@[a-z]+']) {
|
|
const issues = linkRegexCatastrophicBacktrack(mk(ok)) as any[];
|
|
expect(issues.length).toBe(0);
|
|
}
|
|
});
|
|
|
|
test('no regex → no issue', () => {
|
|
const manifest = { name: 'p', page_types: [], link_types: [{ name: 'x' }] } as unknown as SchemaPackManifest;
|
|
expect((linkRegexCatastrophicBacktrack(manifest) as any[]).length).toBe(0);
|
|
});
|
|
});
|
|
|
|
describe('#1569 --no-schema-pack + heartbeat wiring (structural)', () => {
|
|
const SYNC = readFileSync(join(import.meta.dir, '..', 'src', 'commands', 'sync.ts'), 'utf-8');
|
|
|
|
test('SyncOpts carries noSchemaPack and it gates loadActivePack', () => {
|
|
expect(SYNC).toContain('noSchemaPack?: boolean');
|
|
expect(SYNC).toContain("args.includes('--no-schema-pack')");
|
|
expect(SYNC).toContain('if (opts.noSchemaPack)');
|
|
});
|
|
|
|
test('begin heartbeat fires before importFile (GBRAIN_SYNC_TRACE)', () => {
|
|
const beginIdx = SYNC.indexOf('begin import:');
|
|
const importIdx = SYNC.indexOf('importFile(eng, filePath, path');
|
|
expect(beginIdx).toBeGreaterThan(0);
|
|
expect(importIdx).toBeGreaterThan(0);
|
|
expect(beginIdx).toBeLessThan(importIdx);
|
|
});
|
|
});
|