mirror of
https://github.com/garrytan/gbrain.git
synced 2026-07-27 22:15:33 +00:00
v0.41.37.0 fix: critical fix wave — reindex tag wipe, grandfather hang, Windows migration spawn, sync ReDoS (#1621 #1581 #1605 #1569) (#1665)
* fix(reindex): add-only tag reconciliation + DB-only re-chunk preserves frontmatter (#1621) reindex --markdown and re-import no longer wipe DB-side enrichment tags. Tag reconciliation is now ADD-ONLY (import-file.ts): re-import adds current frontmatter tags and never deletes, so auto/dream/signal-detector tags survive. The reindex DB-only fallback reconstructs full markdown via serializeMarkdown so re-chunking a page with no on-disk source preserves frontmatter/title/timeline. * fix(migrations): v0.13.1 grandfather chunked, source-safe, soft-delete-filtered (#1581) phaseCGrandfather rewritten from a per-page getPage+putPage loop (which hung 70+ min on an 82K-page PGLite brain) to a chunked bulk SQL pass keyed on pages.id (NOT slug — slug isn't globally unique), filtering deleted_at IS NULL, with a batched rollback log carrying source identity. * fix(migrations): run schema phases in-process to fix Windows getaddrinfo ENOTFOUND (#1605) The 9 'gbrain init --migrate-only' execSync spawns died on Windows+bun+Supabase (child DNS resolution). runMigrateOnlyCore (extracted from initMigrateOnly) runs the schema bring-up in-process for all engines, unblocking schema_version advancement. Includes async-call-site audit, a wall-clock guard, and a runGbrainSubprocess stderr-capture wrapper for the remaining backfill spawns. * fix(sync): ReDoS hardening + diagnostics for schema-pack regexes (#1569) Input-length cap in runRegexBounded + route the unbounded link-inference path through it (closes the only no-timeout ReDoS hole); star-height lint rule warns on nested-quantifier patterns; --no-schema-pack sync escape hatch; GBRAIN_SYNC_TRACE per-file begin heartbeat; PGLite serve/sync concurrency doc. Defensive hardening + diagnostics — the deterministic ~3100-file wedge root cause remains open (no repro). * docs(todos): file v0.41.37.0 fix-wave follow-ups (#1621/#1605/#1569) * chore: bump version and changelog (v0.41.37.0) Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com> * docs: sync README + CLAUDE.md for v0.41.37.0 critical fix wave Add reindex add-only tag reconciliation (#1621), v0.13.1 grandfather + Windows in-process migration (#1581/#1605), and schema-pack ReDoS hardening + sync --no-schema-pack / GBRAIN_SYNC_TRACE triage (#1569) to CLAUDE.md key-files annotations and README Troubleshooting. Regenerated llms-full.txt. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com> * fix(ci): bump llms-full.txt budget 700KB→750KB (CLAUDE.md crossed 700KB after master merge) The build-llms size-budget test failed: llms-full.txt is 703,244 bytes after the v0.41.37.0 key-files annotations merged on top of master's v0.41.34/35/36 CLAUDE.md additions. Matches the v0.41.9.0 precedent (600→700); the single-fetch bundle still fits comfortably in modern long-context models. --------- Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
Garry Tan
co-authored by
Claude Opus 4.8
Claude Opus 4.8
parent
ca13f40820
commit
6f26d5e4df
@@ -95,11 +95,16 @@ describe('v0.36.1.x #1077 — admin register-client supports PKCE public clients
|
||||
});
|
||||
});
|
||||
|
||||
describe('v0.36.1.x #1100 — PGLite v0.11.0 phaseASchema routes in-process', () => {
|
||||
test('phaseASchema branches on pglite and calls initSchema directly', () => {
|
||||
describe('v0.41.37.0 #1605 — v0.11.0 phaseASchema routes in-process for ALL engines', () => {
|
||||
test('phaseASchema calls runMigrateOnlyCore (in-process) + is awaited', () => {
|
||||
// Supersedes #1100's PGLite-only in-process branch. v0.41.37.0 #1605 routes
|
||||
// EVERY engine through runMigrateOnlyCore (no execSync subprocess at all),
|
||||
// which is strictly stronger: PGLite still never subprocesses, AND the
|
||||
// Windows+Postgres getaddrinfo-ENOTFOUND spawn bug is closed too.
|
||||
// The eng.initSchema() call moved into src/commands/migrations/in-process.ts.
|
||||
const src = readFileSync('src/commands/migrations/v0_11_0.ts', 'utf8');
|
||||
expect(src).toMatch(/cfg\?\.engine\s*===\s*'pglite'/);
|
||||
expect(src).toMatch(/eng\.initSchema\(\)/);
|
||||
expect(src).toContain('runMigrateOnlyCore()');
|
||||
expect(src).not.toContain("execSync('gbrain init --migrate-only'");
|
||||
expect(src).toMatch(/await\s+phaseASchema/);
|
||||
});
|
||||
|
||||
|
||||
@@ -217,7 +217,11 @@ Same content.
|
||||
expect(putCall).toBeUndefined();
|
||||
});
|
||||
|
||||
test('reconciles tags: removes old, adds new', async () => {
|
||||
test('reconciles tags: ADD-ONLY, never removes (v0.41.37.0 #1621)', async () => {
|
||||
// Add-only reconciliation: re-import adds current frontmatter tags and
|
||||
// NEVER deletes — so DB-side enrichment tags (here simulated as 'old-tag')
|
||||
// survive. The pre-#1621 behavior deleted every existing tag not in the
|
||||
// current frontmatter, wiping enrichment tags on every re-import/reindex.
|
||||
const filePath = join(TMP, 'retag.md');
|
||||
writeFileSync(filePath, `---
|
||||
type: concept
|
||||
@@ -239,9 +243,11 @@ Content here.
|
||||
const removeCalls = calls.filter((c: any) => c.method === 'removeTag');
|
||||
const addCalls = calls.filter((c: any) => c.method === 'addTag');
|
||||
|
||||
expect(removeCalls.length).toBe(1);
|
||||
expect(removeCalls[0].args[1]).toBe('old-tag');
|
||||
// No removals — 'old-tag' (enrichment) is preserved.
|
||||
expect(removeCalls.length).toBe(0);
|
||||
// Both current frontmatter tags are added (idempotent via ON CONFLICT).
|
||||
expect(addCalls.length).toBe(2);
|
||||
expect(addCalls.map((c: any) => c.args[1]).sort()).toEqual(['kept-tag', 'new-tag']);
|
||||
});
|
||||
|
||||
test('chunks compiled_truth and timeline separately', async () => {
|
||||
|
||||
@@ -0,0 +1,112 @@
|
||||
// v0.41.37.0 #1605 — migration schema phases run IN-PROCESS (was a
|
||||
// `gbrain init --migrate-only` subprocess that died with getaddrinfo ENOTFOUND
|
||||
// on Windows+bun+Supabase). runMigrateOnlyCore is the single in-process path;
|
||||
// runGbrainSubprocess captures child stderr for the remaining backfill spawns.
|
||||
import { describe, test, expect } from 'bun:test';
|
||||
import { tmpdir } from 'os';
|
||||
import { mkdtempSync, mkdirSync, writeFileSync, readFileSync, readdirSync } from 'fs';
|
||||
import { join } from 'path';
|
||||
import { withEnv } from './helpers/with-env.ts';
|
||||
import { PGLiteEngine } from '../src/core/pglite-engine.ts';
|
||||
import {
|
||||
runMigrateOnlyCore,
|
||||
runGbrainSubprocess,
|
||||
MigrateOnlyError,
|
||||
} from '../src/commands/migrations/in-process.ts';
|
||||
|
||||
const MIGRATION_DIR = join(import.meta.dir, '..', 'src', 'commands', 'migrations');
|
||||
const SCHEMA_PHASE_FILES = [
|
||||
'v0_11_0', 'v0_12_0', 'v0_12_2', 'v0_13_0', 'v0_16_0',
|
||||
'v0_18_0', 'v0_18_1', 'v0_21_0', 'v0_29_1',
|
||||
];
|
||||
|
||||
describe('#1605 runMigrateOnlyCore (in-process schema)', () => {
|
||||
test('brings a fresh PGLite brain to head without spawning', async () => {
|
||||
const home = mkdtempSync(join(tmpdir(), 'mip-'));
|
||||
const dataDir = join(home, 'data');
|
||||
mkdirSync(join(home, '.gbrain'), { recursive: true });
|
||||
writeFileSync(
|
||||
join(home, '.gbrain', 'config.json'),
|
||||
JSON.stringify({ engine: 'pglite', database_path: dataDir }),
|
||||
);
|
||||
|
||||
const result = await withEnv(
|
||||
{ GBRAIN_HOME: home, DATABASE_URL: undefined, GBRAIN_DATABASE_URL: undefined },
|
||||
() => runMigrateOnlyCore(),
|
||||
);
|
||||
expect(result.engine).toBe('pglite');
|
||||
|
||||
// Verify schema landed: reconnect a fresh engine to the same data dir and
|
||||
// confirm a core table exists (proves initSchema ran in-process).
|
||||
const verify = new PGLiteEngine();
|
||||
await verify.connect({ database_path: dataDir });
|
||||
try {
|
||||
const rows = await verify.executeRaw<{ t: string | null }>(
|
||||
"SELECT to_regclass('public.pages')::text AS t",
|
||||
);
|
||||
expect(rows[0]?.t).toBe('pages');
|
||||
} finally {
|
||||
await verify.disconnect();
|
||||
}
|
||||
});
|
||||
|
||||
test('throws MigrateOnlyError when no brain is configured', async () => {
|
||||
const home = mkdtempSync(join(tmpdir(), 'mip-noconf-'));
|
||||
await expect(
|
||||
withEnv(
|
||||
{ GBRAIN_HOME: home, DATABASE_URL: undefined, GBRAIN_DATABASE_URL: undefined },
|
||||
() => runMigrateOnlyCore(),
|
||||
),
|
||||
).rejects.toBeInstanceOf(MigrateOnlyError);
|
||||
});
|
||||
});
|
||||
|
||||
describe('#1605 runGbrainSubprocess (stderr capture)', () => {
|
||||
test('folds child stderr into the thrown error', () => {
|
||||
let msg = '';
|
||||
try {
|
||||
runGbrainSubprocess("sh -c 'echo BOOM_STDERR 1>&2; exit 1'");
|
||||
} catch (e) {
|
||||
msg = e instanceof Error ? e.message : String(e);
|
||||
}
|
||||
expect(msg).toContain('BOOM_STDERR');
|
||||
});
|
||||
|
||||
test('returns child stdout on success', () => {
|
||||
const out = runGbrainSubprocess("sh -c 'echo hello-stdout'");
|
||||
expect(out).toContain('hello-stdout');
|
||||
});
|
||||
});
|
||||
|
||||
describe('#1605 structural guard: schema phases are in-process', () => {
|
||||
test('no schema phase still execSyncs `gbrain init --migrate-only`', () => {
|
||||
for (const f of SCHEMA_PHASE_FILES) {
|
||||
const src = readFileSync(join(MIGRATION_DIR, `${f}.ts`), 'utf-8');
|
||||
expect(src).not.toContain("execSync('gbrain init --migrate-only'");
|
||||
}
|
||||
});
|
||||
|
||||
test('every schema phase calls runMigrateOnlyCore + is awaited', () => {
|
||||
for (const f of SCHEMA_PHASE_FILES) {
|
||||
const src = readFileSync(join(MIGRATION_DIR, `${f}.ts`), 'utf-8');
|
||||
expect(src).toContain('runMigrateOnlyCore()');
|
||||
// phaseASchema must be async + awaited at its call site.
|
||||
expect(src).toContain('async function phaseASchema');
|
||||
const awaited = src.includes('await phaseASchema(opts)') ||
|
||||
src.includes('push(await phaseASchema(opts))');
|
||||
expect(awaited).toBe(true);
|
||||
}
|
||||
});
|
||||
|
||||
test('NO migration orchestrator anywhere spawns `gbrain init --migrate-only`', () => {
|
||||
// All-files invariant (not just the 9): the subprocess spawn is the
|
||||
// Windows-ENOTFOUND bug class. Other files (v0_22_4, v0_28_0, v0_31_0,
|
||||
// v0_14_0, v0_32_2) define an in-process phaseASchema that never spawned —
|
||||
// those are fine. We only ban the spawn literal.
|
||||
const files = readdirSync(MIGRATION_DIR).filter(n => /^v\d/.test(n) && n.endsWith('.ts'));
|
||||
for (const n of files) {
|
||||
const src = readFileSync(join(MIGRATION_DIR, n), 'utf-8');
|
||||
expect(src).not.toContain("execSync('gbrain init --migrate-only'");
|
||||
}
|
||||
});
|
||||
});
|
||||
@@ -59,12 +59,16 @@ describe('v0.13.0 — Frontmatter relationship indexing migration', () => {
|
||||
expect(src).not.toMatch(/\$\{GBRAIN\}/);
|
||||
});
|
||||
|
||||
test('phase commands invoke bare `gbrain` shell-out (Bug 1 fix)', () => {
|
||||
test('phases use in-process schema + bare `gbrain` subprocess (v0.41.37.0 #1605)', () => {
|
||||
const src = readFileSync(SRC_PATH, 'utf-8');
|
||||
// All three phases shell out to bare `gbrain` so the canonical shim
|
||||
// on PATH wins. This is the shape v0_12_0 has always used.
|
||||
expect(src).toContain("execSync('gbrain init --migrate-only'");
|
||||
expect(src).toContain("execSync('gbrain extract links --source db --include-frontmatter'");
|
||||
// Schema bring-up is now IN-PROCESS (was execSync('gbrain init
|
||||
// --migrate-only'), which died with getaddrinfo ENOTFOUND on Windows).
|
||||
expect(src).toContain('runMigrateOnlyCore()');
|
||||
expect(src).not.toContain("execSync('gbrain init --migrate-only'");
|
||||
// Backfill extract goes through the stderr-capturing wrapper (still bare
|
||||
// `gbrain` so the canonical shim on PATH wins).
|
||||
expect(src).toContain("runGbrainSubprocess('gbrain extract links --source db --include-frontmatter'");
|
||||
// Stats readback still shells out (reads stdout); bare gbrain.
|
||||
expect(src).toContain("execSync('gbrain call get_stats'");
|
||||
});
|
||||
|
||||
|
||||
@@ -0,0 +1,144 @@
|
||||
// v0.41.37.0 #1581 — chunked, source-safe, soft-delete-filtered grandfather pass.
|
||||
//
|
||||
// The pre-#1581 per-page getPage+putPage loop hung CPU-bound for 70+ min on an
|
||||
// 82K-page PGLite brain. The rewrite is a chunked bulk SQL pass keyed on
|
||||
// pages.id (NOT slug — slug isn't globally unique), filtering deleted_at IS NULL.
|
||||
// These tests drive phaseCGrandfather directly against a real PGLite engine.
|
||||
import { describe, test, expect, beforeAll, afterAll, beforeEach } from 'bun:test';
|
||||
import { tmpdir } from 'os';
|
||||
import { mkdtempSync, existsSync, readFileSync } from 'fs';
|
||||
import { join } from 'path';
|
||||
import { PGLiteEngine } from '../src/core/pglite-engine.ts';
|
||||
import { resetPgliteState } from './helpers/reset-pglite.ts';
|
||||
import { withEnv } from './helpers/with-env.ts';
|
||||
import { phaseCGrandfather } from '../src/commands/migrations/v0_13_1.ts';
|
||||
import type { OrchestratorOpts } from '../src/commands/migrations/types.ts';
|
||||
|
||||
let engine: PGLiteEngine;
|
||||
|
||||
beforeAll(async () => {
|
||||
engine = new PGLiteEngine();
|
||||
await engine.connect({});
|
||||
await engine.initSchema();
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
await engine.disconnect();
|
||||
});
|
||||
|
||||
beforeEach(async () => {
|
||||
await resetPgliteState(engine);
|
||||
});
|
||||
|
||||
const ensureSource = (id: string) =>
|
||||
engine.executeRaw('INSERT INTO sources (id, name) VALUES ($1, $1) ON CONFLICT DO NOTHING', [id]);
|
||||
|
||||
const seed = (slug: string, frontmatter: Record<string, unknown>, sourceId = 'default') =>
|
||||
engine.putPage(slug, {
|
||||
type: 'concept', title: slug, compiled_truth: 'body', timeline: '', frontmatter,
|
||||
}, { sourceId });
|
||||
|
||||
const fmOf = async (slug: string, sourceId = 'default') => {
|
||||
const p = await engine.getPage(slug, { sourceId });
|
||||
return p?.frontmatter ?? null;
|
||||
};
|
||||
|
||||
// Run the phase with an isolated GBRAIN_HOME so the rollback log lands in a
|
||||
// tempdir (withEnv keeps R1 test-isolation compliance — no raw process.env writes).
|
||||
async function gf(home: string, opts: Partial<OrchestratorOpts> = {}) {
|
||||
const full: OrchestratorOpts = { yes: true, dryRun: false, noAutopilotInstall: true, ...opts };
|
||||
return withEnv({ GBRAIN_HOME: home }, () => phaseCGrandfather(engine, full));
|
||||
}
|
||||
|
||||
describe('#1581 phaseCGrandfather (chunked, source-safe)', () => {
|
||||
test('absent validate key → validate:false', async () => {
|
||||
const home = mkdtempSync(join(tmpdir(), 'gf-'));
|
||||
await seed('a', { foo: 1 });
|
||||
const { result } = await gf(home);
|
||||
expect(result.status).toBe('complete');
|
||||
expect((await fmOf('a'))?.validate).toBe(false);
|
||||
});
|
||||
|
||||
test('explicit validate (true/false/null) is NOT modified', async () => {
|
||||
const home = mkdtempSync(join(tmpdir(), 'gf-'));
|
||||
await seed('t', { validate: true });
|
||||
await seed('f', { validate: false });
|
||||
await seed('n', { validate: null });
|
||||
await gf(home);
|
||||
expect((await fmOf('t'))?.validate).toBe(true);
|
||||
expect((await fmOf('f'))?.validate).toBe(false);
|
||||
// null value still counts as "key present" → left untouched.
|
||||
expect((await fmOf('n'))?.validate ?? 'WAS_NULL').toBe('WAS_NULL');
|
||||
});
|
||||
|
||||
test('soft-deleted pages are NOT grandfathered', async () => {
|
||||
const home = mkdtempSync(join(tmpdir(), 'gf-'));
|
||||
await seed('live', { foo: 1 });
|
||||
await seed('dead', { foo: 1 });
|
||||
await engine.softDeletePage('dead');
|
||||
await gf(home);
|
||||
|
||||
expect((await fmOf('live'))?.validate).toBe(false);
|
||||
const rows = await engine.executeRaw<{ fm: Record<string, unknown> }>(
|
||||
"SELECT frontmatter AS fm FROM pages WHERE slug = 'dead'",
|
||||
);
|
||||
expect(Object.prototype.hasOwnProperty.call(rows[0]?.fm ?? {}, 'validate')).toBe(false);
|
||||
});
|
||||
|
||||
test('multi-source duplicate slugs: both grandfathered, no cross-contamination', async () => {
|
||||
const home = mkdtempSync(join(tmpdir(), 'gf-'));
|
||||
await ensureSource('src2');
|
||||
await seed('people/dup', { src: 'a' }, 'default');
|
||||
await seed('people/dup', { src: 'b' }, 'src2');
|
||||
await gf(home);
|
||||
|
||||
expect((await fmOf('people/dup', 'default'))?.validate).toBe(false);
|
||||
expect((await fmOf('people/dup', 'src2'))?.validate).toBe(false);
|
||||
expect((await fmOf('people/dup', 'default'))?.src).toBe('a');
|
||||
expect((await fmOf('people/dup', 'src2'))?.src).toBe('b');
|
||||
});
|
||||
|
||||
test('rollback log carries source identity', async () => {
|
||||
const home = mkdtempSync(join(tmpdir(), 'gf-'));
|
||||
await ensureSource('src2');
|
||||
await seed('people/dup', { src: 'a' }, 'default');
|
||||
await seed('people/dup', { src: 'b' }, 'src2');
|
||||
await gf(home);
|
||||
|
||||
// configDir() appends '.gbrain' to GBRAIN_HOME.
|
||||
const logPath = join(home, '.gbrain', 'migrations', 'v0_13_1-rollback.jsonl');
|
||||
expect(existsSync(logPath)).toBe(true);
|
||||
const lines = readFileSync(logPath, 'utf-8').trim().split('\n').map(l => JSON.parse(l));
|
||||
expect(lines.map(l => l.source_id).sort()).toEqual(['default', 'src2']);
|
||||
expect(lines.every(l => typeof l.id === 'number' && l.slug === 'people/dup')).toBe(true);
|
||||
});
|
||||
|
||||
test('re-run is a no-op (idempotent)', async () => {
|
||||
const home = mkdtempSync(join(tmpdir(), 'gf-'));
|
||||
await seed('a', { foo: 1 });
|
||||
const first = await gf(home);
|
||||
expect(first.detail.touched).toBe(1);
|
||||
const second = await gf(home);
|
||||
expect(second.detail.touched).toBe(0);
|
||||
});
|
||||
|
||||
test('dry-run counts without mutating', async () => {
|
||||
const home = mkdtempSync(join(tmpdir(), 'gf-'));
|
||||
await seed('a', { foo: 1 });
|
||||
const { detail } = await gf(home, { dryRun: true });
|
||||
expect(detail.touched).toBe(1);
|
||||
expect((await fmOf('a'))?.validate ?? 'UNSET').toBe('UNSET');
|
||||
});
|
||||
|
||||
test('large-N (1200 pages) completes via chunked pass (hang regression)', async () => {
|
||||
const home = mkdtempSync(join(tmpdir(), 'gf-'));
|
||||
for (let i = 0; i < 1200; i++) await seed(`bulk/${i}`, { i });
|
||||
const t0 = Date.now();
|
||||
const { result, detail } = await gf(home);
|
||||
expect(result.status).toBe('complete');
|
||||
expect(detail.touched).toBe(1200);
|
||||
expect((await fmOf('bulk/0'))?.validate).toBe(false);
|
||||
expect((await fmOf('bulk/1199'))?.validate).toBe(false);
|
||||
expect(Date.now() - t0).toBeLessThan(30_000);
|
||||
});
|
||||
});
|
||||
@@ -54,8 +54,9 @@ describe('v0.16.0 migration', () => {
|
||||
]);
|
||||
});
|
||||
|
||||
test('phaseASchema skips on dry-run', () => {
|
||||
const r = __testing.phaseASchema({ dryRun: true, yes: true, noAutopilotInstall: true });
|
||||
test('phaseASchema skips on dry-run', async () => {
|
||||
// v0.41.37.0 #1605: phaseASchema is now async (in-process runMigrateOnlyCore).
|
||||
const r = await __testing.phaseASchema({ dryRun: true, yes: true, noAutopilotInstall: true });
|
||||
expect(r.status).toBe('skipped');
|
||||
expect(r.detail).toBe('dry-run');
|
||||
});
|
||||
|
||||
@@ -0,0 +1,96 @@
|
||||
// v0.41.37.0 #1569 — ReDoS hardening + diagnostics for schema-pack regexes.
|
||||
//
|
||||
// The real exposure was link-inference.ts:85 running `new RegExp(pattern)
|
||||
// .test(context)` UNBOUNDED when no PageRegexBudget was passed. Fix: an input-
|
||||
// length cap (the runtime safety net) + routing that path through the bounded
|
||||
// executor + an advisory star-height lint rule.
|
||||
import { describe, test, expect } from 'bun:test';
|
||||
import { readFileSync } from 'fs';
|
||||
import { join } from 'path';
|
||||
import {
|
||||
runRegexBounded,
|
||||
PageRegexBudget,
|
||||
RegexInputTooLargeError,
|
||||
MAX_REGEX_INPUT_CHARS,
|
||||
} from '../src/core/schema-pack/redos-guard.ts';
|
||||
import { inferLinkTypeFromPack } from '../src/core/schema-pack/link-inference.ts';
|
||||
import { linkRegexCatastrophicBacktrack } from '../src/core/schema-pack/lint-rules.ts';
|
||||
import type { SchemaPackManifest } from '../src/core/schema-pack/manifest-v1.ts';
|
||||
|
||||
describe('#1569 input-length cap', () => {
|
||||
test('runRegexBounded throws RegexInputTooLargeError over the cap', () => {
|
||||
const big = 'a'.repeat(MAX_REGEX_INPUT_CHARS + 1);
|
||||
expect(() => runRegexBounded('a', big)).toThrow(RegexInputTooLargeError);
|
||||
});
|
||||
|
||||
test('runRegexBounded works normally under the cap', () => {
|
||||
const m = runRegexBounded('wor', 'hello world');
|
||||
expect(m).not.toBeNull();
|
||||
expect(runRegexBounded('zzz', 'hello world')).toBeNull();
|
||||
});
|
||||
|
||||
test('PageRegexBudget.runBounded degrades (null) on oversize input', () => {
|
||||
const budget = new PageRegexBudget();
|
||||
const big = 'a'.repeat(MAX_REGEX_INPUT_CHARS + 1);
|
||||
expect(budget.runBounded('verb', 'a', big)).toBeNull();
|
||||
});
|
||||
|
||||
test('inferLinkTypeFromPack (no budget) does NOT run regex unbounded on huge input', () => {
|
||||
// Pre-#1569 this ran new RegExp().test() with no length cap → ReDoS risk.
|
||||
// A catastrophic pattern + a long input must NOT hang; the cap skips it.
|
||||
const pack = {
|
||||
link_types: [{ name: 'founded', inference: { regex: '(a+)+$' } }],
|
||||
} as unknown as Pick<SchemaPackManifest, 'link_types'>;
|
||||
const huge = 'a'.repeat(MAX_REGEX_INPUT_CHARS + 100) + '!';
|
||||
const t0 = Date.now();
|
||||
const result = inferLinkTypeFromPack(pack, 'company', huge);
|
||||
// Skipped via the cap → no match, and fast (no catastrophic backtrack).
|
||||
expect(result).toBeNull();
|
||||
expect(Date.now() - t0).toBeLessThan(2_000);
|
||||
});
|
||||
});
|
||||
|
||||
describe('#1569 star-height lint rule', () => {
|
||||
const mk = (regex: string): SchemaPackManifest =>
|
||||
({ name: 'testpack', page_types: [], link_types: [{ name: 'founded', inference: { regex } }] }) as unknown as SchemaPackManifest;
|
||||
|
||||
test('flags classic nested-quantifier shapes as warnings', () => {
|
||||
for (const bad of ['(a+)+', '(a*)*', '(a+)*', '(\\w+)+$', '(.*)+']) {
|
||||
const issues = linkRegexCatastrophicBacktrack(mk(bad)) as ReturnType<typeof linkRegexCatastrophicBacktrack> & any[];
|
||||
expect(issues.length).toBe(1);
|
||||
expect(issues[0].severity).toBe('warning');
|
||||
expect(issues[0].rule).toBe('link_regex_catastrophic_backtrack');
|
||||
expect(issues[0].link).toBe('founded');
|
||||
}
|
||||
});
|
||||
|
||||
test('does NOT flag benign patterns', () => {
|
||||
for (const ok of ['a+', '(abc)+', '(a|b)+', 'founded\\s+\\w+', '[a-z]+@[a-z]+']) {
|
||||
const issues = linkRegexCatastrophicBacktrack(mk(ok)) as any[];
|
||||
expect(issues.length).toBe(0);
|
||||
}
|
||||
});
|
||||
|
||||
test('no regex → no issue', () => {
|
||||
const manifest = { name: 'p', page_types: [], link_types: [{ name: 'x' }] } as unknown as SchemaPackManifest;
|
||||
expect((linkRegexCatastrophicBacktrack(manifest) as any[]).length).toBe(0);
|
||||
});
|
||||
});
|
||||
|
||||
describe('#1569 --no-schema-pack + heartbeat wiring (structural)', () => {
|
||||
const SYNC = readFileSync(join(import.meta.dir, '..', 'src', 'commands', 'sync.ts'), 'utf-8');
|
||||
|
||||
test('SyncOpts carries noSchemaPack and it gates loadActivePack', () => {
|
||||
expect(SYNC).toContain('noSchemaPack?: boolean');
|
||||
expect(SYNC).toContain("args.includes('--no-schema-pack')");
|
||||
expect(SYNC).toContain('if (opts.noSchemaPack)');
|
||||
});
|
||||
|
||||
test('begin heartbeat fires before importFile (GBRAIN_SYNC_TRACE)', () => {
|
||||
const beginIdx = SYNC.indexOf('begin import:');
|
||||
const importIdx = SYNC.indexOf('importFile(eng, filePath, path');
|
||||
expect(beginIdx).toBeGreaterThan(0);
|
||||
expect(importIdx).toBeGreaterThan(0);
|
||||
expect(beginIdx).toBeLessThan(importIdx);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,75 @@
|
||||
// v0.41.37.0 #1621 — reindex / re-import must NOT wipe DB-side enrichment tags.
|
||||
//
|
||||
// Root cause: import-file.ts reconciled tags from frontmatter only via
|
||||
// DELETE-then-INSERT. The tags table has no provenance column and frontmatter
|
||||
// tags are stripped from stored frontmatter (markdown.ts:118), so every
|
||||
// re-import (notably `gbrain reindex --markdown`, which re-imports with
|
||||
// forceRechunk) deleted all enrichment / dream / signal-detector tags.
|
||||
//
|
||||
// Fix: ADD-ONLY reconciliation. Re-import adds current frontmatter tags and
|
||||
// never deletes. Accepted trade-off: removing a frontmatter tag no longer
|
||||
// removes it from the DB (additive metadata; far better than wiping enrichment).
|
||||
import { describe, test, expect, beforeAll, afterAll, beforeEach } from 'bun:test';
|
||||
import { PGLiteEngine } from '../src/core/pglite-engine.ts';
|
||||
import { resetPgliteState } from './helpers/reset-pglite.ts';
|
||||
import { importFromContent } from '../src/core/import-file.ts';
|
||||
|
||||
let engine: PGLiteEngine;
|
||||
|
||||
beforeAll(async () => {
|
||||
engine = new PGLiteEngine();
|
||||
await engine.connect({});
|
||||
await engine.initSchema();
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
await engine.disconnect();
|
||||
});
|
||||
|
||||
beforeEach(async () => {
|
||||
await resetPgliteState(engine);
|
||||
});
|
||||
|
||||
const page = (tags: string[]) =>
|
||||
`---\ntype: concept\ntitle: Xavi\ntags: [${tags.join(', ')}]\n---\nBody text for the page.\n`;
|
||||
|
||||
describe('#1621 add-only tag reconciliation', () => {
|
||||
test('re-import (reindex) preserves DB-side enrichment tags', async () => {
|
||||
await importFromContent(engine, 'people/xavi', page(['founder', 'yc']), { forceRechunk: true, noEmbed: true });
|
||||
// Simulate DB-side enrichment writing a tag not present in frontmatter.
|
||||
await engine.addTag('people/xavi', 'enrichment-tag');
|
||||
|
||||
// Re-import the same content (what reindex --markdown does).
|
||||
await importFromContent(engine, 'people/xavi', page(['founder', 'yc']), { forceRechunk: true, noEmbed: true });
|
||||
|
||||
const tags = await engine.getTags('people/xavi');
|
||||
expect(tags.sort()).toEqual(['enrichment-tag', 'founder', 'yc']);
|
||||
});
|
||||
|
||||
test('frontmatter tags are still added on import', async () => {
|
||||
await importFromContent(engine, 'people/xavi', page(['founder', 'yc']), { forceRechunk: true, noEmbed: true });
|
||||
const tags = await engine.getTags('people/xavi');
|
||||
expect(tags.sort()).toEqual(['founder', 'yc']);
|
||||
});
|
||||
|
||||
test('add-only: removing a frontmatter tag does NOT remove it (accepted trade-off)', async () => {
|
||||
await importFromContent(engine, 'people/xavi', page(['founder', 'yc']), { forceRechunk: true, noEmbed: true });
|
||||
await engine.addTag('people/xavi', 'enrichment-tag');
|
||||
|
||||
// User removes "yc" from frontmatter and re-imports.
|
||||
await importFromContent(engine, 'people/xavi', page(['founder']), { forceRechunk: true, noEmbed: true });
|
||||
|
||||
const tags = await engine.getTags('people/xavi');
|
||||
// "yc" lingers (add-only); enrichment-tag preserved; founder present.
|
||||
expect(tags.sort()).toEqual(['enrichment-tag', 'founder', 'yc']);
|
||||
});
|
||||
|
||||
test('adding a new frontmatter tag on re-import works (idempotent add)', async () => {
|
||||
await importFromContent(engine, 'people/xavi', page(['founder']), { forceRechunk: true, noEmbed: true });
|
||||
await engine.addTag('people/xavi', 'enrichment-tag');
|
||||
await importFromContent(engine, 'people/xavi', page(['founder', 'growth']), { forceRechunk: true, noEmbed: true });
|
||||
|
||||
const tags = await engine.getTags('people/xavi');
|
||||
expect(tags.sort()).toEqual(['enrichment-tag', 'founder', 'growth']);
|
||||
});
|
||||
});
|
||||
@@ -332,12 +332,13 @@ describe('runAllLintRules — composition', () => {
|
||||
});
|
||||
|
||||
describe('rule registry shape', () => {
|
||||
it('ALL_LINT_RULES contains 11 rules', () => {
|
||||
expect(ALL_LINT_RULES.length).toBe(11);
|
||||
it('ALL_LINT_RULES contains 12 rules', () => {
|
||||
// v0.41.37.0 #1569 added link_regex_catastrophic_backtrack (file-plane).
|
||||
expect(ALL_LINT_RULES.length).toBe(12);
|
||||
});
|
||||
|
||||
it('FILE_PLANE_LINT_RULES excludes the 2 DB-aware rules', () => {
|
||||
expect(FILE_PLANE_LINT_RULES.length).toBe(9);
|
||||
expect(FILE_PLANE_LINT_RULES.length).toBe(10);
|
||||
expect(FILE_PLANE_LINT_RULES.every((r) => !r.planeAware)).toBe(true);
|
||||
});
|
||||
|
||||
|
||||
Reference in New Issue
Block a user