Files
gbrain/test/import-checkpoint.test.ts
T
3325b405bb v0.34.2.0 fix(import): path-based checkpoint resume — kills parallel-drop + failed-file-skip + sort-flip bugs (#988)
* feat(sync): sort files newest-first for faster salience on recent content

Problem: sync processes files in git-diff order (alphabetical), so
meetings/2020-* embeds before meetings/2026-*. After a burst of writes,
new pages can be invisible to search for hours while older pages process first.

Fix: sort addsAndMods descending in both incremental sync and full import.
Brain paths are date-prefixed by convention, so lexicographic descending
naturally prioritizes recent content.

This ensures the most relevant pages become searchable first.

* feat(import): path-based checkpoint resume + sort-newest-first helper

Replace gbrain import's positional `processedIndex` checkpoint with a
path-set checkpoint via `src/core/import-checkpoint.ts`. A file is only
"done" when its processFile returns success — failed files never enter
the set, parallel workers can't lose slow files, and sort-order changes
don't drop the newest N files on resume.

Three bug classes fixed:
- Parallel import + slow worker = silent file drop on crash-resume
- Failed file = checkpoint advanced past it, never retried until manual clear
- Sort-order flip (v0.33.x) = cross-version resume drops newest N files

Old positional checkpoints are detected on first resume and discarded
with a stderr log line. Re-walking is cheap because content_hash
short-circuits unchanged files.

Also extracts the descending-lex sort into src/core/sort-newest-first.ts
so import.ts and sync.ts share a single source of truth.

Tests:
- test/sort-newest-first.test.ts (5 hermetic cases)
- test/import-checkpoint.test.ts (18 unit cases over the helpers)
- test/import-resume.test.ts (refactored — GBRAIN_HOME isolation,
  drives runImport against PGLite, 5 integration cases including
  SLUG_MISMATCH retry regression)

Includes the original sort-newest-first contribution from
@garrytan-agents's PR #964 (commit 8dbcf6a5).

* chore: bump version and changelog (v0.34.2.0)

Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>

* docs: update project documentation for v0.34.2.0

Add CLAUDE.md Key Files entries for the path-based import checkpoint
work: new entries for src/core/import-checkpoint.ts and
src/core/sort-newest-first.ts, plus a dedicated src/commands/import.ts
entry covering the v0.34.2.0 refactor. Update src/commands/sync.ts
entry to reference sortNewestFirst. Regenerate llms-full.txt.

Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>

* fix(tests): swap banned /data/brain placeholder for /tmp/example-brain

scripts/check-privacy.sh banlist includes /data/brain/ (legacy private
OpenClaw fork layout). New test files must not use it — CI privacy
guard caught this on PR #988's first push.

No behavior change. test/import-checkpoint.test.ts is unit-level with
no fs access; the dir string is just an identity marker for the
loadCheckpoint dir-mismatch guard.

---------

Co-authored-by: garrytan-agents <garrytan-agents@users.noreply.github.com>
Co-authored-by: Claude Opus 4.7 <noreply@anthropic.com>
2026-05-14 20:51:11 -07:00

213 lines
7.0 KiB
TypeScript

import { describe, test, expect, beforeEach, afterEach } from 'bun:test';
import { writeFileSync, readFileSync, existsSync, mkdtempSync, rmSync } from 'fs';
import { tmpdir } from 'os';
import { join } from 'path';
import {
loadCheckpoint,
saveCheckpoint,
resumeFilter,
clearCheckpoint,
type ImportCheckpoint,
} from '../src/core/import-checkpoint.ts';
let workDir: string;
let cpPath: string;
let stderrCaptured = '';
let originalConsoleError: typeof console.error;
function captureStderr() {
stderrCaptured = '';
originalConsoleError = console.error.bind(console);
console.error = (...args: unknown[]) => {
stderrCaptured += args.map(a => (typeof a === 'string' ? a : JSON.stringify(a))).join(' ') + '\n';
};
}
function restoreStderr() {
if (originalConsoleError) {
console.error = originalConsoleError;
originalConsoleError = undefined as unknown as typeof console.error;
}
}
beforeEach(() => {
workDir = mkdtempSync(join(tmpdir(), 'gbrain-checkpoint-'));
cpPath = join(workDir, 'import-checkpoint.json');
});
afterEach(() => {
restoreStderr();
rmSync(workDir, { recursive: true, force: true });
});
describe('loadCheckpoint', () => {
test('returns null when file is missing', () => {
expect(loadCheckpoint(cpPath, '/some/dir')).toBeNull();
});
test('returns null when JSON is malformed', () => {
writeFileSync(cpPath, 'not json at all');
expect(loadCheckpoint(cpPath, '/some/dir')).toBeNull();
});
test('returns null when dir mismatches the current run', () => {
const cp: ImportCheckpoint = {
dir: '/other/brain',
completedPaths: ['a.md'],
timestamp: '2026-05-14T00:00:00Z',
};
writeFileSync(cpPath, JSON.stringify(cp));
expect(loadCheckpoint(cpPath, '/different/brain')).toBeNull();
});
test('returns null and logs to stderr for old positional format', () => {
captureStderr();
writeFileSync(cpPath, JSON.stringify({
dir: '/tmp/example-brain',
totalFiles: 13768,
processedIndex: 5000,
timestamp: '2026-01-01T00:00:00Z',
}));
const result = loadCheckpoint(cpPath, '/tmp/example-brain');
expect(result).toBeNull();
expect(stderrCaptured).toContain('Older checkpoint format detected');
});
test('returns null silently for missing completedPaths without processedIndex', () => {
// Not the v0.33.2 schema and not the old positional schema either —
// probably a manually-edited file or third-party tooling. Discard
// without the migration log line.
captureStderr();
writeFileSync(cpPath, JSON.stringify({
dir: '/tmp/example-brain',
timestamp: '2026-01-01T00:00:00Z',
}));
expect(loadCheckpoint(cpPath, '/tmp/example-brain')).toBeNull();
expect(stderrCaptured).not.toContain('Older checkpoint format');
});
test('returns null when completedPaths contains non-strings', () => {
writeFileSync(cpPath, JSON.stringify({
dir: '/tmp/example-brain',
completedPaths: ['a.md', 42, 'b.md'],
timestamp: '2026-01-01T00:00:00Z',
}));
expect(loadCheckpoint(cpPath, '/tmp/example-brain')).toBeNull();
});
test('returns the checkpoint for valid v0.33.2 payload', () => {
const cp: ImportCheckpoint = {
dir: '/tmp/example-brain',
completedPaths: ['meetings/2026-05-13.md', 'concepts/foo.md'],
timestamp: '2026-05-14T12:34:56Z',
};
writeFileSync(cpPath, JSON.stringify(cp));
const loaded = loadCheckpoint(cpPath, '/tmp/example-brain');
expect(loaded).not.toBeNull();
expect(loaded?.dir).toBe('/tmp/example-brain');
expect(loaded?.completedPaths).toEqual(['meetings/2026-05-13.md', 'concepts/foo.md']);
expect(loaded?.timestamp).toBe('2026-05-14T12:34:56Z');
});
});
describe('saveCheckpoint', () => {
test('round-trips through loadCheckpoint', () => {
const cp: ImportCheckpoint = {
dir: '/tmp/example-brain',
completedPaths: ['a.md', 'b.md', 'c.md'],
timestamp: '2026-05-14T00:00:00Z',
};
saveCheckpoint(cpPath, cp);
const loaded = loadCheckpoint(cpPath, '/tmp/example-brain');
expect(loaded?.completedPaths).toEqual(['a.md', 'b.md', 'c.md']);
expect(loaded?.dir).toBe('/tmp/example-brain');
});
test('serializes completedPaths sorted (deterministic output)', () => {
saveCheckpoint(cpPath, {
dir: '/tmp/example-brain',
completedPaths: ['z.md', 'a.md', 'm.md'],
timestamp: '2026-05-14T00:00:00Z',
});
const onDisk = JSON.parse(readFileSync(cpPath, 'utf-8'));
expect(onDisk.completedPaths).toEqual(['a.md', 'm.md', 'z.md']);
});
test('atomic-ish write — no stray .tmp file after success', () => {
saveCheckpoint(cpPath, {
dir: '/tmp/example-brain',
completedPaths: ['a.md'],
timestamp: '2026-05-14T00:00:00Z',
});
expect(existsSync(cpPath)).toBe(true);
expect(existsSync(`${cpPath}.tmp`)).toBe(false);
});
test('non-fatal on write failure (path under non-existent dir)', () => {
// Should NOT throw, just silently skip the write.
const badPath = join(workDir, 'does-not-exist', 'cp.json');
expect(() =>
saveCheckpoint(badPath, {
dir: '/tmp/example-brain',
completedPaths: ['a.md'],
timestamp: '2026-05-14T00:00:00Z',
}),
).not.toThrow();
expect(existsSync(badPath)).toBe(false);
});
});
describe('resumeFilter', () => {
test('empty completed set returns all files unchanged', () => {
const all = ['a.md', 'b.md', 'c.md'];
expect(resumeFilter(all, '/tmp/example-brain', new Set())).toEqual(all);
});
test('completed set filters matching paths out', () => {
const all = ['a.md', 'b.md', 'c.md'];
const completed = new Set(['b.md']);
expect(resumeFilter(all, '/tmp/example-brain', completed)).toEqual(['a.md', 'c.md']);
});
test('absolute paths get normalized to relative for lookup', () => {
const all = [
'/tmp/example-brain/meetings/2026-05-13.md',
'/tmp/example-brain/concepts/a.md',
];
const completed = new Set(['meetings/2026-05-13.md']);
expect(resumeFilter(all, '/tmp/example-brain', completed)).toEqual([
'/tmp/example-brain/concepts/a.md',
]);
});
test('mixed absolute and relative inputs both work', () => {
const all = [
'/tmp/example-brain/a.md',
'b.md',
'/tmp/example-brain/c.md',
];
const completed = new Set(['a.md', 'c.md']);
expect(resumeFilter(all, '/tmp/example-brain', completed)).toEqual(['b.md']);
});
test('full match returns empty array', () => {
const all = ['a.md', 'b.md'];
const completed = new Set(['a.md', 'b.md']);
expect(resumeFilter(all, '/tmp/example-brain', completed)).toEqual([]);
});
});
describe('clearCheckpoint', () => {
test('removes the checkpoint file when present', () => {
writeFileSync(cpPath, '{}');
expect(existsSync(cpPath)).toBe(true);
clearCheckpoint(cpPath);
expect(existsSync(cpPath)).toBe(false);
});
test('is a no-op when the checkpoint file is missing', () => {
expect(existsSync(cpPath)).toBe(false);
expect(() => clearCheckpoint(cpPath)).not.toThrow();
});
});