mirror of
https://github.com/garrytan/gbrain.git
synced 2026-07-27 22:15:33 +00:00
* fix(import): coerce non-string frontmatter title/slug/type (#1939) YAML `title: 2024-06-01` parses to a Date and `title: 1458` to a number; the old `(frontmatter.X as string)` cast was a compile-time lie, so downstream `.toLowerCase()` threw and (via the importer failure gate) could wedge sync indefinitely. parseMarkdown now coerces via coerceFrontmatterString (Date -> UTC ISO date, deterministic), and the pure assessContentSanity self-protects against a non-string title. * feat(sync): bounded auto-skip failure ledger; poison file can't wedge indexing (#1939) New src/core/sync-failure-ledger.ts owns the failure store + a crash-safe, multi-source, concurrent bounded auto-skip valve. A file that fails N consecutive syncs (GBRAIN_SYNC_AUTOSKIP_AFTER, default 3) auto-skips so it can't freeze all indexing forever, while fresh failures still fail-closed and a `<head>` history-rewrite sentinel hard-blocks even with --skip-failed. - (source_id, path) keying — failures never merge across sources - success clears a path so attempts are truly consecutive - advance-before-ack ordering (a crash can't mark a file skipped while wedged) - shared applySyncFailureGate used by BOTH the incremental and full-sync gates - legacy-row normalization + duplicate collapse on load - cross-process lock + atomic temp-rename, age-based stale-lock break sync.ts re-exports the ledger for existing callers; import.ts records source-scoped and defers the bookmark to the gate under managedBookmark. * fix(doctor): sync_failures severity via one shared decision on both surfaces (#1939) Local buildChecks and remote doctorReportRemote now both route through decideSyncFailureSeverity, so a stuck bookmark escalates WARN -> FAIL consistently (oldest-open age > fail cadence, or large unresolved count), auto-skipped pages stay visible (WARN, not hidden), and the acknowledged/acknowledged_at field-split that caused drift is gone. The remote surface stays subprocess-free (file read + Date.parse only). * chore(test): add trailing newline to e5-lease-cap-ab baseline fixture * fix(sync): address adversarial review findings on the failure ledger (#1939) - #1: a parse-failed file that is later deleted/renamed-away no longer leaves a permanent open ledger row. Removed paths (filtered.deleted, renamed-from, and the "gone from disk" forward-delete skip branch) are treated as resolved so the ledger self-heals instead of aging doctor to a stuck FAIL. - #3: decideSyncFailureSeverity escalates to FAIL on OPEN (blocking) failures only — auto_skipped rows already advanced the bookmark, so they stay WARN-visible regardless of count, matching the state-machine contract. * chore: bump version and changelog (v0.42.30.0) Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com> * docs: document sync-failure ledger + auto-skip valve for v0.42.30.0 KEY_FILES.md: new src/core/sync-failure-ledger.ts entry (bounded auto-skip state machine, decideGateAction/decideSyncFailureSeverity/applySyncFailureGate, GBRAIN_SYNC_AUTOSKIP_AFTER); update sync.ts (failure store moved to ledger, re-exported), doctor.ts (sync_failures severity via shared rule on both surfaces), markdown.ts (coerceFrontmatterString), import.ts (managedBookmark). live-sync.md: poison-file auto-skip tricky-spot. Regenerated llms-full.txt. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com> * chore: re-bump to v0.42.31.0 (queue collision on 0.42.30.0) Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com> * chore: re-bump to v0.42.32.0 (queue collision) Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com> --------- Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
658 lines
26 KiB
TypeScript
658 lines
26 KiB
TypeScript
/**
|
|
* E2E Sync Tests — Tier 1 (no API keys required)
|
|
*
|
|
* Tests the full git-to-DB sync pipeline: create a git repo, commit
|
|
* markdown files, run gbrain sync, verify pages appear in the database.
|
|
* Covers first sync, incremental add/modify/delete, and the critical
|
|
* "edit → sync → search returns corrected text" flow.
|
|
*
|
|
* Run: DATABASE_URL=... bun test test/e2e/sync.test.ts
|
|
*/
|
|
|
|
import { describe, test, expect, beforeAll, afterAll } from 'bun:test';
|
|
import { mkdtempSync, writeFileSync, rmSync, mkdirSync, unlinkSync, existsSync, readFileSync } from 'fs';
|
|
import { join } from 'path';
|
|
import { execSync } from 'child_process';
|
|
import { tmpdir, homedir } from 'os';
|
|
import {
|
|
hasDatabase, setupDB, teardownDB, getEngine,
|
|
} from './helpers.ts';
|
|
|
|
const skip = !hasDatabase();
|
|
const describeE2E = skip ? describe.skip : describe;
|
|
|
|
if (skip) {
|
|
console.log('Skipping E2E sync tests (DATABASE_URL not set)');
|
|
}
|
|
|
|
/** Create a temp git repo with initial markdown files */
|
|
function createTestRepo(): string {
|
|
const dir = mkdtempSync(join(tmpdir(), 'gbrain-sync-e2e-'));
|
|
execSync('git init', { cwd: dir, stdio: 'pipe' });
|
|
execSync('git config user.email "test@test.com"', { cwd: dir, stdio: 'pipe' });
|
|
execSync('git config user.name "Test"', { cwd: dir, stdio: 'pipe' });
|
|
|
|
// Create initial structure
|
|
mkdirSync(join(dir, 'people'), { recursive: true });
|
|
mkdirSync(join(dir, 'concepts'), { recursive: true });
|
|
|
|
writeFileSync(join(dir, 'people/alice.md'), [
|
|
'---',
|
|
'type: person',
|
|
'title: Alice Smith',
|
|
'tags: [engineer, frontend]',
|
|
'---',
|
|
'',
|
|
'Alice is a frontend engineer at Acme Corp.',
|
|
'',
|
|
'---',
|
|
'',
|
|
'- 2026-01-15: Joined Acme Corp',
|
|
].join('\n'));
|
|
|
|
writeFileSync(join(dir, 'concepts/testing.md'), [
|
|
'---',
|
|
'type: concept',
|
|
'title: Testing Philosophy',
|
|
'tags: [engineering]',
|
|
'---',
|
|
'',
|
|
'Every untested path is a path where bugs hide.',
|
|
].join('\n'));
|
|
|
|
// Initial commit
|
|
execSync('git add -A && git commit -m "initial commit"', { cwd: dir, stdio: 'pipe' });
|
|
|
|
return dir;
|
|
}
|
|
|
|
function gitCommit(repoPath: string, message: string) {
|
|
execSync(`git add -A && git commit -m "${message}"`, { cwd: repoPath, stdio: 'pipe' });
|
|
}
|
|
|
|
describeE2E('E2E: Git-to-DB Sync Pipeline', () => {
|
|
let repoPath: string;
|
|
|
|
beforeAll(async () => {
|
|
await setupDB();
|
|
repoPath = createTestRepo();
|
|
}, 30_000);
|
|
|
|
afterAll(async () => {
|
|
await teardownDB();
|
|
if (repoPath) rmSync(repoPath, { recursive: true, force: true });
|
|
});
|
|
|
|
test('first sync imports all pages from git repo', async () => {
|
|
const { performSync } = await import('../../src/commands/sync.ts');
|
|
const engine = getEngine();
|
|
|
|
const result = await performSync(engine, {
|
|
repoPath,
|
|
noPull: true,
|
|
noEmbed: true,
|
|
});
|
|
|
|
expect(result.status).toBe('first_sync');
|
|
// performFullSync delegates to runImport which doesn't populate pagesAffected
|
|
// Verify pages exist in DB directly instead
|
|
const alice = await engine.getPage('people/alice');
|
|
expect(alice).not.toBeNull();
|
|
expect(alice!.title).toBe('Alice Smith');
|
|
|
|
const testing = await engine.getPage('concepts/testing');
|
|
expect(testing).not.toBeNull();
|
|
expect(testing!.title).toBe('Testing Philosophy');
|
|
});
|
|
|
|
test('second sync with no changes returns up_to_date', async () => {
|
|
const { performSync } = await import('../../src/commands/sync.ts');
|
|
const engine = getEngine();
|
|
|
|
const result = await performSync(engine, {
|
|
repoPath,
|
|
noPull: true,
|
|
noEmbed: true,
|
|
});
|
|
|
|
expect(result.status).toBe('up_to_date');
|
|
expect(result.added).toBe(0);
|
|
expect(result.modified).toBe(0);
|
|
expect(result.deleted).toBe(0);
|
|
});
|
|
|
|
test('incremental sync picks up new files', async () => {
|
|
const { performSync } = await import('../../src/commands/sync.ts');
|
|
const engine = getEngine();
|
|
|
|
// Add a new file
|
|
writeFileSync(join(repoPath, 'people/bob.md'), [
|
|
'---',
|
|
'type: person',
|
|
'title: Bob Jones',
|
|
'tags: [designer]',
|
|
'---',
|
|
'',
|
|
'Bob is a product designer who loves typography.',
|
|
].join('\n'));
|
|
gitCommit(repoPath, 'add bob');
|
|
|
|
const result = await performSync(engine, {
|
|
repoPath,
|
|
noPull: true,
|
|
noEmbed: true,
|
|
});
|
|
|
|
expect(result.status).toBe('synced');
|
|
expect(result.added).toBe(1);
|
|
expect(result.pagesAffected).toContain('people/bob');
|
|
|
|
const bob = await engine.getPage('people/bob');
|
|
expect(bob).not.toBeNull();
|
|
expect(bob!.title).toBe('Bob Jones');
|
|
expect(bob!.compiled_truth).toContain('typography');
|
|
});
|
|
|
|
test('incremental sync picks up modifications — corrected text appears', async () => {
|
|
const { performSync } = await import('../../src/commands/sync.ts');
|
|
const engine = getEngine();
|
|
|
|
// Modify alice's page — this is the critical "correction" test
|
|
writeFileSync(join(repoPath, 'people/alice.md'), [
|
|
'---',
|
|
'type: person',
|
|
'title: Alice Smith',
|
|
'tags: [engineer, frontend]',
|
|
'---',
|
|
'',
|
|
'Alice is a staff frontend engineer at Acme Corp, leading the design system team.',
|
|
'',
|
|
'---',
|
|
'',
|
|
'- 2026-04-01: Promoted to staff engineer',
|
|
'- 2026-01-15: Joined Acme Corp',
|
|
].join('\n'));
|
|
gitCommit(repoPath, 'update alice - promotion');
|
|
|
|
const result = await performSync(engine, {
|
|
repoPath,
|
|
noPull: true,
|
|
noEmbed: true,
|
|
});
|
|
|
|
expect(result.status).toBe('synced');
|
|
expect(result.modified).toBe(1);
|
|
expect(result.pagesAffected).toContain('people/alice');
|
|
|
|
// THE CRITICAL CHECK: corrected text appears in the DB
|
|
const alice = await engine.getPage('people/alice');
|
|
expect(alice!.compiled_truth).toContain('staff frontend engineer');
|
|
expect(alice!.compiled_truth).toContain('design system team');
|
|
// Old text should be replaced, not appended
|
|
expect(alice!.compiled_truth).not.toBe('Alice is a frontend engineer at Acme Corp.');
|
|
});
|
|
|
|
test('keyword search finds corrected text after sync', async () => {
|
|
const engine = getEngine();
|
|
|
|
// Search for the new text
|
|
const results = await engine.searchKeyword('design system team');
|
|
expect(results.length).toBeGreaterThanOrEqual(1);
|
|
|
|
const aliceResult = results.find((r: any) => r.slug === 'people/alice');
|
|
expect(aliceResult).toBeDefined();
|
|
});
|
|
|
|
test('incremental sync handles deletes', async () => {
|
|
const { performSync } = await import('../../src/commands/sync.ts');
|
|
const engine = getEngine();
|
|
|
|
// Delete bob's page
|
|
unlinkSync(join(repoPath, 'people/bob.md'));
|
|
gitCommit(repoPath, 'remove bob');
|
|
|
|
const result = await performSync(engine, {
|
|
repoPath,
|
|
noPull: true,
|
|
noEmbed: true,
|
|
});
|
|
|
|
expect(result.status).toBe('synced');
|
|
expect(result.deleted).toBe(1);
|
|
|
|
const bob = await engine.getPage('people/bob');
|
|
expect(bob).toBeNull();
|
|
});
|
|
|
|
test('sync skips non-syncable files (README, hidden, .raw)', async () => {
|
|
const { performSync } = await import('../../src/commands/sync.ts');
|
|
const engine = getEngine();
|
|
|
|
// Add files that should be excluded
|
|
writeFileSync(join(repoPath, 'README.md'), '# Brain Repo\nThis is the readme.');
|
|
mkdirSync(join(repoPath, '.raw'), { recursive: true });
|
|
writeFileSync(join(repoPath, '.raw/data.md'), '---\ntitle: Raw\n---\nRaw data.');
|
|
mkdirSync(join(repoPath, 'ops'), { recursive: true });
|
|
writeFileSync(join(repoPath, 'ops/deploy.md'), '---\ntitle: Deploy\n---\nOps stuff.');
|
|
gitCommit(repoPath, 'add non-syncable files');
|
|
|
|
const result = await performSync(engine, {
|
|
repoPath,
|
|
noPull: true,
|
|
noEmbed: true,
|
|
});
|
|
|
|
// These should not create pages
|
|
const readme = await engine.getPage('README');
|
|
expect(readme).toBeNull();
|
|
|
|
const raw = await engine.getPage('.raw/data');
|
|
expect(raw).toBeNull();
|
|
|
|
const ops = await engine.getPage('ops/deploy');
|
|
expect(ops).toBeNull();
|
|
});
|
|
|
|
test('sync stores last_commit and last_run in config', async () => {
|
|
const engine = getEngine();
|
|
|
|
const lastCommit = await engine.getConfig('sync.last_commit');
|
|
const lastRun = await engine.getConfig('sync.last_run');
|
|
const repoPathConfig = await engine.getConfig('sync.repo_path');
|
|
|
|
expect(lastCommit).toBeTruthy();
|
|
expect(lastCommit!.length).toBe(40); // full SHA
|
|
expect(lastRun).toBeTruthy();
|
|
expect(repoPathConfig).toBe(repoPath);
|
|
});
|
|
|
|
test('sync logs to ingest_log', async () => {
|
|
const engine = getEngine();
|
|
|
|
const logs = await engine.getIngestLog();
|
|
const syncLogs = logs.filter((l: any) => l.source_type === 'git_sync');
|
|
|
|
expect(syncLogs.length).toBeGreaterThanOrEqual(1);
|
|
expect(syncLogs[0].source_ref).toContain(repoPath);
|
|
});
|
|
|
|
test('--full reimports everything regardless of last_commit', async () => {
|
|
const { performSync } = await import('../../src/commands/sync.ts');
|
|
const engine = getEngine();
|
|
|
|
const result = await performSync(engine, {
|
|
repoPath,
|
|
noPull: true,
|
|
noEmbed: true,
|
|
full: true,
|
|
});
|
|
|
|
expect(result.status).toBe('first_sync');
|
|
// performFullSync delegates to runImport — verify pages exist instead
|
|
const alice = await engine.getPage('people/alice');
|
|
expect(alice).not.toBeNull();
|
|
const testing = await engine.getPage('concepts/testing');
|
|
expect(testing).not.toBeNull();
|
|
});
|
|
|
|
test('dry-run shows changes without applying them', async () => {
|
|
const { performSync } = await import('../../src/commands/sync.ts');
|
|
const engine = getEngine();
|
|
|
|
// Add a new file
|
|
writeFileSync(join(repoPath, 'concepts/dry-run-test.md'), [
|
|
'---',
|
|
'type: concept',
|
|
'title: Dry Run Test',
|
|
'---',
|
|
'',
|
|
'This should not be imported.',
|
|
].join('\n'));
|
|
gitCommit(repoPath, 'add dry run test');
|
|
|
|
const result = await performSync(engine, {
|
|
repoPath,
|
|
noPull: true,
|
|
noEmbed: true,
|
|
dryRun: true,
|
|
});
|
|
|
|
expect(result.status).toBe('dry_run');
|
|
expect(result.added).toBe(1);
|
|
|
|
// Page should NOT exist in DB
|
|
const page = await engine.getPage('concepts/dry-run-test');
|
|
expect(page).toBeNull();
|
|
|
|
// Clean up: do a real sync so the commit is consumed
|
|
await performSync(engine, {
|
|
repoPath,
|
|
noPull: true,
|
|
noEmbed: true,
|
|
});
|
|
});
|
|
|
|
test('files with spaces in names get slugified slugs', async () => {
|
|
const { performSync } = await import('../../src/commands/sync.ts');
|
|
const engine = getEngine();
|
|
|
|
// Add a file with spaces (Apple Notes style)
|
|
mkdirSync(join(repoPath, 'Apple Notes'), { recursive: true });
|
|
writeFileSync(join(repoPath, 'Apple Notes/2017-05-03 ohmygreen.md'), [
|
|
'---',
|
|
'title: Ohmygreen Notes',
|
|
'---',
|
|
'',
|
|
'Notes about ohmygreen lunch service.',
|
|
].join('\n'));
|
|
gitCommit(repoPath, 'add apple notes file with spaces');
|
|
|
|
const result = await performSync(engine, {
|
|
repoPath,
|
|
noPull: true,
|
|
noEmbed: true,
|
|
});
|
|
|
|
expect(result.status).toBe('synced');
|
|
expect(result.added).toBe(1);
|
|
|
|
// Slug should be slugified (lowercase, spaces → hyphens)
|
|
const page = await engine.getPage('apple-notes/2017-05-03-ohmygreen');
|
|
expect(page).not.toBeNull();
|
|
expect(page!.title).toBe('Ohmygreen Notes');
|
|
|
|
// Original space-based slug should NOT exist
|
|
const rawSlug = await engine.getPage('Apple Notes/2017-05-03 ohmygreen');
|
|
expect(rawSlug).toBeNull();
|
|
});
|
|
|
|
test('incremental sync adds file with special characters', async () => {
|
|
const { performSync } = await import('../../src/commands/sync.ts');
|
|
const engine = getEngine();
|
|
|
|
// Add a file with parens and special chars
|
|
writeFileSync(join(repoPath, 'Apple Notes/meeting notes (draft).md'), [
|
|
'---',
|
|
'title: Draft Meeting Notes',
|
|
'---',
|
|
'',
|
|
'Some draft notes from the meeting.',
|
|
].join('\n'));
|
|
gitCommit(repoPath, 'add file with parens');
|
|
|
|
const result = await performSync(engine, {
|
|
repoPath,
|
|
noPull: true,
|
|
noEmbed: true,
|
|
});
|
|
|
|
expect(result.status).toBe('synced');
|
|
|
|
// Slug should have parens stripped, spaces → hyphens
|
|
const page = await engine.getPage('apple-notes/meeting-notes-draft');
|
|
expect(page).not.toBeNull();
|
|
expect(page!.title).toBe('Draft Meeting Notes');
|
|
});
|
|
});
|
|
|
|
/**
|
|
* E2E: --skip-failed loop with structured error code summary.
|
|
*
|
|
* Closes the v0.22.12 ship-blocker gap from issue #500 — the whole code path
|
|
* (record → classify → block → skip → doctor render → second cycle) had only
|
|
* mocked-JSONL unit coverage. This is the integration test that proves the
|
|
* chain holds together with a real Postgres engine, real git history, and
|
|
* real frontmatter validation.
|
|
*
|
|
* Owns its own repo + sync-failures.jsonl lifecycle so it can't leak state
|
|
* into the shared describeE2E above. Saves and restores the user's real
|
|
* ~/.gbrain/sync-failures.jsonl so running E2E on a developer machine
|
|
* doesn't trash their local sync state.
|
|
*/
|
|
describeE2E('E2E: sync --skip-failed structured summary loop (v0.22.12, issue #500)', () => {
|
|
let repoPath: string;
|
|
const realFailuresPath = join(homedir(), '.gbrain', 'sync-failures.jsonl');
|
|
let savedFailuresContent: string | null = null;
|
|
|
|
beforeAll(async () => {
|
|
await setupDB();
|
|
|
|
// Save+clear the real ~/.gbrain/sync-failures.jsonl so the test starts from
|
|
// a known-empty state. Restored in afterAll. This file is per-machine, NOT
|
|
// per-repo, so we have to be defensive about a developer running this
|
|
// suite on their actual brain machine.
|
|
if (existsSync(realFailuresPath)) {
|
|
savedFailuresContent = readFileSync(realFailuresPath, 'utf-8');
|
|
unlinkSync(realFailuresPath);
|
|
}
|
|
|
|
// Fresh git repo with one valid file. Mirrors createTestRepo above but
|
|
// scoped to this describe block.
|
|
repoPath = mkdtempSync(join(tmpdir(), 'gbrain-skipfailed-e2e-'));
|
|
execSync('git init', { cwd: repoPath, stdio: 'pipe' });
|
|
execSync('git config user.email "test@test.com"', { cwd: repoPath, stdio: 'pipe' });
|
|
execSync('git config user.name "Test"', { cwd: repoPath, stdio: 'pipe' });
|
|
mkdirSync(join(repoPath, 'people'), { recursive: true });
|
|
writeFileSync(join(repoPath, 'people/alice.md'), [
|
|
'---', 'type: person', 'title: Alice', '---', '', 'Body.',
|
|
].join('\n'));
|
|
execSync('git add -A && git commit -m "initial"', { cwd: repoPath, stdio: 'pipe' });
|
|
}, 30_000);
|
|
|
|
afterAll(async () => {
|
|
await teardownDB();
|
|
if (repoPath) rmSync(repoPath, { recursive: true, force: true });
|
|
|
|
// Restore the user's real sync-failures.jsonl, if any.
|
|
if (savedFailuresContent !== null) {
|
|
mkdirSync(join(homedir(), '.gbrain'), { recursive: true });
|
|
writeFileSync(realFailuresPath, savedFailuresContent);
|
|
} else if (existsSync(realFailuresPath)) {
|
|
// Test wrote one but there was none before. Clean up.
|
|
unlinkSync(realFailuresPath);
|
|
}
|
|
});
|
|
|
|
test('full --skip-failed loop: blocks on bad file, skip advances bookmark, doctor shows code breakdown', async () => {
|
|
const { performSync } = await import('../../src/commands/sync.ts');
|
|
const { loadSyncFailures, summarizeFailuresByCode } = await import('../../src/core/sync.ts');
|
|
const engine = getEngine();
|
|
|
|
// Step 1: First sync of the clean repo — should succeed.
|
|
let result = await performSync(engine, { repoPath, noPull: true, noEmbed: true });
|
|
expect(result.status).toBe('first_sync');
|
|
const firstCommit = await engine.getConfig('sync.last_commit');
|
|
expect(firstCommit).toBeTruthy();
|
|
|
|
// Step 2: Add a broken file — frontmatter slug doesn't match path-derived slug.
|
|
// The file path is people/bob.md so the path-derived slug is "people/bob",
|
|
// but we declare slug: "wrong-slug" in frontmatter. import-file.ts:368-377
|
|
// raises "Frontmatter slug ... does not match path-derived slug ..." which
|
|
// classifier hits as SLUG_MISMATCH.
|
|
writeFileSync(join(repoPath, 'people/bob.md'), [
|
|
'---', 'type: person', 'title: Bob', 'slug: wrong-slug', '---', '', 'Body.',
|
|
].join('\n'));
|
|
execSync('git add -A && git commit -m "add broken bob"', { cwd: repoPath, stdio: 'pipe' });
|
|
|
|
// Step 3: Sync should block. Bookmark must NOT advance.
|
|
result = await performSync(engine, { repoPath, noPull: true, noEmbed: true });
|
|
expect(result.status).toBe('blocked_by_failures');
|
|
const afterBlockedCommit = await engine.getConfig('sync.last_commit');
|
|
expect(afterBlockedCommit).toBe(firstCommit); // bookmark stuck at the pre-broken commit
|
|
|
|
// JSONL has one unacked entry with code SLUG_MISMATCH.
|
|
let failures = loadSyncFailures();
|
|
expect(failures.length).toBe(1);
|
|
expect(failures[0].code).toBe('SLUG_MISMATCH');
|
|
expect(failures[0].acknowledged).toBeFalsy();
|
|
// Group summary aggregates correctly across the unacked set.
|
|
expect(summarizeFailuresByCode(failures)).toEqual([{ code: 'SLUG_MISMATCH', count: 1 }]);
|
|
|
|
// Step 4: Run with skipFailed — bookmark advances, entry gets acked.
|
|
result = await performSync(engine, { repoPath, noPull: true, noEmbed: true, skipFailed: true });
|
|
expect(result.status).toBe('synced');
|
|
const afterSkipCommit = await engine.getConfig('sync.last_commit');
|
|
expect(afterSkipCommit).not.toBe(firstCommit); // bookmark moved past the broken commit
|
|
failures = loadSyncFailures();
|
|
expect(failures.length).toBe(1);
|
|
expect(failures[0].acknowledged).toBe(true);
|
|
expect(typeof failures[0].acknowledged_at).toBe('string');
|
|
|
|
// Step 5: Verify what doctor would render for the historical entry.
|
|
// We call the same primitives doctor's `sync_failures` check uses
|
|
// (src/commands/doctor.ts:252-275) — loadSyncFailures + summarizeFailuresByCode —
|
|
// and assert the rendering string. Directly invoking runDoctor() here is a CLI
|
|
// entrypoint with stdout/exit side effects that would truncate this test mid-flow.
|
|
{
|
|
const all = loadSyncFailures();
|
|
const ackedSummary = summarizeFailuresByCode(all);
|
|
const ackedBreakdown = ackedSummary.map(s => `${s.code}=${s.count}`).join(', ');
|
|
// This is the literal string interpolation doctor.ts:271-274 produces.
|
|
const doctorMessage = `${all.length} historical sync failure(s), all acknowledged [${ackedBreakdown}].`;
|
|
expect(doctorMessage).toContain('SLUG_MISMATCH=1');
|
|
expect(doctorMessage).toContain('1 historical');
|
|
}
|
|
|
|
// Step 6: Add a second broken file — this one with a different failure code
|
|
// (also SLUG_MISMATCH but on a different file) so the JSONL has 2 entries
|
|
// with DIFFERENT paths but the same code. This proves both: per-file dedup
|
|
// honors path identity, and summary aggregation sums across files.
|
|
//
|
|
// We'd ideally test a different code class here, but the sync path uses
|
|
// parseMarkdown WITHOUT {validate:true}, so the markdown.ts validation
|
|
// codes (MISSING_OPEN/CLOSE, NESTED_QUOTES, EMPTY_FRONTMATTER, NULL_BYTES)
|
|
// don't naturally surface — they'd need {validate:true} plumbed in. That
|
|
// plumbing is the v0.22.13+ follow-up. For v0.22.12, two SLUG_MISMATCH
|
|
// entries from different files still proves the dedup + aggregation chain.
|
|
writeFileSync(join(repoPath, 'people/carol.md'), [
|
|
'---', 'type: person', 'title: Carol', 'slug: also-wrong-slug', '---', '', 'Body.',
|
|
].join('\n'));
|
|
execSync('git add -A && git commit -m "add carol with bad slug"', { cwd: repoPath, stdio: 'pipe' });
|
|
|
|
// Step 7: Sync blocks again on the new failure. Old entry stays acked.
|
|
result = await performSync(engine, { repoPath, noPull: true, noEmbed: true });
|
|
expect(result.status).toBe('blocked_by_failures');
|
|
failures = loadSyncFailures();
|
|
expect(failures.length).toBe(2);
|
|
const acked = failures.filter(f => f.acknowledged);
|
|
const unacked = failures.filter(f => !f.acknowledged);
|
|
expect(acked.length).toBe(1);
|
|
expect(acked[0].code).toBe('SLUG_MISMATCH');
|
|
expect(acked[0].path).toContain('bob');
|
|
expect(unacked.length).toBe(1);
|
|
expect(unacked[0].code).toBe('SLUG_MISMATCH');
|
|
expect(unacked[0].path).toContain('carol');
|
|
|
|
// Step 8: Skip again — both entries acked, summary aggregates the count.
|
|
result = await performSync(engine, { repoPath, noPull: true, noEmbed: true, skipFailed: true });
|
|
expect(result.status).toBe('synced');
|
|
failures = loadSyncFailures();
|
|
expect(failures.length).toBe(2);
|
|
expect(failures.every(f => f.acknowledged)).toBe(true);
|
|
|
|
const finalSummary = summarizeFailuresByCode(failures);
|
|
expect(finalSummary).toEqual([{ code: 'SLUG_MISMATCH', count: 2 }]);
|
|
});
|
|
|
|
// issue #1939 CRITICAL REGRESSION: a page whose YAML title parses to a Date
|
|
// (or number) must import cleanly — pre-fix it threw in assessContentSanity
|
|
// and wedged the bookmark. This mirrors the apple-notes repro
|
|
// (sources/apple-notes/.../2024-06-01 8189165238.md).
|
|
test('date/number-titled page imports cleanly; bookmark advances; get returns it', async () => {
|
|
const { performSync } = await import('../../src/commands/sync.ts');
|
|
const { loadSyncFailures } = await import('../../src/core/sync.ts');
|
|
const engine = getEngine();
|
|
|
|
const beforeCommit = await engine.getConfig('sync.last_commit');
|
|
// Bare-date title (→ Date) and bare-number title (→ number).
|
|
writeFileSync(join(repoPath, 'people/datey.md'), [
|
|
'---', 'type: note', 'title: 2024-06-01', '---', '', 'Apple note body.',
|
|
].join('\n'));
|
|
writeFileSync(join(repoPath, 'people/numbery.md'), [
|
|
'---', 'type: note', 'title: 1458', '---', '', 'Another note.',
|
|
].join('\n'));
|
|
execSync('git add -A && git commit -m "add date/number titled notes"', { cwd: repoPath, stdio: 'pipe' });
|
|
|
|
const result = await performSync(engine, { repoPath, noPull: true, noEmbed: true });
|
|
expect(result.status).not.toBe('blocked_by_failures');
|
|
|
|
// Neither file landed as a failure.
|
|
const fails = loadSyncFailures().filter(f => f.path.includes('datey') || f.path.includes('numbery'));
|
|
expect(fails.length).toBe(0);
|
|
|
|
// Bookmark advanced past the broken-but-now-fixed commit.
|
|
const afterCommit = await engine.getConfig('sync.last_commit');
|
|
expect(afterCommit).not.toBe(beforeCommit);
|
|
|
|
// The pages are retrievable, with deterministic coerced titles.
|
|
const datey = await engine.getPage('people/datey');
|
|
expect(datey).not.toBeNull();
|
|
expect(datey!.title).toBe('2024-06-01');
|
|
const numbery = await engine.getPage('people/numbery');
|
|
expect(numbery).not.toBeNull();
|
|
expect(numbery!.title).toBe('1458');
|
|
});
|
|
|
|
// issue #1939 valve: a genuinely un-importable file blocks for (threshold-1)
|
|
// syncs, then auto-skips on the Nth so it can't wedge indexing forever.
|
|
test('bounded auto-skip: poison file blocks then auto-skips, advancing the bookmark', async () => {
|
|
const { performSync } = await import('../../src/commands/sync.ts');
|
|
const { loadSyncFailures } = await import('../../src/core/sync.ts');
|
|
const engine = getEngine();
|
|
const prevThreshold = process.env.GBRAIN_SYNC_AUTOSKIP_AFTER;
|
|
process.env.GBRAIN_SYNC_AUTOSKIP_AFTER = '2';
|
|
try {
|
|
const beforeCommit = await engine.getConfig('sync.last_commit');
|
|
// slug-mismatch = a reliable per-file import failure.
|
|
writeFileSync(join(repoPath, 'people/poison.md'), [
|
|
'---', 'type: person', 'title: Poison', 'slug: not-the-path-slug', '---', '', 'Body.',
|
|
].join('\n'));
|
|
execSync('git add -A && git commit -m "add poison file"', { cwd: repoPath, stdio: 'pipe' });
|
|
|
|
// Attempt 1: blocks (attempts=1 < 2).
|
|
let result = await performSync(engine, { repoPath, noPull: true, noEmbed: true });
|
|
expect(result.status).toBe('blocked_by_failures');
|
|
expect(await engine.getConfig('sync.last_commit')).toBe(beforeCommit);
|
|
let poison = loadSyncFailures().find(f => f.path.includes('poison'))!;
|
|
expect(poison.state).toBe('open');
|
|
expect(poison.attempts).toBe(1);
|
|
|
|
// Attempt 2: attempts hits threshold, fresh==0 → auto-skip + advance.
|
|
result = await performSync(engine, { repoPath, noPull: true, noEmbed: true });
|
|
expect(result.status).toBe('synced');
|
|
expect(await engine.getConfig('sync.last_commit')).not.toBe(beforeCommit);
|
|
poison = loadSyncFailures().find(f => f.path.includes('poison'))!;
|
|
expect(poison.state).toBe('auto_skipped');
|
|
} finally {
|
|
if (prevThreshold === undefined) delete process.env.GBRAIN_SYNC_AUTOSKIP_AFTER;
|
|
else process.env.GBRAIN_SYNC_AUTOSKIP_AFTER = prevThreshold;
|
|
}
|
|
});
|
|
|
|
// issue #1939 adversarial finding #1: a parse-failed file that is later DELETED
|
|
// from the repo must not leave a permanent open ledger row (which would age
|
|
// doctor to FAIL forever). The incremental gate treats removed paths as resolved.
|
|
test('deleting a failed file clears its ledger row (self-heal, no stuck FAIL)', async () => {
|
|
const { performSync } = await import('../../src/commands/sync.ts');
|
|
const { loadSyncFailures } = await import('../../src/core/sync.ts');
|
|
const engine = getEngine();
|
|
|
|
writeFileSync(join(repoPath, 'people/gonepoison.md'), [
|
|
'---', 'type: person', 'title: Gone', 'slug: wrong-derived-slug', '---', '', 'Body.',
|
|
].join('\n'));
|
|
execSync('git add -A && git commit -m "add a file that fails to parse"', { cwd: repoPath, stdio: 'pipe' });
|
|
|
|
// Sync blocks; an open ledger row exists for the bad file.
|
|
let result = await performSync(engine, { repoPath, noPull: true, noEmbed: true });
|
|
expect(result.status).toBe('blocked_by_failures');
|
|
expect(loadSyncFailures().some(f => f.path.includes('gonepoison'))).toBe(true);
|
|
|
|
// Delete the file and sync. The removed path is treated as resolved, so the
|
|
// ledger row is cleared and the bookmark advances.
|
|
execSync('git rm people/gonepoison.md && git commit -m "delete the bad file"', { cwd: repoPath, stdio: 'pipe' });
|
|
result = await performSync(engine, { repoPath, noPull: true, noEmbed: true });
|
|
expect(result.status).not.toBe('blocked_by_failures');
|
|
expect(loadSyncFailures().some(f => f.path.includes('gonepoison'))).toBe(false);
|
|
});
|
|
});
|