fix(sync): stop the sync data-loss family — ops/ prune, DB-only write-through, full-sync gate drift (#2404, #2426, #2607)

Three verified-open defects, one family: sync silently destroying or
diverging on content it should preserve.

#2404 (P0) — 'ops' was hardcoded in PRUNE_DIR_NAMES (a v0.2.0-era
carve-out), so any path with an ops segment was 'pruned-dir': committed
ops/*.md never imported, and modified ops/* files hit the
unsyncableModified delete loop (whose #1433 guard only spared
'metafile'), silently deleting put-created pages like the bundled
daily-task-manager's canonical ops/tasks on every sync. Fix: remove
'ops' from the prune list (ordinary user content; the vendor/generated
entries stay), and harden the delete loop to also skip 'pruned-dir' —
a page under a pruned dir can only exist via a deliberate put_page.

#2426 (P0) — write-through content stayed DB-only and was deleted by
sync --full. All three compounding bugs fixed:
 1. writePageThrough now best-effort commits the artifact (path-limited
    git commit) on durability-hardened repos, so the post-commit hook
    can push it; result carries committed?: boolean.
 2. scripts/brain-commit-push.sh stages+commits BEFORE any pull — the
    old fetch+pull-rebase-first order aborted on any dirty tree, so the
    helper could never commit a MODIFIED page; brain_push's
    rebase-on-reject already handles an advanced remote.
 3. The full-sync delete-reconcile partitions stale pages by git
    history (listEverCommittedPaths): never-committed source_paths are
    DB-only write-through — pages are KEPT and re-exported to the
    working tree instead of soft-deleted. Builds on the #2828
    mass-delete valve (covers the below-valve cases).

#2607 — the sync --full git ls-files fast path bypassed pruneDir, so a
full pass imported (and resurrected soft-deleted) pages under dot-dirs
and vendored trees that incremental sync excludes. Fix:
isCollectibleForWalker applies the same segment-level pruneDir gate as
classifySync, so full and incremental enumeration agree.

One regression test per defect (all verified failing against master
src): test/sync-ops-pages.serial.test.ts,
test/write-through-commit.serial.test.ts, the #2426 helper-order test
in test/brain-durability-hook.serial.test.ts,
test/sync-reconcile-db-only.serial.test.ts,
test/import-git-fastpath-prune.test.ts.

Fixes #2404
Fixes #2426
Fixes #2607

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Sinabina
2026-07-17 14:24:28 -07:00
co-authored by Claude Fable 5
parent 3aeb622dc7
commit bff3066985
17 changed files with 712 additions and 32 deletions
File diff suppressed because one or more lines are too long
+1 -1
View File
@@ -188,7 +188,7 @@ export function walkMarkdownFiles(dir: string): { path: string; relPath: string
// Pre-fix, this walker had only an ad-hoc dot-prefix exclusion and didn't
// call isSyncable at all — so it descended into `node_modules/`, emitted
// markdown files from there, AND ignored the canonical exclusion list
// (`.raw/`, `ops/`, README.md, etc.). Now: pruneDir skips entire vendor
// (`.raw/`, README.md, etc.). Now: pruneDir skips entire vendor
// subtrees before recursion (saving IO), and isSyncable filters the emit
// set against the canonical markdown-strategy rules.
const files: { path: string; relPath: string }[] = [];
+12 -1
View File
@@ -510,10 +510,21 @@ function isCollectibleForWalker(
strategy: SyncStrategy,
multimodalOn: boolean,
): boolean {
// #2607: apply the SAME segment-level prune gate as incremental sync's
// `classifySync` (core/sync.ts). The FS walk below prunes at descent time,
// but the git fast path enumerates via `git ls-files` and historically
// filtered only by extension — so `sync --full` imported (and resurrected
// previously-deleted) pages under dot-dirs / vendored trees that incremental
// sync excludes. Full and incremental must agree on the exclusion set.
// (In the FS-walk route `path` is a basename, so this is the same dot-file
// check pruneDir already applied there — no behavior change on that route.)
const segments = path.split('/');
if (segments.some((seg) => !pruneDir(seg))) return false;
// Metafiles are directory scaffolding (READMEs / index / log / schema /
// resolver), not typed brain pages — same exclusion `sync`'s `isSyncable`
// applies. Guards both the FS-walk and the git-fast-path collection routes.
const basename = path.split('/').pop() || '';
const basename = segments[segments.length - 1] || '';
if ((SYNC_SKIP_FILES as readonly string[]).includes(basename)) return false;
switch (strategy) {
+73 -3
View File
@@ -1897,7 +1897,13 @@ async function performSyncInner(engine: BrainEngine, opts: SyncOpts): Promise<Sy
const pageOpts = opts.sourceId ? { sourceId: opts.sourceId } : undefined;
for (const path of unsyncableModified) {
// v0.41.13 #1433: never delete on metafile classification.
if (unsyncableReason(path, syncOpts) === 'metafile') continue;
// #2404 hardening: same for 'pruned-dir' — a page under a pruned
// directory can only exist via a deliberate put_page (sync never
// imports those paths), so "the file was modified" is not evidence
// the page is stale. Deleting here silently destroyed put-created
// pages every time their materialized file landed in a commit.
const reason = unsyncableReason(path, syncOpts);
if (reason === 'metafile' || reason === 'pruned-dir') continue;
const slug = await resolveSlugByPathOrSourcePath(engine, path, opts.sourceId);
try {
const existing = await engine.getPage(slug, pageOpts);
@@ -3169,9 +3175,45 @@ async function performFullSync(
`GBRAIN_ALLOW_MASS_RECONCILE=1 to restore the old behavior.`,
);
} else if (plan.staleSlugs.length > 0) {
// #2426: a stale page whose source_path was NEVER committed to git is
// DB-only write-through (the file was written into the clone but never
// committed/pushed, then lost — e.g. a fresh clone). "Absent from git"
// is the SYMPTOM of that bug, not evidence the content is disposable.
// Keep those pages and re-export their markdown to the working tree so
// they're file-backed again; only pages whose file once existed in git
// history (i.e. was genuinely deleted) are reconcile-deleted.
const everCommitted = listEverCommittedPaths(repoPath);
const pathBySlug = new Map(rows.map(r => [r.slug, r.source_path]));
let deletableSlugs = plan.staleSlugs;
const dbOnlySlugs: string[] = [];
if (everCommitted) {
deletableSlugs = [];
for (const slug of plan.staleSlugs) {
const sp = pathBySlug.get(slug);
if (sp && !everCommitted.has(sp.replace(/\\/g, '/'))) dbOnlySlugs.push(slug);
else deletableSlugs.push(slug);
}
}
if (dbOnlySlugs.length > 0) {
let reExported = 0;
try {
const { writePageThrough } = await import('../core/write-through.ts');
for (const slug of dbOnlySlugs) {
const r = await writePageThrough(engine, slug, { sourceId: sid });
if (r.written) reExported++;
}
} catch { /* best-effort — pages are preserved either way */ }
serr(
`\n Kept ${dbOnlySlugs.length} page(s) whose markdown was never committed to git ` +
`(DB-only write-through — not deleting).` +
(reExported > 0 ? ` Re-exported ${reExported} of them to the working tree.` : '') +
`\n Commit + push them (e.g. scripts/brain-commit-push.sh, or 'gbrain sources harden') ` +
`so the next sync sees them as file-backed.`,
);
}
const deleteScopedOpts = { sourceId: sid };
for (let i = 0; i < plan.staleSlugs.length; i += DELETE_BATCH_SIZE) {
const batch = plan.staleSlugs.slice(i, i + DELETE_BATCH_SIZE);
for (let i = 0; i < deletableSlugs.length; i += DELETE_BATCH_SIZE) {
const batch = deletableSlugs.slice(i, i + DELETE_BATCH_SIZE);
try {
const deleted = await engine.deletePages(batch, deleteScopedOpts);
reconciledDeletes += deleted.length;
@@ -3290,6 +3332,34 @@ export function planReconcileDeletes(
return { staleSlugs, reconcilableCount: reconcilable.length, massDelete };
}
/**
* #2426: every repo-relative path that ever appeared as an ADD in git history
* (rename detection off, so a `git mv` destination still counts as an add).
* Used by the full-sync reconcile to distinguish "file was committed and later
* deleted" (genuine delete → reconcile) from "file was NEVER committed"
* (DB-only write-through → preserve). Returns null when `repoPath` isn't a git
* work tree or git is unavailable — callers keep the plain-directory behavior.
* Forward-slash-normalized to match `normalizeReconcilePath` membership tests.
*/
export function listEverCommittedPaths(repoPath: string): Set<string> | null {
let stdout: string;
try {
stdout = execFileSync(
'git',
['-C', repoPath, '-c', 'core.quotepath=off', 'log', '--all', '--no-renames',
'--diff-filter=A', '--format=', '--name-only'],
{ encoding: 'utf8', maxBuffer: 512 * 1024 * 1024, stdio: ['ignore', 'pipe', 'ignore'] },
);
} catch {
return null;
}
const set = new Set<string>();
for (const line of stdout.split('\n')) {
if (line) set.add(line.replace(/\\/g, '/'));
}
return set;
}
/**
* #2828 escape hatch: `GBRAIN_ALLOW_MASS_RECONCILE=1` restores the pre-valve
* behavior for the rare intentional bulk removal. Env-only (an incident-time
+46 -3
View File
@@ -183,15 +183,17 @@ if [ "\${1:-}" = "--push-only" ]; then
fi
_msg="\${1:?usage: brain-commit-push.sh <message> <path> [paths...]}"; shift || true
# Pull first so the local tree is current before we stage.
git fetch origin >/dev/null 2>&1 || true
git pull --rebase origin "$_branch" || { git rebase --abort >/dev/null 2>&1 || true; echo "rebase conflict: manual attention needed" >&2; exit 3; }
# EXPLICIT paths only — never a blind 'git add -A' (would risk committing
# secrets, temp files, or unrelated edits).
if [ "$#" -eq 0 ]; then
echo "refusing blind 'git add -A' — pass explicit path(s) to commit" >&2; exit 2
fi
# COMMIT BEFORE PULL (#2426): the old order (fetch + pull --rebase, THEN stage)
# aborted on any dirty tree — 'cannot pull with rebase: You have unstaged
# changes' — so the helper could never commit a MODIFIED page (exactly the
# write-through case). Stage + commit first; brain_push below already handles
# a remote that advanced (push -> rejected -> pull --rebase -> push).
git add -- "$@"
if git diff --cached --quiet; then echo "nothing to commit"; exit 0; fi
git commit -m "$_msg"
@@ -337,6 +339,47 @@ function uninstallLocalHook(repoPath: string): boolean {
return true;
}
/**
* True when the gbrain durability post-commit hook is installed — i.e. the
* user opted this repo into push-durability via `gbrain sources harden`.
* Cheap (one git-config read + one file read); used as the gate for
* write-through auto-commit (#2426).
*/
export function isDurabilityHardened(repoPath: string): boolean {
try {
const { dir } = resolveHooksDir(repoPath);
const hookPath = join(dir, 'post-commit');
return existsSync(hookPath) && readFileSync(hookPath, 'utf-8').includes(HOOK_BANNER);
} catch {
return false;
}
}
/**
* #2426: best-effort commit of a single write-through artifact so DB writes
* reach git (the post-commit hook then background-pushes). Pre-fix,
* write-through `.md` accumulated uncommitted forever: it never reached the
* remote, froze `last_sync_at` (HEAD never moved), and a later `sync --full`
* delete-reconcile treated the never-committed pages as disposable.
*
* Path-limited (`git commit -- <path>`) so unrelated staged/dirty edits are
* never swept into the commit. Never throws; returns false on any failure
* (index.lock contention, nothing changed, detached states) — the DB row and
* the on-disk file remain the durable sinks either way.
*/
export function commitWriteThroughFile(repoPath: string, absPath: string, slug: string): boolean {
try {
const rel = relative(repoPath, absPath);
if (!rel || rel.startsWith('..') || isAbsolute(rel)) return false;
const gitOpts = { stdio: 'ignore', timeout: 30_000, env: { ...process.env, ...GIT_ENV } } as const;
execFileSync('git', ['-C', repoPath, 'add', '--', rel], gitOpts);
execFileSync('git', ['-C', repoPath, 'commit', '-m', `gbrain: write-through ${slug}`, '--', rel], gitOpts);
return true;
} catch {
return false;
}
}
// ── Committed helper ────────────────────────────────────────────────────────
function installHelper(repoPath: string, dryRun: boolean): { status: StepStatus; detail: string } {
+8 -3
View File
@@ -255,7 +255,12 @@ const PRUNE_DIR_NAMES = new Set<string>([
// with the first-sync walker in commands/import.ts.
'venv',
'.raw',
'ops',
// NOTE (#2404): `'ops'` used to be in this list (a v0.2.0-era carve-out for
// one brain layout). Matching the bare segment pruned EVERY user `ops/`
// directory at any depth — sync silently deleted `ops/*` pages and never
// imported `ops/*` files, while the bundled daily-task-manager skill
// prescribes `ops/tasks` as its canonical page. `ops/` is ordinary content;
// do NOT re-add it. Only generated/vendored trees belong here.
]);
/**
@@ -352,8 +357,8 @@ function classifySync(path: string, opts: SyncableOptions = {}): SyncableReason
if (!isAllowedByStrategy(path, strategy)) return 'strategy';
// Skip every path segment that pruneDir would block walkers from descending
// into. Catches hidden dirs (`.git`, `.obsidian`), `.raw/` sidecars,
// `node_modules/` (latent bug fix), and `ops/` at any depth.
// into. Catches hidden dirs (`.git`, `.obsidian`), `.raw/` sidecars, and
// vendor/generated trees (`node_modules/`, `vendor/`, …) at any depth.
const segments = path.split('/');
if (segments.some(p => !pruneDir(p))) return 'pruned-dir';
+22 -1
View File
@@ -27,6 +27,7 @@ import { randomBytes } from 'crypto';
import type { BrainEngine } from './engine.ts';
import { serializePageToMarkdown, resolvePageFilePath } from './markdown.ts';
import { isWriteTargetContained } from './path-confine.ts';
import { isDurabilityHardened, commitWriteThroughFile } from './brain-repo-durability.ts';
/** Minimal logger surface — structurally compatible with operations.ts `Logger`. */
export interface WriteThroughLogger {
@@ -36,6 +37,13 @@ export interface WriteThroughLogger {
export interface WriteThroughResult {
written: boolean;
path?: string;
/**
* True when the write was also committed to git (#2426). Only attempted on
* repos hardened via `gbrain sources harden` (durability hook installed);
* the hook then background-pushes the commit. Best-effort — a false/absent
* value never blocks the write.
*/
committed?: boolean;
/**
* Non-error reasons the file was not written:
* - no_repo_configured: the resolved target (source `local_path` or, for a
@@ -157,7 +165,20 @@ export async function writePageThrough(
throw writeErr;
}
return { written: true, path: filePath };
// #2426: on a durability-hardened repo (user ran `gbrain sources harden`),
// commit the artifact so it reaches git — pre-fix, write-through content
// stayed uncommitted forever: never pushed, `last_sync_at` frozen, and
// silently deleted by a later `sync --full` delete-reconcile. The local
// post-commit hook background-pushes the commit. Best-effort: a commit
// failure never fails the write (the DB row + file are the durable sinks).
let committed = false;
try {
if (isDurabilityHardened(writeRoot)) {
committed = commitWriteThroughFile(writeRoot, filePath, slug);
}
} catch { /* best-effort */ }
return { written: true, path: filePath, ...(committed ? { committed } : {}) };
} catch (e) {
const msg = e instanceof Error ? e.message : String(e);
opts.logger?.warn(`[write-through] failed for ${slug}: ${msg}`);
+30
View File
@@ -90,6 +90,36 @@ describe('brain-commit-push.sh (D13 guarantee)', () => {
} catch (e: any) { code = e.status ?? 1; }
expect(code).toBe(2);
});
test('#2426 — commits a MODIFIED tracked file even when the remote advanced (commit before pull)', () => {
// Pre-fix, the helper ran `git pull --rebase` BEFORE staging, so any dirty
// tree (a modified/enriched page — exactly the write-through case) aborted
// with 'cannot pull with rebase: You have unstaged changes' (exit 3). The
// helper could only ever commit untracked-NEW files, never modifications.
// Remove the post-commit hook so its background push can't race the
// helper's own push (macOS has no flock to serialize them) — this test
// targets the HELPER's ordering; hook behavior is covered below.
rmSync(join(work, '.git', 'hooks', 'post-commit'));
// Advance the remote from a second clone so a pull is genuinely needed.
const other = mkdtempSync(join(root, 'other-'));
execFileSync('git', ['-c', 'protocol.file.allow=always', 'clone', '-q', bare, other], { stdio: 'ignore' });
git(other, 'config', 'user.email', 'o@o.o'); git(other, 'config', 'user.name', 'other');
writeFileSync(join(other, 'remote.md'), 'from other\n');
git(other, 'add', 'remote.md'); git(other, 'commit', '-qm', 'remote change'); git(other, 'push', '-q', 'origin', 'main');
// Dirty MODIFICATION of a tracked file in the hardened clone (write-through shape).
writeFileSync(join(work, 'README.md'), 'modified by write-through\n');
execFileSync('bash', [join(work, 'scripts', 'brain-commit-push.sh'), 'wt: README', 'README.md'], {
cwd: work, stdio: ['ignore', 'pipe', 'pipe'], env: process.env,
});
// Both the remote's commit and ours are on origin/main.
const subjects = git(bare, 'log', '--format=%s', 'main');
expect(subjects).toContain('wt: README');
expect(subjects).toContain('remote change');
// Working tree is clean — the modification was committed, not stranded.
expect(git(work, 'status', '--porcelain', 'README.md')).toBe('');
});
});
describe('post-commit hook (D9 local, D7 self-contained)', () => {
+16 -4
View File
@@ -43,8 +43,10 @@ beforeAll(() => {
writeFileSync(join(root, '.obsidian', 'workspace.json'), '{}');
mkdirSync(join(root, 'people', 'pedro.raw'), { recursive: true });
writeFileSync(join(root, 'people', 'pedro.raw', 'source.md'), '---\ntitle: should not visit\n---\n');
// ops/ is ORDINARY content (#2404) — walker MUST descend (it used to be
// wrongly pruned, silently excluding user runbooks / ops/tasks).
mkdirSync(join(root, 'ops', 'logs'), { recursive: true });
writeFileSync(join(root, 'ops', 'logs', 'run.md'), '# nope\n');
writeFileSync(join(root, 'ops', 'logs', 'run.md'), '---\ntitle: Run\n---\n\nbody\n');
// Nested node_modules — must also be pruned, not just at the root.
mkdirSync(join(root, 'people', 'tools', 'node_modules', 'inner'), { recursive: true });
writeFileSync(join(root, 'people', 'tools', 'node_modules', 'inner', 'a.md'), '---\ntitle: nope\n---\n');
@@ -95,8 +97,11 @@ describe('walkDir (brain-writer.ts) — descent-time pruning', () => {
walkDir(root, (f) => { files.push(f); }, (dir) => visited.push(dir));
expect(visited.some(d => d.endsWith('/people'))).toBe(true);
expect(visited.some(d => d.endsWith('/concepts/subdir'))).toBe(true);
// ops/ is ordinary content — descended, not pruned (#2404).
expect(visited.some(d => d.endsWith('/ops/logs'))).toBe(true);
expect(files.some(f => f.endsWith('/people/alice.md'))).toBe(true);
expect(files.some(f => f.endsWith('/concepts/subdir/thing.md'))).toBe(true);
expect(files.some(f => f.endsWith('/ops/logs/run.md'))).toBe(true);
// And explicitly does NOT visit the file under node_modules.
expect(files.some(f => f.includes('/node_modules/'))).toBe(false);
});
@@ -107,7 +112,7 @@ describe('walkDir (brain-writer.ts) — descent-time pruning', () => {
// visitDir would be called with node_modules paths.
const descents: string[] = [];
walkDir(root, () => {}, (d) => descents.push(d));
const vendor = descents.filter(d => /\/(node_modules|\.git|\.obsidian|ops)(\/|$)/.test(d) || /\.raw$/.test(d));
const vendor = descents.filter(d => /\/(node_modules|\.git|\.obsidian)(\/|$)/.test(d) || /\.raw$/.test(d));
expect(vendor).toEqual([]);
});
});
@@ -119,13 +124,20 @@ describe('collectFiles (frontmatter.ts) — descent-time pruning parity', () =>
expect(visited.some(d => d.includes('/node_modules'))).toBe(false);
});
test('does NOT descend into .git, .obsidian, *.raw, or ops', () => {
test('does NOT descend into .git, .obsidian, or *.raw', () => {
const visited: string[] = [];
collectFiles(root, (dir) => visited.push(dir));
expect(visited.some(d => d.includes('/.git'))).toBe(false);
expect(visited.some(d => d.includes('/.obsidian'))).toBe(false);
expect(visited.some(d => d.endsWith('.raw'))).toBe(false);
expect(visited.some(d => d.endsWith('/ops') || d.includes('/ops/'))).toBe(false);
});
test('DOES descend into ops/ — ordinary content, not a vendor tree (#2404)', () => {
const visited: string[] = [];
collectFiles(root, (dir) => visited.push(dir));
expect(visited.some(d => d.endsWith('/ops') || d.includes('/ops/'))).toBe(true);
const files = collectFiles(root);
expect(files.some(f => f.endsWith('/ops/logs/run.md'))).toBe(true);
});
test('does NOT descend into git submodule directories', () => {
+3 -2
View File
@@ -224,7 +224,7 @@ describeE2E('E2E: Git-to-DB Sync Pipeline', () => {
expect(bob).toBeNull();
});
test('sync skips non-syncable files (README, hidden, .raw)', async () => {
test('sync skips non-syncable files (README, hidden, .raw) but imports ops/ (#2404)', async () => {
const { performSync } = await import('../../src/commands/sync.ts');
const engine = getEngine();
@@ -249,8 +249,9 @@ describeE2E('E2E: Git-to-DB Sync Pipeline', () => {
const raw = await engine.getPage('.raw/data');
expect(raw).toBeNull();
// ops/ is ordinary content and DOES sync (#2404).
const ops = await engine.getPage('ops/deploy');
expect(ops).toBeNull();
expect(ops).not.toBeNull();
});
test('sync stores last_commit and last_run in config', async () => {
+90
View File
@@ -0,0 +1,90 @@
/**
* #2607 — the `sync --full` git fast path applies the same prune gate as
* incremental sync.
*
* Bug class: `collectSyncableFiles` on a git work tree takes the
* `git ls-files` fast path, which historically filtered ONLY by
* strategy/extension + .gitignore — no `pruneDir`, so `sync --full`
* imported (and resurrected previously-soft-deleted) pages under dot-dirs
* and vendored trees that incremental sync's `isSyncable` excludes. The two
* enumeration modes cycled content in and out depending on which ran last.
*
* Fix: `isCollectibleForWalker` (shared by the git fast path AND the FS-walk
* emit filter) now rejects any path with a segment `pruneDir` would block —
* the same segment rule `classifySync` applies on the incremental path.
*
* No PGLite needed: `collectSyncableFiles` is pure filesystem + git.
*/
import { describe, test, expect, beforeAll, afterAll } from 'bun:test';
import { mkdtempSync, writeFileSync, rmSync, mkdirSync } from 'fs';
import { execSync } from 'child_process';
import { tmpdir } from 'os';
import { join, relative } from 'path';
import { collectSyncableFiles } from '../src/commands/import.ts';
import { isSyncable } from '../src/core/sync.ts';
let repo: string;
function rel(files: string[]): string[] {
return files.map((f) => relative(repo, f));
}
beforeAll(() => {
repo = mkdtempSync(join(tmpdir(), 'gbrain-fastpath-'));
execSync('git init', { cwd: repo, stdio: 'pipe' });
execSync('git config user.email "t@t.t"', { cwd: repo, stdio: 'pipe' });
execSync('git config user.name "T"', { cwd: repo, stdio: 'pipe' });
// Ordinary content — must be collected.
mkdirSync(join(repo, 'notes'), { recursive: true });
writeFileSync(join(repo, 'notes/real.md'), '---\ntitle: Real\n---\nbody\n');
mkdirSync(join(repo, 'ops'), { recursive: true });
writeFileSync(join(repo, 'ops/tasks.md'), '---\ntitle: Tasks\n---\nbody\n');
// TRACKED files under excluded trees — `git ls-files` returns these, so
// only the prune gate keeps them out (this is the #2607 divergence).
mkdirSync(join(repo, '.obsidian'), { recursive: true });
writeFileSync(join(repo, '.obsidian/plugin-notes.md'), 'not a page\n');
mkdirSync(join(repo, 'vendor/pkg'), { recursive: true });
writeFileSync(join(repo, 'vendor/pkg/notes.md'), 'vendored\n');
mkdirSync(join(repo, 'node_modules/dep'), { recursive: true });
writeFileSync(join(repo, 'node_modules/dep/CHANGELOG.md'), 'dep changelog\n');
mkdirSync(join(repo, 'people/pedro.raw'), { recursive: true });
writeFileSync(join(repo, 'people/pedro.raw/source.md'), 'raw sidecar\n');
// Metafiles — excluded on both routes (pre-existing #345 behavior).
writeFileSync(join(repo, 'README.md'), '# repo\n');
writeFileSync(join(repo, 'notes/index.md'), '# index\n');
execSync('git add -A -f && git commit -m "fixture"', { cwd: repo, stdio: 'pipe' });
});
afterAll(() => {
if (repo) rmSync(repo, { recursive: true, force: true });
});
describe('#2607 — git fast path excludes what incremental sync excludes', () => {
test('tracked files under pruned dirs are NOT collected', () => {
const files = rel(collectSyncableFiles(repo, { strategy: 'markdown' }));
expect(files).toContain('notes/real.md');
expect(files).toContain('ops/tasks.md'); // ordinary content (#2404)
expect(files).not.toContain('.obsidian/plugin-notes.md');
expect(files).not.toContain('vendor/pkg/notes.md');
expect(files).not.toContain('node_modules/dep/CHANGELOG.md');
expect(files).not.toContain('people/pedro.raw/source.md');
// Metafiles stay excluded too.
expect(files).not.toContain('README.md');
expect(files).not.toContain('notes/index.md');
});
test('full-sync enumeration agrees with incremental isSyncable for every collected file', () => {
// The single-source-of-truth contract: nothing the full path collects may
// be something the incremental path would refuse to sync.
const files = rel(collectSyncableFiles(repo, { strategy: 'markdown' }));
for (const f of files) {
expect({ path: f, syncable: isSyncable(f) }).toEqual({ path: f, syncable: true });
}
expect(files.length).toBeGreaterThan(0);
});
});
+2 -1
View File
@@ -28,7 +28,8 @@ describe('#1433 — isSyncable / unsyncableReason are duals of one classifier',
{ path: 'RESOLVER.md', expected: 'metafile', note: 'top-level master routing config (closes #345)' },
{ path: 'brain/RESOLVER.md', expected: 'metafile', note: 'RESOLVER.md anywhere is metafile (closes #345)' },
{ path: 'people/alice.txt', expected: 'strategy', note: '.txt rejected by markdown strategy' },
{ path: 'ops/scratch/note.md', expected: 'pruned-dir', note: 'ops/ is pruned' },
{ path: 'ops/scratch/note.md', expected: null, note: 'ops/ is ordinary content, not pruned (#2404)' },
{ path: 'vendor/pkg/note.md', expected: 'pruned-dir', note: 'vendor/ is pruned' },
{ path: '.git/notes.md', expected: 'pruned-dir', note: 'hidden dir pruned' },
{ path: 'node_modules/foo/README.md', expected: 'pruned-dir', note: 'node_modules pruned' },
];
+129
View File
@@ -0,0 +1,129 @@
/**
* #2404 — `ops/` is ordinary content: sync imports `ops/*` files and never
* deletes `ops/*` pages.
*
* Bug class: `'ops'` was hardcoded in PRUNE_DIR_NAMES (a v0.2.0-era carve-out),
* so `classifySync` treated ANY path with an `ops` segment as 'pruned-dir':
* - committed `ops/*.md` files were never imported (even by `sync --full`);
* - a modified `ops/*` file fell into the unsyncableModified delete loop,
* whose #1433 guard only skipped 'metafile' — so put-created `ops/*` pages
* (e.g. the bundled daily-task-manager's canonical `ops/tasks`) were
* silently deleted on every sync.
*
* Fix: remove `'ops'` from PRUNE_DIR_NAMES, and harden the delete loop to also
* skip 'pruned-dir' classifications (a page under a genuinely-pruned dir can
* only exist via a deliberate put_page — never delete it on a file edit).
*
* Modeled on test/sync-metafile-skip.serial.test.ts (the #1433 iron rule).
*/
import { describe, test, expect, beforeAll, afterAll, beforeEach, afterEach } from 'bun:test';
import { mkdtempSync, writeFileSync, rmSync, mkdirSync } from 'fs';
import { execSync } from 'child_process';
import { tmpdir } from 'os';
import { join } from 'path';
import { PGLiteEngine } from '../src/core/pglite-engine.ts';
import { resetPgliteState } from './helpers/reset-pglite.ts';
let engine: PGLiteEngine;
let repoPath: string;
function gitInit(repo: string): void {
execSync('git init', { cwd: repo, stdio: 'pipe' });
execSync('git config user.email "test@test.com"', { cwd: repo, stdio: 'pipe' });
execSync('git config user.name "Test"', { cwd: repo, stdio: 'pipe' });
}
describe('#2404 — ops/ pages sync like any other content', () => {
beforeAll(async () => {
engine = new PGLiteEngine();
await engine.connect({});
await engine.initSchema();
}, 60_000);
afterAll(async () => {
if (engine) await engine.disconnect();
}, 60_000);
beforeEach(async () => {
await resetPgliteState(engine);
repoPath = mkdtempSync(join(tmpdir(), 'gbrain-ops-'));
gitInit(repoPath);
mkdirSync(join(repoPath, 'topics'), { recursive: true });
writeFileSync(join(repoPath, 'topics/foo.md'), [
'---', 'type: concept', 'title: Foo', '---', '', 'Baseline content.',
].join('\n'));
execSync('git add -A && git commit -m "initial"', { cwd: repoPath, stdio: 'pipe' });
});
afterEach(() => {
if (repoPath) rmSync(repoPath, { recursive: true, force: true });
});
test('a committed ops/*.md file is imported by sync', async () => {
const { performSync } = await import('../src/commands/sync.ts');
mkdirSync(join(repoPath, 'ops'), { recursive: true });
writeFileSync(join(repoPath, 'ops/tasks.md'), [
'---', 'type: concept', 'title: Tasks', '---', '', 'Open tasks live here.',
].join('\n'));
execSync('git add -A && git commit -m "add ops/tasks"', { cwd: repoPath, stdio: 'pipe' });
const result = await performSync(engine, { repoPath, full: true, noPull: true, noEmbed: true });
expect(['first_sync', 'synced']).toContain(result.status);
// Pre-fix: ops/* was 'pruned-dir' → imported=0 for it, even on --full.
const page = await engine.getPage('ops/tasks');
expect(page).not.toBeNull();
expect(page?.compiled_truth).toContain('Open tasks');
}, 60_000);
test('an edited ops/*.md updates its page instead of deleting it', async () => {
const { performSync } = await import('../src/commands/sync.ts');
mkdirSync(join(repoPath, 'ops'), { recursive: true });
writeFileSync(join(repoPath, 'ops/tasks.md'), [
'---', 'type: concept', 'title: Tasks', '---', '', 'v1',
].join('\n'));
execSync('git add -A && git commit -m "add ops/tasks"', { cwd: repoPath, stdio: 'pipe' });
await performSync(engine, { repoPath, full: true, noPull: true, noEmbed: true });
writeFileSync(join(repoPath, 'ops/tasks.md'), [
'---', 'type: concept', 'title: Tasks', '---', '', 'v2 with a new task',
].join('\n'));
execSync('git add -A && git commit -m "edit ops/tasks"', { cwd: repoPath, stdio: 'pipe' });
// Pre-fix: this incremental sync hit the unsyncableModified delete loop
// ("Deleted un-syncable page: ops/tasks" — the autopilot kill-loop).
const second = await performSync(engine, { repoPath, noPull: true, noEmbed: true });
expect(['synced', 'up_to_date', 'first_sync']).toContain(second.status);
const page = await engine.getPage('ops/tasks');
expect(page).not.toBeNull();
expect(page?.compiled_truth).toContain('v2');
}, 60_000);
test('hardening: a put-created page under a STILL-pruned dir survives a file edit (pruned-dir delete guard)', async () => {
const { performSync } = await import('../src/commands/sync.ts');
// node_modules stays in PRUNE_DIR_NAMES. Commit a file there, then seed a
// same-path page via putPage — the deliberate-put precondition.
mkdirSync(join(repoPath, 'node_modules/pkg'), { recursive: true });
writeFileSync(join(repoPath, 'node_modules/pkg/notes.md'), 'v1\n');
execSync('git add -A -f && git commit -m "vendor file"', { cwd: repoPath, stdio: 'pipe' });
await performSync(engine, { repoPath, full: true, noPull: true, noEmbed: true });
await engine.putPage('node_modules/pkg/notes', {
type: 'concept',
title: 'Deliberate put page',
compiled_truth: 'Created via put_page; must survive sync.',
timeline: '',
frontmatter: { type: 'concept' },
});
writeFileSync(join(repoPath, 'node_modules/pkg/notes.md'), 'v2\n');
execSync('git add -A -f && git commit -m "edit vendor file"', { cwd: repoPath, stdio: 'pipe' });
await performSync(engine, { repoPath, noPull: true, noEmbed: true });
// Pre-fix: reason 'pruned-dir' was not guarded → page deleted.
const survivor = await engine.getPage('node_modules/pkg/notes');
expect(survivor).not.toBeNull();
}, 60_000);
});
+142
View File
@@ -0,0 +1,142 @@
/**
* #2426 (bug 3) — `sync --full` delete-reconcile preserves DB-only pages.
*
* Bug class: the full-sync reconcile soft-deleted ANY file-backed page whose
* `source_path` was absent from the working tree — including pages whose
* markdown was NEVER committed to git (write-through that never made it to
* the remote, then a fresh clone). "Absent from git" is the SYMPTOM of the
* missing write-through commit, not evidence the content is disposable; one
* production pass soft-deleted thousands of genuine pages this way.
*
* Fix: the reconcile partitions stale pages by git history — a path that ever
* appeared as an ADD was genuinely deleted (reconcile as before); a path with
* NO history is DB-only write-through: keep the page and re-export its
* markdown to the working tree so it's file-backed again.
*
* Builds on the #2828 mass-delete valve (this guard covers the below-valve
* cases the ratio check can't see).
*/
import { describe, test, expect, beforeAll, afterAll, beforeEach, afterEach } from 'bun:test';
import { mkdtempSync, writeFileSync, rmSync, mkdirSync, existsSync } from 'fs';
import { execSync } from 'child_process';
import { tmpdir } from 'os';
import { join } from 'path';
import { PGLiteEngine } from '../src/core/pglite-engine.ts';
import { resetPgliteState } from './helpers/reset-pglite.ts';
import { listEverCommittedPaths } from '../src/commands/sync.ts';
let engine: PGLiteEngine;
let repoPath: string;
function gitInit(repo: string): void {
execSync('git init', { cwd: repo, stdio: 'pipe' });
execSync('git config user.email "t@t.t"', { cwd: repo, stdio: 'pipe' });
execSync('git config user.name "T"', { cwd: repo, stdio: 'pipe' });
}
describe('listEverCommittedPaths (#2426)', () => {
test('returns every path ever added, including later-deleted ones; null for non-git dirs', () => {
const repo = mkdtempSync(join(tmpdir(), 'gbrain-ecp-'));
try {
gitInit(repo);
writeFileSync(join(repo, 'kept.md'), 'kept\n');
writeFileSync(join(repo, 'gone.md'), 'gone\n');
execSync('git add -A && git commit -m add', { cwd: repo, stdio: 'pipe' });
execSync('git rm -q gone.md && git commit -m rm', { cwd: repo, stdio: 'pipe' });
const set = listEverCommittedPaths(repo);
expect(set).not.toBeNull();
expect(set!.has('kept.md')).toBe(true);
expect(set!.has('gone.md')).toBe(true); // deleted, but WAS committed
expect(set!.has('never-committed.md')).toBe(false);
const plain = mkdtempSync(join(tmpdir(), 'gbrain-ecp-plain-'));
try {
expect(listEverCommittedPaths(plain)).toBeNull();
} finally {
rmSync(plain, { recursive: true, force: true });
}
} finally {
rmSync(repo, { recursive: true, force: true });
}
});
});
describe('#2426 — full-sync reconcile keeps never-committed (DB-only) pages', () => {
beforeAll(async () => {
engine = new PGLiteEngine();
await engine.connect({});
await engine.initSchema();
}, 60_000);
afterAll(async () => {
if (engine) await engine.disconnect();
}, 60_000);
beforeEach(async () => {
await resetPgliteState(engine);
repoPath = mkdtempSync(join(tmpdir(), 'gbrain-dbonly-'));
gitInit(repoPath);
mkdirSync(join(repoPath, 'topics'), { recursive: true });
writeFileSync(join(repoPath, 'topics/keep.md'), [
'---', 'type: concept', 'title: Keep', '---', '', 'still here',
].join('\n'));
writeFileSync(join(repoPath, 'topics/gone.md'), [
'---', 'type: concept', 'title: Gone', '---', '', 'will be git-rm-ed',
].join('\n'));
execSync('git add -A && git commit -m initial', { cwd: repoPath, stdio: 'pipe' });
});
afterEach(() => {
if (repoPath) rmSync(repoPath, { recursive: true, force: true });
});
test('genuinely-deleted pages reconcile; never-committed pages are kept and re-exported', async () => {
const { performSync } = await import('../src/commands/sync.ts');
// Full sync #1: both file-backed pages land.
const first = await performSync(engine, {
repoPath, full: true, sourceId: 'default', noPull: true, noEmbed: true,
});
expect(['first_sync', 'synced']).toContain(first.status);
expect(await engine.getPage('topics/keep')).not.toBeNull();
expect(await engine.getPage('topics/gone')).not.toBeNull();
// A DB-only write-through casualty: the page row exists with a
// source_path, but its file was never committed and is absent from the
// clone (e.g. write-through was never pushed, then the repo was re-cloned).
await engine.putPage('memories/lost', {
type: 'concept',
title: 'Lost write-through',
compiled_truth: 'Years of content that must not be reconciled away.',
timeline: '',
frontmatter: { type: 'concept' },
});
await engine.executeRaw(
`UPDATE pages SET source_path = $1 WHERE slug = $2 AND source_id = $3`,
['memories/lost.md', 'memories/lost', 'default'],
);
// A genuine deletion: topics/gone.md removed via git.
execSync('git rm -q topics/gone.md && git commit -m "rm gone"', { cwd: repoPath, stdio: 'pipe' });
await engine.setConfig('sync.repo_path', repoPath);
// Full sync #2 runs the delete-reconcile.
const second = await performSync(engine, {
repoPath, full: true, sourceId: 'default', noPull: true, noEmbed: true,
});
expect(['first_sync', 'synced']).toContain(second.status);
// The genuinely-deleted page is reconciled away…
expect(await engine.getPage('topics/gone')).toBeNull();
// …the still-present page survives…
expect(await engine.getPage('topics/keep')).not.toBeNull();
// …and the DB-only page is PRESERVED (pre-fix: soft-deleted here)…
const lost = await engine.getPage('memories/lost');
expect(lost).not.toBeNull();
expect(lost?.compiled_truth).toContain('must not be reconciled');
// …and re-exported to the working tree so it is file-backed again.
expect(existsSync(join(repoPath, 'memories/lost.md'))).toBe(true);
}, 120_000);
});
+4 -2
View File
@@ -56,8 +56,10 @@ describe('isSyncable with strategy', () => {
expect(isSyncable('.git/config.js', { strategy: 'code' })).toBe(false);
// README.md is skipped under markdown
expect(isSyncable('README.md', { strategy: 'markdown' })).toBe(false);
// ops/ directory always skipped
expect(isSyncable('ops/migrate.py', { strategy: 'code' })).toBe(false);
// ops/ is ordinary content — NOT skipped (#2404)
expect(isSyncable('ops/migrate.py', { strategy: 'code' })).toBe(true);
// vendored trees always skipped
expect(isSyncable('vendor/pkg/migrate.py', { strategy: 'code' })).toBe(false);
// .raw/ sidecar always skipped
expect(isSyncable('dir/.raw/code.ts', { strategy: 'code' })).toBe(false);
});
+13 -5
View File
@@ -95,9 +95,10 @@ describe('isSyncable', () => {
expect(isSyncable('people/README.md')).toBe(false);
});
test('rejects ops/ directory', () => {
expect(isSyncable('ops/deploy-log.md')).toBe(false);
expect(isSyncable('ops/config.md')).toBe(false);
test('accepts ops/ — ordinary content directory, not pruned (#2404)', () => {
expect(isSyncable('ops/deploy-log.md')).toBe(true);
expect(isSyncable('ops/config.md')).toBe(true);
expect(isSyncable('ops/tasks.md')).toBe(true);
});
// ────────────────────────────────────────────────────────────────
@@ -128,8 +129,15 @@ describe('pruneDir', () => {
expect(pruneDir('.vscode')).toBe(false);
});
test('blocks ops (gbrain operational dir)', () => {
expect(pruneDir('ops')).toBe(false);
test('allows ops — ordinary content dir, not a vendor tree (#2404)', () => {
expect(pruneDir('ops')).toBe(true);
});
test('blocks vendored / generated trees', () => {
expect(pruneDir('vendor')).toBe(false);
expect(pruneDir('dist')).toBe(false);
expect(pruneDir('build')).toBe(false);
expect(pruneDir('venv')).toBe(false);
});
test('blocks *.raw sidecar dirs (gbrain convention)', () => {
+115
View File
@@ -0,0 +1,115 @@
/**
* #2426 (bug 1) — write-through reaches git on durability-hardened repos.
*
* Bug class: `put_page` / capture / enrichment wrote `.md` into
* `sync.repo_path` but NOTHING ever committed it. The post-commit hook only
* fires after a commit — and write-through never made one — so write-through
* content accumulated uncommitted forever: never pushed, `last_sync_at`
* frozen (HEAD never moved), and silently deleted by a later `sync --full`
* delete-reconcile.
*
* Fix: `writePageThrough` best-effort commits the artifact (path-limited)
* when the repo carries the gbrain durability post-commit hook (i.e. the
* user opted in via `gbrain sources harden`); the hook then background-pushes.
* Unhardened repos keep the old write-only behavior.
*/
import { describe, test, expect, beforeAll, afterAll, beforeEach, afterEach } from 'bun:test';
import { mkdtempSync, writeFileSync, rmSync, mkdirSync, chmodSync } from 'fs';
import { execSync, execFileSync } from 'child_process';
import { tmpdir } from 'os';
import { join } from 'path';
import { PGLiteEngine } from '../src/core/pglite-engine.ts';
import { resetPgliteState } from './helpers/reset-pglite.ts';
import { writePageThrough } from '../src/core/write-through.ts';
let engine: PGLiteEngine;
let repo: string;
function git(cwd: string, ...args: string[]): string {
return execFileSync('git', ['-C', cwd, ...args], {
stdio: ['ignore', 'pipe', 'pipe'], encoding: 'utf-8',
}).trim();
}
/** Install a hook file carrying the gbrain durability banner (the detection
* key `isDurabilityHardened` looks for) with a no-op body so tests never
* attempt a real push. */
function installFakeDurabilityHook(repoPath: string): void {
const hooksDir = join(repoPath, '.git', 'hooks');
mkdirSync(hooksDir, { recursive: true });
const hookPath = join(hooksDir, 'post-commit');
writeFileSync(hookPath, [
'#!/usr/bin/env bash',
'# gbrain brain-durability post-commit hook (v0.42.44+)',
'exit 0',
'',
].join('\n'));
chmodSync(hookPath, 0o755);
}
async function seedPage(slug: string): Promise<void> {
await engine.putPage(slug, {
type: 'concept',
title: 'Write-through page',
compiled_truth: 'Content that must reach git.',
timeline: '',
frontmatter: { type: 'concept' },
});
}
describe('#2426 — writePageThrough auto-commit', () => {
beforeAll(async () => {
engine = new PGLiteEngine();
await engine.connect({});
await engine.initSchema();
}, 60_000);
afterAll(async () => {
if (engine) await engine.disconnect();
}, 60_000);
beforeEach(async () => {
await resetPgliteState(engine);
repo = mkdtempSync(join(tmpdir(), 'gbrain-wt-'));
execSync('git init', { cwd: repo, stdio: 'pipe' });
execSync('git config user.email "t@t.t"', { cwd: repo, stdio: 'pipe' });
execSync('git config user.name "T"', { cwd: repo, stdio: 'pipe' });
writeFileSync(join(repo, 'seed.md'), 'seed\n');
execSync('git add -A && git commit -m init', { cwd: repo, stdio: 'pipe' });
await engine.setConfig('sync.repo_path', repo);
});
afterEach(() => {
if (repo) rmSync(repo, { recursive: true, force: true });
});
test('on a hardened repo, the write-through artifact is committed (path-limited)', async () => {
installFakeDurabilityHook(repo);
// Unrelated dirty edit — must NOT be swept into the write-through commit.
writeFileSync(join(repo, 'seed.md'), 'dirty unrelated edit\n');
await seedPage('notes/hello');
const result = await writePageThrough(engine, 'notes/hello');
expect(result.written).toBe(true);
expect(result.committed).toBe(true);
// The artifact is committed…
expect(git(repo, 'log', '-1', '--format=%s')).toBe('gbrain: write-through notes/hello');
expect(git(repo, 'log', '-1', '--name-only', '--format=')).toBe('notes/hello.md');
expect(git(repo, 'status', '--porcelain', 'notes/hello.md')).toBe('');
// …and the unrelated edit stays uncommitted (explicit-path discipline).
expect(git(repo, 'status', '--porcelain', 'seed.md')).not.toBe('');
}, 60_000);
test('on an unhardened repo, the file is written but NOT committed (no behavior change)', async () => {
await seedPage('notes/plain');
const result = await writePageThrough(engine, 'notes/plain');
expect(result.written).toBe(true);
expect(result.committed).toBeUndefined();
// Untracked, uncommitted — the pre-existing contract.
expect(git(repo, 'status', '--porcelain', 'notes/plain.md')).toContain('?? notes/plain.md');
expect(git(repo, 'log', '-1', '--format=%s')).toBe('init');
}, 60_000);
});