feat(sync): --src-subpath + --exclude for monorepo subdir-source support (#753)

Rebased port of PR #774 onto current master. A single git repo can hold N
logical sources at subdirectories: git operations run at the discovered
repo root (git rev-parse --show-toplevel — worktrees and submodules
resolve natively) while file walking, imports, deletes and renames are
scoped to the subpath. Passing the subdirectory directly as the repo path
(auto-discovery) works through the same code path.

Path-containment guards (the point of the feature):
- NAV-1/NAV-2: the realpath-resolved scope must live inside the
  realpath-resolved git root — ../-traversal and symlinked subdirs
  pointing outside the repo are rejected before any git op.
- NAV-1 TOCTOU: per-file realpath re-validation during the incremental
  import drain and rename reimport; symlink-escape files are recorded as
  failures (fail-closed — the bookmark cannot advance past an escape).
- NAV-4: an --exclude set that filters out every candidate warns loudly.

Scoped syncs use git-root-relative slugs + source_path in BOTH the full
and incremental paths (runImport gains slugRoot), fixing the original
PR's full/incremental slug divergence in the auto-discovery flow.
--exclude matches scope-relative paths in both paths; exclusion never
deletes previously-imported pages. The full-sync delete reconcile is
scope-restricted and relativizes against the slug base so a healthy
scoped source can't trip the #2828 mass-delete valve. .gitignore
management resolves to the git root at every call site.

Preserves all master-side sync work since the original branch: the #2828
mass-delete safety valve, #1794 resumable checkpoints + pinned targets,
#1950 stall watchdog, #2335 heartbeat bump, #1970 bookmark reachability,
and the git-ls-files walker (#2315/#2462/#2678, whose symlink/cycle
hardening is untouched).

Co-authored-by: Jeremy Knows <jeremy@veefriends.com>
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Garry Tan
2026-07-17 14:46:11 -07:00
co-authored by Jeremy Knows Claude Fable 5
parent e1cefd0654
commit 14d20048c3
5 changed files with 678 additions and 62 deletions
File diff suppressed because one or more lines are too long
+50 -7
View File
@@ -11,6 +11,7 @@ import {
isCodeFilePath,
isMarkdownFilePath,
isImageFilePath as isImageFilePathFromSync,
matchesAnyGlob,
pruneDir,
SYNC_SKIP_FILES,
type SyncStrategy,
@@ -47,7 +48,25 @@ export interface RunImportResult {
export async function runImport(
engine: BrainEngine,
args: string[],
opts: { commit?: string; strategy?: SyncStrategy; sourceId?: string; managedBookmark?: boolean } = {},
opts: {
commit?: string;
strategy?: SyncStrategy;
sourceId?: string;
managedBookmark?: boolean;
/**
* #753/#774: glob patterns to exclude from the import (same semantics as
* `isSyncable`'s `exclude` — matched against the dir-relative path).
* Threaded by performFullSync for `gbrain sync --exclude`.
*/
exclude?: string[];
/**
* #753/#774 monorepo subdir-source support: when set, slugs and
* `source_path` are computed relative to this root (the git repo root)
* instead of `dir` (the sync scope), so `wiki/page1.md` lands as slug
* `wiki/page1` consistently across full and incremental sync.
*/
slugRoot?: string;
} = {},
): Promise<RunImportResult> {
const noEmbed = args.includes('--no-embed');
const fresh = args.includes('--fresh');
@@ -190,13 +209,30 @@ export async function runImport(
const strategy: SyncStrategy = opts.strategy ?? 'markdown';
const _walkT0 = Date.now();
console.error(`[gbrain phase] import.collect_files start dir=${dir} strategy=${strategy}`);
const allFiles = collectSyncableFiles(dir, { strategy });
let allFiles = collectSyncableFiles(dir, { strategy });
console.error(
`[gbrain phase] import.collect_files done ${Date.now() - _walkT0}ms files=${allFiles.length}`,
);
const fileTypeLabel = strategy === 'code' ? 'code'
: strategy === 'auto' ? 'syncable' : 'markdown';
console.log(`Found ${allFiles.length} ${fileTypeLabel} files`);
// #753/#774: apply --exclude glob patterns (threaded by performFullSync).
if (opts.exclude && opts.exclude.length > 0) {
const beforeExclude = allFiles.length;
allFiles = allFiles.filter(abs => !matchesAnyGlob(relative(dir, abs), opts.exclude));
console.log(
`Found ${allFiles.length} ${fileTypeLabel} files ` +
`(${beforeExclude - allFiles.length} excluded by --exclude patterns)`,
);
// NAV-4: everything excluded is almost always a mistyped pattern — warn.
if (beforeExclude > 0 && allFiles.length === 0) {
console.warn(
`[gbrain sync] No files matched after applying ${opts.exclude.length} --exclude pattern(s). ` +
`Check your --exclude flags. Patterns: ${JSON.stringify(opts.exclude)}`,
);
}
} else {
console.log(`Found ${allFiles.length} ${fileTypeLabel} files`);
}
// Sort newest-first so date-prefixed brain paths get embedded before older ones.
// See src/core/sort-newest-first.ts for the policy.
@@ -242,6 +278,11 @@ export async function runImport(
async function processFile(eng: BrainEngine, filePath: string) {
const relativePath = relative(dir, filePath);
// #753/#774: slug + source_path base. When performFullSync syncs a
// monorepo subdir, slugRoot is the git root so slugs stay git-root-
// relative (matching the incremental path's git-diff paths). The
// checkpoint (`completed`) stays dir-relative — resumeFilter's contract.
const importRelPath = opts.slugRoot ? relative(opts.slugRoot, filePath) : relativePath;
// v0.31.2 (D5): per-file slow-path log. Fires only when a single
// file takes >5s. The user's hang surfaces as one file taking
// forever — without this, the agent can't see which file.
@@ -252,8 +293,8 @@ export async function runImport(
// up images when GBRAIN_EMBEDDING_MULTIMODAL=true so this branch is
// unreachable when the gate is off; defense-in-depth check anyway.
const result = isImageFilePath(relativePath) && process.env.GBRAIN_EMBEDDING_MULTIMODAL === 'true'
? await importImageFile(eng, filePath, relativePath, { noEmbed, sourceId })
: await importFile(eng, filePath, relativePath, { noEmbed, sourceId, activePack: importActivePack });
? await importImageFile(eng, filePath, importRelPath, { noEmbed, sourceId })
: await importFile(eng, filePath, importRelPath, { noEmbed, sourceId, activePack: importActivePack });
const _fileMs = Date.now() - _fileT0;
if (_fileMs > 5000) {
console.error(`[gbrain phase] import.process_file slow ${_fileMs}ms ${relativePath}`);
@@ -269,7 +310,9 @@ export async function runImport(
if (result.error && result.error !== 'unchanged') {
console.error(` Skipped ${relativePath}: ${result.error}`);
// Bug 9 — non-"unchanged" skips carry a real error reason.
failures.push({ path: relativePath, error: result.error });
// #774: ledger paths use the slug base so an incremental sync's
// success at the same (git-root-relative) path clears the row.
failures.push({ path: importRelPath, error: result.error });
} else {
// 'unchanged' or no-error skip: content_hash matched a prior
// successful import, so this file IS done for checkpoint purposes.
@@ -287,7 +330,7 @@ export async function runImport(
}
errors++;
skipped++;
failures.push({ path: relativePath, error: msg });
failures.push({ path: importRelPath, error: msg });
}
processed++;
tickProgress();
+240 -52
View File
@@ -1,4 +1,4 @@
import { existsSync, readFileSync, writeFileSync, statSync } from 'fs';
import { existsSync, readFileSync, writeFileSync, statSync, realpathSync } from 'fs';
import { execFileSync } from 'child_process';
import { join, relative } from 'path';
import type { BrainEngine } from '../core/engine.ts';
@@ -9,6 +9,7 @@ import { createInterface } from 'readline';
import {
isSyncable,
unsyncableReason,
matchesAnyGlob,
resolveSlugForPath,
unacknowledgedSyncFailures,
acknowledgeFailures,
@@ -742,6 +743,27 @@ export interface SyncOpts {
sourceId?: string;
/** Multi-repo: sync strategy override (markdown, code, auto). */
strategy?: 'markdown' | 'code' | 'auto';
/**
* #753/#774 — sync only files under this subdirectory of the git repo.
* Git operations (pull, diff, rev-parse) still run against the repo root
* (discovered via `git rev-parse --show-toplevel`); file walking, imports,
* deletes and renames are scoped to the subpath. Slugs are git-root-relative
* (`wiki/page1.md` → slug `wiki/page1`) so full and incremental syncs of
* the same scope agree. Enables N logical sources in one git repo.
*
* SECURITY (NAV-1/NAV-2): the resolved subpath must realpath-resolve inside
* the git root — `../escape` and symlinked subdirs pointing outside the repo
* are rejected before any git op runs.
*/
srcSubpath?: string;
/**
* #753/#774 — glob patterns for files to exclude from sync (repeatable
* `--exclude` on the CLI). Matched against the scope-relative path in both
* the full-sync and incremental paths. Excluded files are never imported;
* exclusion does NOT delete previously-imported pages (conservative,
* matching the #1433 metafile posture).
*/
exclude?: string[];
/**
* Number of parallel workers for the import phase. When > 1, each worker
* gets its own small Postgres connection pool and files are dispatched via
@@ -905,6 +927,37 @@ function git(repoPath: string, args: string[], configs: string[] = []): string {
}).trim();
}
/**
* #753/#774: walk up from inputPath to the nearest git repo root via
* `git -C <path> rev-parse --show-toplevel`. Handles worktrees and submodules
* natively (git itself resolves them). Throws a user-friendly error when no
* git repo is found.
*/
export function discoverGitRoot(inputPath: string): string {
try {
return git(inputPath, ['rev-parse', '--show-toplevel']);
} catch {
throw new Error(
`Not inside a git repository: ${inputPath}. GBrain sync requires a git-initialized repo (or a subdirectory of one).`,
);
}
}
/**
* #774 NAV-1 TOCTOU: true only if filePath realpath-resolves inside gitRoot.
* Guards symlink escape at the per-file level (a committed symlink whose
* target lives outside the repo), not just at scope entry.
*/
function isPathSafe(filePath: string, gitRoot: string): boolean {
try {
const real = realpathSync(filePath);
const rootReal = realpathSync(gitRoot);
return real === rootReal || real.startsWith(rootReal + '/');
} catch {
return false;
}
}
function hasOriginRemote(repoPath: string): boolean {
try {
execFileSync('git', buildGitInvocation(repoPath, ['remote', 'get-url', 'origin']), {
@@ -1567,17 +1620,46 @@ async function performSyncInner(engine: BrainEngine, opts: SyncOpts): Promise<Sy
}
}
// Validate git repo
if (!existsSync(join(repoPath, '.git'))) {
throw new Error(`Not a git repository: ${repoPath}. GBrain sync requires a git-initialized repo.`);
// #753/#774: discover the git root instead of requiring `.git` at repoPath
// directly. Supports subdir-of-git-repo sources (monorepo pattern): either
// an explicit `--src-subpath` under a git-root repoPath, or a repoPath that
// IS a subdirectory (auto-discovery). Two axes fall out:
// - gitContextRoot: ALL git operations (pull, rev-parse, diff, cat-file)
// - syncScopeRoot: file walking, imports, deletes, renames
// In the common case (repoPath == git root, no subpath) they are identical.
serr(`[gbrain phase] sync.discover_git_root`);
const gitContextRoot = realpathSync(discoverGitRoot(repoPath));
const rawScopeRoot = opts.srcSubpath ? join(repoPath, opts.srcSubpath) : repoPath;
if (!existsSync(rawScopeRoot)) {
throw new Error(`Sync scope does not exist: ${rawScopeRoot}`);
}
const syncScopeRoot = realpathSync(rawScopeRoot);
// NAV-1/NAV-2 scope-entry guard: the realpath-resolved scope must live
// inside the realpath-resolved git root. Catches `--src-subpath ../escape`
// AND a symlinked subdir pointing outside the repo, before any git op runs.
if (syncScopeRoot !== gitContextRoot && !syncScopeRoot.startsWith(gitContextRoot + '/')) {
throw new Error(
`Sync scope ${syncScopeRoot} resolves outside git repo ${gitContextRoot}. ` +
`Refusing to sync: possible path traversal via --src-subpath.`,
);
}
// Relative path from git root to sync scope ('' when scope == root).
const syncScopeRelPath = syncScopeRoot === gitContextRoot ? '' : relative(gitContextRoot, syncScopeRoot);
const scoped = syncScopeRelPath !== '';
// Anchor written back to sync state (sources.local_path / sync.repo_path):
// the SCOPE path, so a follow-up bare `gbrain sync` auto-discovers the same
// scope. Unchanged (the caller's repoPath spelling) when no --src-subpath.
const anchorPath = opts.srcSubpath ? rawScopeRoot : repoPath;
const fullSyncRoots = { gitContextRoot, syncScopeRoot, anchorPath };
serr(`[gbrain phase] sync.detect_head`);
// Detect detached HEAD up front so the working-tree fallback fires for both
// the default sync and `--no-pull` callers. Only the actual git pull is
// gated on opts.noPull.
const detachedHead = isDetachedHead(repoPath);
const detachedHead = isDetachedHead(gitContextRoot);
if (detachedHead && !opts.noPull) {
// Print the caller's repoPath spelling (not the realpathed git root) —
// it's what the operator recognizes, and tests pin it.
serr(`Detached HEAD on ${repoPath}; skipping git pull. Syncing from local working tree.`);
}
@@ -1587,7 +1669,7 @@ async function performSyncInner(engine: BrainEngine, opts: SyncOpts): Promise<Sy
// hardening that cloneRepo applies. Route through pullRepo from
// git-remote.ts so the flag set is consistent across initial clone and
// ongoing pulls — single source of truth for the defensive flags.
const originRemotePresent = !opts.noPull && !detachedHead ? hasOriginRemote(repoPath) : false;
const originRemotePresent = !opts.noPull && !detachedHead ? hasOriginRemote(gitContextRoot) : false;
if (!opts.noPull && !detachedHead && !originRemotePresent) {
serr(`No origin remote on ${repoPath}; skipping git pull. Syncing from local working tree.`);
}
@@ -1626,8 +1708,9 @@ async function performSyncInner(engine: BrainEngine, opts: SyncOpts): Promise<Sy
// We pass a safe default (the operator's full --timeout if set, else
// pullRepo's own 300s default). The catch below distinguishes
// timeout (ETIMEDOUT / SIGTERM on err.cause) from ordinary pull
// failure.
pullRepo(repoPath);
// failure. Pull applies to the whole git repo (gitContextRoot), not
// just the sync scope — git has no per-subdir pull.
pullRepo(gitContextRoot);
serr(`[gbrain phase] sync.git_pull done ${Date.now() - _t0}ms`);
} catch (e: unknown) {
const msg = e instanceof Error ? e.message : String(e);
@@ -1668,7 +1751,7 @@ async function performSyncInner(engine: BrainEngine, opts: SyncOpts): Promise<Sy
// Get current HEAD
let headCommit: string;
try {
headCommit = git(repoPath, ['rev-parse', 'HEAD']);
headCommit = git(gitContextRoot, ['rev-parse', 'HEAD']);
} catch {
throw new Error(`No commits in repo ${repoPath}. Make at least one commit before syncing.`);
}
@@ -1690,7 +1773,7 @@ async function performSyncInner(engine: BrainEngine, opts: SyncOpts): Promise<Sy
if (lastCommit) {
let objectPresent = true;
try {
git(repoPath, ['cat-file', '-t', lastCommit]);
git(gitContextRoot, ['cat-file', '-t', lastCommit]);
} catch {
objectPresent = false;
}
@@ -1699,7 +1782,7 @@ async function performSyncInner(engine: BrainEngine, opts: SyncOpts): Promise<Sy
// back to the authoritative full reconcile (which now also purges stale
// pages for deleted files; see performFullSync's delete-reconcile pass).
serr(`Sync anchor ${lastCommit.slice(0, 8)} object missing (gc'd after history rewrite). Running full reimport.`);
return performFullSync(engine, repoPath, headCommit, opts);
return performFullSync(engine, fullSyncRoots, headCommit, opts);
}
// Observability only — NOT control flow. A non-ancestor bookmark is still
@@ -1707,7 +1790,7 @@ async function performSyncInner(engine: BrainEngine, opts: SyncOpts): Promise<Sy
// failure mode (#1970) is visible in the logs.
let isAncestor = true;
try {
git(repoPath, ['merge-base', '--is-ancestor', lastCommit, headCommit]);
git(gitContextRoot, ['merge-base', '--is-ancestor', lastCommit, headCommit]);
} catch {
isAncestor = false;
}
@@ -1722,7 +1805,7 @@ async function performSyncInner(engine: BrainEngine, opts: SyncOpts): Promise<Sy
// First sync
if (!lastCommit) {
return performFullSync(engine, repoPath, headCommit, opts);
return performFullSync(engine, fullSyncRoots, headCommit, opts);
}
// v0.42.x (#1794): resumable incremental sync — resolve the PINNED target.
@@ -1744,7 +1827,7 @@ async function performSyncInner(engine: BrainEngine, opts: SyncOpts): Promise<Sy
if (storedTarget) {
let pinReachable = false;
try {
git(repoPath, ['merge-base', '--is-ancestor', storedTarget, headCommit]);
git(gitContextRoot, ['merge-base', '--is-ancestor', storedTarget, headCommit]);
pinReachable = true;
} catch {
pinReachable = false;
@@ -1778,7 +1861,7 @@ async function performSyncInner(engine: BrainEngine, opts: SyncOpts): Promise<Sy
const currentVersion = String(CHUNKER_VERSION);
const versionMismatch = storedVersion !== null && storedVersion !== currentVersion;
const versionNeverSet = storedVersion === null && opts.sourceId !== undefined;
const detachedWorkingTreeManifest = detachedHead ? buildDetachedWorkingTreeManifest(repoPath) : null;
const detachedWorkingTreeManifest = detachedHead ? buildDetachedWorkingTreeManifest(gitContextRoot) : null;
const hasDetachedWorkingTreeChanges = detachedWorkingTreeManifest !== null &&
(detachedWorkingTreeManifest.added.length > 0 ||
detachedWorkingTreeManifest.modified.length > 0 ||
@@ -1814,7 +1897,7 @@ async function performSyncInner(engine: BrainEngine, opts: SyncOpts): Promise<Sy
`[sync] chunker_version gate: stored=${storedVersion ?? 'unset'}, current=${currentVersion}. ` +
`Forcing full re-chunk pass (git HEAD unchanged but pipeline version advanced).`,
);
const result = await performFullSync(engine, repoPath, headCommit, opts);
const result = await performFullSync(engine, fullSyncRoots, headCommit, opts);
await writeChunkerVersion(engine, opts.sourceId, currentVersion);
return result;
}
@@ -1835,7 +1918,7 @@ async function performSyncInner(engine: BrainEngine, opts: SyncOpts): Promise<Sy
// buffer, and a gc'd anchor object can't be diffed at all. On either
// `unavailable`, fall back to the authoritative full reconcile instead of
// throwing — a slow correct reconcile beats a hard error or a silent walk.
const delta = computeSyncDelta(repoPath, lastCommit, pin, {
const delta = computeSyncDelta(gitContextRoot, lastCommit, pin, {
detachedManifest: detachedWorkingTreeManifest,
});
if (delta.status === 'unavailable') {
@@ -1843,30 +1926,60 @@ async function performSyncInner(engine: BrainEngine, opts: SyncOpts): Promise<Sy
`[sync] delta ${lastCommit.slice(0, 8)}..${pin.slice(0, 8)} unavailable ` +
`(${delta.reason}) — falling back to full reconcile.`,
);
return performFullSync(engine, repoPath, headCommit, opts);
return performFullSync(engine, fullSyncRoots, headCommit, opts);
}
const manifest = delta.manifest;
// Filter to syncable files (strategy-aware)
// #753/#774 scope filter: git-diff paths are git-root-relative; when a
// subpath scope is active, only paths under it participate. Back-compat:
// syncScopeRelPath is '' when scope == root, so inScope is always true and
// the filters below reduce to the pre-#774 behavior exactly.
const inScope = (p: string): boolean =>
!scoped || p === syncScopeRelPath || p.startsWith(syncScopeRelPath + '/');
// --exclude patterns match the SCOPE-relative path (what the user of a
// scoped source thinks in), same form runImport matches on full sync.
const scopeRel = (p: string): string =>
scoped && p.startsWith(syncScopeRelPath + '/') ? p.slice(syncScopeRelPath.length + 1) : p;
const excluded = (p: string): boolean =>
opts.exclude !== undefined && opts.exclude.length > 0 && matchesAnyGlob(scopeRel(p), opts.exclude);
// Filter to syncable files (strategy-aware + scope-aware + exclude-aware)
const syncOpts = opts.strategy ? { strategy: opts.strategy } : undefined;
// #1970 (F-C): a rename whose DESTINATION is unsyncable drops out of BOTH
// `renamed` (only `r.to` is kept below) AND `deleted` (git emits it as `R`,
// not `D`), leaving the OLD page stale. Fold the source side into the delete
// set. isSyncable(r.from) excludes metafiles automatically, so a rename of a
// metafile is left untouched (matching the #1433 metafile-skip invariant).
// #774: a rename whose destination LEFT the scope is the same class — the
// old page's backing file is gone from this source's slice of the repo.
const renamedToUnsyncable = manifest.renamed
.filter(r => isSyncable(r.from, syncOpts) && !isSyncable(r.to, syncOpts))
.filter(r => inScope(r.from) && isSyncable(r.from, syncOpts) &&
!(inScope(r.to) && isSyncable(r.to, syncOpts)))
.map(r => r.from);
const filtered: SyncManifest = {
added: manifest.added.filter(p => isSyncable(p, syncOpts)),
modified: manifest.modified.filter(p => isSyncable(p, syncOpts)),
added: manifest.added.filter(p => inScope(p) && !excluded(p) && isSyncable(p, syncOpts)),
modified: manifest.modified.filter(p => inScope(p) && !excluded(p) && isSyncable(p, syncOpts)),
deleted: unique([
...manifest.deleted.filter(p => isSyncable(p, syncOpts)),
...manifest.deleted.filter(p => inScope(p) && isSyncable(p, syncOpts)),
...renamedToUnsyncable,
]),
renamed: manifest.renamed.filter(r => isSyncable(r.to, syncOpts)),
renamed: manifest.renamed.filter(r => inScope(r.to) && !excluded(r.to) && isSyncable(r.to, syncOpts)),
};
// NAV-4: warn when --exclude filtered out every candidate change — almost
// always a mistyped pattern, and otherwise indistinguishable from
// "up to date" in the output.
if (opts.exclude && opts.exclude.length > 0) {
const excludeCandidates = [...manifest.added, ...manifest.modified]
.filter(p => inScope(p) && isSyncable(p, syncOpts));
if (excludeCandidates.length > 0 && excludeCandidates.every(excluded)) {
console.warn(
`[gbrain sync] No files matched after applying ${opts.exclude.length} --exclude pattern(s). ` +
`Check your --exclude flags. Patterns: ${JSON.stringify(opts.exclude)}`,
);
}
}
// Delete pages that became un-syncable (modified but filtered out).
// v0.20.0 Cathedral II SP-5: resolveSlugForPath picks the right slug shape
// (markdown vs code) based on the chunker's classifier, so a Rust file that
@@ -1890,7 +2003,7 @@ async function performSyncInner(engine: BrainEngine, opts: SyncOpts): Promise<Sy
// delete the page. That's the same pre-fix behavior — removing the
// page requires `gbrain pages purge-deleted` or a direct MCP delete.
// Filed as v0.42+ follow-up for a `gbrain pages remove <slug>` surface.
const unsyncableModified = manifest.modified.filter(p => !isSyncable(p, syncOpts));
const unsyncableModified = manifest.modified.filter(p => inScope(p) && !isSyncable(p, syncOpts));
// v0.18.0+ multi-source: scope getPage + deletePage to opts.sourceId so
// unsyncable cleanup in source A doesn't accidentally sweep same-slug
// pages in sources B/C/D.
@@ -1938,7 +2051,7 @@ async function performSyncInner(engine: BrainEngine, opts: SyncOpts): Promise<Sy
// (#1794): advance to the PINNED target, and clear any checkpoint (a resume
// whose remaining range turned out to have no syncable changes still
// completes cleanly here).
await writeSyncAnchor(engine, opts.sourceId, 'last_commit', pin, commitTimeMs(repoPath, pin));
await writeSyncAnchor(engine, opts.sourceId, 'last_commit', pin, commitTimeMs(gitContextRoot, pin));
await engine.setConfig('sync.last_run', new Date().toISOString());
await writeChunkerVersion(engine, opts.sourceId, String(CHUNKER_VERSION));
await clearOpCheckpoint(engine, ckpt.paths);
@@ -2325,8 +2438,11 @@ async function performSyncInner(engine: BrainEngine, opts: SyncOpts): Promise<Sy
// throw here crashes the whole sync mid-run and freezes the checkpoint,
// defeating --skip-failed. A `skipped` result carrying an error is also
// captured so the failure is recorded rather than silently dropped.
const filePath = join(repoPath, to);
if (existsSync(filePath)) {
// Paths from git diff are relative to gitContextRoot; join from there.
// NAV-1 TOCTOU: refuse a destination that realpath-resolves outside the
// repo (committed symlink pointing out).
const filePath = join(gitContextRoot, to);
if (existsSync(filePath) && isPathSafe(filePath, gitContextRoot)) {
try {
const result = await importFile(engine, filePath, to, { noEmbed, sourceId: opts.sourceId, activePack: syncActivePack });
if (result.status === 'imported') chunksCreated += result.chunks;
@@ -2411,8 +2527,8 @@ async function performSyncInner(engine: BrainEngine, opts: SyncOpts): Promise<Sy
progress.start('sync.imports', importsToDo.length);
// Core import logic shared by serial and parallel paths.
// repoPath is validated non-null at the top of performSyncInner; narrow for TS.
const syncRepoPath = repoPath!;
// Paths from git diff are relative to gitContextRoot; join from there.
const syncRepoPath = gitContextRoot;
// paced-backfill (T3 / C9 / CX4): ONE shared pacer across all worker
// engines. This is the multi-pool permit case — each parallel worker owns a
// separate PostgresEngine, so a single worker count can't bound TOTAL
@@ -2500,6 +2616,16 @@ async function performSyncInner(engine: BrainEngine, opts: SyncOpts): Promise<Sy
progress.tick(1, `skip:${path}`);
return;
}
// #774 NAV-1 TOCTOU: re-validate the file's realpath at import time so a
// committed symlink pointing outside the repo (or one swapped in after
// the scope-entry check) is never read. Recorded as a failure —
// fail-closed: the bookmark won't advance past a symlink escape.
if (!isPathSafe(filePath, gitContextRoot)) {
failedFiles.push({ path, error: 'path resolves outside git repo (symlink escape)' });
progressAt.last = Date.now();
progress.tick(1, `skip:${path}`);
return;
}
// v0.41.37.0 #1569: per-file BEGIN heartbeat, emitted BEFORE importFile so a
// hang names the stalling file (the progress.tick below only fires AFTER
// importFile returns — useless when one file wedges). Off by default
@@ -2703,11 +2829,11 @@ async function performSyncInner(engine: BrainEngine, opts: SyncOpts): Promise<Sy
// - pin NOT an ancestor of HEAD (history REWRITE / reset / force-push) →
// the tree we imported against is gone. Block; do not advance.
try {
const currentHead = git(repoPath, ['rev-parse', 'HEAD']);
const currentHead = git(gitContextRoot, ['rev-parse', 'HEAD']);
if (currentHead !== pin) {
let pinStillReachable = false;
try {
git(repoPath, ['merge-base', '--is-ancestor', pin, currentHead]);
git(gitContextRoot, ['merge-base', '--is-ancestor', pin, currentHead]);
pinStillReachable = true;
} catch {
pinStillReachable = false;
@@ -2748,9 +2874,9 @@ async function performSyncInner(engine: BrainEngine, opts: SyncOpts): Promise<Sy
// "fresh". The checkpoint rows clear here — CONVERGENCE CONTRACT: sync
// convergence == IMPORT convergence; downstream extract/facts/embed is
// decoupled (its own resumable stale sweeps).
await writeSyncAnchor(engine, opts.sourceId, 'last_commit', pin, commitTimeMs(repoPath, pin));
await writeSyncAnchor(engine, opts.sourceId, 'last_commit', pin, commitTimeMs(gitContextRoot, pin));
await engine.setConfig('sync.last_run', new Date().toISOString());
await writeSyncAnchor(engine, opts.sourceId, 'repo_path', repoPath);
await writeSyncAnchor(engine, opts.sourceId, 'repo_path', anchorPath);
await writeChunkerVersion(engine, opts.sourceId, String(CHUNKER_VERSION));
await clearOpCheckpoint(engine, ckpt.paths);
await clearOpCheckpoint(engine, ckpt.target);
@@ -2799,7 +2925,7 @@ async function performSyncInner(engine: BrainEngine, opts: SyncOpts): Promise<Sy
// checkpoint is INTENTIONALLY left in place — the banked completed set lets
// the next run skip the drained files and re-attempt only the failures.
await engine.setConfig('sync.last_run', new Date().toISOString());
await writeSyncAnchor(engine, opts.sourceId, 'repo_path', repoPath);
await writeSyncAnchor(engine, opts.sourceId, 'repo_path', anchorPath);
// v0.42.x (#1794): surface banked progress so a blocked run doesn't read as
// total loss (last_commit is unchanged by design; the checkpoint is banked).
serr(
@@ -2870,8 +2996,10 @@ async function performSyncInner(engine: BrainEngine, opts: SyncOpts): Promise<Sy
if (!opts.noExtract && totalChanges <= 100 && pagesAffected.length > 0) {
try {
const { extractLinksForSlugs, extractTimelineForSlugs, stampExtracted } = await import('./extract.ts');
const linksCreated = await extractLinksForSlugs(engine, repoPath, pagesAffected, extractOpts);
const timelineCreated = await extractTimelineForSlugs(engine, repoPath, pagesAffected, extractOpts);
// #774: pages' source_path is git-root-relative, so extract resolves
// files from gitContextRoot (== repoPath realpath when unscoped).
const linksCreated = await extractLinksForSlugs(engine, gitContextRoot, pagesAffected, extractOpts);
const timelineCreated = await extractTimelineForSlugs(engine, gitContextRoot, pagesAffected, extractOpts);
if (linksCreated > 0 || timelineCreated > 0) {
slog(` Extracted: ${linksCreated} links, ${timelineCreated} timeline entries`);
}
@@ -2976,11 +3104,21 @@ async function performSyncInner(engine: BrainEngine, opts: SyncOpts): Promise<Sy
async function performFullSync(
engine: BrainEngine,
repoPath: string,
// #753/#774: the three roots resolved once at the top of performSyncInner.
// gitContextRoot — git repo root (git ops, slug base for scoped syncs)
// syncScopeRoot — where files are walked/imported (== gitContextRoot
// when no subpath scope is active)
// anchorPath — what gets written back to sync.repo_path/local_path
roots: { gitContextRoot: string; syncScopeRoot: string; anchorPath: string },
headCommit: string,
opts: SyncOpts,
): Promise<SyncResult> {
// Dry-run: walk the repo, count syncable files, return without writing.
const { gitContextRoot, syncScopeRoot, anchorPath } = roots;
// Scoped sync → slugs/source_path are git-root-relative (matches the
// incremental path's git-diff paths). Unscoped → undefined (dir-relative,
// the pre-#774 behavior, byte-for-byte).
const slugRoot = syncScopeRoot !== gitContextRoot ? gitContextRoot : undefined;
// Dry-run: walk the scope, count syncable files, return without writing.
// Fixes the silent-write-on-dry-run bug where performFullSync called
// runImport unconditionally regardless of opts.dryRun.
//
@@ -2990,11 +3128,14 @@ async function performFullSync(
// code --dry-run` always reported zero files even when ~1500 code
// files were waiting.
if (opts.dryRun) {
const allFiles = collectSyncableFiles(repoPath, { strategy: opts.strategy ?? 'markdown' });
let allFiles = collectSyncableFiles(syncScopeRoot, { strategy: opts.strategy ?? 'markdown' });
if (opts.exclude && opts.exclude.length > 0) {
allFiles = allFiles.filter(abs => !matchesAnyGlob(relative(syncScopeRoot, abs), opts.exclude));
}
slog(
`Full-sync dry run (strategy=${opts.strategy ?? 'markdown'}): ` +
`${allFiles.length} file(s) would be imported ` +
`from ${repoPath} @ ${headCommit.slice(0, 8)}.`,
`from ${syncScopeRoot} @ ${headCommit.slice(0, 8)}.`,
);
return {
status: 'dry_run',
@@ -3017,21 +3158,24 @@ async function performFullSync(
// sync and the jobs handler.
const FULL_SYNC_LARGE_MARKER = Number.MAX_SAFE_INTEGER;
const fullConcurrency = autoConcurrency(engine, FULL_SYNC_LARGE_MARKER, opts.concurrency);
slog(`Running full import of ${repoPath}${fullConcurrency > 1 ? ` (${fullConcurrency} workers)` : ''}...`);
slog(`Running full import of ${syncScopeRoot}${fullConcurrency > 1 ? ` (${fullConcurrency} workers)` : ''}...`);
const { runImport } = await import('./import.ts');
const importArgs = [repoPath];
const importArgs = [syncScopeRoot];
if (opts.noEmbed) importArgs.push('--no-embed');
if (fullConcurrency > 1) importArgs.push('--workers', String(fullConcurrency));
// v0.31.2: thread strategy through so code-strategy first sync
// actually enumerates code files (closes bug 1).
// v0.30.x: thread sourceId so performFullSync routes pages to the named
// source (incremental path already does this).
// #753/#774: thread exclude (--exclude CLI) + slugRoot (monorepo subdir).
const _fullImportT0 = Date.now();
serr(`[gbrain phase] sync.fullsync.import start strategy=${opts.strategy ?? 'markdown'}`);
const result = await runImport(engine, importArgs, {
commit: headCommit,
strategy: opts.strategy,
sourceId: opts.sourceId,
exclude: opts.exclude,
slugRoot,
// issue #1939: performFullSync owns the failure ledger + bookmark via the
// shared gate below; don't let runImport double-record or write its own.
managedBookmark: true,
@@ -3055,9 +3199,9 @@ async function performFullSync(
const advanceFull = async (): Promise<void> => {
// Persist sync state so the next sync is incremental. Routed through
// writeSyncAnchor so --source pins the right sources row.
await writeSyncAnchor(engine, opts.sourceId, 'last_commit', headCommit, newestCommitMs(repoPath));
await writeSyncAnchor(engine, opts.sourceId, 'last_commit', headCommit, newestCommitMs(gitContextRoot));
await engine.setConfig('sync.last_run', new Date().toISOString());
await writeSyncAnchor(engine, opts.sourceId, 'repo_path', repoPath);
await writeSyncAnchor(engine, opts.sourceId, 'repo_path', anchorPath);
await writeChunkerVersion(engine, opts.sourceId, String(CHUNKER_VERSION));
};
@@ -3084,7 +3228,7 @@ async function performFullSync(
);
}
await engine.setConfig('sync.last_run', new Date().toISOString());
await writeSyncAnchor(engine, opts.sourceId, 'repo_path', repoPath);
await writeSyncAnchor(engine, opts.sourceId, 'repo_path', anchorPath);
return {
status: 'blocked_by_failures',
fromCommit: null,
@@ -3140,16 +3284,24 @@ async function performFullSync(
// backslash paths while a stored source_path can hold git-derived forward
// slashes; without normalization every file-backed page mismatches, looks
// stale, and the reconcile wipes the whole source.
const currentFiles = collectSyncableFiles(repoPath, { strategy: opts.strategy ?? 'markdown' })
.map(abs => relative(repoPath, abs));
// #774: scoped syncs store git-root-relative source_paths (slugRoot), so
// relativize the walk to the same base — otherwise every page mismatches
// and the mass-delete valve trips on a perfectly healthy scoped source.
const currentFiles = collectSyncableFiles(syncScopeRoot, { strategy: opts.strategy ?? 'markdown' })
.map(abs => relative(slugRoot ?? syncScopeRoot, abs));
const rows = await engine.executeRaw<{ slug: string; source_path: string | null }>(
`SELECT slug, source_path FROM pages WHERE source_id = $1 AND source_path IS NOT NULL AND deleted_at IS NULL`,
[sid],
);
// #774: a scoped full sync is authoritative ONLY for its scope — pages
// whose source_path lives outside the subpath (e.g. from an earlier
// root-level sync of this source) are out of this walk's sight and must
// not be treated as stale.
const scopePrefix = slugRoot ? relative(gitContextRoot, syncScopeRoot) + '/' : '';
const plan = planReconcileDeletes(
rows,
currentFiles,
p => isSyncable(p, reconcileSyncOpts),
p => (scopePrefix === '' || p.startsWith(scopePrefix)) && isSyncable(p, reconcileSyncOpts),
);
if (plan.staleSlugs.length > 0 && plan.massDelete && !massReconcileAllowed()) {
// #2828 mass-delete safety valve: a reconcile that would sweep more than
@@ -3400,6 +3552,19 @@ export function composeAbortSignals(
return AbortSignal.any(live);
}
/**
* #753/#774: `.gitignore` must be managed at the git ROOT — when a source's
* local_path (or --repo) points at a monorepo subdirectory, writing ignore
* entries into the subdir would create a stray `.gitignore` git doesn't
* consult for the repo-level db_only rules. Best-effort: falls back to the
* given path when git discovery fails (manageGitignore no-ops on non-repos).
*/
function manageGitignoreAtGitRoot(path: string, engineKind?: 'pglite' | 'postgres'): void {
let root = path;
try { root = discoverGitRoot(path); } catch { /* best-effort */ }
manageGitignore(root, engineKind);
}
export async function runSync(engine: BrainEngine, args: string[]) {
// v0.40 Federated Sync v2: `gbrain sync trigger` subcommand
// Routes to runSyncTrigger which queues a 'sync' minion job with
@@ -3432,6 +3597,13 @@ Options:
--repo <path> Path to the brain repo. Defaults to the path
saved by 'gbrain init'.
--full Force a full re-sync (rare; usually incremental).
--src-subpath <dir> Sync only this subdirectory of the git repo (monorepo
pattern: N logical sources in one repo). Git pull/diff
run at the repo root; imports are scoped to the subdir
and slugs stay root-relative (wiki/page1). Passing the
subdirectory directly as --repo also works.
--exclude <glob> Exclude files matching the glob from sync (repeatable;
matched against the scope-relative path).
--dry-run Show what would be synced without writing.
--skip-failed Acknowledge previously-recorded sync failures so
the bookmark can advance past unparseable files.
@@ -3591,6 +3763,20 @@ See also:
process.exit(1);
}
const strategyArg = args.find((a, i) => args[i - 1] === '--strategy') as SyncOpts['strategy'] | undefined;
// #753/#774: monorepo subdir-source flags. --exclude is repeatable.
const srcSubpath = args.find((a, i) => args[i - 1] === '--src-subpath') || undefined;
const excludePatterns: string[] = [];
for (let i = 0; i < args.length; i++) {
if (args[i] === '--exclude' && i + 1 < args.length) excludePatterns.push(args[i + 1]);
}
if (syncAll && (srcSubpath || excludePatterns.length > 0)) {
console.error(
`--src-subpath/--exclude scope a single sync invocation; they cannot be combined with --all. ` +
`For --all runs, register the subdirectory as the source's local_path instead ` +
`(gbrain sources add <id> --path <repo>/<subdir>).`,
);
process.exit(1);
}
const concurrencyStr = args.find((a, i) => args[i - 1] === '--concurrency' || args[i - 1] === '--workers');
const parallelStr = args.find((a, i) => args[i - 1] === '--parallel');
// v0.22.13 (PR #490 Q2): parseWorkers throws on '0', '-3', 'foo', '1.5' instead
@@ -3850,7 +4036,7 @@ See also:
result.status !== 'blocked_by_failures' &&
result.status !== 'partial'
) {
manageGitignore(src.local_path!, engine.kind);
manageGitignoreAtGitRoot(src.local_path!, engine.kind);
}
// D18: auto-enqueue embed-backfill per source (unless opted out).
// v0.41.13.0 (T7 / D-V3-5): partial excluded — the next clean sync
@@ -4038,6 +4224,8 @@ See also:
const opts: SyncOpts = {
repoPath, dryRun, full, noPull, noEmbed, noExtract, skipFailed, retryFailed, noSchemaPack, sourceId,
strategy: strategyArg, concurrency,
srcSubpath,
exclude: excludePatterns.length > 0 ? excludePatterns : undefined,
signal: composeAbortSignals(singleSourceInterrupt.signal, singleSourceController?.signal),
};
@@ -4121,7 +4309,7 @@ See also:
) {
const effectiveRepoPath = opts.repoPath ?? (await getDefaultSourcePath(engine));
if (effectiveRepoPath) {
manageGitignore(effectiveRepoPath, engine.kind);
manageGitignoreAtGitRoot(effectiveRepoPath, engine.kind);
}
}
// v0.42.42.0 (#2139, Step 4b): the inline gate auto-deferred this run's
@@ -4170,7 +4358,7 @@ See also:
) {
const effectiveRepoPath = opts.repoPath ?? (await getDefaultSourcePath(engine));
if (effectiveRepoPath) {
manageGitignore(effectiveRepoPath, engine.kind);
manageGitignoreAtGitRoot(effectiveRepoPath, engine.kind);
}
}
} catch (e: unknown) {
+1 -1
View File
@@ -219,7 +219,7 @@ function globToRegex(pattern: string): RegExp {
return new RegExp(regex);
}
function matchesAnyGlob(path: string, patterns?: string[]): boolean {
export function matchesAnyGlob(path: string, patterns?: string[]): boolean {
if (!patterns || patterns.length === 0) return false;
const normalized = path.replace(/\\/g, '/');
return patterns.some((pattern) => globToRegex(pattern).test(normalized));
+385
View File
@@ -0,0 +1,385 @@
/**
* #753/#774 — --src-subpath + --exclude monorepo subdir-source support.
*
* A single git repo can hold N logical sources at subdirectories (wiki/,
* memory/, ...). `gbrain sync --src-subpath wiki` (or passing the subdir
* directly as the repo path) scopes file walking + imports to the subdir
* while git operations (pull, rev-parse, diff) run at the discovered repo
* root. Slugs stay git-root-relative (`wiki/page1`) so full and incremental
* syncs of the same scope agree.
*
* Security pins (the point of the feature's guards):
* NAV-1/NAV-2 — `--src-subpath ../escape` and a symlinked subdir pointing
* outside the repo are realpath-checked and rejected before any git op.
* NAV-1 TOCTOU — per-file realpath checks during the incremental import
* drain (see the isPathSafe guard in sync.ts's importOnePath).
* NAV-4 — an --exclude set that filters out everything warns loudly.
*
* Regression note: against pre-#774 master every subdir test fails with
* "Not a git repository".
*/
import { describe, test, expect, beforeAll, afterAll, beforeEach, afterEach } from 'bun:test';
import { mkdtempSync, writeFileSync, rmSync, mkdirSync, symlinkSync } from 'fs';
import { join } from 'path';
import { execSync } from 'child_process';
import { tmpdir } from 'os';
import { PGLiteEngine } from '../src/core/pglite-engine.ts';
import { resetPgliteState } from './helpers/reset-pglite.ts';
// Helper: create a minimal valid markdown file
function mdPage(title: string, body = 'Content.'): string {
return `---\ntype: note\ntitle: ${title}\n---\n\n${body}`;
}
// Helper: init a git repo with author identity
function gitInit(dir: string): void {
execSync('git init', { cwd: dir, stdio: 'pipe' });
execSync('git config user.email "test@test.com"', { cwd: dir, stdio: 'pipe' });
execSync('git config user.name "Test"', { cwd: dir, stdio: 'pipe' });
}
// Helper: stage + commit everything in a git repo
function gitCommit(dir: string, msg = 'initial'): void {
execSync('git add -A', { cwd: dir, stdio: 'pipe' });
execSync(`git commit -m "${msg}"`, { cwd: dir, stdio: 'pipe' });
}
describe('sync monorepo subdir-source support (#753/#774)', () => {
let engine: PGLiteEngine;
let repoPath: string;
beforeAll(async () => {
engine = new PGLiteEngine();
await engine.connect({});
await engine.initSchema();
}, 60_000);
afterAll(async () => {
await engine.disconnect();
}, 60_000);
beforeEach(async () => {
await resetPgliteState(engine);
repoPath = mkdtempSync(join(tmpdir(), 'gbrain-monorepo-'));
gitInit(repoPath);
mkdirSync(join(repoPath, 'wiki'), { recursive: true });
mkdirSync(join(repoPath, 'memory'), { recursive: true });
writeFileSync(join(repoPath, 'wiki', 'page1.md'), mdPage('Wiki Page 1'));
writeFileSync(join(repoPath, 'wiki', 'page2.md'), mdPage('Wiki Page 2'));
writeFileSync(join(repoPath, 'memory', 'note1.md'), mdPage('Memory Note 1'));
writeFileSync(join(repoPath, 'memory', 'note2.md'), mdPage('Memory Note 2'));
gitCommit(repoPath);
});
afterEach(() => {
if (repoPath) rmSync(repoPath, { recursive: true, force: true });
});
// ─────────────────────────────────────────────────────────────────────────
// Back-compat: sync at git root (no srcSubpath) still works
// ─────────────────────────────────────────────────────────────────────────
test('back-compat: sync at git root without srcSubpath imports all files', async () => {
const { performSync } = await import('../src/commands/sync.ts');
const result = await performSync(engine, {
repoPath,
noPull: true,
noEmbed: true,
full: true,
});
expect(result.status).toBe('first_sync');
expect(result.added).toBe(4); // wiki/page1 + wiki/page2 + memory/note1 + memory/note2
// Slug shape unchanged for git-root syncs.
expect(await engine.getPage('wiki/page1')).not.toBeNull();
});
// ─────────────────────────────────────────────────────────────────────────
// Auto-discovery: repoPath IS a non-git-root subdir
// ─────────────────────────────────────────────────────────────────────────
test('auto-discovery: repoPath is a git subdir — discoverGitRoot succeeds', async () => {
const { performSync } = await import('../src/commands/sync.ts');
// Pass the wiki/ subdir directly as repoPath (no explicit srcSubpath).
// Pre-#774: throws "Not a git repository".
// Post-#774: gitContextRoot = repo root, syncScopeRoot = wiki/.
const result = await performSync(engine, {
repoPath: join(repoPath, 'wiki'),
noPull: true,
noEmbed: true,
full: true,
});
expect(result.status).toBe('first_sync');
expect(result.added).toBe(2); // only wiki/page1 + wiki/page2
// Slugs are git-root-relative in BOTH spellings (subdir repoPath and
// --src-subpath) so full and incremental syncs of the same scope agree.
expect(await engine.getPage('wiki/page1')).not.toBeNull();
expect(await engine.getPage('page1')).toBeNull();
});
// ─────────────────────────────────────────────────────────────────────────
// srcSubpath explicit flag: scope to subdir from git root
// ─────────────────────────────────────────────────────────────────────────
test('--src-subpath wiki: only wiki/ files are imported', async () => {
const { performSync } = await import('../src/commands/sync.ts');
const result = await performSync(engine, {
repoPath,
srcSubpath: 'wiki',
noPull: true,
noEmbed: true,
full: true,
});
expect(result.status).toBe('first_sync');
expect(result.added).toBe(2);
// Verify the imported slugs are from wiki/ only (git-root-relative)
const wikiPage = await engine.getPage('wiki/page1');
expect(wikiPage).not.toBeNull();
const memoryPage = await engine.getPage('memory/note1');
expect(memoryPage).toBeNull();
});
test('--src-subpath memory: only memory/ files are imported', async () => {
const { performSync } = await import('../src/commands/sync.ts');
const result = await performSync(engine, {
repoPath,
srcSubpath: 'memory',
noPull: true,
noEmbed: true,
full: true,
});
expect(result.status).toBe('first_sync');
expect(result.added).toBe(2);
const memoryPage = await engine.getPage('memory/note1');
expect(memoryPage).not.toBeNull();
const wikiPage = await engine.getPage('wiki/page1');
expect(wikiPage).toBeNull();
});
// ─────────────────────────────────────────────────────────────────────────
// Two sources in one repo, scoped independently
// ─────────────────────────────────────────────────────────────────────────
test('2 sources in 1 repo: sync each scope independently, no cross-contamination', async () => {
const { performSync } = await import('../src/commands/sync.ts');
const wikiResult = await performSync(engine, {
repoPath,
srcSubpath: 'wiki',
noPull: true,
noEmbed: true,
full: true,
});
expect(wikiResult.status).toBe('first_sync');
expect(wikiResult.added).toBe(2);
// Reset only page state, keep the engine connected for second sync
await resetPgliteState(engine);
const memResult = await performSync(engine, {
repoPath,
srcSubpath: 'memory',
noPull: true,
noEmbed: true,
full: true,
});
expect(memResult.status).toBe('first_sync');
expect(memResult.added).toBe(2);
// After memory sync, memory pages exist and wiki pages don't
expect(await engine.getPage('memory/note1')).not.toBeNull();
expect(await engine.getPage('wiki/page1')).toBeNull();
});
// ─────────────────────────────────────────────────────────────────────────
// Incremental sync respects the scope (the gap #774 left untested)
// ─────────────────────────────────────────────────────────────────────────
test('incremental --src-subpath: only in-scope diff paths are processed', async () => {
const { performSync } = await import('../src/commands/sync.ts');
const first = await performSync(engine, {
repoPath,
srcSubpath: 'wiki',
noPull: true,
noEmbed: true,
full: true,
});
expect(first.status).toBe('first_sync');
// Commit 2: touch one file in each scope + add one wiki file.
writeFileSync(join(repoPath, 'wiki', 'page1.md'), mdPage('Wiki Page 1', 'Updated.'));
writeFileSync(join(repoPath, 'memory', 'note1.md'), mdPage('Memory Note 1', 'Updated.'));
writeFileSync(join(repoPath, 'wiki', 'page3.md'), mdPage('Wiki Page 3'));
gitCommit(repoPath, 'second');
const second = await performSync(engine, {
repoPath,
srcSubpath: 'wiki',
noPull: true,
noEmbed: true,
});
expect(second.status).toBe('synced');
expect(second.added).toBe(1); // wiki/page3 only — memory change filtered by scope
expect(second.modified).toBe(1); // wiki/page1
expect(await engine.getPage('wiki/page3')).not.toBeNull();
expect(await engine.getPage('memory/note1')).toBeNull();
});
// ─────────────────────────────────────────────────────────────────────────
// Path-traversal sanitization (NAV-1 + NAV-2)
// ─────────────────────────────────────────────────────────────────────────
test('path-traversal: --src-subpath ../escape is rejected before any git op', async () => {
const outsideDir = mkdtempSync(join(tmpdir(), 'gbrain-escape-'));
try {
const { performSync } = await import('../src/commands/sync.ts');
await expect(
performSync(engine, {
repoPath,
srcSubpath: '../' + outsideDir.split('/').pop(),
noPull: true,
noEmbed: true,
full: true,
}),
).rejects.toThrow(/outside git repo|does not exist/i);
} finally {
rmSync(outsideDir, { recursive: true, force: true });
}
});
test('path-traversal: symlink subdir pointing outside repo is rejected (NAV-1 TOCTOU)', async () => {
const outsideDir = mkdtempSync(join(tmpdir(), 'gbrain-sym-target-'));
writeFileSync(join(outsideDir, 'secret.md'), mdPage('Secret'));
const symlinkPath = join(repoPath, 'symlink-escape');
try {
symlinkSync(outsideDir, symlinkPath);
const { performSync } = await import('../src/commands/sync.ts');
await expect(
performSync(engine, {
repoPath,
srcSubpath: 'symlink-escape',
noPull: true,
noEmbed: true,
full: true,
}),
).rejects.toThrow(/outside git repo/i);
} finally {
rmSync(outsideDir, { recursive: true, force: true });
}
});
test('path-traversal: absolute --src-subpath outside the repo is rejected', async () => {
const outsideDir = mkdtempSync(join(tmpdir(), 'gbrain-abs-escape-'));
writeFileSync(join(outsideDir, 'secret.md'), mdPage('Secret'));
try {
const { performSync } = await import('../src/commands/sync.ts');
// path.join(repoPath, '/abs/path') keeps the traversal relative, but a
// crafted subpath can still resolve outside via ..-segments; both are
// caught by the same realpath containment check.
await expect(
performSync(engine, {
repoPath,
srcSubpath: join('..', '..', outsideDir.slice(1)),
noPull: true,
noEmbed: true,
full: true,
}),
).rejects.toThrow(/outside git repo|does not exist/i);
} finally {
rmSync(outsideDir, { recursive: true, force: true });
}
});
// ─────────────────────────────────────────────────────────────────────────
// --exclude: repeatable glob pattern flag
// ─────────────────────────────────────────────────────────────────────────
test('--exclude: single pattern excludes matching files from full sync', async () => {
const { performSync } = await import('../src/commands/sync.ts');
// Sync wiki/ but exclude page2.md (patterns are scope-relative)
const result = await performSync(engine, {
repoPath,
srcSubpath: 'wiki',
exclude: ['page2.md'],
noPull: true,
noEmbed: true,
full: true,
});
expect(result.status).toBe('first_sync');
expect(result.added).toBe(1); // only page1 (page2 excluded)
expect(await engine.getPage('wiki/page1')).not.toBeNull();
expect(await engine.getPage('wiki/page2')).toBeNull();
});
test('--exclude: glob pattern with wildcard', async () => {
const { performSync } = await import('../src/commands/sync.ts');
// Exclude all files matching *2.md
const result = await performSync(engine, {
repoPath,
srcSubpath: 'wiki',
exclude: ['*2.md'],
noPull: true,
noEmbed: true,
full: true,
});
expect(result.status).toBe('first_sync');
expect(result.added).toBe(1); // only page1 (page2 excluded by *2.md)
});
test('--exclude applies to the incremental path too', async () => {
const { performSync } = await import('../src/commands/sync.ts');
const first = await performSync(engine, {
repoPath,
srcSubpath: 'wiki',
noPull: true,
noEmbed: true,
full: true,
});
expect(first.status).toBe('first_sync');
writeFileSync(join(repoPath, 'wiki', 'draft-a.md'), mdPage('Draft A'));
writeFileSync(join(repoPath, 'wiki', 'page3.md'), mdPage('Wiki Page 3'));
gitCommit(repoPath, 'drafts');
const second = await performSync(engine, {
repoPath,
srcSubpath: 'wiki',
exclude: ['draft-*.md'],
noPull: true,
noEmbed: true,
});
expect(second.status).toBe('synced');
expect(second.added).toBe(1); // page3 only; draft-a excluded
expect(await engine.getPage('wiki/page3')).not.toBeNull();
expect(await engine.getPage('wiki/draft-a')).toBeNull();
});
// ─────────────────────────────────────────────────────────────────────────
// --exclude '**/*' emits warning (NAV-4)
// ─────────────────────────────────────────────────────────────────────────
test('--exclude **/* emits warning when all files are excluded (NAV-4)', async () => {
const { performSync } = await import('../src/commands/sync.ts');
const warnMessages: string[] = [];
const origWarn = console.warn;
console.warn = (...args: unknown[]) => {
warnMessages.push(args.join(' '));
origWarn(...args);
};
try {
await performSync(engine, {
repoPath,
srcSubpath: 'wiki',
exclude: ['**/*'],
noPull: true,
noEmbed: true,
full: true,
});
} finally {
console.warn = origWarn;
}
const hasExcludeWarn = warnMessages.some(m => m.includes('--exclude') || m.includes('No files matched'));
expect(hasExcludeWarn).toBe(true);
});
});