mirror of
https://github.com/garrytan/gbrain.git
synced 2026-07-27 22:15:33 +00:00
feat(sync): --src-subpath + --exclude for monorepo subdir-source support (#753)
Rebased port of PR #774 onto current master. A single git repo can hold N logical sources at subdirectories: git operations run at the discovered repo root (git rev-parse --show-toplevel — worktrees and submodules resolve natively) while file walking, imports, deletes and renames are scoped to the subpath. Passing the subdirectory directly as the repo path (auto-discovery) works through the same code path. Path-containment guards (the point of the feature): - NAV-1/NAV-2: the realpath-resolved scope must live inside the realpath-resolved git root — ../-traversal and symlinked subdirs pointing outside the repo are rejected before any git op. - NAV-1 TOCTOU: per-file realpath re-validation during the incremental import drain and rename reimport; symlink-escape files are recorded as failures (fail-closed — the bookmark cannot advance past an escape). - NAV-4: an --exclude set that filters out every candidate warns loudly. Scoped syncs use git-root-relative slugs + source_path in BOTH the full and incremental paths (runImport gains slugRoot), fixing the original PR's full/incremental slug divergence in the auto-discovery flow. --exclude matches scope-relative paths in both paths; exclusion never deletes previously-imported pages. The full-sync delete reconcile is scope-restricted and relativizes against the slug base so a healthy scoped source can't trip the #2828 mass-delete valve. .gitignore management resolves to the git root at every call site. Preserves all master-side sync work since the original branch: the #2828 mass-delete safety valve, #1794 resumable checkpoints + pinned targets, #1950 stall watchdog, #2335 heartbeat bump, #1970 bookmark reachability, and the git-ls-files walker (#2315/#2462/#2678, whose symlink/cycle hardening is untouched). Co-authored-by: Jeremy Knows <jeremy@veefriends.com> Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Jeremy Knows
Claude Fable 5
parent
e1cefd0654
commit
14d20048c3
File diff suppressed because one or more lines are too long
+50
-7
@@ -11,6 +11,7 @@ import {
|
||||
isCodeFilePath,
|
||||
isMarkdownFilePath,
|
||||
isImageFilePath as isImageFilePathFromSync,
|
||||
matchesAnyGlob,
|
||||
pruneDir,
|
||||
SYNC_SKIP_FILES,
|
||||
type SyncStrategy,
|
||||
@@ -47,7 +48,25 @@ export interface RunImportResult {
|
||||
export async function runImport(
|
||||
engine: BrainEngine,
|
||||
args: string[],
|
||||
opts: { commit?: string; strategy?: SyncStrategy; sourceId?: string; managedBookmark?: boolean } = {},
|
||||
opts: {
|
||||
commit?: string;
|
||||
strategy?: SyncStrategy;
|
||||
sourceId?: string;
|
||||
managedBookmark?: boolean;
|
||||
/**
|
||||
* #753/#774: glob patterns to exclude from the import (same semantics as
|
||||
* `isSyncable`'s `exclude` — matched against the dir-relative path).
|
||||
* Threaded by performFullSync for `gbrain sync --exclude`.
|
||||
*/
|
||||
exclude?: string[];
|
||||
/**
|
||||
* #753/#774 monorepo subdir-source support: when set, slugs and
|
||||
* `source_path` are computed relative to this root (the git repo root)
|
||||
* instead of `dir` (the sync scope), so `wiki/page1.md` lands as slug
|
||||
* `wiki/page1` consistently across full and incremental sync.
|
||||
*/
|
||||
slugRoot?: string;
|
||||
} = {},
|
||||
): Promise<RunImportResult> {
|
||||
const noEmbed = args.includes('--no-embed');
|
||||
const fresh = args.includes('--fresh');
|
||||
@@ -190,13 +209,30 @@ export async function runImport(
|
||||
const strategy: SyncStrategy = opts.strategy ?? 'markdown';
|
||||
const _walkT0 = Date.now();
|
||||
console.error(`[gbrain phase] import.collect_files start dir=${dir} strategy=${strategy}`);
|
||||
const allFiles = collectSyncableFiles(dir, { strategy });
|
||||
let allFiles = collectSyncableFiles(dir, { strategy });
|
||||
console.error(
|
||||
`[gbrain phase] import.collect_files done ${Date.now() - _walkT0}ms files=${allFiles.length}`,
|
||||
);
|
||||
const fileTypeLabel = strategy === 'code' ? 'code'
|
||||
: strategy === 'auto' ? 'syncable' : 'markdown';
|
||||
console.log(`Found ${allFiles.length} ${fileTypeLabel} files`);
|
||||
// #753/#774: apply --exclude glob patterns (threaded by performFullSync).
|
||||
if (opts.exclude && opts.exclude.length > 0) {
|
||||
const beforeExclude = allFiles.length;
|
||||
allFiles = allFiles.filter(abs => !matchesAnyGlob(relative(dir, abs), opts.exclude));
|
||||
console.log(
|
||||
`Found ${allFiles.length} ${fileTypeLabel} files ` +
|
||||
`(${beforeExclude - allFiles.length} excluded by --exclude patterns)`,
|
||||
);
|
||||
// NAV-4: everything excluded is almost always a mistyped pattern — warn.
|
||||
if (beforeExclude > 0 && allFiles.length === 0) {
|
||||
console.warn(
|
||||
`[gbrain sync] No files matched after applying ${opts.exclude.length} --exclude pattern(s). ` +
|
||||
`Check your --exclude flags. Patterns: ${JSON.stringify(opts.exclude)}`,
|
||||
);
|
||||
}
|
||||
} else {
|
||||
console.log(`Found ${allFiles.length} ${fileTypeLabel} files`);
|
||||
}
|
||||
|
||||
// Sort newest-first so date-prefixed brain paths get embedded before older ones.
|
||||
// See src/core/sort-newest-first.ts for the policy.
|
||||
@@ -242,6 +278,11 @@ export async function runImport(
|
||||
|
||||
async function processFile(eng: BrainEngine, filePath: string) {
|
||||
const relativePath = relative(dir, filePath);
|
||||
// #753/#774: slug + source_path base. When performFullSync syncs a
|
||||
// monorepo subdir, slugRoot is the git root so slugs stay git-root-
|
||||
// relative (matching the incremental path's git-diff paths). The
|
||||
// checkpoint (`completed`) stays dir-relative — resumeFilter's contract.
|
||||
const importRelPath = opts.slugRoot ? relative(opts.slugRoot, filePath) : relativePath;
|
||||
// v0.31.2 (D5): per-file slow-path log. Fires only when a single
|
||||
// file takes >5s. The user's hang surfaces as one file taking
|
||||
// forever — without this, the agent can't see which file.
|
||||
@@ -252,8 +293,8 @@ export async function runImport(
|
||||
// up images when GBRAIN_EMBEDDING_MULTIMODAL=true so this branch is
|
||||
// unreachable when the gate is off; defense-in-depth check anyway.
|
||||
const result = isImageFilePath(relativePath) && process.env.GBRAIN_EMBEDDING_MULTIMODAL === 'true'
|
||||
? await importImageFile(eng, filePath, relativePath, { noEmbed, sourceId })
|
||||
: await importFile(eng, filePath, relativePath, { noEmbed, sourceId, activePack: importActivePack });
|
||||
? await importImageFile(eng, filePath, importRelPath, { noEmbed, sourceId })
|
||||
: await importFile(eng, filePath, importRelPath, { noEmbed, sourceId, activePack: importActivePack });
|
||||
const _fileMs = Date.now() - _fileT0;
|
||||
if (_fileMs > 5000) {
|
||||
console.error(`[gbrain phase] import.process_file slow ${_fileMs}ms ${relativePath}`);
|
||||
@@ -269,7 +310,9 @@ export async function runImport(
|
||||
if (result.error && result.error !== 'unchanged') {
|
||||
console.error(` Skipped ${relativePath}: ${result.error}`);
|
||||
// Bug 9 — non-"unchanged" skips carry a real error reason.
|
||||
failures.push({ path: relativePath, error: result.error });
|
||||
// #774: ledger paths use the slug base so an incremental sync's
|
||||
// success at the same (git-root-relative) path clears the row.
|
||||
failures.push({ path: importRelPath, error: result.error });
|
||||
} else {
|
||||
// 'unchanged' or no-error skip: content_hash matched a prior
|
||||
// successful import, so this file IS done for checkpoint purposes.
|
||||
@@ -287,7 +330,7 @@ export async function runImport(
|
||||
}
|
||||
errors++;
|
||||
skipped++;
|
||||
failures.push({ path: relativePath, error: msg });
|
||||
failures.push({ path: importRelPath, error: msg });
|
||||
}
|
||||
processed++;
|
||||
tickProgress();
|
||||
|
||||
+240
-52
@@ -1,4 +1,4 @@
|
||||
import { existsSync, readFileSync, writeFileSync, statSync } from 'fs';
|
||||
import { existsSync, readFileSync, writeFileSync, statSync, realpathSync } from 'fs';
|
||||
import { execFileSync } from 'child_process';
|
||||
import { join, relative } from 'path';
|
||||
import type { BrainEngine } from '../core/engine.ts';
|
||||
@@ -9,6 +9,7 @@ import { createInterface } from 'readline';
|
||||
import {
|
||||
isSyncable,
|
||||
unsyncableReason,
|
||||
matchesAnyGlob,
|
||||
resolveSlugForPath,
|
||||
unacknowledgedSyncFailures,
|
||||
acknowledgeFailures,
|
||||
@@ -742,6 +743,27 @@ export interface SyncOpts {
|
||||
sourceId?: string;
|
||||
/** Multi-repo: sync strategy override (markdown, code, auto). */
|
||||
strategy?: 'markdown' | 'code' | 'auto';
|
||||
/**
|
||||
* #753/#774 — sync only files under this subdirectory of the git repo.
|
||||
* Git operations (pull, diff, rev-parse) still run against the repo root
|
||||
* (discovered via `git rev-parse --show-toplevel`); file walking, imports,
|
||||
* deletes and renames are scoped to the subpath. Slugs are git-root-relative
|
||||
* (`wiki/page1.md` → slug `wiki/page1`) so full and incremental syncs of
|
||||
* the same scope agree. Enables N logical sources in one git repo.
|
||||
*
|
||||
* SECURITY (NAV-1/NAV-2): the resolved subpath must realpath-resolve inside
|
||||
* the git root — `../escape` and symlinked subdirs pointing outside the repo
|
||||
* are rejected before any git op runs.
|
||||
*/
|
||||
srcSubpath?: string;
|
||||
/**
|
||||
* #753/#774 — glob patterns for files to exclude from sync (repeatable
|
||||
* `--exclude` on the CLI). Matched against the scope-relative path in both
|
||||
* the full-sync and incremental paths. Excluded files are never imported;
|
||||
* exclusion does NOT delete previously-imported pages (conservative,
|
||||
* matching the #1433 metafile posture).
|
||||
*/
|
||||
exclude?: string[];
|
||||
/**
|
||||
* Number of parallel workers for the import phase. When > 1, each worker
|
||||
* gets its own small Postgres connection pool and files are dispatched via
|
||||
@@ -905,6 +927,37 @@ function git(repoPath: string, args: string[], configs: string[] = []): string {
|
||||
}).trim();
|
||||
}
|
||||
|
||||
/**
|
||||
* #753/#774: walk up from inputPath to the nearest git repo root via
|
||||
* `git -C <path> rev-parse --show-toplevel`. Handles worktrees and submodules
|
||||
* natively (git itself resolves them). Throws a user-friendly error when no
|
||||
* git repo is found.
|
||||
*/
|
||||
export function discoverGitRoot(inputPath: string): string {
|
||||
try {
|
||||
return git(inputPath, ['rev-parse', '--show-toplevel']);
|
||||
} catch {
|
||||
throw new Error(
|
||||
`Not inside a git repository: ${inputPath}. GBrain sync requires a git-initialized repo (or a subdirectory of one).`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* #774 NAV-1 TOCTOU: true only if filePath realpath-resolves inside gitRoot.
|
||||
* Guards symlink escape at the per-file level (a committed symlink whose
|
||||
* target lives outside the repo), not just at scope entry.
|
||||
*/
|
||||
function isPathSafe(filePath: string, gitRoot: string): boolean {
|
||||
try {
|
||||
const real = realpathSync(filePath);
|
||||
const rootReal = realpathSync(gitRoot);
|
||||
return real === rootReal || real.startsWith(rootReal + '/');
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
function hasOriginRemote(repoPath: string): boolean {
|
||||
try {
|
||||
execFileSync('git', buildGitInvocation(repoPath, ['remote', 'get-url', 'origin']), {
|
||||
@@ -1567,17 +1620,46 @@ async function performSyncInner(engine: BrainEngine, opts: SyncOpts): Promise<Sy
|
||||
}
|
||||
}
|
||||
|
||||
// Validate git repo
|
||||
if (!existsSync(join(repoPath, '.git'))) {
|
||||
throw new Error(`Not a git repository: ${repoPath}. GBrain sync requires a git-initialized repo.`);
|
||||
// #753/#774: discover the git root instead of requiring `.git` at repoPath
|
||||
// directly. Supports subdir-of-git-repo sources (monorepo pattern): either
|
||||
// an explicit `--src-subpath` under a git-root repoPath, or a repoPath that
|
||||
// IS a subdirectory (auto-discovery). Two axes fall out:
|
||||
// - gitContextRoot: ALL git operations (pull, rev-parse, diff, cat-file)
|
||||
// - syncScopeRoot: file walking, imports, deletes, renames
|
||||
// In the common case (repoPath == git root, no subpath) they are identical.
|
||||
serr(`[gbrain phase] sync.discover_git_root`);
|
||||
const gitContextRoot = realpathSync(discoverGitRoot(repoPath));
|
||||
const rawScopeRoot = opts.srcSubpath ? join(repoPath, opts.srcSubpath) : repoPath;
|
||||
if (!existsSync(rawScopeRoot)) {
|
||||
throw new Error(`Sync scope does not exist: ${rawScopeRoot}`);
|
||||
}
|
||||
const syncScopeRoot = realpathSync(rawScopeRoot);
|
||||
// NAV-1/NAV-2 scope-entry guard: the realpath-resolved scope must live
|
||||
// inside the realpath-resolved git root. Catches `--src-subpath ../escape`
|
||||
// AND a symlinked subdir pointing outside the repo, before any git op runs.
|
||||
if (syncScopeRoot !== gitContextRoot && !syncScopeRoot.startsWith(gitContextRoot + '/')) {
|
||||
throw new Error(
|
||||
`Sync scope ${syncScopeRoot} resolves outside git repo ${gitContextRoot}. ` +
|
||||
`Refusing to sync: possible path traversal via --src-subpath.`,
|
||||
);
|
||||
}
|
||||
// Relative path from git root to sync scope ('' when scope == root).
|
||||
const syncScopeRelPath = syncScopeRoot === gitContextRoot ? '' : relative(gitContextRoot, syncScopeRoot);
|
||||
const scoped = syncScopeRelPath !== '';
|
||||
// Anchor written back to sync state (sources.local_path / sync.repo_path):
|
||||
// the SCOPE path, so a follow-up bare `gbrain sync` auto-discovers the same
|
||||
// scope. Unchanged (the caller's repoPath spelling) when no --src-subpath.
|
||||
const anchorPath = opts.srcSubpath ? rawScopeRoot : repoPath;
|
||||
const fullSyncRoots = { gitContextRoot, syncScopeRoot, anchorPath };
|
||||
|
||||
serr(`[gbrain phase] sync.detect_head`);
|
||||
// Detect detached HEAD up front so the working-tree fallback fires for both
|
||||
// the default sync and `--no-pull` callers. Only the actual git pull is
|
||||
// gated on opts.noPull.
|
||||
const detachedHead = isDetachedHead(repoPath);
|
||||
const detachedHead = isDetachedHead(gitContextRoot);
|
||||
if (detachedHead && !opts.noPull) {
|
||||
// Print the caller's repoPath spelling (not the realpathed git root) —
|
||||
// it's what the operator recognizes, and tests pin it.
|
||||
serr(`Detached HEAD on ${repoPath}; skipping git pull. Syncing from local working tree.`);
|
||||
}
|
||||
|
||||
@@ -1587,7 +1669,7 @@ async function performSyncInner(engine: BrainEngine, opts: SyncOpts): Promise<Sy
|
||||
// hardening that cloneRepo applies. Route through pullRepo from
|
||||
// git-remote.ts so the flag set is consistent across initial clone and
|
||||
// ongoing pulls — single source of truth for the defensive flags.
|
||||
const originRemotePresent = !opts.noPull && !detachedHead ? hasOriginRemote(repoPath) : false;
|
||||
const originRemotePresent = !opts.noPull && !detachedHead ? hasOriginRemote(gitContextRoot) : false;
|
||||
if (!opts.noPull && !detachedHead && !originRemotePresent) {
|
||||
serr(`No origin remote on ${repoPath}; skipping git pull. Syncing from local working tree.`);
|
||||
}
|
||||
@@ -1626,8 +1708,9 @@ async function performSyncInner(engine: BrainEngine, opts: SyncOpts): Promise<Sy
|
||||
// We pass a safe default (the operator's full --timeout if set, else
|
||||
// pullRepo's own 300s default). The catch below distinguishes
|
||||
// timeout (ETIMEDOUT / SIGTERM on err.cause) from ordinary pull
|
||||
// failure.
|
||||
pullRepo(repoPath);
|
||||
// failure. Pull applies to the whole git repo (gitContextRoot), not
|
||||
// just the sync scope — git has no per-subdir pull.
|
||||
pullRepo(gitContextRoot);
|
||||
serr(`[gbrain phase] sync.git_pull done ${Date.now() - _t0}ms`);
|
||||
} catch (e: unknown) {
|
||||
const msg = e instanceof Error ? e.message : String(e);
|
||||
@@ -1668,7 +1751,7 @@ async function performSyncInner(engine: BrainEngine, opts: SyncOpts): Promise<Sy
|
||||
// Get current HEAD
|
||||
let headCommit: string;
|
||||
try {
|
||||
headCommit = git(repoPath, ['rev-parse', 'HEAD']);
|
||||
headCommit = git(gitContextRoot, ['rev-parse', 'HEAD']);
|
||||
} catch {
|
||||
throw new Error(`No commits in repo ${repoPath}. Make at least one commit before syncing.`);
|
||||
}
|
||||
@@ -1690,7 +1773,7 @@ async function performSyncInner(engine: BrainEngine, opts: SyncOpts): Promise<Sy
|
||||
if (lastCommit) {
|
||||
let objectPresent = true;
|
||||
try {
|
||||
git(repoPath, ['cat-file', '-t', lastCommit]);
|
||||
git(gitContextRoot, ['cat-file', '-t', lastCommit]);
|
||||
} catch {
|
||||
objectPresent = false;
|
||||
}
|
||||
@@ -1699,7 +1782,7 @@ async function performSyncInner(engine: BrainEngine, opts: SyncOpts): Promise<Sy
|
||||
// back to the authoritative full reconcile (which now also purges stale
|
||||
// pages for deleted files; see performFullSync's delete-reconcile pass).
|
||||
serr(`Sync anchor ${lastCommit.slice(0, 8)} object missing (gc'd after history rewrite). Running full reimport.`);
|
||||
return performFullSync(engine, repoPath, headCommit, opts);
|
||||
return performFullSync(engine, fullSyncRoots, headCommit, opts);
|
||||
}
|
||||
|
||||
// Observability only — NOT control flow. A non-ancestor bookmark is still
|
||||
@@ -1707,7 +1790,7 @@ async function performSyncInner(engine: BrainEngine, opts: SyncOpts): Promise<Sy
|
||||
// failure mode (#1970) is visible in the logs.
|
||||
let isAncestor = true;
|
||||
try {
|
||||
git(repoPath, ['merge-base', '--is-ancestor', lastCommit, headCommit]);
|
||||
git(gitContextRoot, ['merge-base', '--is-ancestor', lastCommit, headCommit]);
|
||||
} catch {
|
||||
isAncestor = false;
|
||||
}
|
||||
@@ -1722,7 +1805,7 @@ async function performSyncInner(engine: BrainEngine, opts: SyncOpts): Promise<Sy
|
||||
|
||||
// First sync
|
||||
if (!lastCommit) {
|
||||
return performFullSync(engine, repoPath, headCommit, opts);
|
||||
return performFullSync(engine, fullSyncRoots, headCommit, opts);
|
||||
}
|
||||
|
||||
// v0.42.x (#1794): resumable incremental sync — resolve the PINNED target.
|
||||
@@ -1744,7 +1827,7 @@ async function performSyncInner(engine: BrainEngine, opts: SyncOpts): Promise<Sy
|
||||
if (storedTarget) {
|
||||
let pinReachable = false;
|
||||
try {
|
||||
git(repoPath, ['merge-base', '--is-ancestor', storedTarget, headCommit]);
|
||||
git(gitContextRoot, ['merge-base', '--is-ancestor', storedTarget, headCommit]);
|
||||
pinReachable = true;
|
||||
} catch {
|
||||
pinReachable = false;
|
||||
@@ -1778,7 +1861,7 @@ async function performSyncInner(engine: BrainEngine, opts: SyncOpts): Promise<Sy
|
||||
const currentVersion = String(CHUNKER_VERSION);
|
||||
const versionMismatch = storedVersion !== null && storedVersion !== currentVersion;
|
||||
const versionNeverSet = storedVersion === null && opts.sourceId !== undefined;
|
||||
const detachedWorkingTreeManifest = detachedHead ? buildDetachedWorkingTreeManifest(repoPath) : null;
|
||||
const detachedWorkingTreeManifest = detachedHead ? buildDetachedWorkingTreeManifest(gitContextRoot) : null;
|
||||
const hasDetachedWorkingTreeChanges = detachedWorkingTreeManifest !== null &&
|
||||
(detachedWorkingTreeManifest.added.length > 0 ||
|
||||
detachedWorkingTreeManifest.modified.length > 0 ||
|
||||
@@ -1814,7 +1897,7 @@ async function performSyncInner(engine: BrainEngine, opts: SyncOpts): Promise<Sy
|
||||
`[sync] chunker_version gate: stored=${storedVersion ?? 'unset'}, current=${currentVersion}. ` +
|
||||
`Forcing full re-chunk pass (git HEAD unchanged but pipeline version advanced).`,
|
||||
);
|
||||
const result = await performFullSync(engine, repoPath, headCommit, opts);
|
||||
const result = await performFullSync(engine, fullSyncRoots, headCommit, opts);
|
||||
await writeChunkerVersion(engine, opts.sourceId, currentVersion);
|
||||
return result;
|
||||
}
|
||||
@@ -1835,7 +1918,7 @@ async function performSyncInner(engine: BrainEngine, opts: SyncOpts): Promise<Sy
|
||||
// buffer, and a gc'd anchor object can't be diffed at all. On either
|
||||
// `unavailable`, fall back to the authoritative full reconcile instead of
|
||||
// throwing — a slow correct reconcile beats a hard error or a silent walk.
|
||||
const delta = computeSyncDelta(repoPath, lastCommit, pin, {
|
||||
const delta = computeSyncDelta(gitContextRoot, lastCommit, pin, {
|
||||
detachedManifest: detachedWorkingTreeManifest,
|
||||
});
|
||||
if (delta.status === 'unavailable') {
|
||||
@@ -1843,30 +1926,60 @@ async function performSyncInner(engine: BrainEngine, opts: SyncOpts): Promise<Sy
|
||||
`[sync] delta ${lastCommit.slice(0, 8)}..${pin.slice(0, 8)} unavailable ` +
|
||||
`(${delta.reason}) — falling back to full reconcile.`,
|
||||
);
|
||||
return performFullSync(engine, repoPath, headCommit, opts);
|
||||
return performFullSync(engine, fullSyncRoots, headCommit, opts);
|
||||
}
|
||||
const manifest = delta.manifest;
|
||||
|
||||
// Filter to syncable files (strategy-aware)
|
||||
// #753/#774 scope filter: git-diff paths are git-root-relative; when a
|
||||
// subpath scope is active, only paths under it participate. Back-compat:
|
||||
// syncScopeRelPath is '' when scope == root, so inScope is always true and
|
||||
// the filters below reduce to the pre-#774 behavior exactly.
|
||||
const inScope = (p: string): boolean =>
|
||||
!scoped || p === syncScopeRelPath || p.startsWith(syncScopeRelPath + '/');
|
||||
// --exclude patterns match the SCOPE-relative path (what the user of a
|
||||
// scoped source thinks in), same form runImport matches on full sync.
|
||||
const scopeRel = (p: string): string =>
|
||||
scoped && p.startsWith(syncScopeRelPath + '/') ? p.slice(syncScopeRelPath.length + 1) : p;
|
||||
const excluded = (p: string): boolean =>
|
||||
opts.exclude !== undefined && opts.exclude.length > 0 && matchesAnyGlob(scopeRel(p), opts.exclude);
|
||||
|
||||
// Filter to syncable files (strategy-aware + scope-aware + exclude-aware)
|
||||
const syncOpts = opts.strategy ? { strategy: opts.strategy } : undefined;
|
||||
// #1970 (F-C): a rename whose DESTINATION is unsyncable drops out of BOTH
|
||||
// `renamed` (only `r.to` is kept below) AND `deleted` (git emits it as `R`,
|
||||
// not `D`), leaving the OLD page stale. Fold the source side into the delete
|
||||
// set. isSyncable(r.from) excludes metafiles automatically, so a rename of a
|
||||
// metafile is left untouched (matching the #1433 metafile-skip invariant).
|
||||
// #774: a rename whose destination LEFT the scope is the same class — the
|
||||
// old page's backing file is gone from this source's slice of the repo.
|
||||
const renamedToUnsyncable = manifest.renamed
|
||||
.filter(r => isSyncable(r.from, syncOpts) && !isSyncable(r.to, syncOpts))
|
||||
.filter(r => inScope(r.from) && isSyncable(r.from, syncOpts) &&
|
||||
!(inScope(r.to) && isSyncable(r.to, syncOpts)))
|
||||
.map(r => r.from);
|
||||
const filtered: SyncManifest = {
|
||||
added: manifest.added.filter(p => isSyncable(p, syncOpts)),
|
||||
modified: manifest.modified.filter(p => isSyncable(p, syncOpts)),
|
||||
added: manifest.added.filter(p => inScope(p) && !excluded(p) && isSyncable(p, syncOpts)),
|
||||
modified: manifest.modified.filter(p => inScope(p) && !excluded(p) && isSyncable(p, syncOpts)),
|
||||
deleted: unique([
|
||||
...manifest.deleted.filter(p => isSyncable(p, syncOpts)),
|
||||
...manifest.deleted.filter(p => inScope(p) && isSyncable(p, syncOpts)),
|
||||
...renamedToUnsyncable,
|
||||
]),
|
||||
renamed: manifest.renamed.filter(r => isSyncable(r.to, syncOpts)),
|
||||
renamed: manifest.renamed.filter(r => inScope(r.to) && !excluded(r.to) && isSyncable(r.to, syncOpts)),
|
||||
};
|
||||
|
||||
// NAV-4: warn when --exclude filtered out every candidate change — almost
|
||||
// always a mistyped pattern, and otherwise indistinguishable from
|
||||
// "up to date" in the output.
|
||||
if (opts.exclude && opts.exclude.length > 0) {
|
||||
const excludeCandidates = [...manifest.added, ...manifest.modified]
|
||||
.filter(p => inScope(p) && isSyncable(p, syncOpts));
|
||||
if (excludeCandidates.length > 0 && excludeCandidates.every(excluded)) {
|
||||
console.warn(
|
||||
`[gbrain sync] No files matched after applying ${opts.exclude.length} --exclude pattern(s). ` +
|
||||
`Check your --exclude flags. Patterns: ${JSON.stringify(opts.exclude)}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// Delete pages that became un-syncable (modified but filtered out).
|
||||
// v0.20.0 Cathedral II SP-5: resolveSlugForPath picks the right slug shape
|
||||
// (markdown vs code) based on the chunker's classifier, so a Rust file that
|
||||
@@ -1890,7 +2003,7 @@ async function performSyncInner(engine: BrainEngine, opts: SyncOpts): Promise<Sy
|
||||
// delete the page. That's the same pre-fix behavior — removing the
|
||||
// page requires `gbrain pages purge-deleted` or a direct MCP delete.
|
||||
// Filed as v0.42+ follow-up for a `gbrain pages remove <slug>` surface.
|
||||
const unsyncableModified = manifest.modified.filter(p => !isSyncable(p, syncOpts));
|
||||
const unsyncableModified = manifest.modified.filter(p => inScope(p) && !isSyncable(p, syncOpts));
|
||||
// v0.18.0+ multi-source: scope getPage + deletePage to opts.sourceId so
|
||||
// unsyncable cleanup in source A doesn't accidentally sweep same-slug
|
||||
// pages in sources B/C/D.
|
||||
@@ -1938,7 +2051,7 @@ async function performSyncInner(engine: BrainEngine, opts: SyncOpts): Promise<Sy
|
||||
// (#1794): advance to the PINNED target, and clear any checkpoint (a resume
|
||||
// whose remaining range turned out to have no syncable changes still
|
||||
// completes cleanly here).
|
||||
await writeSyncAnchor(engine, opts.sourceId, 'last_commit', pin, commitTimeMs(repoPath, pin));
|
||||
await writeSyncAnchor(engine, opts.sourceId, 'last_commit', pin, commitTimeMs(gitContextRoot, pin));
|
||||
await engine.setConfig('sync.last_run', new Date().toISOString());
|
||||
await writeChunkerVersion(engine, opts.sourceId, String(CHUNKER_VERSION));
|
||||
await clearOpCheckpoint(engine, ckpt.paths);
|
||||
@@ -2325,8 +2438,11 @@ async function performSyncInner(engine: BrainEngine, opts: SyncOpts): Promise<Sy
|
||||
// throw here crashes the whole sync mid-run and freezes the checkpoint,
|
||||
// defeating --skip-failed. A `skipped` result carrying an error is also
|
||||
// captured so the failure is recorded rather than silently dropped.
|
||||
const filePath = join(repoPath, to);
|
||||
if (existsSync(filePath)) {
|
||||
// Paths from git diff are relative to gitContextRoot; join from there.
|
||||
// NAV-1 TOCTOU: refuse a destination that realpath-resolves outside the
|
||||
// repo (committed symlink pointing out).
|
||||
const filePath = join(gitContextRoot, to);
|
||||
if (existsSync(filePath) && isPathSafe(filePath, gitContextRoot)) {
|
||||
try {
|
||||
const result = await importFile(engine, filePath, to, { noEmbed, sourceId: opts.sourceId, activePack: syncActivePack });
|
||||
if (result.status === 'imported') chunksCreated += result.chunks;
|
||||
@@ -2411,8 +2527,8 @@ async function performSyncInner(engine: BrainEngine, opts: SyncOpts): Promise<Sy
|
||||
progress.start('sync.imports', importsToDo.length);
|
||||
|
||||
// Core import logic shared by serial and parallel paths.
|
||||
// repoPath is validated non-null at the top of performSyncInner; narrow for TS.
|
||||
const syncRepoPath = repoPath!;
|
||||
// Paths from git diff are relative to gitContextRoot; join from there.
|
||||
const syncRepoPath = gitContextRoot;
|
||||
// paced-backfill (T3 / C9 / CX4): ONE shared pacer across all worker
|
||||
// engines. This is the multi-pool permit case — each parallel worker owns a
|
||||
// separate PostgresEngine, so a single worker count can't bound TOTAL
|
||||
@@ -2500,6 +2616,16 @@ async function performSyncInner(engine: BrainEngine, opts: SyncOpts): Promise<Sy
|
||||
progress.tick(1, `skip:${path}`);
|
||||
return;
|
||||
}
|
||||
// #774 NAV-1 TOCTOU: re-validate the file's realpath at import time so a
|
||||
// committed symlink pointing outside the repo (or one swapped in after
|
||||
// the scope-entry check) is never read. Recorded as a failure —
|
||||
// fail-closed: the bookmark won't advance past a symlink escape.
|
||||
if (!isPathSafe(filePath, gitContextRoot)) {
|
||||
failedFiles.push({ path, error: 'path resolves outside git repo (symlink escape)' });
|
||||
progressAt.last = Date.now();
|
||||
progress.tick(1, `skip:${path}`);
|
||||
return;
|
||||
}
|
||||
// v0.41.37.0 #1569: per-file BEGIN heartbeat, emitted BEFORE importFile so a
|
||||
// hang names the stalling file (the progress.tick below only fires AFTER
|
||||
// importFile returns — useless when one file wedges). Off by default
|
||||
@@ -2703,11 +2829,11 @@ async function performSyncInner(engine: BrainEngine, opts: SyncOpts): Promise<Sy
|
||||
// - pin NOT an ancestor of HEAD (history REWRITE / reset / force-push) →
|
||||
// the tree we imported against is gone. Block; do not advance.
|
||||
try {
|
||||
const currentHead = git(repoPath, ['rev-parse', 'HEAD']);
|
||||
const currentHead = git(gitContextRoot, ['rev-parse', 'HEAD']);
|
||||
if (currentHead !== pin) {
|
||||
let pinStillReachable = false;
|
||||
try {
|
||||
git(repoPath, ['merge-base', '--is-ancestor', pin, currentHead]);
|
||||
git(gitContextRoot, ['merge-base', '--is-ancestor', pin, currentHead]);
|
||||
pinStillReachable = true;
|
||||
} catch {
|
||||
pinStillReachable = false;
|
||||
@@ -2748,9 +2874,9 @@ async function performSyncInner(engine: BrainEngine, opts: SyncOpts): Promise<Sy
|
||||
// "fresh". The checkpoint rows clear here — CONVERGENCE CONTRACT: sync
|
||||
// convergence == IMPORT convergence; downstream extract/facts/embed is
|
||||
// decoupled (its own resumable stale sweeps).
|
||||
await writeSyncAnchor(engine, opts.sourceId, 'last_commit', pin, commitTimeMs(repoPath, pin));
|
||||
await writeSyncAnchor(engine, opts.sourceId, 'last_commit', pin, commitTimeMs(gitContextRoot, pin));
|
||||
await engine.setConfig('sync.last_run', new Date().toISOString());
|
||||
await writeSyncAnchor(engine, opts.sourceId, 'repo_path', repoPath);
|
||||
await writeSyncAnchor(engine, opts.sourceId, 'repo_path', anchorPath);
|
||||
await writeChunkerVersion(engine, opts.sourceId, String(CHUNKER_VERSION));
|
||||
await clearOpCheckpoint(engine, ckpt.paths);
|
||||
await clearOpCheckpoint(engine, ckpt.target);
|
||||
@@ -2799,7 +2925,7 @@ async function performSyncInner(engine: BrainEngine, opts: SyncOpts): Promise<Sy
|
||||
// checkpoint is INTENTIONALLY left in place — the banked completed set lets
|
||||
// the next run skip the drained files and re-attempt only the failures.
|
||||
await engine.setConfig('sync.last_run', new Date().toISOString());
|
||||
await writeSyncAnchor(engine, opts.sourceId, 'repo_path', repoPath);
|
||||
await writeSyncAnchor(engine, opts.sourceId, 'repo_path', anchorPath);
|
||||
// v0.42.x (#1794): surface banked progress so a blocked run doesn't read as
|
||||
// total loss (last_commit is unchanged by design; the checkpoint is banked).
|
||||
serr(
|
||||
@@ -2870,8 +2996,10 @@ async function performSyncInner(engine: BrainEngine, opts: SyncOpts): Promise<Sy
|
||||
if (!opts.noExtract && totalChanges <= 100 && pagesAffected.length > 0) {
|
||||
try {
|
||||
const { extractLinksForSlugs, extractTimelineForSlugs, stampExtracted } = await import('./extract.ts');
|
||||
const linksCreated = await extractLinksForSlugs(engine, repoPath, pagesAffected, extractOpts);
|
||||
const timelineCreated = await extractTimelineForSlugs(engine, repoPath, pagesAffected, extractOpts);
|
||||
// #774: pages' source_path is git-root-relative, so extract resolves
|
||||
// files from gitContextRoot (== repoPath realpath when unscoped).
|
||||
const linksCreated = await extractLinksForSlugs(engine, gitContextRoot, pagesAffected, extractOpts);
|
||||
const timelineCreated = await extractTimelineForSlugs(engine, gitContextRoot, pagesAffected, extractOpts);
|
||||
if (linksCreated > 0 || timelineCreated > 0) {
|
||||
slog(` Extracted: ${linksCreated} links, ${timelineCreated} timeline entries`);
|
||||
}
|
||||
@@ -2976,11 +3104,21 @@ async function performSyncInner(engine: BrainEngine, opts: SyncOpts): Promise<Sy
|
||||
|
||||
async function performFullSync(
|
||||
engine: BrainEngine,
|
||||
repoPath: string,
|
||||
// #753/#774: the three roots resolved once at the top of performSyncInner.
|
||||
// gitContextRoot — git repo root (git ops, slug base for scoped syncs)
|
||||
// syncScopeRoot — where files are walked/imported (== gitContextRoot
|
||||
// when no subpath scope is active)
|
||||
// anchorPath — what gets written back to sync.repo_path/local_path
|
||||
roots: { gitContextRoot: string; syncScopeRoot: string; anchorPath: string },
|
||||
headCommit: string,
|
||||
opts: SyncOpts,
|
||||
): Promise<SyncResult> {
|
||||
// Dry-run: walk the repo, count syncable files, return without writing.
|
||||
const { gitContextRoot, syncScopeRoot, anchorPath } = roots;
|
||||
// Scoped sync → slugs/source_path are git-root-relative (matches the
|
||||
// incremental path's git-diff paths). Unscoped → undefined (dir-relative,
|
||||
// the pre-#774 behavior, byte-for-byte).
|
||||
const slugRoot = syncScopeRoot !== gitContextRoot ? gitContextRoot : undefined;
|
||||
// Dry-run: walk the scope, count syncable files, return without writing.
|
||||
// Fixes the silent-write-on-dry-run bug where performFullSync called
|
||||
// runImport unconditionally regardless of opts.dryRun.
|
||||
//
|
||||
@@ -2990,11 +3128,14 @@ async function performFullSync(
|
||||
// code --dry-run` always reported zero files even when ~1500 code
|
||||
// files were waiting.
|
||||
if (opts.dryRun) {
|
||||
const allFiles = collectSyncableFiles(repoPath, { strategy: opts.strategy ?? 'markdown' });
|
||||
let allFiles = collectSyncableFiles(syncScopeRoot, { strategy: opts.strategy ?? 'markdown' });
|
||||
if (opts.exclude && opts.exclude.length > 0) {
|
||||
allFiles = allFiles.filter(abs => !matchesAnyGlob(relative(syncScopeRoot, abs), opts.exclude));
|
||||
}
|
||||
slog(
|
||||
`Full-sync dry run (strategy=${opts.strategy ?? 'markdown'}): ` +
|
||||
`${allFiles.length} file(s) would be imported ` +
|
||||
`from ${repoPath} @ ${headCommit.slice(0, 8)}.`,
|
||||
`from ${syncScopeRoot} @ ${headCommit.slice(0, 8)}.`,
|
||||
);
|
||||
return {
|
||||
status: 'dry_run',
|
||||
@@ -3017,21 +3158,24 @@ async function performFullSync(
|
||||
// sync and the jobs handler.
|
||||
const FULL_SYNC_LARGE_MARKER = Number.MAX_SAFE_INTEGER;
|
||||
const fullConcurrency = autoConcurrency(engine, FULL_SYNC_LARGE_MARKER, opts.concurrency);
|
||||
slog(`Running full import of ${repoPath}${fullConcurrency > 1 ? ` (${fullConcurrency} workers)` : ''}...`);
|
||||
slog(`Running full import of ${syncScopeRoot}${fullConcurrency > 1 ? ` (${fullConcurrency} workers)` : ''}...`);
|
||||
const { runImport } = await import('./import.ts');
|
||||
const importArgs = [repoPath];
|
||||
const importArgs = [syncScopeRoot];
|
||||
if (opts.noEmbed) importArgs.push('--no-embed');
|
||||
if (fullConcurrency > 1) importArgs.push('--workers', String(fullConcurrency));
|
||||
// v0.31.2: thread strategy through so code-strategy first sync
|
||||
// actually enumerates code files (closes bug 1).
|
||||
// v0.30.x: thread sourceId so performFullSync routes pages to the named
|
||||
// source (incremental path already does this).
|
||||
// #753/#774: thread exclude (--exclude CLI) + slugRoot (monorepo subdir).
|
||||
const _fullImportT0 = Date.now();
|
||||
serr(`[gbrain phase] sync.fullsync.import start strategy=${opts.strategy ?? 'markdown'}`);
|
||||
const result = await runImport(engine, importArgs, {
|
||||
commit: headCommit,
|
||||
strategy: opts.strategy,
|
||||
sourceId: opts.sourceId,
|
||||
exclude: opts.exclude,
|
||||
slugRoot,
|
||||
// issue #1939: performFullSync owns the failure ledger + bookmark via the
|
||||
// shared gate below; don't let runImport double-record or write its own.
|
||||
managedBookmark: true,
|
||||
@@ -3055,9 +3199,9 @@ async function performFullSync(
|
||||
const advanceFull = async (): Promise<void> => {
|
||||
// Persist sync state so the next sync is incremental. Routed through
|
||||
// writeSyncAnchor so --source pins the right sources row.
|
||||
await writeSyncAnchor(engine, opts.sourceId, 'last_commit', headCommit, newestCommitMs(repoPath));
|
||||
await writeSyncAnchor(engine, opts.sourceId, 'last_commit', headCommit, newestCommitMs(gitContextRoot));
|
||||
await engine.setConfig('sync.last_run', new Date().toISOString());
|
||||
await writeSyncAnchor(engine, opts.sourceId, 'repo_path', repoPath);
|
||||
await writeSyncAnchor(engine, opts.sourceId, 'repo_path', anchorPath);
|
||||
await writeChunkerVersion(engine, opts.sourceId, String(CHUNKER_VERSION));
|
||||
};
|
||||
|
||||
@@ -3084,7 +3228,7 @@ async function performFullSync(
|
||||
);
|
||||
}
|
||||
await engine.setConfig('sync.last_run', new Date().toISOString());
|
||||
await writeSyncAnchor(engine, opts.sourceId, 'repo_path', repoPath);
|
||||
await writeSyncAnchor(engine, opts.sourceId, 'repo_path', anchorPath);
|
||||
return {
|
||||
status: 'blocked_by_failures',
|
||||
fromCommit: null,
|
||||
@@ -3140,16 +3284,24 @@ async function performFullSync(
|
||||
// backslash paths while a stored source_path can hold git-derived forward
|
||||
// slashes; without normalization every file-backed page mismatches, looks
|
||||
// stale, and the reconcile wipes the whole source.
|
||||
const currentFiles = collectSyncableFiles(repoPath, { strategy: opts.strategy ?? 'markdown' })
|
||||
.map(abs => relative(repoPath, abs));
|
||||
// #774: scoped syncs store git-root-relative source_paths (slugRoot), so
|
||||
// relativize the walk to the same base — otherwise every page mismatches
|
||||
// and the mass-delete valve trips on a perfectly healthy scoped source.
|
||||
const currentFiles = collectSyncableFiles(syncScopeRoot, { strategy: opts.strategy ?? 'markdown' })
|
||||
.map(abs => relative(slugRoot ?? syncScopeRoot, abs));
|
||||
const rows = await engine.executeRaw<{ slug: string; source_path: string | null }>(
|
||||
`SELECT slug, source_path FROM pages WHERE source_id = $1 AND source_path IS NOT NULL AND deleted_at IS NULL`,
|
||||
[sid],
|
||||
);
|
||||
// #774: a scoped full sync is authoritative ONLY for its scope — pages
|
||||
// whose source_path lives outside the subpath (e.g. from an earlier
|
||||
// root-level sync of this source) are out of this walk's sight and must
|
||||
// not be treated as stale.
|
||||
const scopePrefix = slugRoot ? relative(gitContextRoot, syncScopeRoot) + '/' : '';
|
||||
const plan = planReconcileDeletes(
|
||||
rows,
|
||||
currentFiles,
|
||||
p => isSyncable(p, reconcileSyncOpts),
|
||||
p => (scopePrefix === '' || p.startsWith(scopePrefix)) && isSyncable(p, reconcileSyncOpts),
|
||||
);
|
||||
if (plan.staleSlugs.length > 0 && plan.massDelete && !massReconcileAllowed()) {
|
||||
// #2828 mass-delete safety valve: a reconcile that would sweep more than
|
||||
@@ -3400,6 +3552,19 @@ export function composeAbortSignals(
|
||||
return AbortSignal.any(live);
|
||||
}
|
||||
|
||||
/**
|
||||
* #753/#774: `.gitignore` must be managed at the git ROOT — when a source's
|
||||
* local_path (or --repo) points at a monorepo subdirectory, writing ignore
|
||||
* entries into the subdir would create a stray `.gitignore` git doesn't
|
||||
* consult for the repo-level db_only rules. Best-effort: falls back to the
|
||||
* given path when git discovery fails (manageGitignore no-ops on non-repos).
|
||||
*/
|
||||
function manageGitignoreAtGitRoot(path: string, engineKind?: 'pglite' | 'postgres'): void {
|
||||
let root = path;
|
||||
try { root = discoverGitRoot(path); } catch { /* best-effort */ }
|
||||
manageGitignore(root, engineKind);
|
||||
}
|
||||
|
||||
export async function runSync(engine: BrainEngine, args: string[]) {
|
||||
// v0.40 Federated Sync v2: `gbrain sync trigger` subcommand
|
||||
// Routes to runSyncTrigger which queues a 'sync' minion job with
|
||||
@@ -3432,6 +3597,13 @@ Options:
|
||||
--repo <path> Path to the brain repo. Defaults to the path
|
||||
saved by 'gbrain init'.
|
||||
--full Force a full re-sync (rare; usually incremental).
|
||||
--src-subpath <dir> Sync only this subdirectory of the git repo (monorepo
|
||||
pattern: N logical sources in one repo). Git pull/diff
|
||||
run at the repo root; imports are scoped to the subdir
|
||||
and slugs stay root-relative (wiki/page1). Passing the
|
||||
subdirectory directly as --repo also works.
|
||||
--exclude <glob> Exclude files matching the glob from sync (repeatable;
|
||||
matched against the scope-relative path).
|
||||
--dry-run Show what would be synced without writing.
|
||||
--skip-failed Acknowledge previously-recorded sync failures so
|
||||
the bookmark can advance past unparseable files.
|
||||
@@ -3591,6 +3763,20 @@ See also:
|
||||
process.exit(1);
|
||||
}
|
||||
const strategyArg = args.find((a, i) => args[i - 1] === '--strategy') as SyncOpts['strategy'] | undefined;
|
||||
// #753/#774: monorepo subdir-source flags. --exclude is repeatable.
|
||||
const srcSubpath = args.find((a, i) => args[i - 1] === '--src-subpath') || undefined;
|
||||
const excludePatterns: string[] = [];
|
||||
for (let i = 0; i < args.length; i++) {
|
||||
if (args[i] === '--exclude' && i + 1 < args.length) excludePatterns.push(args[i + 1]);
|
||||
}
|
||||
if (syncAll && (srcSubpath || excludePatterns.length > 0)) {
|
||||
console.error(
|
||||
`--src-subpath/--exclude scope a single sync invocation; they cannot be combined with --all. ` +
|
||||
`For --all runs, register the subdirectory as the source's local_path instead ` +
|
||||
`(gbrain sources add <id> --path <repo>/<subdir>).`,
|
||||
);
|
||||
process.exit(1);
|
||||
}
|
||||
const concurrencyStr = args.find((a, i) => args[i - 1] === '--concurrency' || args[i - 1] === '--workers');
|
||||
const parallelStr = args.find((a, i) => args[i - 1] === '--parallel');
|
||||
// v0.22.13 (PR #490 Q2): parseWorkers throws on '0', '-3', 'foo', '1.5' instead
|
||||
@@ -3850,7 +4036,7 @@ See also:
|
||||
result.status !== 'blocked_by_failures' &&
|
||||
result.status !== 'partial'
|
||||
) {
|
||||
manageGitignore(src.local_path!, engine.kind);
|
||||
manageGitignoreAtGitRoot(src.local_path!, engine.kind);
|
||||
}
|
||||
// D18: auto-enqueue embed-backfill per source (unless opted out).
|
||||
// v0.41.13.0 (T7 / D-V3-5): partial excluded — the next clean sync
|
||||
@@ -4038,6 +4224,8 @@ See also:
|
||||
const opts: SyncOpts = {
|
||||
repoPath, dryRun, full, noPull, noEmbed, noExtract, skipFailed, retryFailed, noSchemaPack, sourceId,
|
||||
strategy: strategyArg, concurrency,
|
||||
srcSubpath,
|
||||
exclude: excludePatterns.length > 0 ? excludePatterns : undefined,
|
||||
signal: composeAbortSignals(singleSourceInterrupt.signal, singleSourceController?.signal),
|
||||
};
|
||||
|
||||
@@ -4121,7 +4309,7 @@ See also:
|
||||
) {
|
||||
const effectiveRepoPath = opts.repoPath ?? (await getDefaultSourcePath(engine));
|
||||
if (effectiveRepoPath) {
|
||||
manageGitignore(effectiveRepoPath, engine.kind);
|
||||
manageGitignoreAtGitRoot(effectiveRepoPath, engine.kind);
|
||||
}
|
||||
}
|
||||
// v0.42.42.0 (#2139, Step 4b): the inline gate auto-deferred this run's
|
||||
@@ -4170,7 +4358,7 @@ See also:
|
||||
) {
|
||||
const effectiveRepoPath = opts.repoPath ?? (await getDefaultSourcePath(engine));
|
||||
if (effectiveRepoPath) {
|
||||
manageGitignore(effectiveRepoPath, engine.kind);
|
||||
manageGitignoreAtGitRoot(effectiveRepoPath, engine.kind);
|
||||
}
|
||||
}
|
||||
} catch (e: unknown) {
|
||||
|
||||
+1
-1
@@ -219,7 +219,7 @@ function globToRegex(pattern: string): RegExp {
|
||||
return new RegExp(regex);
|
||||
}
|
||||
|
||||
function matchesAnyGlob(path: string, patterns?: string[]): boolean {
|
||||
export function matchesAnyGlob(path: string, patterns?: string[]): boolean {
|
||||
if (!patterns || patterns.length === 0) return false;
|
||||
const normalized = path.replace(/\\/g, '/');
|
||||
return patterns.some((pattern) => globToRegex(pattern).test(normalized));
|
||||
|
||||
@@ -0,0 +1,385 @@
|
||||
/**
|
||||
* #753/#774 — --src-subpath + --exclude monorepo subdir-source support.
|
||||
*
|
||||
* A single git repo can hold N logical sources at subdirectories (wiki/,
|
||||
* memory/, ...). `gbrain sync --src-subpath wiki` (or passing the subdir
|
||||
* directly as the repo path) scopes file walking + imports to the subdir
|
||||
* while git operations (pull, rev-parse, diff) run at the discovered repo
|
||||
* root. Slugs stay git-root-relative (`wiki/page1`) so full and incremental
|
||||
* syncs of the same scope agree.
|
||||
*
|
||||
* Security pins (the point of the feature's guards):
|
||||
* NAV-1/NAV-2 — `--src-subpath ../escape` and a symlinked subdir pointing
|
||||
* outside the repo are realpath-checked and rejected before any git op.
|
||||
* NAV-1 TOCTOU — per-file realpath checks during the incremental import
|
||||
* drain (see the isPathSafe guard in sync.ts's importOnePath).
|
||||
* NAV-4 — an --exclude set that filters out everything warns loudly.
|
||||
*
|
||||
* Regression note: against pre-#774 master every subdir test fails with
|
||||
* "Not a git repository".
|
||||
*/
|
||||
|
||||
import { describe, test, expect, beforeAll, afterAll, beforeEach, afterEach } from 'bun:test';
|
||||
import { mkdtempSync, writeFileSync, rmSync, mkdirSync, symlinkSync } from 'fs';
|
||||
import { join } from 'path';
|
||||
import { execSync } from 'child_process';
|
||||
import { tmpdir } from 'os';
|
||||
import { PGLiteEngine } from '../src/core/pglite-engine.ts';
|
||||
import { resetPgliteState } from './helpers/reset-pglite.ts';
|
||||
|
||||
// Helper: create a minimal valid markdown file
|
||||
function mdPage(title: string, body = 'Content.'): string {
|
||||
return `---\ntype: note\ntitle: ${title}\n---\n\n${body}`;
|
||||
}
|
||||
|
||||
// Helper: init a git repo with author identity
|
||||
function gitInit(dir: string): void {
|
||||
execSync('git init', { cwd: dir, stdio: 'pipe' });
|
||||
execSync('git config user.email "test@test.com"', { cwd: dir, stdio: 'pipe' });
|
||||
execSync('git config user.name "Test"', { cwd: dir, stdio: 'pipe' });
|
||||
}
|
||||
|
||||
// Helper: stage + commit everything in a git repo
|
||||
function gitCommit(dir: string, msg = 'initial'): void {
|
||||
execSync('git add -A', { cwd: dir, stdio: 'pipe' });
|
||||
execSync(`git commit -m "${msg}"`, { cwd: dir, stdio: 'pipe' });
|
||||
}
|
||||
|
||||
describe('sync monorepo subdir-source support (#753/#774)', () => {
|
||||
let engine: PGLiteEngine;
|
||||
let repoPath: string;
|
||||
|
||||
beforeAll(async () => {
|
||||
engine = new PGLiteEngine();
|
||||
await engine.connect({});
|
||||
await engine.initSchema();
|
||||
}, 60_000);
|
||||
|
||||
afterAll(async () => {
|
||||
await engine.disconnect();
|
||||
}, 60_000);
|
||||
|
||||
beforeEach(async () => {
|
||||
await resetPgliteState(engine);
|
||||
repoPath = mkdtempSync(join(tmpdir(), 'gbrain-monorepo-'));
|
||||
gitInit(repoPath);
|
||||
mkdirSync(join(repoPath, 'wiki'), { recursive: true });
|
||||
mkdirSync(join(repoPath, 'memory'), { recursive: true });
|
||||
writeFileSync(join(repoPath, 'wiki', 'page1.md'), mdPage('Wiki Page 1'));
|
||||
writeFileSync(join(repoPath, 'wiki', 'page2.md'), mdPage('Wiki Page 2'));
|
||||
writeFileSync(join(repoPath, 'memory', 'note1.md'), mdPage('Memory Note 1'));
|
||||
writeFileSync(join(repoPath, 'memory', 'note2.md'), mdPage('Memory Note 2'));
|
||||
gitCommit(repoPath);
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
if (repoPath) rmSync(repoPath, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
// ─────────────────────────────────────────────────────────────────────────
|
||||
// Back-compat: sync at git root (no srcSubpath) still works
|
||||
// ─────────────────────────────────────────────────────────────────────────
|
||||
|
||||
test('back-compat: sync at git root without srcSubpath imports all files', async () => {
|
||||
const { performSync } = await import('../src/commands/sync.ts');
|
||||
const result = await performSync(engine, {
|
||||
repoPath,
|
||||
noPull: true,
|
||||
noEmbed: true,
|
||||
full: true,
|
||||
});
|
||||
expect(result.status).toBe('first_sync');
|
||||
expect(result.added).toBe(4); // wiki/page1 + wiki/page2 + memory/note1 + memory/note2
|
||||
// Slug shape unchanged for git-root syncs.
|
||||
expect(await engine.getPage('wiki/page1')).not.toBeNull();
|
||||
});
|
||||
|
||||
// ─────────────────────────────────────────────────────────────────────────
|
||||
// Auto-discovery: repoPath IS a non-git-root subdir
|
||||
// ─────────────────────────────────────────────────────────────────────────
|
||||
|
||||
test('auto-discovery: repoPath is a git subdir — discoverGitRoot succeeds', async () => {
|
||||
const { performSync } = await import('../src/commands/sync.ts');
|
||||
// Pass the wiki/ subdir directly as repoPath (no explicit srcSubpath).
|
||||
// Pre-#774: throws "Not a git repository".
|
||||
// Post-#774: gitContextRoot = repo root, syncScopeRoot = wiki/.
|
||||
const result = await performSync(engine, {
|
||||
repoPath: join(repoPath, 'wiki'),
|
||||
noPull: true,
|
||||
noEmbed: true,
|
||||
full: true,
|
||||
});
|
||||
expect(result.status).toBe('first_sync');
|
||||
expect(result.added).toBe(2); // only wiki/page1 + wiki/page2
|
||||
// Slugs are git-root-relative in BOTH spellings (subdir repoPath and
|
||||
// --src-subpath) so full and incremental syncs of the same scope agree.
|
||||
expect(await engine.getPage('wiki/page1')).not.toBeNull();
|
||||
expect(await engine.getPage('page1')).toBeNull();
|
||||
});
|
||||
|
||||
// ─────────────────────────────────────────────────────────────────────────
|
||||
// srcSubpath explicit flag: scope to subdir from git root
|
||||
// ─────────────────────────────────────────────────────────────────────────
|
||||
|
||||
test('--src-subpath wiki: only wiki/ files are imported', async () => {
|
||||
const { performSync } = await import('../src/commands/sync.ts');
|
||||
const result = await performSync(engine, {
|
||||
repoPath,
|
||||
srcSubpath: 'wiki',
|
||||
noPull: true,
|
||||
noEmbed: true,
|
||||
full: true,
|
||||
});
|
||||
expect(result.status).toBe('first_sync');
|
||||
expect(result.added).toBe(2);
|
||||
// Verify the imported slugs are from wiki/ only (git-root-relative)
|
||||
const wikiPage = await engine.getPage('wiki/page1');
|
||||
expect(wikiPage).not.toBeNull();
|
||||
const memoryPage = await engine.getPage('memory/note1');
|
||||
expect(memoryPage).toBeNull();
|
||||
});
|
||||
|
||||
test('--src-subpath memory: only memory/ files are imported', async () => {
|
||||
const { performSync } = await import('../src/commands/sync.ts');
|
||||
const result = await performSync(engine, {
|
||||
repoPath,
|
||||
srcSubpath: 'memory',
|
||||
noPull: true,
|
||||
noEmbed: true,
|
||||
full: true,
|
||||
});
|
||||
expect(result.status).toBe('first_sync');
|
||||
expect(result.added).toBe(2);
|
||||
const memoryPage = await engine.getPage('memory/note1');
|
||||
expect(memoryPage).not.toBeNull();
|
||||
const wikiPage = await engine.getPage('wiki/page1');
|
||||
expect(wikiPage).toBeNull();
|
||||
});
|
||||
|
||||
// ─────────────────────────────────────────────────────────────────────────
|
||||
// Two sources in one repo, scoped independently
|
||||
// ─────────────────────────────────────────────────────────────────────────
|
||||
|
||||
test('2 sources in 1 repo: sync each scope independently, no cross-contamination', async () => {
|
||||
const { performSync } = await import('../src/commands/sync.ts');
|
||||
|
||||
const wikiResult = await performSync(engine, {
|
||||
repoPath,
|
||||
srcSubpath: 'wiki',
|
||||
noPull: true,
|
||||
noEmbed: true,
|
||||
full: true,
|
||||
});
|
||||
expect(wikiResult.status).toBe('first_sync');
|
||||
expect(wikiResult.added).toBe(2);
|
||||
|
||||
// Reset only page state, keep the engine connected for second sync
|
||||
await resetPgliteState(engine);
|
||||
|
||||
const memResult = await performSync(engine, {
|
||||
repoPath,
|
||||
srcSubpath: 'memory',
|
||||
noPull: true,
|
||||
noEmbed: true,
|
||||
full: true,
|
||||
});
|
||||
expect(memResult.status).toBe('first_sync');
|
||||
expect(memResult.added).toBe(2);
|
||||
|
||||
// After memory sync, memory pages exist and wiki pages don't
|
||||
expect(await engine.getPage('memory/note1')).not.toBeNull();
|
||||
expect(await engine.getPage('wiki/page1')).toBeNull();
|
||||
});
|
||||
|
||||
// ─────────────────────────────────────────────────────────────────────────
|
||||
// Incremental sync respects the scope (the gap #774 left untested)
|
||||
// ─────────────────────────────────────────────────────────────────────────
|
||||
|
||||
test('incremental --src-subpath: only in-scope diff paths are processed', async () => {
|
||||
const { performSync } = await import('../src/commands/sync.ts');
|
||||
|
||||
const first = await performSync(engine, {
|
||||
repoPath,
|
||||
srcSubpath: 'wiki',
|
||||
noPull: true,
|
||||
noEmbed: true,
|
||||
full: true,
|
||||
});
|
||||
expect(first.status).toBe('first_sync');
|
||||
|
||||
// Commit 2: touch one file in each scope + add one wiki file.
|
||||
writeFileSync(join(repoPath, 'wiki', 'page1.md'), mdPage('Wiki Page 1', 'Updated.'));
|
||||
writeFileSync(join(repoPath, 'memory', 'note1.md'), mdPage('Memory Note 1', 'Updated.'));
|
||||
writeFileSync(join(repoPath, 'wiki', 'page3.md'), mdPage('Wiki Page 3'));
|
||||
gitCommit(repoPath, 'second');
|
||||
|
||||
const second = await performSync(engine, {
|
||||
repoPath,
|
||||
srcSubpath: 'wiki',
|
||||
noPull: true,
|
||||
noEmbed: true,
|
||||
});
|
||||
expect(second.status).toBe('synced');
|
||||
expect(second.added).toBe(1); // wiki/page3 only — memory change filtered by scope
|
||||
expect(second.modified).toBe(1); // wiki/page1
|
||||
expect(await engine.getPage('wiki/page3')).not.toBeNull();
|
||||
expect(await engine.getPage('memory/note1')).toBeNull();
|
||||
});
|
||||
|
||||
// ─────────────────────────────────────────────────────────────────────────
|
||||
// Path-traversal sanitization (NAV-1 + NAV-2)
|
||||
// ─────────────────────────────────────────────────────────────────────────
|
||||
|
||||
test('path-traversal: --src-subpath ../escape is rejected before any git op', async () => {
|
||||
const outsideDir = mkdtempSync(join(tmpdir(), 'gbrain-escape-'));
|
||||
try {
|
||||
const { performSync } = await import('../src/commands/sync.ts');
|
||||
await expect(
|
||||
performSync(engine, {
|
||||
repoPath,
|
||||
srcSubpath: '../' + outsideDir.split('/').pop(),
|
||||
noPull: true,
|
||||
noEmbed: true,
|
||||
full: true,
|
||||
}),
|
||||
).rejects.toThrow(/outside git repo|does not exist/i);
|
||||
} finally {
|
||||
rmSync(outsideDir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test('path-traversal: symlink subdir pointing outside repo is rejected (NAV-1 TOCTOU)', async () => {
|
||||
const outsideDir = mkdtempSync(join(tmpdir(), 'gbrain-sym-target-'));
|
||||
writeFileSync(join(outsideDir, 'secret.md'), mdPage('Secret'));
|
||||
const symlinkPath = join(repoPath, 'symlink-escape');
|
||||
try {
|
||||
symlinkSync(outsideDir, symlinkPath);
|
||||
const { performSync } = await import('../src/commands/sync.ts');
|
||||
await expect(
|
||||
performSync(engine, {
|
||||
repoPath,
|
||||
srcSubpath: 'symlink-escape',
|
||||
noPull: true,
|
||||
noEmbed: true,
|
||||
full: true,
|
||||
}),
|
||||
).rejects.toThrow(/outside git repo/i);
|
||||
} finally {
|
||||
rmSync(outsideDir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test('path-traversal: absolute --src-subpath outside the repo is rejected', async () => {
|
||||
const outsideDir = mkdtempSync(join(tmpdir(), 'gbrain-abs-escape-'));
|
||||
writeFileSync(join(outsideDir, 'secret.md'), mdPage('Secret'));
|
||||
try {
|
||||
const { performSync } = await import('../src/commands/sync.ts');
|
||||
// path.join(repoPath, '/abs/path') keeps the traversal relative, but a
|
||||
// crafted subpath can still resolve outside via ..-segments; both are
|
||||
// caught by the same realpath containment check.
|
||||
await expect(
|
||||
performSync(engine, {
|
||||
repoPath,
|
||||
srcSubpath: join('..', '..', outsideDir.slice(1)),
|
||||
noPull: true,
|
||||
noEmbed: true,
|
||||
full: true,
|
||||
}),
|
||||
).rejects.toThrow(/outside git repo|does not exist/i);
|
||||
} finally {
|
||||
rmSync(outsideDir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
// ─────────────────────────────────────────────────────────────────────────
|
||||
// --exclude: repeatable glob pattern flag
|
||||
// ─────────────────────────────────────────────────────────────────────────
|
||||
|
||||
test('--exclude: single pattern excludes matching files from full sync', async () => {
|
||||
const { performSync } = await import('../src/commands/sync.ts');
|
||||
// Sync wiki/ but exclude page2.md (patterns are scope-relative)
|
||||
const result = await performSync(engine, {
|
||||
repoPath,
|
||||
srcSubpath: 'wiki',
|
||||
exclude: ['page2.md'],
|
||||
noPull: true,
|
||||
noEmbed: true,
|
||||
full: true,
|
||||
});
|
||||
expect(result.status).toBe('first_sync');
|
||||
expect(result.added).toBe(1); // only page1 (page2 excluded)
|
||||
expect(await engine.getPage('wiki/page1')).not.toBeNull();
|
||||
expect(await engine.getPage('wiki/page2')).toBeNull();
|
||||
});
|
||||
|
||||
test('--exclude: glob pattern with wildcard', async () => {
|
||||
const { performSync } = await import('../src/commands/sync.ts');
|
||||
// Exclude all files matching *2.md
|
||||
const result = await performSync(engine, {
|
||||
repoPath,
|
||||
srcSubpath: 'wiki',
|
||||
exclude: ['*2.md'],
|
||||
noPull: true,
|
||||
noEmbed: true,
|
||||
full: true,
|
||||
});
|
||||
expect(result.status).toBe('first_sync');
|
||||
expect(result.added).toBe(1); // only page1 (page2 excluded by *2.md)
|
||||
});
|
||||
|
||||
test('--exclude applies to the incremental path too', async () => {
|
||||
const { performSync } = await import('../src/commands/sync.ts');
|
||||
const first = await performSync(engine, {
|
||||
repoPath,
|
||||
srcSubpath: 'wiki',
|
||||
noPull: true,
|
||||
noEmbed: true,
|
||||
full: true,
|
||||
});
|
||||
expect(first.status).toBe('first_sync');
|
||||
|
||||
writeFileSync(join(repoPath, 'wiki', 'draft-a.md'), mdPage('Draft A'));
|
||||
writeFileSync(join(repoPath, 'wiki', 'page3.md'), mdPage('Wiki Page 3'));
|
||||
gitCommit(repoPath, 'drafts');
|
||||
|
||||
const second = await performSync(engine, {
|
||||
repoPath,
|
||||
srcSubpath: 'wiki',
|
||||
exclude: ['draft-*.md'],
|
||||
noPull: true,
|
||||
noEmbed: true,
|
||||
});
|
||||
expect(second.status).toBe('synced');
|
||||
expect(second.added).toBe(1); // page3 only; draft-a excluded
|
||||
expect(await engine.getPage('wiki/page3')).not.toBeNull();
|
||||
expect(await engine.getPage('wiki/draft-a')).toBeNull();
|
||||
});
|
||||
|
||||
// ─────────────────────────────────────────────────────────────────────────
|
||||
// --exclude '**/*' emits warning (NAV-4)
|
||||
// ─────────────────────────────────────────────────────────────────────────
|
||||
|
||||
test('--exclude **/* emits warning when all files are excluded (NAV-4)', async () => {
|
||||
const { performSync } = await import('../src/commands/sync.ts');
|
||||
const warnMessages: string[] = [];
|
||||
const origWarn = console.warn;
|
||||
console.warn = (...args: unknown[]) => {
|
||||
warnMessages.push(args.join(' '));
|
||||
origWarn(...args);
|
||||
};
|
||||
try {
|
||||
await performSync(engine, {
|
||||
repoPath,
|
||||
srcSubpath: 'wiki',
|
||||
exclude: ['**/*'],
|
||||
noPull: true,
|
||||
noEmbed: true,
|
||||
full: true,
|
||||
});
|
||||
} finally {
|
||||
console.warn = origWarn;
|
||||
}
|
||||
const hasExcludeWarn = warnMessages.some(m => m.includes('--exclude') || m.includes('No files matched'));
|
||||
expect(hasExcludeWarn).toBe(true);
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user