fix(import): walker skips SYNC_SKIP_FILES metafiles so import and sync agree (#2315)

Closes #345.

The bulk-import walker isCollectibleForWalker filtered admitted files by
extension only, while incremental sync excludes README/index/log/schema via
isSyncable -> SYNC_SKIP_FILES. A directory import therefore ingested every
directory README as a folder-titled ghost page that trigram-corrupts fuzzy
entity resolution and inflates orphan count. Apply SYNC_SKIP_FILES (basename
guard) at the top of isCollectibleForWalker so both the FS-walk and the
git-fast-path collection routes agree with sync.

Also add RESOLVER.md to SYNC_SKIP_FILES: a structural routing metafile
(docs-aligned with schema.md/index.md/log.md/README.md), not indexable content.

Co-authored-by: ElliotDrel <ElliotDrel@users.noreply.github.com>
Co-authored-by: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
Elliot Drel
2026-07-17 11:32:42 -07:00
committed by GitHub
co-authored by ElliotDrel Claude Opus 4.8
parent ad1fe25e61
commit ff8ce4d764
4 changed files with 99 additions and 5 deletions
+17
View File
@@ -12,6 +12,7 @@ import {
isMarkdownFilePath,
isImageFilePath as isImageFilePathFromSync,
pruneDir,
SYNC_SKIP_FILES,
type SyncStrategy,
} from '../core/sync.ts';
import { sortNewestFirst } from '../core/sort-newest-first.ts';
@@ -493,12 +494,28 @@ interface CollectOpts {
* The first-sync walker historically admitted them on markdown too when
* `GBRAIN_EMBEDDING_MULTIMODAL=true`. Codex (C5) flagged the contradiction
* — preserve the walker semantic explicitly.
*
* Closes #345: exclude `SYNC_SKIP_FILES` metafiles
* (`README.md` / `index.md` / `log.md` / `schema.md` / `RESOLVER.md`).
* Incremental `sync` skips these via `isSyncable`, but the bulk-import
* walker only filtered by extension — so a directory import imported every
* directory README as a page, titled by its folder ("People", "Companies",
* …). Those index-titled pages then trigram-corrupt fuzzy entity resolution
* (any `people/X` slug matches the "People" page) and inflate orphan count.
* Funnel both admission paths through the same metafile exclusion so import
* and sync agree on what is a page.
*/
function isCollectibleForWalker(
path: string,
strategy: SyncStrategy,
multimodalOn: boolean,
): boolean {
// Metafiles are directory scaffolding (READMEs / index / log / schema /
// resolver), not typed brain pages — same exclusion `sync`'s `isSyncable`
// applies. Guards both the FS-walk and the git-fast-path collection routes.
const basename = path.split('/').pop() || '';
if ((SYNC_SKIP_FILES as readonly string[]).includes(basename)) return false;
switch (strategy) {
case 'code':
return isCodeFilePath(path);
+11 -3
View File
@@ -324,10 +324,18 @@ export type SyncableReason =
* surface them in user-facing logs / docs without re-declaring the list.
*
* These files are append-only domain logs / index pages / boilerplate
* READMEs — not typed brain pages — by convention. A user who genuinely
* wants to index one of these basenames as a page should rename it.
* READMEs / the master filing decision-tree — not typed brain pages — by
* convention. A user who genuinely wants to index one of these basenames as
* a page should rename it.
*
* `RESOLVER.md` is the brain's master routing/decision-tree config file. The
* recommended-schema docs group it with `schema.md` / `index.md` / `log.md`
* as a structural document ("a document … plus schema.md and RESOLVER.md …
* that tells the agent how the brain is structured"), NOT searchable content.
* It was the lone structural sibling missing from this list, so it leaked
* into the index as a content page (slug `resolver`).
*/
export const SYNC_SKIP_FILES = ['schema.md', 'index.md', 'log.md', 'README.md'] as const;
export const SYNC_SKIP_FILES = ['schema.md', 'index.md', 'log.md', 'README.md', 'RESOLVER.md'] as const;
/**
* Internal classifier. Returns null when the path IS syncable, or a tagged
+67
View File
@@ -0,0 +1,67 @@
/**
* Closes #345: the bulk-import walker must skip SYNC_SKIP_FILES metafiles
* (README.md / index.md / log.md / schema.md / RESOLVER.md), the same way
* incremental `sync` (isSyncable) does.
*
* Root cause this locks: a directory-import pass imported every directory
* README.md as a page (titled "People", "Companies", …), because
* collectSyncableFiles only filtered by extension. Those index-titled pages
* then trigram-corrupted fuzzy entity resolution and inflated orphan count.
*/
import { describe, test, expect, beforeEach, afterEach } from 'bun:test';
import { mkdtempSync, mkdirSync, rmSync, writeFileSync } from 'fs';
import { execFileSync } from 'child_process';
import { join, basename } from 'path';
import { tmpdir } from 'os';
import { collectSyncableFiles } from '../src/commands/import.ts';
let tmp: string;
function write(relPath: string, content: string): void {
const full = join(tmp, relPath);
mkdirSync(join(full, '..'), { recursive: true });
writeFileSync(full, content);
}
beforeEach(() => {
tmp = mkdtempSync(join(tmpdir(), 'gbrain-import-metafile-'));
});
afterEach(() => {
rmSync(tmp, { recursive: true, force: true });
});
describe('collectSyncableFiles metafile exclusion (closes #345)', () => {
function seed(): void {
write('people/example-person.md', '# Example Person\n');
write('people/README.md', '# People\n\nOne page per person.\n');
write('companies/README.md', '# Companies\n');
write('README.md', '# Brain\n');
write('index.md', '# Brain Index\n');
write('log.md', '# Brain Log\n');
write('schema.md', '# Brain Schema\n');
write('RESOLVER.md', '# Brain Resolver\n');
}
test('FS-walk path excludes README/index/log/schema/RESOLVER, keeps real pages', () => {
seed();
const got = collectSyncableFiles(tmp).map(f => basename(f));
expect(got).toContain('example-person.md');
for (const meta of ['README.md', 'index.md', 'log.md', 'schema.md', 'RESOLVER.md']) {
expect(got).not.toContain(meta);
}
});
test('git-fast-path also excludes metafiles', () => {
seed();
execFileSync('git', ['-C', tmp, 'init', '-q'], { stdio: 'ignore' });
execFileSync('git', ['-C', tmp, 'add', '-A'], { stdio: 'ignore' });
const got = collectSyncableFiles(tmp).map(f => basename(f));
expect(got).toContain('example-person.md');
expect(got.filter(n => n === 'README.md')).toHaveLength(0);
expect(got).not.toContain('index.md');
expect(got).not.toContain('log.md');
expect(got).not.toContain('schema.md');
expect(got).not.toContain('RESOLVER.md');
});
});
+4 -2
View File
@@ -25,6 +25,8 @@ describe('#1433 — isSyncable / unsyncableReason are duals of one classifier',
{ path: 'index.md', expected: 'metafile', note: 'top-level index.md' },
{ path: 'README.md', expected: 'metafile', note: 'top-level README' },
{ path: 'docs/README.md', expected: 'metafile', note: 'nested README' },
{ path: 'RESOLVER.md', expected: 'metafile', note: 'top-level master routing config (closes #345)' },
{ path: 'brain/RESOLVER.md', expected: 'metafile', note: 'RESOLVER.md anywhere is metafile (closes #345)' },
{ path: 'people/alice.txt', expected: 'strategy', note: '.txt rejected by markdown strategy' },
{ path: 'ops/scratch/note.md', expected: 'pruned-dir', note: 'ops/ is pruned' },
{ path: '.git/notes.md', expected: 'pruned-dir', note: 'hidden dir pruned' },
@@ -50,8 +52,8 @@ describe('#1433 — isSyncable / unsyncableReason are duals of one classifier',
expect(isSyncable('drafts/wip.md', { exclude: ['drafts/**'] })).toBe(false);
});
test('SYNC_SKIP_FILES export contains the canonical four basenames', () => {
expect([...SYNC_SKIP_FILES]).toEqual(['schema.md', 'index.md', 'log.md', 'README.md']);
test('SYNC_SKIP_FILES export contains the canonical structural metafiles', () => {
expect([...SYNC_SKIP_FILES]).toEqual(['schema.md', 'index.md', 'log.md', 'README.md', 'RESOLVER.md']);
});
test('isSyncable(p) === (unsyncableReason(p) === null) — duality holds for all canonical cases', () => {