mirror of
https://github.com/garrytan/gbrain.git
synced 2026-07-30 11:22:34 +00:00
slugifySegment stripped every character outside [a-z0-9._-] + four CJK
ranges, so filenames in Hebrew, Arabic, Cyrillic, Greek, Thai (and every
other script) collapsed to empty segments. Distinct files then mapped to
the SAME slug (their shared directory prefix) and silently overwrote each
other, last-writer-wins, with import reporting 0 errors.
All three slug grammars move together so sync never emits a slug that
put_page rejects:
- sync.ts SLUGIFY_KEEP_RE + SLUG_SEGMENT_PATTERN — now keep
\p{Ll}\p{Lm}\p{Lo}\p{M}\p{N} (new single-source SLUG_WORD_CHARS in
cjk.ts), with the u flag
- cjk.ts PAGE_SLUG_SEG — rebuilt on SLUG_WORD_CHARS; consumers
(validatePageSlug, SlugRegistry SLUG_RE, dream-cycle SUMMARY_SLUG_RE,
takes-fence HOLDER_REGEX) all gain the required u flag
CJK_SLUG_CHARS is untouched — it also drives the countCJKAwareWords
chunking density heuristic and must not move with slug grammar.
Kebab-casing, lowercasing, Latin accent-strip (cafe), dots/underscores,
path handling, and the NFC re-normalize (NFD macOS filenames converge
with NFC git filenames) are all unchanged and regression-pinned in
test/slug-unicode-scripts.test.ts. No data migration: the collapse was
N-to-1, so only one row per group ever persisted; a normal gbrain sync
recreates the lost pages under their real slugs.
Co-authored-by: Garry Tan <garrytan@gmail.com>
Co-authored-by: Claude Opus 5 <noreply@anthropic.com>
849 lines
29 KiB
TypeScript
849 lines
29 KiB
TypeScript
import { describe, test, expect, beforeAll, afterAll } from 'bun:test';
|
|
import { writeFileSync, mkdirSync, rmSync, symlinkSync, readdirSync, readFileSync, existsSync } from 'fs';
|
|
import { join } from 'path';
|
|
import { importFile, importFromContent } from '../src/core/import-file.ts';
|
|
import type { BrainEngine } from '../src/core/engine.ts';
|
|
import { MARKDOWN_CHUNKER_VERSION } from '../src/core/chunkers/recursive.ts';
|
|
|
|
const TMP = join(import.meta.dir, '.tmp-import-test');
|
|
|
|
// Minimal mock engine that tracks calls and supports transaction()
|
|
//
|
|
// Post-write read-back guard: the mock now simulates a real DB by storing pages written
|
|
// via putPage so getPage can read them back. The post-write read-back guard
|
|
// in importFromContent calls getPage after the transaction commits — a mock
|
|
// that always returns null (the pre-fix default) triggers the guard's
|
|
// "page not found after write" path. This is the correct loud-failure
|
|
// behavior for a real desync, but for the existing unit tests we need the
|
|
// mock to behave like a working DB where writes are readable.
|
|
function mockEngine(overrides: Partial<Record<string, any>> = {}): BrainEngine {
|
|
const calls: { method: string; args: any[] }[] = [];
|
|
// In-memory page store: slug → page row (simulates a real DB index).
|
|
const pageStore = new Map<string, { slug: string; content_hash: string; title: string; type: string; frontmatter: Record<string, unknown> }>();
|
|
const track = (method: string) => (...args: any[]) => {
|
|
calls.push({ method, args });
|
|
if (overrides[method]) return overrides[method](...args);
|
|
return Promise.resolve(null);
|
|
};
|
|
|
|
const engine = new Proxy({} as any, {
|
|
get(_, prop: string) {
|
|
if (prop === '_calls') return calls;
|
|
if (prop === 'getTags') return overrides.getTags || (() => Promise.resolve([]));
|
|
if (prop === 'getPage') {
|
|
return async (slug: string, opts?: { sourceId?: string }) => {
|
|
// If the test provides a custom getPage, call it first (for the
|
|
// "existing" check at the top of importFromContent). If it returns
|
|
// a value, use that. If it returns null, fall back to the in-memory
|
|
// store (for the read-back guard after the write).
|
|
//
|
|
// Read-back guard: when the override returns a non-null value,
|
|
// we still need the read-back to see the page as written by putPage.
|
|
// The override simulates the "existing page" state; the store has
|
|
// the post-write state. If the override and the store disagree on
|
|
// content_hash, the store wins (the write already committed).
|
|
if (overrides.getPage) {
|
|
const overrideResult = await overrides.getPage(slug, opts);
|
|
if (overrideResult) {
|
|
// If the page was written (store has it), the store's hash
|
|
// is the post-write hash. Merge: return the override's shape
|
|
// but with the store's content_hash (the committed value).
|
|
const stored = pageStore.get(slug);
|
|
if (stored) {
|
|
return { ...overrideResult, content_hash: stored.content_hash };
|
|
}
|
|
return overrideResult;
|
|
}
|
|
}
|
|
return pageStore.get(slug) ?? null;
|
|
};
|
|
}
|
|
if (prop === 'putPage') {
|
|
return async (slug: string, page: { content_hash?: string; title?: string; type?: string; frontmatter?: Record<string, unknown> }, _opts?: { sourceId?: string }) => {
|
|
calls.push({ method: 'putPage', args: [slug, page, _opts] });
|
|
if (overrides.putPage) overrides.putPage(slug, page, _opts);
|
|
// Always store the page so getPage can read it back (simulates DB index).
|
|
pageStore.set(slug, {
|
|
slug,
|
|
content_hash: page.content_hash ?? '',
|
|
title: page.title ?? '',
|
|
type: page.type ?? '',
|
|
frontmatter: page.frontmatter ?? {},
|
|
});
|
|
return Promise.resolve(undefined);
|
|
};
|
|
}
|
|
// transaction: just call the fn with the same engine (no real DB transaction in tests)
|
|
if (prop === 'transaction') return async (fn: (tx: BrainEngine) => Promise<any>) => fn(engine);
|
|
return track(prop);
|
|
},
|
|
});
|
|
return engine;
|
|
}
|
|
|
|
beforeAll(() => {
|
|
mkdirSync(TMP, { recursive: true });
|
|
});
|
|
|
|
afterAll(() => {
|
|
rmSync(TMP, { recursive: true, force: true });
|
|
});
|
|
|
|
describe('importFile', () => {
|
|
test('imports a valid markdown file', async () => {
|
|
const filePath = join(TMP, 'test-page.md');
|
|
writeFileSync(filePath, `---
|
|
type: concept
|
|
title: Test Page
|
|
tags: [alpha, beta]
|
|
---
|
|
|
|
This is the compiled truth.
|
|
|
|
---
|
|
|
|
- 2024-01-01: Something happened.
|
|
`);
|
|
|
|
const engine = mockEngine();
|
|
const result = await importFile(engine, filePath, 'concepts/test-page.md', { noEmbed: true });
|
|
|
|
expect(result.status).toBe('imported');
|
|
expect(result.slug).toBe('concepts/test-page');
|
|
expect(result.chunks).toBeGreaterThan(0);
|
|
|
|
// Verify engine was called correctly
|
|
const calls = (engine as any)._calls;
|
|
const putCall = calls.find((c: any) => c.method === 'putPage');
|
|
expect(putCall).toBeTruthy();
|
|
expect(putCall.args[0]).toBe('concepts/test-page');
|
|
|
|
// Tags were added
|
|
const tagCalls = calls.filter((c: any) => c.method === 'addTag');
|
|
expect(tagCalls.length).toBe(2);
|
|
|
|
// Chunks were upserted
|
|
const chunkCall = calls.find((c: any) => c.method === 'upsertChunks');
|
|
expect(chunkCall).toBeTruthy();
|
|
});
|
|
|
|
test('skips files larger than MAX_FILE_SIZE (5MB)', async () => {
|
|
const filePath = join(TMP, 'big-file.md');
|
|
const bigContent = '---\ntitle: Big\n---\n' + 'x'.repeat(5_100_000);
|
|
writeFileSync(filePath, bigContent);
|
|
|
|
const engine = mockEngine();
|
|
const result = await importFile(engine, filePath, 'big-file.md', { noEmbed: true });
|
|
|
|
expect(result.status).toBe('skipped');
|
|
expect(result.error).toContain('too large');
|
|
expect((engine as any)._calls.length).toBe(0);
|
|
});
|
|
|
|
test('rejects frontmatter slug that does not match the file path', async () => {
|
|
// In a shared brain where contributors can land PRs, this prevents a
|
|
// poisoned notes/random.md from declaring `slug: people/elon` in its
|
|
// frontmatter and overwriting the legitimate people/elon page on sync.
|
|
const filePath = join(TMP, 'hijack.md');
|
|
writeFileSync(filePath, `---
|
|
type: person
|
|
title: Elon Musk
|
|
slug: people/elon
|
|
---
|
|
|
|
Poisoned content that would overwrite people/elon.
|
|
`);
|
|
|
|
const engine = mockEngine();
|
|
const result = await importFile(engine, filePath, 'notes/random.md', { noEmbed: true });
|
|
|
|
expect(result.status).toBe('skipped');
|
|
expect(result.error).toContain('people/elon');
|
|
expect(result.error).toContain('notes/random');
|
|
// No writes to the DB — the hijack never reaches putPage/createVersion.
|
|
expect((engine as any)._calls.length).toBe(0);
|
|
});
|
|
|
|
test('accepts frontmatter slug that matches the file path', async () => {
|
|
// Sanity: a legitimate file whose frontmatter slug happens to equal the
|
|
// path-derived slug must still import.
|
|
const filePath = join(TMP, 'alice.md');
|
|
writeFileSync(filePath, `---
|
|
type: person
|
|
title: Alice
|
|
slug: people/alice-smith
|
|
---
|
|
|
|
Legit content.
|
|
`);
|
|
|
|
const engine = mockEngine();
|
|
const result = await importFile(engine, filePath, 'people/alice-smith.md', { noEmbed: true });
|
|
|
|
expect(result.status).toBe('imported');
|
|
expect(result.slug).toBe('people/alice-smith');
|
|
});
|
|
|
|
test('uses path-derived slug when no frontmatter slug is set', async () => {
|
|
// The common case: no frontmatter.slug, so the path determines the slug.
|
|
const filePath = join(TMP, 'concept-path.md');
|
|
writeFileSync(filePath, `---
|
|
type: concept
|
|
title: From Path
|
|
---
|
|
|
|
Content.
|
|
`);
|
|
|
|
const engine = mockEngine();
|
|
const result = await importFile(engine, filePath, 'concepts/from-path.md', { noEmbed: true });
|
|
|
|
expect(result.status).toBe('imported');
|
|
expect(result.slug).toBe('concepts/from-path');
|
|
});
|
|
|
|
test('skips symlinks in importFromFile (defense-in-depth)', async () => {
|
|
// Even if the walker somehow passes a symlink through, importFromFile
|
|
// should catch it and return skipped.
|
|
const realFile = join(TMP, 'real-target.md');
|
|
writeFileSync(realFile, `---
|
|
type: concept
|
|
title: Real
|
|
---
|
|
|
|
Content.
|
|
`);
|
|
const linkPath = join(TMP, 'symlink-file.md');
|
|
try { rmSync(linkPath); } catch { /* may not exist */ }
|
|
symlinkSync(realFile, linkPath);
|
|
|
|
const engine = mockEngine();
|
|
const result = await importFile(engine, linkPath, 'symlink-file.md', { noEmbed: true });
|
|
|
|
expect(result.status).toBe('skipped');
|
|
expect(result.error).toContain('symlink');
|
|
expect((engine as any)._calls.length).toBe(0);
|
|
});
|
|
|
|
test('skips file when content hash matches (idempotent)', async () => {
|
|
const filePath = join(TMP, 'unchanged.md');
|
|
writeFileSync(filePath, `---
|
|
type: concept
|
|
title: Unchanged
|
|
---
|
|
|
|
Same content.
|
|
`);
|
|
|
|
// Hash now includes ALL fields (title, type, frontmatter, tags)
|
|
const { createHash } = await import('crypto');
|
|
const { parseMarkdown } = await import('../src/core/markdown.ts');
|
|
const content = `---
|
|
type: concept
|
|
title: Unchanged
|
|
---
|
|
|
|
Same content.
|
|
`;
|
|
const parsed = parseMarkdown(content, 'concepts/unchanged.md');
|
|
const hash = createHash('sha256')
|
|
.update(JSON.stringify({
|
|
title: parsed.title,
|
|
type: parsed.type,
|
|
compiled_truth: parsed.compiled_truth,
|
|
timeline: parsed.timeline,
|
|
frontmatter: parsed.frontmatter,
|
|
tags: parsed.tags.sort(),
|
|
}))
|
|
.digest('hex');
|
|
|
|
const engine = mockEngine({
|
|
getPage: () => Promise.resolve({ content_hash: hash }),
|
|
});
|
|
|
|
const result = await importFile(engine, filePath, 'concepts/unchanged.md', { noEmbed: true });
|
|
expect(result.status).toBe('skipped');
|
|
|
|
const calls = (engine as any)._calls;
|
|
const putCall = calls.find((c: any) => c.method === 'putPage');
|
|
expect(putCall).toBeUndefined();
|
|
});
|
|
|
|
test('reconciles tags: ADD-ONLY, never removes (v0.41.37.0 #1621)', async () => {
|
|
// Add-only reconciliation: re-import adds current frontmatter tags and
|
|
// NEVER deletes — so DB-side enrichment tags (here simulated as 'old-tag')
|
|
// survive. The pre-#1621 behavior deleted every existing tag not in the
|
|
// current frontmatter, wiping enrichment tags on every re-import/reindex.
|
|
const filePath = join(TMP, 'retag.md');
|
|
writeFileSync(filePath, `---
|
|
type: concept
|
|
title: Retagged
|
|
tags: [new-tag, kept-tag]
|
|
---
|
|
|
|
Content here.
|
|
`);
|
|
|
|
const engine = mockEngine({
|
|
getTags: () => Promise.resolve(['old-tag', 'kept-tag']),
|
|
getPage: () => Promise.resolve(null),
|
|
});
|
|
|
|
await importFile(engine, filePath, 'concepts/retag.md', { noEmbed: true });
|
|
|
|
const calls = (engine as any)._calls;
|
|
const removeCalls = calls.filter((c: any) => c.method === 'removeTag');
|
|
const addCalls = calls.filter((c: any) => c.method === 'addTag');
|
|
|
|
// No removals — 'old-tag' (enrichment) is preserved.
|
|
expect(removeCalls.length).toBe(0);
|
|
// Both current frontmatter tags are added (idempotent via ON CONFLICT).
|
|
expect(addCalls.length).toBe(2);
|
|
expect(addCalls.map((c: any) => c.args[1]).sort()).toEqual(['kept-tag', 'new-tag']);
|
|
});
|
|
|
|
test('chunks compiled_truth and timeline separately', async () => {
|
|
const filePath = join(TMP, 'chunked.md');
|
|
writeFileSync(filePath, `---
|
|
type: concept
|
|
title: Chunked
|
|
---
|
|
|
|
This is compiled truth content that should be chunked as compiled_truth source.
|
|
|
|
<!-- timeline -->
|
|
|
|
- 2024-01-01: This is timeline content that should be chunked as timeline source.
|
|
`);
|
|
|
|
const engine = mockEngine();
|
|
const result = await importFile(engine, filePath, 'concepts/chunked.md', { noEmbed: true });
|
|
|
|
expect(result.status).toBe('imported');
|
|
expect(result.chunks).toBeGreaterThanOrEqual(2);
|
|
|
|
const calls = (engine as any)._calls;
|
|
const chunkCall = calls.find((c: any) => c.method === 'upsertChunks');
|
|
const chunks = chunkCall.args[1];
|
|
|
|
const ctChunks = chunks.filter((c: any) => c.chunk_source === 'compiled_truth');
|
|
const tlChunks = chunks.filter((c: any) => c.chunk_source === 'timeline');
|
|
expect(ctChunks.length).toBeGreaterThan(0);
|
|
expect(tlChunks.length).toBeGreaterThan(0);
|
|
});
|
|
|
|
test('handles file with minimal content', async () => {
|
|
const filePath = join(TMP, 'minimal.md');
|
|
writeFileSync(filePath, `---
|
|
type: concept
|
|
title: Minimal
|
|
---
|
|
|
|
One line.
|
|
`);
|
|
|
|
const engine = mockEngine();
|
|
const result = await importFile(engine, filePath, 'concepts/minimal.md', { noEmbed: true });
|
|
|
|
expect(result.status).toBe('imported');
|
|
expect(result.chunks).toBeGreaterThanOrEqual(1);
|
|
});
|
|
|
|
test('skips chunking for empty timeline', async () => {
|
|
const filePath = join(TMP, 'empty-tl.md');
|
|
writeFileSync(filePath, `---
|
|
type: concept
|
|
title: No Timeline
|
|
---
|
|
|
|
Just compiled truth, no timeline separator.
|
|
`);
|
|
|
|
const engine = mockEngine();
|
|
const result = await importFile(engine, filePath, 'concepts/empty-tl.md', { noEmbed: true });
|
|
|
|
expect(result.status).toBe('imported');
|
|
|
|
const calls = (engine as any)._calls;
|
|
const chunkCall = calls.find((c: any) => c.method === 'upsertChunks');
|
|
if (chunkCall) {
|
|
const chunks = chunkCall.args[1];
|
|
const tlChunks = chunks.filter((c: any) => c.chunk_source === 'timeline');
|
|
expect(tlChunks.length).toBe(0);
|
|
}
|
|
});
|
|
|
|
test('noEmbed: true skips embedding', async () => {
|
|
const filePath = join(TMP, 'no-embed.md');
|
|
writeFileSync(filePath, `---
|
|
type: concept
|
|
title: No Embed
|
|
---
|
|
|
|
Content to chunk but not embed.
|
|
`);
|
|
|
|
const engine = mockEngine();
|
|
const result = await importFile(engine, filePath, 'concepts/no-embed.md', { noEmbed: true });
|
|
|
|
expect(result.status).toBe('imported');
|
|
const calls = (engine as any)._calls;
|
|
const chunkCall = calls.find((c: any) => c.method === 'upsertChunks');
|
|
if (chunkCall) {
|
|
for (const chunk of chunkCall.args[1]) {
|
|
expect(chunk.embedding).toBeUndefined();
|
|
}
|
|
}
|
|
});
|
|
|
|
test('rejects in-memory content larger than MAX_FILE_SIZE', async () => {
|
|
// The remote MCP put_page operation hands user-supplied content straight
|
|
// to importFromContent, which is the path this guard defends. The guard
|
|
// must trigger BEFORE parseMarkdown / chunkText / embedBatch — if it doesn't,
|
|
// an authenticated attacker can force the owner to pay for embedding a
|
|
// multi-megabyte string.
|
|
const bigContent = '---\ntitle: Big\n---\n' + 'x'.repeat(5_100_000);
|
|
|
|
const engine = mockEngine();
|
|
const result = await importFromContent(engine, 'big-slug', bigContent, { noEmbed: true });
|
|
|
|
expect(result.status).toBe('skipped');
|
|
expect(result.error).toContain('too large');
|
|
// No engine work at all — confirms the guard short-circuits before any
|
|
// parsing or chunking allocation.
|
|
expect((engine as any)._calls.length).toBe(0);
|
|
});
|
|
|
|
test('uses UTF-8 byte length, not JS string length, for the size check', async () => {
|
|
// 2.6M 4-byte codepoints = ~10.4 MB UTF-8 but only 2.6M JS UTF-16 code units.
|
|
// A length-based check would let this through; a byteLength check catches it.
|
|
const fourByteChar = '\u{1F600}'; // emoji, 4 bytes in UTF-8
|
|
const bigContent = fourByteChar.repeat(2_600_000);
|
|
|
|
const engine = mockEngine();
|
|
const result = await importFromContent(engine, 'emoji-slug', bigContent, { noEmbed: true });
|
|
|
|
expect(result.status).toBe('skipped');
|
|
expect(result.error).toContain('too large');
|
|
expect((engine as any)._calls.length).toBe(0);
|
|
});
|
|
|
|
test('accepts in-memory content just under MAX_FILE_SIZE', async () => {
|
|
// Sanity: content exactly at the limit must still import. If this test
|
|
// fails, the guard is off-by-one and will break legitimate large imports.
|
|
const content = '---\ntitle: Borderline\n---\n' + 'x'.repeat(4_900_000);
|
|
|
|
const engine = mockEngine();
|
|
const result = await importFromContent(engine, 'borderline-slug', content, { noEmbed: true });
|
|
|
|
expect(result.status).toBe('imported');
|
|
});
|
|
|
|
test('assigns sequential chunk_index values', async () => {
|
|
const filePath = join(TMP, 'indexed.md');
|
|
const longText = Array(50).fill('This is a sentence that adds length to the content.').join(' ');
|
|
writeFileSync(filePath, `---
|
|
type: concept
|
|
title: Indexed
|
|
---
|
|
|
|
${longText}
|
|
|
|
---
|
|
|
|
${longText}
|
|
`);
|
|
|
|
const engine = mockEngine();
|
|
await importFile(engine, filePath, 'concepts/indexed.md', { noEmbed: true });
|
|
|
|
const calls = (engine as any)._calls;
|
|
const chunkCall = calls.find((c: any) => c.method === 'upsertChunks');
|
|
if (chunkCall) {
|
|
const chunks = chunkCall.args[1];
|
|
for (let i = 0; i < chunks.length; i++) {
|
|
expect(chunks[i].chunk_index).toBe(i);
|
|
}
|
|
}
|
|
});
|
|
});
|
|
|
|
describe('importFile — CJK wave (v0.32.7)', () => {
|
|
test('REGRESSION: pure-CJK filename with NO frontmatter slug imports cleanly as CJK slug', async () => {
|
|
// After #115, slugifyPath('小米.md') = '小米' (CJK preserved). The
|
|
// anti-spoof rule is content with no frontmatter slug present.
|
|
const filePath = join(TMP, '小米.md');
|
|
writeFileSync(filePath, `---
|
|
type: company
|
|
title: Xiaomi
|
|
---
|
|
|
|
Body text.
|
|
`);
|
|
const engine = mockEngine();
|
|
const result = await importFile(engine, filePath, '小米.md', { noEmbed: true });
|
|
expect(result.status).toBe('imported');
|
|
expect(result.slug).toBe('小米');
|
|
const putCall = (engine as any)._calls.find((c: any) => c.method === 'putPage');
|
|
expect(putCall.args[1].chunker_version).toBe(MARKDOWN_CHUNKER_VERSION);
|
|
expect(putCall.args[1].source_path).toBe('小米.md');
|
|
});
|
|
|
|
test('empty-path-slug + frontmatter slug → fallback path fires (emoji filename)', async () => {
|
|
// 🚀.md slugifies empty even after #115 (emoji not in CJK ranges).
|
|
// Frontmatter slug must take over. logSlugFallback fires.
|
|
const filePath = join(TMP, '🚀.md');
|
|
writeFileSync(filePath, `---
|
|
type: project
|
|
title: Launch
|
|
slug: projects/launch
|
|
---
|
|
|
|
Lifting off.
|
|
`);
|
|
const engine = mockEngine();
|
|
const result = await importFile(engine, filePath, '🚀.md', { noEmbed: true });
|
|
expect(result.status).toBe('imported');
|
|
expect(result.slug).toBe('projects/launch');
|
|
const putCall = (engine as any)._calls.find((c: any) => c.method === 'putPage');
|
|
expect(putCall.args[0]).toBe('projects/launch');
|
|
expect(putCall.args[1].source_path).toBe('🚀.md');
|
|
});
|
|
|
|
test('empty-path-slug + NO frontmatter slug → friendly D6=B error message', async () => {
|
|
// 🌟🚀 slugifies to '' (both emoji stripped, no remaining chars).
|
|
// No frontmatter slug to fall back on → friendly error.
|
|
const filePath = join(TMP, '🌟🚀.md');
|
|
writeFileSync(filePath, `# Bare body without frontmatter slug
|
|
|
|
just content.
|
|
`);
|
|
const engine = mockEngine();
|
|
const result = await importFile(engine, filePath, '🌟🚀.md', { noEmbed: true });
|
|
expect(result.status).toBe('skipped');
|
|
expect(result.error).toContain('no usable slug');
|
|
expect(result.error).toContain('at least one letter or number (any script)');
|
|
expect((engine as any)._calls.length).toBe(0);
|
|
});
|
|
|
|
test('REGRESSION: anti-spoof still rejects when path DOES derive a slug', async () => {
|
|
// notes/random.md derives slug `notes/random`. Frontmatter `slug: people/elon`
|
|
// is a mismatch and MUST still be rejected (the original PR #598 + C1 test
|
|
// fixture contradiction concern).
|
|
const filePath = join(TMP, 'antispoof-cjk-wave.md');
|
|
writeFileSync(filePath, `---
|
|
type: person
|
|
title: Elon
|
|
slug: people/elon
|
|
---
|
|
|
|
Hijack.
|
|
`);
|
|
const engine = mockEngine();
|
|
const result = await importFile(engine, filePath, 'notes/antispoof-cjk-wave.md', { noEmbed: true });
|
|
expect(result.status).toBe('skipped');
|
|
expect(result.error).toContain('does not match');
|
|
expect((engine as any)._calls.length).toBe(0);
|
|
});
|
|
|
|
test('chunker_version + source_path populated on every import', async () => {
|
|
const filePath = join(TMP, 'cjk-source-path.md');
|
|
writeFileSync(filePath, `---
|
|
type: concept
|
|
title: Has source path
|
|
---
|
|
|
|
Content.
|
|
`);
|
|
const engine = mockEngine();
|
|
await importFile(engine, filePath, 'concepts/cjk-source-path.md', { noEmbed: true });
|
|
const putCall = (engine as any)._calls.find((c: any) => c.method === 'putPage');
|
|
expect(putCall).toBeTruthy();
|
|
expect(putCall.args[1].chunker_version).toBe(MARKDOWN_CHUNKER_VERSION);
|
|
expect(putCall.args[1].source_path).toBe('concepts/cjk-source-path.md');
|
|
});
|
|
});
|
|
|
|
// v0.39.3.0 CV8 Phase 3d — DB content_hash excludes timestamp-bearing
|
|
// frontmatter keys (captured_at, ingested_at) so identical body content
|
|
// from capture-cli produces a stable hash across multiple captures.
|
|
// Pre-fix, every capture invocation produced a fresh hash because the
|
|
// captured_at timestamp changed, defeating both the existing.content_hash
|
|
// short-circuit AND the daemon's 24h LRU dedup.
|
|
describe('importFromContent — CV8 DB content_hash stability', () => {
|
|
test('captured_at differences produce IDENTICAL hash (capture-cli dedup)', async () => {
|
|
// Capture #1 at one timestamp
|
|
const t1 = '2026-05-22T10:00:00.000Z';
|
|
const content1 = `---
|
|
type: note
|
|
title: Same Text
|
|
captured_at: ${t1}
|
|
captured_via: capture-cli
|
|
---
|
|
|
|
# Same Text
|
|
|
|
remember to follow up
|
|
`;
|
|
// Capture #2 at a different timestamp (same body)
|
|
const t2 = '2026-05-22T11:00:00.000Z';
|
|
const content2 = `---
|
|
type: note
|
|
title: Same Text
|
|
captured_at: ${t2}
|
|
captured_via: capture-cli
|
|
---
|
|
|
|
# Same Text
|
|
|
|
remember to follow up
|
|
`;
|
|
|
|
let firstHash: string | undefined;
|
|
let secondHash: string | undefined;
|
|
let firstStatus: string | undefined;
|
|
let secondStatus: string | undefined;
|
|
|
|
// First call: no existing page; hash is computed and written
|
|
const engine1 = mockEngine({
|
|
getPage: () => Promise.resolve(null),
|
|
putPage: (_slug: string, page: any) => {
|
|
firstHash = page.content_hash;
|
|
return Promise.resolve(null);
|
|
},
|
|
});
|
|
const r1 = await importFromContent(engine1, 'inbox/test', content1, { noEmbed: true });
|
|
firstStatus = r1.status;
|
|
expect(firstStatus).toBe('imported');
|
|
expect(firstHash).toBeTruthy();
|
|
|
|
// Second call: existing page has the first hash; the second capture's
|
|
// hash must match so the short-circuit fires and status === 'skipped'.
|
|
const engine2 = mockEngine({
|
|
getPage: () => Promise.resolve({ content_hash: firstHash } as any),
|
|
putPage: (_slug: string, page: any) => {
|
|
secondHash = page.content_hash;
|
|
return Promise.resolve(null);
|
|
},
|
|
});
|
|
const r2 = await importFromContent(engine2, 'inbox/test', content2, { noEmbed: true });
|
|
secondStatus = r2.status;
|
|
expect(secondStatus).toBe('skipped'); // hash matched
|
|
expect(secondHash).toBeUndefined(); // putPage NOT called (short-circuited)
|
|
});
|
|
|
|
test('body change DOES change the hash (real edits not silently swallowed)', async () => {
|
|
const t = '2026-05-22T10:00:00.000Z';
|
|
const content1 = `---
|
|
type: note
|
|
captured_at: ${t}
|
|
---
|
|
|
|
original body
|
|
`;
|
|
const content2 = `---
|
|
type: note
|
|
captured_at: ${t}
|
|
---
|
|
|
|
edited body
|
|
`;
|
|
|
|
let firstHash: string | undefined;
|
|
let secondHash: string | undefined;
|
|
|
|
const engine1 = mockEngine({
|
|
getPage: () => Promise.resolve(null),
|
|
putPage: (_slug: string, page: any) => {
|
|
firstHash = page.content_hash;
|
|
return Promise.resolve(null);
|
|
},
|
|
});
|
|
await importFromContent(engine1, 'inbox/test', content1, { noEmbed: true });
|
|
|
|
const engine2 = mockEngine({
|
|
getPage: () => Promise.resolve({ content_hash: firstHash } as any),
|
|
putPage: (_slug: string, page: any) => {
|
|
secondHash = page.content_hash;
|
|
return Promise.resolve(null);
|
|
},
|
|
});
|
|
const r2 = await importFromContent(engine2, 'inbox/test', content2, { noEmbed: true });
|
|
expect(r2.status).toBe('imported'); // body changed, hash differs, re-imported
|
|
expect(secondHash).toBeTruthy();
|
|
expect(secondHash).not.toBe(firstHash);
|
|
});
|
|
|
|
test('tag change DOES change the hash (sync tag-add not silently swallowed)', async () => {
|
|
// Regression guard: stripping captured_at from the hash input must NOT
|
|
// also strip tags. A user editing a markdown file to add a tag still
|
|
// expects tag reconciliation to fire.
|
|
const content1 = `---
|
|
type: concept
|
|
tags: [alpha]
|
|
---
|
|
|
|
body
|
|
`;
|
|
const content2 = `---
|
|
type: concept
|
|
tags: [alpha, beta]
|
|
---
|
|
|
|
body
|
|
`;
|
|
|
|
let firstHash: string | undefined;
|
|
let secondHash: string | undefined;
|
|
|
|
const engine1 = mockEngine({
|
|
getPage: () => Promise.resolve(null),
|
|
putPage: (_slug: string, page: any) => {
|
|
firstHash = page.content_hash;
|
|
return Promise.resolve(null);
|
|
},
|
|
});
|
|
await importFromContent(engine1, 'concepts/x', content1, { noEmbed: true });
|
|
|
|
const engine2 = mockEngine({
|
|
getPage: () => Promise.resolve({ content_hash: firstHash } as any),
|
|
putPage: (_slug: string, page: any) => {
|
|
secondHash = page.content_hash;
|
|
return Promise.resolve(null);
|
|
},
|
|
});
|
|
const r2 = await importFromContent(engine2, 'concepts/x', content2, { noEmbed: true });
|
|
expect(r2.status).toBe('imported'); // tags changed, hash differs
|
|
expect(secondHash).not.toBe(firstHash);
|
|
});
|
|
|
|
test('ingested_at differences produce IDENTICAL hash (server-stamp dedup)', async () => {
|
|
// Provenance write-through stamps `ingested_at` server-side per CV6;
|
|
// a put_page that's just refreshing provenance (e.g. capture re-runs
|
|
// the same file later) must not invalidate the chunk cache.
|
|
const content1 = `---
|
|
type: note
|
|
ingested_at: '2026-05-22T10:00:00.000Z'
|
|
---
|
|
|
|
body unchanged
|
|
`;
|
|
const content2 = `---
|
|
type: note
|
|
ingested_at: '2026-05-22T11:00:00.000Z'
|
|
---
|
|
|
|
body unchanged
|
|
`;
|
|
|
|
let firstHash: string | undefined;
|
|
let shortCircuited = false;
|
|
|
|
const engine1 = mockEngine({
|
|
getPage: () => Promise.resolve(null),
|
|
putPage: (_slug: string, page: any) => {
|
|
firstHash = page.content_hash;
|
|
return Promise.resolve(null);
|
|
},
|
|
});
|
|
await importFromContent(engine1, 'inbox/y', content1, { noEmbed: true });
|
|
|
|
const engine2 = mockEngine({
|
|
getPage: () => Promise.resolve({ content_hash: firstHash } as any),
|
|
putPage: () => {
|
|
shortCircuited = false;
|
|
return Promise.resolve(null);
|
|
},
|
|
});
|
|
const r2 = await importFromContent(engine2, 'inbox/y', content2, { noEmbed: true });
|
|
shortCircuited = r2.status === 'skipped';
|
|
expect(shortCircuited).toBe(true);
|
|
});
|
|
});
|
|
|
|
// ────────────────────────────────────────────────────────────────
|
|
// #1035 — type round-trip preservation
|
|
//
|
|
// put_page → importFromContent → parseMarkdown infers type from the file
|
|
// path when frontmatter omits `type:`, and bare slugs infer 'concept'.
|
|
// Pre-fix, a round-trip put (get_page → edit body → put_page WITHOUT a
|
|
// type: line) silently regressed a curated type to 'concept'. Absence of
|
|
// an explicit frontmatter type on an EXISTING page must preserve the
|
|
// stored type; an explicit type stays an override; new pages still infer.
|
|
// ────────────────────────────────────────────────────────────────
|
|
|
|
describe('importFromContent type round-trip (#1035)', () => {
|
|
test('re-put without type: preserves the existing page type', async () => {
|
|
let putType: string | undefined;
|
|
const engine = mockEngine({
|
|
getPage: () => Promise.resolve({
|
|
slug: 'founder-notes-example',
|
|
type: 'person',
|
|
content_hash: 'different-hash',
|
|
updated_at: new Date(),
|
|
created_at: new Date(),
|
|
} as any),
|
|
putPage: (_slug: string, page: any) => {
|
|
putType = page.type;
|
|
return Promise.resolve(null);
|
|
},
|
|
});
|
|
const result = await importFromContent(engine, 'founder-notes-example', [
|
|
'---',
|
|
'title: Notes',
|
|
'---',
|
|
'',
|
|
'Edited body, no type in frontmatter.',
|
|
].join('\n'), { noEmbed: true });
|
|
expect(result.status).toBe('imported');
|
|
expect(putType).toBe('person'); // pre-fix: 'concept' (bare-slug inference default)
|
|
});
|
|
|
|
test('explicit frontmatter type: still overrides the existing type', async () => {
|
|
let putType: string | undefined;
|
|
const engine = mockEngine({
|
|
getPage: () => Promise.resolve({
|
|
slug: 'founder-notes-example',
|
|
type: 'person',
|
|
content_hash: 'different-hash',
|
|
updated_at: new Date(),
|
|
created_at: new Date(),
|
|
} as any),
|
|
putPage: (_slug: string, page: any) => {
|
|
putType = page.type;
|
|
return Promise.resolve(null);
|
|
},
|
|
});
|
|
const result = await importFromContent(engine, 'founder-notes-example', [
|
|
'---',
|
|
'type: note',
|
|
'title: Notes',
|
|
'---',
|
|
'',
|
|
'Explicit type wins.',
|
|
].join('\n'), { noEmbed: true });
|
|
expect(result.status).toBe('imported');
|
|
expect(putType).toBe('note');
|
|
});
|
|
|
|
test('new page without type: still infers from path', async () => {
|
|
let putType: string | undefined;
|
|
const engine = mockEngine({
|
|
getPage: () => Promise.resolve(null),
|
|
putPage: (_slug: string, page: any) => {
|
|
putType = page.type;
|
|
return Promise.resolve(null);
|
|
},
|
|
});
|
|
const result = await importFromContent(engine, 'people/alice-example', [
|
|
'---',
|
|
'title: Alice',
|
|
'---',
|
|
'',
|
|
'A new person page.',
|
|
].join('\n'), { noEmbed: true });
|
|
expect(result.status).toBe('imported');
|
|
expect(putType).toBe('person'); // /people/ path-prefix inference intact
|
|
});
|
|
});
|