mirror of
https://github.com/garrytan/gbrain.git
synced 2026-07-30 11:22:34 +00:00
88287775e5
Takeover of #2847 (rebased onto current master). Fixes #2826. - cjk.ts: estimateEmbeddingTokens() — conservative per-char-class token estimate (CJK 1.0, other 0.75, whitespace 0.1 per code unit). - recursive.ts (MARKDOWN_CHUNKER_VERSION 3→4): countWords floored at ceil(nonWhitespaceChars/6); capByEstimatedTokens() final pass with ChunkOptions.maxTokens (default 1500). - code.ts (CHUNKER_VERSION 4→5): capCodeChunks() applies the same cap to AST-path chunks that splitLargeNode can't subdivide. Co-authored-by: paul-0320 <paul-0320@users.noreply.github.com> Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
52 lines
2.4 KiB
TypeScript
52 lines
2.4 KiB
TypeScript
/**
|
|
* v0.20.0 Cathedral II Layer 12 — CHUNKER_VERSION 3→4 bump + SP-1 gate.
|
|
*
|
|
* Codex's second-pass review caught that bumping CHUNKER_VERSION alone
|
|
* does nothing on an unchanged repo: `performSync` short-circuits at
|
|
* `up_to_date` before reaching `importCodeFile`'s content_hash check.
|
|
* Layer 12 adds a sources.chunker_version gate that forces a full
|
|
* re-walk when the version mismatches, regardless of git HEAD equality.
|
|
*
|
|
* These tests validate the constant value, the gate logic, and the
|
|
* write-after-sync behavior. Full e2e covered by test/e2e if DB present.
|
|
*/
|
|
|
|
import { describe, test, expect } from 'bun:test';
|
|
import { CHUNKER_VERSION } from '../src/core/chunkers/code.ts';
|
|
|
|
describe('Layer 12 — CHUNKER_VERSION constant', () => {
|
|
test('bumped to 5 for the estimated-token hard cap', () => {
|
|
// v3: v0.19.0 Chonkie parity (tokenizer + small-sibling merge).
|
|
// v4: v0.20.0 Cathedral II (qualified names + parent scope + doc_comment
|
|
// + fence extraction + chunk-grain FTS). Folded into content_hash
|
|
// so any bump forces clean re-chunks on next sync.
|
|
// v5: estimated-token hard cap on AST-path chunks (capCodeChunks) so
|
|
// un-subdividable giant nodes can't overflow strict embedding
|
|
// server token limits.
|
|
expect(CHUNKER_VERSION).toBe(5);
|
|
});
|
|
|
|
test('is stable across imports (not recomputed at call time)', async () => {
|
|
const a = (await import('../src/core/chunkers/code.ts')).CHUNKER_VERSION;
|
|
const b = (await import('../src/core/chunkers/code.ts')).CHUNKER_VERSION;
|
|
expect(a).toBe(b);
|
|
expect(a).toBe(5);
|
|
});
|
|
});
|
|
|
|
describe('Layer 12 — sources.chunker_version column from v27 migration', () => {
|
|
test('v27 foundation migration adds chunker_version to sources', async () => {
|
|
const { MIGRATIONS } = await import('../src/core/migrate.ts');
|
|
const v27 = MIGRATIONS.find(m => m.version === 27);
|
|
expect(v27).toBeDefined();
|
|
expect(v27!.sql).toMatch(/ALTER TABLE sources/);
|
|
expect(v27!.sql).toMatch(/ADD COLUMN IF NOT EXISTS chunker_version TEXT/);
|
|
});
|
|
});
|
|
|
|
// Full integration test: would run an e2e sync twice against a real git
|
|
// repo fixture and assert that the second sync (with HEAD unchanged but
|
|
// chunker_version advanced) re-walks. That lives in
|
|
// test/e2e/cathedral-ii.test.ts (future Layer 5 pilot). This file pins
|
|
// the constant + migration shape so any accidental revert surfaces in CI.
|