diff --git a/CHANGELOG.md b/CHANGELOG.md index 49598a0c0..78b30e60d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,125 @@ All notable changes to GBrain will be documented in this file. +## [0.32.7] - 2026-05-11 + +**CJK users get a working brain end-to-end on PGLite.** +**Six layers of ASCII-only assumptions fixed in one wave.** + +For the past month Chinese / Japanese / Korean gbrain users have been hitting silent data loss at six different points in the pipeline: `git diff` dropping CJK paths, slugify collapsing them to empty, import rejecting them, the chunker treating a whole Chinese paragraph as one word and exceeding the OpenAI embedding token limit, search returning nothing on PGLite, and adjacent slug validators rejecting CJK even after the slugify fix. Five PRs from @vinsew over April 14 to May 3 plus one extracted from @313094319-sudo's #765 land together as one coherent collector. Codex's outside-voice review on the plan caught four critical bugs the eng review missed — the original "fold chunker version into content_hash" idea was a no-op, the cost prompt referenced phantom data fields, the LIKE SQL needed two distinct param bindings, and `countCJKAwareWords` would have over-split English-heavy docs with one foreign term. + +After this release, `gbrain sync` of `品牌圣经.md` actually creates a page at slug `品牌圣经`, the chunker produces ≤ 6000-char chunks regardless of script, `gbrain search "测试"` returns hits on a PGLite brain, and an Apple Notes export with `2026-04-14 22_38 記録-個人智能体_原文.md` lands cleanly through incremental sync. Postgres CJK keyword search needs an extension (pgroonga or zhparser); pgroonga + ngram trigram support is the next v0.33+ TODO. + +### The numbers that matter + +Every fix layer has a concrete observable change. Counts from a fresh PGLite brain with a 4-page Chinese/Japanese/Korean fixture: + +| Layer | Pre-fix | Post-fix | +|---|---|---| +| `git diff --name-status` on CJK path | quoted octal escapes (`\345\223\201`); dropped from manifest | UTF-8 path literal; included in manifest | +| `slugifySegment("品牌圣经")` | `""` (silent collision with `"销售论证文档" → ""`) | `"品牌圣经"` | +| `importFromFile` of root-level `小米.md` | `Invalid slug: ""` | imported as slug `小米` | +| `countWords` on a 1000-char Chinese paragraph | `1` (treated as single token; embedder rejects with `400 maximum input length 8192 tokens`) | `1000` (char count via 30% density threshold) | +| `gbrain search "测试"` on PGLite | empty results (English FTS tokenizer can't segment CJK) | bigram-count-ranked hits | +| `pages.chunker_version` post-upgrade reindex | n/a (re-embed never fired) | `gbrain upgrade` prints cost estimate + reindex sweep brings every markdown page to v2 | + +`gbrain upgrade` prints a stderr line before the sweep starts: + +``` +[chunker-bump] Will re-embed ~1386 markdown pages via openai:text-embedding-3-large, est. ~$0.50, ~23min. Press Ctrl-C within 10s to abort. +``` + +On a 1386-page brain (the maintainer's own deployment) the cost is roughly $0.50 + 3 minutes wall-clock. On a 100K-page brain it scales linearly to ~$36 + 30 minutes. Headless installs (CI, cron) skip the wait; `GBRAIN_NO_REEMBED=1` opts out entirely with a doctor warning marker. + +### What this means for CJK users + +If you imported a Chinese / Japanese / Korean brain pre-v0.32.7 and saw silently empty search or `Invalid slug` errors, run `gbrain upgrade` and the wave heals you up automatically. The reindex sweep bumps `chunker_version` from 1 → 2 on every markdown page; on the next sync, files like `小米.md` that previously failed with `Invalid slug: ""` import cleanly via the frontmatter-slug fallback path. The audit trail at `~/.gbrain/audit/slug-fallback-YYYY-Www.jsonl` shows you every file where the fallback fired. + +### Itemized changes + +**Six layers of CJK fixes (contributed by @vinsew via PRs #114 / #115 / #119 / #598 / #599 + @313094319-sudo via #765, extracted CJK piece):** + +- New `src/core/cjk.ts` module — single source of truth for CJK detection (Han, Hiragana, Katakana, Hangul Syllables), the slug-char string used by adjacent validators, sentence + clause delimiter sets, and the 30% density threshold heuristic. Replaces the inline regex in `expansion.ts:58` so four-place drift becomes impossible. +- `gbrain sync` git helper refactored: `git()` now takes `configs?: string[]` as a separate parameter and always prepends `core.quotepath=false`. CJK paths arrive as UTF-8 literals through `diff`, `log`, `rev-parse`. Hardened so no future call site can put `-c` after the subcommand and silently break path emission. New invariant test `test/sync.test.ts:git() helper invocation order`. +- `slugifySegment` extended to preserve CJK characters with NFC re-normalization for Hangul Jamo recomposition. `SLUG_SEGMENT_PATTERN` (used by takes-holder validation) extended in the same commit so CJK slugs don't get rejected by adjacent validators. Audit pass also extended `validatePageSlug`, `validateFilename` in `src/core/operations.ts`. Existing `café` → `cafe` Latin-accent regression preserved. +- Recursive chunker fixes: CJK-aware `countWords` via 30% density threshold (English docs with one foreign term stay whitespace-tokenized; Chinese-dominant docs get char-counted), CJK sentence delimiters `。！？` at L2 + clause delimiters `；：，、` at L3, char-slice fallback in `splitOnWhitespace` when a single "word" exceeds target, and `maxChars` hard cap (default 6000) with sliding-window `splitByChars` and 500-char overlap. `MARKDOWN_CHUNKER_VERSION = 2` exported. +- `gbrain import` of CJK / emoji / Thai / Arabic root-level files: when `slugifyPath` returns empty AND frontmatter has a `slug:`, the frontmatter slug becomes authoritative (anti-spoof rule preserved when path slug is non-empty). Audit trail at `~/.gbrain/audit/slug-fallback-YYYY-Www.jsonl` records every fallback fire; `gbrain doctor`'s new `slug_fallback_audit` check surfaces a 7-day rolling count as an `ok` line. +- PGLite CJK keyword fallback in `searchKeyword` + `searchKeywordChunks`: detects `hasCJK(query)` and switches the SQL strategy to `ILIKE ... ESCAPE '\'` over `chunk_text` with bigram-frequency-count ranking. Two distinct parameter bindings ($qLike escaped for the ILIKE, $qRaw raw for the position/replace arithmetic) per codex's C8 catch. Source-boost, hard-exclude, visibility, and DISTINCT-ON survival all preserved. ASCII queries continue through `websearch_to_tsquery('english')` unchanged. + +**Migration v54 (`cjk_wave_pages_chunker_version_and_source_path`):** + +- `pages.chunker_version SMALLINT NOT NULL DEFAULT 1` — set to `MARKDOWN_CHUNKER_VERSION` on every import going forward. +- `pages.source_path TEXT` — captures the import-time repo-relative path so sync's delete/rename paths can resolve frontmatter-fallback slugs. +- Partial indexes on both (markdown-only / non-null) so the post-upgrade sweep query is fast. +- PGLite + Postgres parity via the standard `ALTER TABLE ... IF NOT EXISTS` shape. + +**New `gbrain reindex --markdown` command:** + +- Walks `WHERE chunker_version < 2 AND page_kind = 'markdown'` in 100-row batches. For rows with non-null `source_path` re-imports via `importFromFile`; rows without fall back to `importFromContent` from the stored markdown body. +- Idempotent: re-runs after partial completion pick up where they left off via id-ordered batches. +- Flags: `--limit N`, `--dry-run`, `--json`, `--no-embed` (offline / CI), `--repo PATH`. +- Wired into `gbrain upgrade`'s post-upgrade hook automatically. + +**Cost-estimate prompt in `gbrain upgrade`:** + +- Computes pending page count + char totals from real SQL (`COUNT(*)` + `SUM(LENGTH(compiled_truth)) + SUM(LENGTH(timeline))` against the chunker_version-filtered query — no phantom `markdown_body` column). +- Resolves the gateway's embedding model + dollar cost via the new `src/core/embedding-pricing.ts` map keyed `provider:model` (OpenAI text-embedding-3-large + 3-small + ada-002, Voyage 3-large + 3). Unknown providers degrade to `estimate unavailable for ` instead of fabricating numbers. +- TTY-only 10-second Ctrl-C window; non-TTY auto-proceeds; `GBRAIN_REEMBED_GRACE_SECONDS=0` skips wait; `GBRAIN_NO_REEMBED=1` exits with doctor-warning marker. + +**`gbrain doctor` gains `slug_fallback_audit` check:** + +- Reads the latest weekly `~/.gbrain/audit/slug-fallback-*.jsonl`, counts info-severity entries in the last 7 days, and surfaces the count as an `ok` line. No health-score docking; no warning. `sync-failures.jsonl` (which gates bookmark advancement) stays untouched — info rows live in their own surface per codex C7. + +**Tests:** + +- 16 new unit cases in `test/cjk.test.ts` covering `hasCJK`, `countCJKAwareWords` (30% density threshold), `escapeLikePattern`, the four CJK constants. +- 9 new chunker cases including long Chinese paragraph splits, Japanese `。` delimiter, Korean Hangul, mixed CJK+English, maxChars sliding window, and a pure-English regression. +- 16 new slug-validation cases for the CJK ranges + a SLUG_SEGMENT_PATTERN test that confirms `café` still works and Vietnamese (out of scope) stays rejected. +- 5 new migration-v54 cases (column presence, default, indexes, default inheritance). +- 5 new reindex cases (dry-run, idempotent re-run, --limit cap, skip-already-bumped, version bump). +- 11 new upgrade-prompt cases (real-data estimate, unknown provider fallback, TTY / non-TTY paths, GBRAIN_NO_REEMBED, GBRAIN_REEMBED_GRACE_SECONDS=0). +- 6 new audit-jsonl cases (logSlugFallback, readRecentSlugFallbacks, 7-day window, corrupt-row tolerance). +- 8 new pglite-engine cases (CJK detection routes to LIKE branch, bigram ranking, LIKE-meta escape, regression on ASCII FTS). +- 3 new sync git-helper invariant cases pinning the `-c` flag order. +- 2 new E2E files: `test/e2e/sync-cjk-git.test.ts` (real git CLI emits UTF-8 paths) and `test/e2e/cjk-roundtrip.test.ts` (PGLite-in-memory import → chunk → search). + +**NOT in scope (filed as v0.33+ TODOs):** + +- Postgres-side CJK FTS (pgroonga / zhparser / ngram trigrams). Multi-tenant Postgres deployments still can't search Chinese; defer until users complain. +- Widening CJK ranges to Unicode property escapes (`\p{Script=Han}` etc.) for Han Extensions A/B/C, halfwidth katakana, compatibility ideographs, iteration marks (`々` / `〇`). BMP set covers >99% of real content. +- `git diff --name-status -z` + NUL framing for the path-encoding completeness pass (handles tabs/newlines/quotes that `core.quotepath=false` doesn't cover). +- `extractTrailingContext` CJK-aware overlap. Real bug but the maxChars hard cap is protective; the v0.33+ fix gives normal-size CJK chunks proper overlap context. +- Other non-Latin scripts (Thai, Arabic, Cyrillic, Devanagari). + +## To take advantage of v0.32.7 + +`gbrain upgrade` should do this automatically. If it didn't, or if `gbrain doctor` warns about a partial migration: + +1. **Run the orchestrator manually:** + ```bash + gbrain apply-migrations --yes + ``` +2. **Run the markdown reindex sweep:** + ```bash + gbrain reindex --markdown + ``` + Or skip the embedding cost and let the next `gbrain embed --stale` pass fill in vectors: + ```bash + gbrain reindex --markdown --no-embed + gbrain embed --stale + ``` +3. **Verify the outcome:** + ```bash + gbrain doctor + gbrain search "测试" # or your favorite CJK substring; should return hits on a PGLite brain + ``` +4. **If any step fails or the numbers look wrong,** please file an issue: + https://github.com/garrytan/gbrain/issues with: + - output of `gbrain doctor` + - contents of `~/.gbrain/audit/slug-fallback-*.jsonl` if it exists + - which step broke + + This feedback loop is how gbrain maintainers find fragile upgrade paths. Thank you @vinsew + @313094319-sudo for filing the originating PRs. ## [0.32.6] - 2026-05-11 **Your brain learns to detect its own integrity drift.** diff --git a/CLAUDE.md b/CLAUDE.md index 5da07cf6a..2f0681b75 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -46,6 +46,12 @@ strict behavior when unset. - `src/core/pglite-engine.ts` — PGLite (embedded Postgres 17.5 via WASM) implementation, all 40 BrainEngine methods. `addLinksBatch` / `addTimelineEntriesBatch` use multi-row `unnest()` with manual `$N` placeholders. As of v0.13.1, `connect()` wraps `PGlite.create()` in a try/catch that emits an actionable error naming the macOS 26.3 WASM bug (#223) and pointing at `gbrain doctor`; the lock is released on failure so the next process can retry cleanly. v0.22.0: `searchKeyword` and `searchKeywordChunks` multiply `ts_rank` by the source-factor CASE expression at the chunk-grain level; `searchVector` becomes a two-stage CTE — inner CTE keeps `ORDER BY cc.embedding <=> vec` so HNSW stays usable, outer SELECT re-ranks by `raw_score * source_factor`. Inner LIMIT scales with offset to preserve pagination contract. As of v0.22.6.1, `initSchema()` calls `applyForwardReferenceBootstrap()` BEFORE replaying SCHEMA_SQL — probes for the specific forward-referenced state the embedded schema blob needs (`pages.source_id`, `links.link_source`, `links.origin_page_id`, `content_chunks.symbol_name`, `content_chunks.language`, `sources` FK target table) and adds only what's missing. Closes the upgrade-wedge bug class that bit users 10+ times across 6 schema versions over 2 years (#239/#243/#266/#357/#366/#374/#375/#378/#395/#396). No-op on fresh installs and modern brains. - `src/core/pglite-schema.ts` — PGLite-specific DDL (pgvector, pg_trgm, triggers) - `src/core/postgres-engine.ts` — Postgres + pgvector implementation (Supabase / self-hosted). `addLinksBatch` / `addTimelineEntriesBatch` use `INSERT ... SELECT FROM unnest($1::text[], ...) JOIN pages ON CONFLICT DO NOTHING RETURNING 1` — 4-5 array params regardless of batch size, sidesteps the 65535-parameter cap. As of v0.12.3, `searchKeyword` / `searchVector` scope `statement_timeout` via `sql.begin` + `SET LOCAL` so the GUC dies with the transaction instead of leaking across the pooled postgres.js connection (contributed by @garagon). `getEmbeddingsByChunkIds` uses `tryParseEmbedding` so one corrupt row skips+warns instead of killing the query. v0.22.0: `searchKeyword`, `searchKeywordChunks`, and `searchVector` apply source-aware ranking by inlining the source-factor CASE and `NOT (col LIKE …)` hard-exclude clause from `src/core/search/sql-ranking.ts`. `searchVector` switches to a two-stage CTE (HNSW-safe inner ORDER BY, source-boost re-rank in the outer SELECT) and carries `p.source_id` through inner→outer for v0.18 multi-source callers. v0.22.1 (#406): `_savedConfig` retains the connect config; `reconnect()` tears down + recreates the pool from saved config (called by supervisor watchdog after 3 consecutive health-check failures). `executeRaw` is a single-statement passthrough — no per-call retry (D3 dropped that as unsound for non-idempotent statements; recovery is supervisor-driven). v0.22.1 (#363, contributed by @orendi84): `connect()` applies `resolveSessionTimeouts()` from `db.ts` as connection-time startup parameters (`statement_timeout`, `idle_in_transaction_session_timeout`) so orphan pgbouncer backends can't hold locks for hours. v0.22.1 (#409, contributed by @atrevino47): `countStaleChunks()` + `listStaleChunks()` server-side-filter on `embedding IS NULL` for `embed --stale`, eliminating ~76 MB/call client-side pull on a fully-embedded brain; `upsertChunks()` resets both `embedding` AND `embedded_at` to NULL when chunk_text changes without a new embedding (consistency). As of v0.22.6.1, `initSchema()` calls `applyForwardReferenceBootstrap()` BEFORE replaying SCHEMA_SQL on the same forward-reference probe set as the PGLite engine, so old Postgres brains pinned at v0.13/v0.18/v0.19 walk forward cleanly instead of wedging on `column "..." does not exist`. **v0.28.1:** `disconnect()` is now idempotent. New `_connectionStyle` instance field tracks whether the engine owns its pool (worker engines) or shares the module-level singleton; second call on an instance-pool engine is a no-op rather than falling through to `db.disconnect()` and clobbering the singleton. Pinned by `test/e2e/postgres-engine-disconnect-idempotency.test.ts` (2 cases). Closes the bug class where any test sharing an engine across multiple `worker.start()` / `worker.stop()` cycles silently broke its own DB connectivity. +- `src/core/cjk.ts` (v0.32.7 CJK wave) — Single source of truth for CJK detection across the codebase. Exports `CJK_RANGES_REGEX`, `CJK_SLUG_CHARS` (character-class fragment for embedding inside other regexes), `CJK_SENTENCE_DELIMITERS` (`。！？`), `CJK_CLAUSE_DELIMITERS` (`；：，、`), `CJK_DENSITY_THRESHOLD = 0.30`, `hasCJK(s)`, `countCJKAwareWords(s)` (30% density threshold — English docs with one Japanese term stay whitespace-tokenized; Chinese-dominant docs get char-counted), and `escapeLikePattern(s)` (escapes `%`, `_`, `\\` for `ILIKE ... ESCAPE '\\'`). Replaces the inline hasCJK regex previously duplicated at `expansion.ts:58`. BMP-only ranges (Han / Hiragana / Katakana / Hangul Syllables); widening to Unicode property escapes is a v0.33+ TODO. Consumers: `expansion.ts`, `sync.ts:slugifySegment`, `operations.ts:validatePageSlug + validateFilename`, `chunkers/recursive.ts:countWords + DELIMITERS`, `pglite-engine.ts:searchKeyword + searchKeywordChunks`. +- `src/core/audit-slug-fallback.ts` (v0.32.7 CJK wave) — Weekly ISO-week-rotated audit JSONL at `~/.gbrain/audit/slug-fallback-YYYY-Www.jsonl`. `logSlugFallback(slug, sourcePath)` fires when `importFromFile` falls back to a frontmatter slug because `slugifyPath` returned empty (emoji / Thai / Arabic / non-CJK exotic-script filenames). `readRecentSlugFallbacks(days)` reads the last N days for `gbrain doctor`'s `slug_fallback_audit` check. Honors `GBRAIN_AUDIT_DIR` via the shared `resolveAuditDir()` from shell-audit.ts. Separate surface from `sync-failures.jsonl` per codex outside-voice review — that file carries bookmark-gating semantics that info events shouldn't trigger. +- `src/core/embedding-pricing.ts` (v0.32.7 CJK wave) — `EMBEDDING_PRICING` map keyed `provider:model` for the post-upgrade reindex cost estimate. Sibling to `anthropic-pricing.ts`. Entries: OpenAI text-embedding-3-large ($0.13/1M), 3-small ($0.02/1M), ada-002 ($0.10/1M), Voyage 3-large ($0.18/1M), 3 ($0.06/1M). `lookupEmbeddingPrice(modelString)` returns a tagged union (`known` with price + `unknown` with provider name); `estimateCostFromChars(charCount, pricePerMTok)` uses 3.5 chars/token approximation. Unknown providers degrade gracefully to "estimate unavailable" instead of fabricating numbers. +- `src/core/post-upgrade-reembed.ts` (v0.32.7 CJK wave) — Pure functions backing the `gbrain upgrade` chunker-bump cost prompt. `computeReembedEstimate(engine, model)` queries real SQL (`COUNT(*)` + `COALESCE(SUM(LENGTH(compiled_truth)) + SUM(LENGTH(timeline)), 0)`) on `pages WHERE chunker_version < MARKDOWN_CHUNKER_VERSION`. `formatReembedPrompt(est, graceSeconds)` is the stderr-line formatter. `runPostUpgradeReembedPrompt(engine, model, opts)` orchestrates the 10-second Ctrl-C window; TTY-only wait (non-TTY auto-proceeds for CI / cron); `GBRAIN_NO_REEMBED=1` bails out with a doctor-warning marker; `GBRAIN_REEMBED_GRACE_SECONDS=0` skips the wait. +- `src/commands/reindex.ts` (v0.32.7 CJK wave) — `gbrain reindex --markdown [--limit N] [--dry-run] [--json] [--no-embed] [--repo PATH]`. Walks `pages WHERE page_kind = 'markdown' AND chunker_version < MARKDOWN_CHUNKER_VERSION` in 100-row batches, ordered by id. Rows with non-null `source_path` re-import via `importFromFile`; rows without fall back to `importFromContent` against the stored `compiled_truth`. **Both paths pass `forceRechunk: true`** to bypass `importFromContent`'s `content_hash` short-circuit — without that flag (codex post-merge F1), the chunker version bump never reaches pages whose source content hasn't changed since last sync, AND master's v0.32.2 stripFactsFence privacy strip never applies to pre-strip chunks. Idempotent — partial-completion re-runs pick up where they left off via id-ordered batches. Wired into `src/commands/upgrade.ts:runPostUpgrade` after `apply-migrations`. +- `src/commands/sync.ts:resolveSlugByPathOrSourcePath` (v0.32.7 CJK wave, codex post-merge F4) — Resolves a slug by `pages.source_path` first (returns the stored slug for frontmatter-fallback pages whose path doesn't derive a slug), then falls back to `resolveSlugForPath(path)`. Threaded into all 4 delete/rename call sites (`performSync`'s un-syncable cleanup at ~:531, deletes at ~:603, rename oldSlug at ~:622). Without this, emoji-only / Thai / Arabic filenames whose slug came from frontmatter would orphan on delete/rename (the delete path would compute the wrong path-derived slug). Best-effort query — pre-migration brains fall through to the legacy path. - `src/core/utils.ts` — Shared SQL utilities extracted from postgres-engine.ts. Exports `parseEmbedding(value)` (throws on unknown input, used by migration + ingest paths where data integrity matters) and as of v0.12.3 `tryParseEmbedding(value)` (returns `null` + warns once per process, used by search/rescore paths where availability matters more than strictness). **v0.26.9 (D14):** adds `isUndefinedColumnError(err)` predicate — pattern-matches Postgres SQLSTATE 42703 / "column ... does not exist" with engine-driver shape variation tolerated. Replaces bare `catch {}` blocks in `oauth-provider.ts` so genuine errors (lock timeout, network blip, permission denied) propagate while column-missing falls through to the legacy fallback path. Reusable from any future code that needs the same column-existence probe semantics. - `src/core/db.ts` — Connection management, schema initialization. v0.22.1 (#363, contributed by @orendi84): `resolveSessionTimeouts()` returns `statement_timeout` + `idle_in_transaction_session_timeout` (defaults: 5min each, env-overridable via `GBRAIN_STATEMENT_TIMEOUT` / `GBRAIN_IDLE_TX_TIMEOUT` / `GBRAIN_CLIENT_CHECK_INTERVAL`). Both `connect()` (module singleton) and `PostgresEngine.connect()` (worker pool) consume the result via postgres.js's `connection` option, sending GUCs as startup parameters that survive PgBouncer transaction mode (unlike the prior `setSessionDefaults` post-pool SET, kept as a back-compat no-op shim). - `src/commands/migrate-engine.ts` — Bidirectional engine migration (`gbrain migrate --to supabase/pglite`) @@ -334,6 +340,12 @@ Key commands added for Minions (job queue): - `gbrain jobs smoke [--sigkill-rescue]` — health smoke test. `--sigkill-rescue` is the v0.13.1 regression guard for #219: simulates a killed worker and asserts the stalled job is requeued instead of dead-lettered on first stall. - `gbrain jobs work [--queue Q] [--concurrency N]` — start worker daemon (Postgres only) +Key commands added in v0.32.7 (CJK fix wave): +- `gbrain reindex --markdown [--limit N] [--dry-run] [--json] [--no-embed] [--repo PATH]` — operator-facing markdown re-chunk sweep. Walks pages with `chunker_version < MARKDOWN_CHUNKER_VERSION` (currently 2) and re-imports each with `forceRechunk: true` so the new chunker shape actually applies. Run automatically by `gbrain upgrade`'s post-upgrade hook; available manually for triage. +- `gbrain doctor` learns a new `slug_fallback_audit` check: surfaces info-severity entries from `~/.gbrain/audit/slug-fallback-YYYY-Www.jsonl` (last 7 days) as an `ok` count when CJK / emoji / exotic-script filenames imported via the frontmatter-slug fallback path. +- `gbrain search ""` on PGLite brains now uses an `ILIKE`-based fallback with bigram-frequency-count ranking when the query contains Han / Hiragana / Katakana / Hangul Syllables. ASCII queries continue through `websearch_to_tsquery('english')` unchanged. Postgres-side CJK FTS still requires an extension (pgroonga / zhparser) — see v0.33+ TODO. +- `gbrain upgrade` post-upgrade flow now prints a cost estimate before re-embedding: `[chunker-bump] Will re-embed ~N markdown pages via , est. ~$X.XX, ~Ymin. Press Ctrl-C within 10s to abort.` Sourced from real SQL counts + char totals; TTY-only wait (non-TTY auto-proceeds for CI / cron). Env overrides: `GBRAIN_NO_REEMBED=1` bails out entirely with a doctor-warning marker; `GBRAIN_REEMBED_GRACE_SECONDS=0` skips the wait. + Key commands added in v0.31.12 (model tier system + routing CLI): - `gbrain models [--json]` — read-only routing dashboard. Prints the four tier defaults (`utility`/`reasoning`/`deep`/`subagent`), the resolved value for each (after re-walking `models.default` → `models.tier.` → env → `TIER_DEFAULTS`), every per-task override (`models.dream.synthesize`, `models.dream.patterns`, `models.drift`, `models.auto_think`, `models.think`, `models.subagent`, `facts.extraction_model`, `models.eval.longmemeval`, `models.expansion`, `models.chat`, `models.dream.synthesize_verdict`), the alias map (defaults + user overrides), and a source-of-truth column (`default` / `config: ` / `env: `). - `gbrain models doctor [--skip=] [--json]` — 1-token reachability probe against each configured chat + expansion model. Classifies failures into `{model_not_found, auth, rate_limit, network, unknown}`. The structural fix for the bug class that motivated v0.31.12 (v0.31.6's `claude-sonnet-4-6-20250929` chat default 404'd silently on every install). diff --git a/TODOS.md b/TODOS.md index 7809db123..46e5950a2 100644 --- a/TODOS.md +++ b/TODOS.md @@ -45,16 +45,62 @@ infra: `~/.gbrain/oauth/.json` + `gbrain auth login `. Build alongside #691 in one OAuth-subsystem wave. -- [ ] **v0.32.x: CJK PGLite keyword fallback (#765 extracted).** 313094319-sudo - hit a real gap: PGLite's FTS doesn't tokenize CJK well, so Chinese queries - return empty results even with proper embeddings. Their PR added a - hasCJK detection branch in `searchKeyword` that switches to LIKE-based - fuzzy matching with a custom scoring function. ~150 lines of new SQL + - scoring + tests. Worth its own focused PR rather than folded into the - v0.32 wave's adjacent-fix lane. Extract `extractSearchTokens`, - `normalizeSearchText`, `hasCJK` helpers + the CJK branch in - `pglite-engine.ts:searchKeyword`. Includes tests for romaji + Korean - Hangul + traditional/simplified Chinese. +- [x] **v0.32.7: CJK PGLite keyword fallback (#765 extracted).** Landed + in the CJK fix wave. `hasCJK` + `escapeLikePattern` live in + `src/core/cjk.ts`; the CJK branch in `pglite-engine.ts:searchKeyword` + uses ILIKE + bigram-frequency-count ranking. Postgres path deferred + (see new follow-up below). + +- [ ] **v0.33+: Postgres CJK FTS via pgroonga / zhparser / ngram trigrams.** + v0.32.7 only fixed CJK keyword search on PGLite. Multi-tenant Postgres + deployments still hit empty results for CJK queries because + `to_tsvector('english', ...)` can't segment Chinese / Japanese / Korean. + Installing pgroonga or zhparser is an operator decision (extension + install permission, multi-tenant rollout), so gbrain can't default it. + Plan: doctor advisory pointing at the relevant extension docs; + searchKeyword / searchKeywordChunks fall through to PGLite-style ILIKE + when the extension isn't installed. Defer until users complain. + +- [ ] **v0.33+: widen CJK ranges to Unicode property escapes.** v0.32.7 + uses BMP-only ranges (Han `4e00-9fff`, Hiragana `3040-309f`, Katakana + `30a0-30ff`, Hangul Syllables `ac00-d7af`). Misses Han Extensions A/B/C, + halfwidth katakana, compatibility ideographs, compatibility Jamo, and + iteration marks `々` / `〇`. Switch to `\p{Script=Han}` / `\p{Script=Hiragana}` / + `\p{Script=Katakana}` / `\p{Script=Hangul}` (TS supports unicode property + escapes with the `u` flag). Astral-plane support also requires + `Array.from(str)`-style codepoint iteration in the chunker's char-slice + fallback (current `String.prototype.slice` splits surrogate pairs). + Defer until first user hits the gap. + +- [ ] **v0.33+: `git diff --name-status -z` + NUL framing.** v0.32.7 + added `core.quotepath=false` which handles non-ASCII paths but doesn't + cover tabs, newlines, or quotes in filenames. The `-z` flag with + NUL-byte path framing is the robust fix for the whole encoding class. + Affects `src/commands/sync.ts:buildDetachedWorkingTreeManifest` + + `buildSyncManifest`. Defer until someone files a tab-in-filename issue. + +- [ ] **v0.33+: CJK-aware overlap context in chunker.** v0.32.7 + `extractTrailingContext` is still whitespace-token-based, so CJK chunks + under the maxChars cap have no useful overlap with the previous chunk. + Search continuity across chunk boundaries degrades for pure CJK content. + The maxChars sliding-window in v0.32.7 IS overlap-protected for the + hard-cap path, so this only affects normal-size chunks. Plan: switch + `extractTrailingContext` to char-count when `countCJKAwareWords` would + have triggered the CJK branch. + +- [ ] **v0.33+: other non-Latin scripts (Thai, Arabic, Cyrillic, + Devanagari).** Same five-layer fix pattern as CJK applies: slugify + needs the script range, chunker needs density-threshold counting, + PGLite keyword fallback would benefit from script-aware tokenization. + Defer until first issue. + +- [ ] **v0.33+: embedding pricing refresh mechanism.** v0.32.7 added + `src/core/embedding-pricing.ts` as a static lookup table sibling to + `anthropic-pricing.ts`. Both drift when providers change rates. Plan: + a `gbrain prices refresh` skill that diffs against a published canonical + source (OpenAI pricing page, Anthropic pricing page) and proposes an + update PR. Or a release-cadence audit checklist item. Today: when the + estimate looks off, hand-edit the constants. - [ ] **v0.32.x: interactive provider chooser in `gbrain init`.** The full wizard piece of the v0.32 discoverability lane was deferred. Today diff --git a/VERSION b/VERSION index 8327fb077..44105ac2e 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.32.6 \ No newline at end of file +0.32.7 \ No newline at end of file diff --git a/llms-full.txt b/llms-full.txt index e7dc59a2d..d53f90c89 100644 --- a/llms-full.txt +++ b/llms-full.txt @@ -146,6 +146,12 @@ strict behavior when unset. - `src/core/pglite-engine.ts` — PGLite (embedded Postgres 17.5 via WASM) implementation, all 40 BrainEngine methods. `addLinksBatch` / `addTimelineEntriesBatch` use multi-row `unnest()` with manual `$N` placeholders. As of v0.13.1, `connect()` wraps `PGlite.create()` in a try/catch that emits an actionable error naming the macOS 26.3 WASM bug (#223) and pointing at `gbrain doctor`; the lock is released on failure so the next process can retry cleanly. v0.22.0: `searchKeyword` and `searchKeywordChunks` multiply `ts_rank` by the source-factor CASE expression at the chunk-grain level; `searchVector` becomes a two-stage CTE — inner CTE keeps `ORDER BY cc.embedding <=> vec` so HNSW stays usable, outer SELECT re-ranks by `raw_score * source_factor`. Inner LIMIT scales with offset to preserve pagination contract. As of v0.22.6.1, `initSchema()` calls `applyForwardReferenceBootstrap()` BEFORE replaying SCHEMA_SQL — probes for the specific forward-referenced state the embedded schema blob needs (`pages.source_id`, `links.link_source`, `links.origin_page_id`, `content_chunks.symbol_name`, `content_chunks.language`, `sources` FK target table) and adds only what's missing. Closes the upgrade-wedge bug class that bit users 10+ times across 6 schema versions over 2 years (#239/#243/#266/#357/#366/#374/#375/#378/#395/#396). No-op on fresh installs and modern brains. - `src/core/pglite-schema.ts` — PGLite-specific DDL (pgvector, pg_trgm, triggers) - `src/core/postgres-engine.ts` — Postgres + pgvector implementation (Supabase / self-hosted). `addLinksBatch` / `addTimelineEntriesBatch` use `INSERT ... SELECT FROM unnest($1::text[], ...) JOIN pages ON CONFLICT DO NOTHING RETURNING 1` — 4-5 array params regardless of batch size, sidesteps the 65535-parameter cap. As of v0.12.3, `searchKeyword` / `searchVector` scope `statement_timeout` via `sql.begin` + `SET LOCAL` so the GUC dies with the transaction instead of leaking across the pooled postgres.js connection (contributed by @garagon). `getEmbeddingsByChunkIds` uses `tryParseEmbedding` so one corrupt row skips+warns instead of killing the query. v0.22.0: `searchKeyword`, `searchKeywordChunks`, and `searchVector` apply source-aware ranking by inlining the source-factor CASE and `NOT (col LIKE …)` hard-exclude clause from `src/core/search/sql-ranking.ts`. `searchVector` switches to a two-stage CTE (HNSW-safe inner ORDER BY, source-boost re-rank in the outer SELECT) and carries `p.source_id` through inner→outer for v0.18 multi-source callers. v0.22.1 (#406): `_savedConfig` retains the connect config; `reconnect()` tears down + recreates the pool from saved config (called by supervisor watchdog after 3 consecutive health-check failures). `executeRaw` is a single-statement passthrough — no per-call retry (D3 dropped that as unsound for non-idempotent statements; recovery is supervisor-driven). v0.22.1 (#363, contributed by @orendi84): `connect()` applies `resolveSessionTimeouts()` from `db.ts` as connection-time startup parameters (`statement_timeout`, `idle_in_transaction_session_timeout`) so orphan pgbouncer backends can't hold locks for hours. v0.22.1 (#409, contributed by @atrevino47): `countStaleChunks()` + `listStaleChunks()` server-side-filter on `embedding IS NULL` for `embed --stale`, eliminating ~76 MB/call client-side pull on a fully-embedded brain; `upsertChunks()` resets both `embedding` AND `embedded_at` to NULL when chunk_text changes without a new embedding (consistency). As of v0.22.6.1, `initSchema()` calls `applyForwardReferenceBootstrap()` BEFORE replaying SCHEMA_SQL on the same forward-reference probe set as the PGLite engine, so old Postgres brains pinned at v0.13/v0.18/v0.19 walk forward cleanly instead of wedging on `column "..." does not exist`. **v0.28.1:** `disconnect()` is now idempotent. New `_connectionStyle` instance field tracks whether the engine owns its pool (worker engines) or shares the module-level singleton; second call on an instance-pool engine is a no-op rather than falling through to `db.disconnect()` and clobbering the singleton. Pinned by `test/e2e/postgres-engine-disconnect-idempotency.test.ts` (2 cases). Closes the bug class where any test sharing an engine across multiple `worker.start()` / `worker.stop()` cycles silently broke its own DB connectivity. +- `src/core/cjk.ts` (v0.32.7 CJK wave) — Single source of truth for CJK detection across the codebase. Exports `CJK_RANGES_REGEX`, `CJK_SLUG_CHARS` (character-class fragment for embedding inside other regexes), `CJK_SENTENCE_DELIMITERS` (`。！？`), `CJK_CLAUSE_DELIMITERS` (`；：，、`), `CJK_DENSITY_THRESHOLD = 0.30`, `hasCJK(s)`, `countCJKAwareWords(s)` (30% density threshold — English docs with one Japanese term stay whitespace-tokenized; Chinese-dominant docs get char-counted), and `escapeLikePattern(s)` (escapes `%`, `_`, `\\` for `ILIKE ... ESCAPE '\\'`). Replaces the inline hasCJK regex previously duplicated at `expansion.ts:58`. BMP-only ranges (Han / Hiragana / Katakana / Hangul Syllables); widening to Unicode property escapes is a v0.33+ TODO. Consumers: `expansion.ts`, `sync.ts:slugifySegment`, `operations.ts:validatePageSlug + validateFilename`, `chunkers/recursive.ts:countWords + DELIMITERS`, `pglite-engine.ts:searchKeyword + searchKeywordChunks`. +- `src/core/audit-slug-fallback.ts` (v0.32.7 CJK wave) — Weekly ISO-week-rotated audit JSONL at `~/.gbrain/audit/slug-fallback-YYYY-Www.jsonl`. `logSlugFallback(slug, sourcePath)` fires when `importFromFile` falls back to a frontmatter slug because `slugifyPath` returned empty (emoji / Thai / Arabic / non-CJK exotic-script filenames). `readRecentSlugFallbacks(days)` reads the last N days for `gbrain doctor`'s `slug_fallback_audit` check. Honors `GBRAIN_AUDIT_DIR` via the shared `resolveAuditDir()` from shell-audit.ts. Separate surface from `sync-failures.jsonl` per codex outside-voice review — that file carries bookmark-gating semantics that info events shouldn't trigger. +- `src/core/embedding-pricing.ts` (v0.32.7 CJK wave) — `EMBEDDING_PRICING` map keyed `provider:model` for the post-upgrade reindex cost estimate. Sibling to `anthropic-pricing.ts`. Entries: OpenAI text-embedding-3-large ($0.13/1M), 3-small ($0.02/1M), ada-002 ($0.10/1M), Voyage 3-large ($0.18/1M), 3 ($0.06/1M). `lookupEmbeddingPrice(modelString)` returns a tagged union (`known` with price + `unknown` with provider name); `estimateCostFromChars(charCount, pricePerMTok)` uses 3.5 chars/token approximation. Unknown providers degrade gracefully to "estimate unavailable" instead of fabricating numbers. +- `src/core/post-upgrade-reembed.ts` (v0.32.7 CJK wave) — Pure functions backing the `gbrain upgrade` chunker-bump cost prompt. `computeReembedEstimate(engine, model)` queries real SQL (`COUNT(*)` + `COALESCE(SUM(LENGTH(compiled_truth)) + SUM(LENGTH(timeline)), 0)`) on `pages WHERE chunker_version < MARKDOWN_CHUNKER_VERSION`. `formatReembedPrompt(est, graceSeconds)` is the stderr-line formatter. `runPostUpgradeReembedPrompt(engine, model, opts)` orchestrates the 10-second Ctrl-C window; TTY-only wait (non-TTY auto-proceeds for CI / cron); `GBRAIN_NO_REEMBED=1` bails out with a doctor-warning marker; `GBRAIN_REEMBED_GRACE_SECONDS=0` skips the wait. +- `src/commands/reindex.ts` (v0.32.7 CJK wave) — `gbrain reindex --markdown [--limit N] [--dry-run] [--json] [--no-embed] [--repo PATH]`. Walks `pages WHERE page_kind = 'markdown' AND chunker_version < MARKDOWN_CHUNKER_VERSION` in 100-row batches, ordered by id. Rows with non-null `source_path` re-import via `importFromFile`; rows without fall back to `importFromContent` against the stored `compiled_truth`. **Both paths pass `forceRechunk: true`** to bypass `importFromContent`'s `content_hash` short-circuit — without that flag (codex post-merge F1), the chunker version bump never reaches pages whose source content hasn't changed since last sync, AND master's v0.32.2 stripFactsFence privacy strip never applies to pre-strip chunks. Idempotent — partial-completion re-runs pick up where they left off via id-ordered batches. Wired into `src/commands/upgrade.ts:runPostUpgrade` after `apply-migrations`. +- `src/commands/sync.ts:resolveSlugByPathOrSourcePath` (v0.32.7 CJK wave, codex post-merge F4) — Resolves a slug by `pages.source_path` first (returns the stored slug for frontmatter-fallback pages whose path doesn't derive a slug), then falls back to `resolveSlugForPath(path)`. Threaded into all 4 delete/rename call sites (`performSync`'s un-syncable cleanup at ~:531, deletes at ~:603, rename oldSlug at ~:622). Without this, emoji-only / Thai / Arabic filenames whose slug came from frontmatter would orphan on delete/rename (the delete path would compute the wrong path-derived slug). Best-effort query — pre-migration brains fall through to the legacy path. - `src/core/utils.ts` — Shared SQL utilities extracted from postgres-engine.ts. Exports `parseEmbedding(value)` (throws on unknown input, used by migration + ingest paths where data integrity matters) and as of v0.12.3 `tryParseEmbedding(value)` (returns `null` + warns once per process, used by search/rescore paths where availability matters more than strictness). **v0.26.9 (D14):** adds `isUndefinedColumnError(err)` predicate — pattern-matches Postgres SQLSTATE 42703 / "column ... does not exist" with engine-driver shape variation tolerated. Replaces bare `catch {}` blocks in `oauth-provider.ts` so genuine errors (lock timeout, network blip, permission denied) propagate while column-missing falls through to the legacy fallback path. Reusable from any future code that needs the same column-existence probe semantics. - `src/core/db.ts` — Connection management, schema initialization. v0.22.1 (#363, contributed by @orendi84): `resolveSessionTimeouts()` returns `statement_timeout` + `idle_in_transaction_session_timeout` (defaults: 5min each, env-overridable via `GBRAIN_STATEMENT_TIMEOUT` / `GBRAIN_IDLE_TX_TIMEOUT` / `GBRAIN_CLIENT_CHECK_INTERVAL`). Both `connect()` (module singleton) and `PostgresEngine.connect()` (worker pool) consume the result via postgres.js's `connection` option, sending GUCs as startup parameters that survive PgBouncer transaction mode (unlike the prior `setSessionDefaults` post-pool SET, kept as a back-compat no-op shim). - `src/commands/migrate-engine.ts` — Bidirectional engine migration (`gbrain migrate --to supabase/pglite`) @@ -434,6 +440,12 @@ Key commands added for Minions (job queue): - `gbrain jobs smoke [--sigkill-rescue]` — health smoke test. `--sigkill-rescue` is the v0.13.1 regression guard for #219: simulates a killed worker and asserts the stalled job is requeued instead of dead-lettered on first stall. - `gbrain jobs work [--queue Q] [--concurrency N]` — start worker daemon (Postgres only) +Key commands added in v0.32.7 (CJK fix wave): +- `gbrain reindex --markdown [--limit N] [--dry-run] [--json] [--no-embed] [--repo PATH]` — operator-facing markdown re-chunk sweep. Walks pages with `chunker_version < MARKDOWN_CHUNKER_VERSION` (currently 2) and re-imports each with `forceRechunk: true` so the new chunker shape actually applies. Run automatically by `gbrain upgrade`'s post-upgrade hook; available manually for triage. +- `gbrain doctor` learns a new `slug_fallback_audit` check: surfaces info-severity entries from `~/.gbrain/audit/slug-fallback-YYYY-Www.jsonl` (last 7 days) as an `ok` count when CJK / emoji / exotic-script filenames imported via the frontmatter-slug fallback path. +- `gbrain search ""` on PGLite brains now uses an `ILIKE`-based fallback with bigram-frequency-count ranking when the query contains Han / Hiragana / Katakana / Hangul Syllables. ASCII queries continue through `websearch_to_tsquery('english')` unchanged. Postgres-side CJK FTS still requires an extension (pgroonga / zhparser) — see v0.33+ TODO. +- `gbrain upgrade` post-upgrade flow now prints a cost estimate before re-embedding: `[chunker-bump] Will re-embed ~N markdown pages via , est. ~$X.XX, ~Ymin. Press Ctrl-C within 10s to abort.` Sourced from real SQL counts + char totals; TTY-only wait (non-TTY auto-proceeds for CI / cron). Env overrides: `GBRAIN_NO_REEMBED=1` bails out entirely with a doctor-warning marker; `GBRAIN_REEMBED_GRACE_SECONDS=0` skips the wait. + Key commands added in v0.31.12 (model tier system + routing CLI): - `gbrain models [--json]` — read-only routing dashboard. Prints the four tier defaults (`utility`/`reasoning`/`deep`/`subagent`), the resolved value for each (after re-walking `models.default` → `models.tier.` → env → `TIER_DEFAULTS`), every per-task override (`models.dream.synthesize`, `models.dream.patterns`, `models.drift`, `models.auto_think`, `models.think`, `models.subagent`, `facts.extraction_model`, `models.eval.longmemeval`, `models.expansion`, `models.chat`, `models.dream.synthesize_verdict`), the alias map (defaults + user overrides), and a source-of-truth column (`default` / `config: ` / `env: `). - `gbrain models doctor [--skip=] [--json]` — 1-token reachability probe against each configured chat + expansion model. Classifies failures into `{model_not_found, auth, rate_limit, network, unknown}`. The structural fix for the bug class that motivated v0.31.12 (v0.31.6's `claude-sonnet-4-6-20250929` chat default 404'd silently on every install). diff --git a/package.json b/package.json index b8d871991..dcb2df9d4 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "gbrain", - "version": "0.32.6", + "version": "0.32.7", "description": "Postgres-native personal knowledge brain with hybrid RAG search", "type": "module", "main": "src/core/index.ts", diff --git a/src/cli.ts b/src/cli.ts index 5b82a5c2f..a2f7f35af 100755 --- a/src/cli.ts +++ b/src/cli.ts @@ -1056,6 +1056,12 @@ async function handleCliOnly(command: string, args: string[]) { await runOrphans(engine, args); break; } + // v0.32.7 CJK wave — post-upgrade markdown re-chunk sweep. + case 'reindex': { + const { runReindex } = await import('./commands/reindex.ts'); + await runReindex(engine, args); + break; + } // v0.29 — Salience + Anomaly Detection case 'salience': { const { runSalience } = await import('./commands/salience.ts'); diff --git a/src/commands/doctor.ts b/src/commands/doctor.ts index 01083741c..e287cda97 100644 --- a/src/commands/doctor.ts +++ b/src/commands/doctor.ts @@ -844,6 +844,25 @@ export async function runDoctor(engine: BrainEngine | null, args: string[], dbSo // Best-effort. A broken JSONL should not stop doctor. } + // 3d. Slug-fallback audit (v0.32.7 CJK wave, codex C7). Informational + // count of pages where importFromFile fell back to a frontmatter slug + // because the path slugified empty (emoji / Thai / Arabic / exotic-script + // filenames). NOT routed through sync-failures.jsonl — that surface + // gates bookmark advancement, info rows don't fit there. + try { + const { readRecentSlugFallbacks } = await import('../core/audit-slug-fallback.ts'); + const fallbacks = readRecentSlugFallbacks(7); + if (fallbacks.length > 0) { + checks.push({ + name: 'slug_fallback_audit', + status: 'ok', + message: `info: ${fallbacks.length} slug fallback${fallbacks.length === 1 ? '' : 's'} in the last 7 days (SLUG_FALLBACK_FRONTMATTER).`, + }); + } + } catch { + // Best-effort; audit-log read failure shouldn't stop doctor. + } + // 3b-multi-source. Multi-source drift (v0.31.8 — D8 + D17 + OV12 + OV13). // Pre-v0.30.3 putPage misrouted multi-source writes to (default, slug). // For each non-default source with local_path set, walk the FS and surface diff --git a/src/commands/reindex.ts b/src/commands/reindex.ts new file mode 100644 index 000000000..153f36e55 --- /dev/null +++ b/src/commands/reindex.ts @@ -0,0 +1,230 @@ +/** + * v0.32.7 CJK wave — `gbrain reindex --markdown` sweep. + * + * Walks markdown pages whose `chunker_version` is below + * MARKDOWN_CHUNKER_VERSION and re-imports each through the standard + * `importFromFile` / `importFromContent` path. Bumps `chunker_version` on + * success so re-runs are idempotent and a partial sweep can resume. + * + * Driven by: + * - `gbrain upgrade` post-upgrade hook (after the cost-estimate prompt). + * - Operators running `gbrain reindex --markdown` directly. + * + * Performance: batched 100 at a time so a 50K-page brain reindex doesn't + * hold a single transaction open. `--limit` caps total work for triage + * runs; `--dry-run` reports the count without writing. + * + * Codex outside-voice C2 — the original PR #599 `MARKDOWN_CHUNKER_VERSION` + * fold into content_hash was a no-op because `performSync` only re-imports + * files whose content actually changed, not files whose hash WOULD change + * if recomputed. This sweep + the migration v54 column are how the bump + * actually reaches existing markdown pages. + */ + +import type { BrainEngine } from '../core/engine.ts'; +import { MARKDOWN_CHUNKER_VERSION } from '../core/chunkers/recursive.ts'; +import { importFromContent, importFromFile } from '../core/import-file.ts'; +import { createProgress } from '../core/progress.ts'; +import { getCliOptions, cliOptsToProgressOptions } from '../core/cli-options.ts'; +import { existsSync } from 'fs'; +import { resolve } from 'path'; + +interface ReindexOpts { + /** Cap total pages reindexed. Useful for triage runs on huge brains. */ + limit?: number; + /** Report would-do count; don't actually reindex. */ + dryRun?: boolean; + /** Emit JSON envelope on stdout. */ + json?: boolean; + /** Brain repo path (for reading source files). Falls back to sync.repo_path config or process.cwd(). */ + repoPath?: string; + /** + * Skip the embedding call during re-chunk. New chunks land with NULL + * embedding and the next `gbrain embed --stale` pass fills them in. + * Useful for offline / no-API-key brains and for tests. + */ + noEmbed?: boolean; +} + +export interface ReindexResult { + pending: number; + reindexed: number; + skipped: number; + failed: number; + dryRun: boolean; + chunkerVersion: number; +} + +function parseArgs(args: string[]): ReindexOpts { + const out: ReindexOpts = {}; + for (let i = 0; i < args.length; i++) { + const a = args[i]; + if (a === '--markdown') continue; // routing flag, no value + if (a === '--dry-run') out.dryRun = true; + else if (a === '--json') out.json = true; + else if (a === '--no-embed') out.noEmbed = true; + else if (a === '--limit') { + const v = parseInt(args[++i] ?? '', 10); + if (Number.isFinite(v) && v > 0) out.limit = v; + } else if (a === '--repo') { + out.repoPath = args[++i]; + } + } + return out; +} + +/** + * Count markdown pages that haven't been re-chunked by the current + * MARKDOWN_CHUNKER_VERSION. Cheap; uses the partial index from migration v54. + */ +async function countPending(engine: BrainEngine): Promise { + const rows = await engine.executeRaw<{ count: string | number }>( + `SELECT COUNT(*)::bigint AS count + FROM pages + WHERE page_kind = 'markdown' + AND chunker_version < $1 + AND deleted_at IS NULL`, + [MARKDOWN_CHUNKER_VERSION], + ); + return Number(rows[0]?.count ?? 0); +} + +/** + * Read a single batch of pending rows. Ordered by id so re-runs after + * partial completion pick up where they left off without re-doing pages + * whose chunker_version was already bumped. + */ +async function readBatch(engine: BrainEngine, batchSize: number): Promise> { + return engine.executeRaw( + `SELECT slug, source_path, compiled_truth, source_id + FROM pages + WHERE page_kind = 'markdown' + AND chunker_version < $1 + AND deleted_at IS NULL + ORDER BY id ASC + LIMIT $2`, + [MARKDOWN_CHUNKER_VERSION, batchSize], + ); +} + +export async function runReindex(engine: BrainEngine, args: string[]): Promise { + const opts = parseArgs(args); + + // Require `--markdown` explicitly. Future modes (e.g. --code) get their + // own routing here. + if (!args.includes('--markdown')) { + if (opts.json) { + process.stdout.write(JSON.stringify({ error: 'gbrain reindex requires a target flag, e.g. --markdown' }) + '\n'); + } else { + process.stderr.write('Usage: gbrain reindex --markdown [--limit N] [--dry-run] [--json] [--repo PATH]\n'); + } + process.exitCode = 2; + return { pending: 0, reindexed: 0, skipped: 0, failed: 0, dryRun: !!opts.dryRun, chunkerVersion: MARKDOWN_CHUNKER_VERSION }; + } + + const pending = await countPending(engine); + + if (opts.json && pending === 0) { + process.stdout.write(JSON.stringify({ pending: 0, reindexed: 0, skipped: 0, failed: 0, chunker_version: MARKDOWN_CHUNKER_VERSION }) + '\n'); + return { pending: 0, reindexed: 0, skipped: 0, failed: 0, dryRun: !!opts.dryRun, chunkerVersion: MARKDOWN_CHUNKER_VERSION }; + } + + if (pending === 0) { + process.stderr.write(`[reindex] All markdown pages already at chunker_version ${MARKDOWN_CHUNKER_VERSION}. Nothing to do.\n`); + return { pending: 0, reindexed: 0, skipped: 0, failed: 0, dryRun: !!opts.dryRun, chunkerVersion: MARKDOWN_CHUNKER_VERSION }; + } + + const target = typeof opts.limit === 'number' ? Math.min(opts.limit, pending) : pending; + + if (opts.dryRun) { + if (opts.json) { + process.stdout.write(JSON.stringify({ pending, would_reindex: target, dry_run: true, chunker_version: MARKDOWN_CHUNKER_VERSION }) + '\n'); + } else { + process.stderr.write(`[reindex] DRY-RUN: would re-chunk ${target} of ${pending} pending markdown pages.\n`); + } + return { pending, reindexed: 0, skipped: 0, failed: 0, dryRun: true, chunkerVersion: MARKDOWN_CHUNKER_VERSION }; + } + + const reporter = createProgress(cliOptsToProgressOptions(getCliOptions())); + reporter.start('reindex.markdown', target); + + let reindexed = 0; + let skipped = 0; + let failed = 0; + const BATCH = 100; + const repoPath = opts.repoPath ? resolve(opts.repoPath) : null; + + while (reindexed + skipped + failed < target) { + const remaining = target - (reindexed + skipped + failed); + const batchSize = Math.min(BATCH, remaining); + const batch = await readBatch(engine, batchSize); + if (batch.length === 0) break; + + for (const row of batch) { + reporter.tick(); + try { + // Prefer importFromFile when we have a source_path AND the file + // still exists on disk — re-runs both the path-authoritative slug + // resolution AND the parseMarkdown pipeline on the real file. + // When the file is gone or we never recorded source_path (legacy + // rows pre-migration), fall back to importFromContent which uses + // the stored markdown body. importFromContent doesn't re-parse a + // frontmatter file, so timeline + tags don't refresh — accepted + // tradeoff for the post-upgrade sweep. + if (row.source_path && repoPath) { + const absPath = resolve(repoPath, row.source_path); + if (existsSync(absPath)) { + // importFromFile re-parses the markdown and calls importFromContent + // internally; we route through it with forceRechunk so the + // chunker-version bump actually applies (codex post-merge F1). + await importFromFile(engine, absPath, row.source_path, { + noEmbed: !!opts.noEmbed, + sourceId: row.source_id, + inferFrontmatter: false, + forceRechunk: true, + }); + reindexed++; + continue; + } + } + // Fallback path: re-chunk the stored compiled_truth in place. + // forceRechunk bypasses the content_hash short-circuit so the bumped + // chunker actually applies — without this, every unchanged-source page + // is silently skipped and the version bump never reaches existing + // chunks (codex post-merge F1). + await importFromContent(engine, row.slug, row.compiled_truth, { + sourceId: row.source_id, + noEmbed: !!opts.noEmbed, + forceRechunk: true, + }); + reindexed++; + } catch (err) { + const msg = err instanceof Error ? err.message : String(err); + process.stderr.write(`[reindex] ${row.slug}: ${msg}\n`); + failed++; + } + } + } + + reporter.finish(); + + const result: ReindexResult = { + pending, + reindexed, + skipped, + failed, + dryRun: false, + chunkerVersion: MARKDOWN_CHUNKER_VERSION, + }; + + if (opts.json) { + process.stdout.write(JSON.stringify({ + pending, reindexed, skipped, failed, + chunker_version: MARKDOWN_CHUNKER_VERSION, + }) + '\n'); + } else { + process.stderr.write(`[reindex] Done. reindexed=${reindexed} failed=${failed} pending=${Math.max(0, pending - reindexed - failed)}\n`); + } + + return result; +} diff --git a/src/commands/sync.ts b/src/commands/sync.ts index c3ae78a15..a2c619650 100644 --- a/src/commands/sync.ts +++ b/src/commands/sync.ts @@ -162,8 +162,57 @@ export interface SyncOpts { skipLock?: boolean; } -function git(repoPath: string, ...args: string[]): string { - return execFileSync('git', ['-C', repoPath, ...args], { +/** + * v0.32.7 CJK wave (codex post-merge F4): resolve a slug by `pages.source_path` + * first, falling back to `resolveSlugForPath(path)`. + * + * Frontmatter-fallback pages (emoji-only / Thai / Arabic / exotic-script + * filenames where `slugifyPath` returns empty and the slug came from the + * frontmatter) have a slug that ISN'T derivable from the path. Delete and + * rename operations that only know the path would otherwise orphan these + * pages by trying to delete the path-derived (wrong) slug. + * + * Returns the actual stored slug when source_path matches a row, or the + * path-derived slug when there's no match (normal-case path-derived pages). + */ +export async function resolveSlugByPathOrSourcePath( + engine: BrainEngine, + path: string, + sourceId?: string, +): Promise { + try { + const rows = await engine.executeRaw<{ slug: string }>( + sourceId + ? `SELECT slug FROM pages WHERE source_path = $1 AND source_id = $2 LIMIT 1` + : `SELECT slug FROM pages WHERE source_path = $1 LIMIT 1`, + sourceId ? [path, sourceId] : [path], + ); + if (rows.length > 0 && rows[0].slug) return rows[0].slug; + } catch { + // Fall through — best-effort. Pre-migration brains or query errors + // shouldn't break delete/rename for path-derived pages. + } + return resolveSlugForPath(path); +} + +/** + * git CLI helper. + * + * `configs` flags are emitted as `-c key=val` pairs BEFORE `-C repoPath` and + * BEFORE the subcommand. `core.quotepath=false` is always emitted first so CJK + * (and other non-ASCII) paths arrive as UTF-8 in `diff --name-status` and + * sibling commands. Callers that need additional git config should pass via + * the `configs` parameter; never inline `-c` into `args`. + * + * Exported for `test/sync.test.ts` invariant assertion only. + */ +export function buildGitInvocation(repoPath: string, args: string[], configs: string[] = []): string[] { + const cfg = ['core.quotepath=false', ...configs].flatMap(c => ['-c', c]); + return [...cfg, '-C', repoPath, ...args]; +} + +function git(repoPath: string, args: string[], configs: string[] = []): string { + return execFileSync('git', buildGitInvocation(repoPath, args, configs), { encoding: 'utf-8', timeout: 30000, }).trim(); @@ -171,7 +220,7 @@ function git(repoPath: string, ...args: string[]): string { function isDetachedHead(repoPath: string): boolean { try { - git(repoPath, 'symbolic-ref', '--quiet', 'HEAD'); + git(repoPath, ['symbolic-ref', '--quiet', 'HEAD']); return false; } catch { return true; @@ -183,8 +232,8 @@ function unique(items: T[]): T[] { } function buildDetachedWorkingTreeManifest(repoPath: string): SyncManifest { - const manifest = buildSyncManifest(git(repoPath, 'diff', '--name-status', '-M', 'HEAD')); - const untracked = git(repoPath, 'ls-files', '--others', '--exclude-standard') + const manifest = buildSyncManifest(git(repoPath, ['diff', '--name-status', '-M', 'HEAD'])); + const untracked = git(repoPath, ['ls-files', '--others', '--exclude-standard']) .split('\n') .filter(line => line.length > 0); @@ -410,7 +459,7 @@ async function performSyncInner(engine: BrainEngine, opts: SyncOpts): Promise 0) { progress.start('sync.deletes', filtered.deleted.length); for (const path of filtered.deleted) { - const slug = resolveSlugForPath(path); + const slug = await resolveSlugByPathOrSourcePath(engine, path, opts.sourceId); await engine.deletePage(slug, deleteOpts); pagesAffected.push(slug); progress.tick(1, slug); @@ -603,7 +652,8 @@ async function performSyncInner(engine: BrainEngine, opts: SyncOpts): Promise', diff --git a/src/commands/upgrade.ts b/src/commands/upgrade.ts index b736215be..84ce197c7 100644 --- a/src/commands/upgrade.ts +++ b/src/commands/upgrade.ts @@ -256,6 +256,24 @@ export async function runPostUpgrade(args: string[] = []): Promise { await engine.connect(toCfgSchema(cfgSchema)); await engine.initSchema(); console.log(' Schema up to date.'); + + // v0.32.7 CJK wave: chunker-version bump → re-embed sweep. + // Idempotent — `runReindex` short-circuits when no pages are pending. + try { + const { runPostUpgradeReembedPrompt } = await import('../core/post-upgrade-reembed.ts'); + const { getEmbeddingModel } = await import('../core/ai/gateway.ts'); + let modelString = 'openai:text-embedding-3-large'; + try { modelString = getEmbeddingModel(); } catch { /* gateway not configured — keep default */ } + const promptResult = await runPostUpgradeReembedPrompt(engine, modelString); + if (promptResult.proceeded) { + const { runReindex } = await import('./reindex.ts'); + await runReindex(engine, ['--markdown']); + } + } catch (re) { + const msg = re instanceof Error ? re.message : String(re); + console.warn(`\nChunker-bump reindex skipped: ${msg}`); + console.warn('Run `gbrain reindex --markdown` manually when ready.'); + } } finally { try { await engine.disconnect(); } catch { /* best-effort */ } } diff --git a/src/core/audit-slug-fallback.ts b/src/core/audit-slug-fallback.ts new file mode 100644 index 000000000..345f16846 --- /dev/null +++ b/src/core/audit-slug-fallback.ts @@ -0,0 +1,114 @@ +/** + * v0.32.7 CJK wave — slug-fallback audit trail. + * + * Writes info-severity rows to `~/.gbrain/audit/slug-fallback-YYYY-Www.jsonl` + * (ISO-week rotation, mirrors `subagent-audit.ts`). Fired when import-file's + * empty-path-slug + frontmatter-fallback path resolves a slug that wouldn't + * otherwise derive from the file path (emoji, Thai, Arabic, etc. filenames + * whose slugifyPath() returns empty even after the CJK ranges land). + * + * Why a separate JSONL instead of `~/.gbrain/sync-failures.jsonl`: + * - sync-failures.jsonl carries commit-attribution semantics that gate + * bookmark advancement; importFromFile doesn't know the commit. + * - Fallback events are informational, NOT failures. Routing them through + * the failure surface would force doctor / classifyErrorCode / + * acknowledgeSyncFailures to grow a severity tier they weren't designed + * for. Codex outside-voice C7 caught this drift. + * + * Best-effort writes. Write failures go to stderr but the import continues. + */ + +import * as fs from 'node:fs'; +import * as path from 'node:path'; +import { resolveAuditDir } from './minions/handlers/shell-audit.ts'; + +export interface SlugFallbackAuditEvent { + ts: string; + /** Resolved slug (the frontmatter slug that overrode the empty path slug). */ + slug: string; + /** Repo-relative path that produced an empty slugifyPath(). */ + source_path: string; + /** Always 'info' — keeps the schema explicit for future severity tiers. */ + severity: 'info'; + /** Stable code consumed by `gbrain doctor`'s slug_fallback_audit check. */ + code: 'SLUG_FALLBACK_FRONTMATTER'; +} + +/** ISO-week-rotated filename: `slug-fallback-YYYY-Www.jsonl`. */ +export function computeSlugFallbackAuditFilename(now: Date = new Date()): string { + const d = new Date(Date.UTC(now.getUTCFullYear(), now.getUTCMonth(), now.getUTCDate())); + const dayNum = (d.getUTCDay() + 6) % 7; + d.setUTCDate(d.getUTCDate() - dayNum + 3); + const isoYear = d.getUTCFullYear(); + const firstThursday = new Date(Date.UTC(isoYear, 0, 4)); + const firstThursdayDayNum = (firstThursday.getUTCDay() + 6) % 7; + firstThursday.setUTCDate(firstThursday.getUTCDate() - firstThursdayDayNum + 3); + const weekNum = Math.round((d.getTime() - firstThursday.getTime()) / (7 * 86400000)) + 1; + const ww = String(weekNum).padStart(2, '0'); + return `slug-fallback-${isoYear}-W${ww}.jsonl`; +} + +/** + * Append a slug-fallback event to the current week's audit JSONL. + * + * Also emits one stderr line per call for operator visibility (per D7 dual + * logging). Write failure to the JSONL is logged but does NOT throw — the + * import succeeds either way. + */ +export function logSlugFallback(slug: string, sourcePath: string): void { + process.stderr.write(`[gbrain] slug fallback: ${sourcePath} → ${slug} (frontmatter slug; path slugified empty)\n`); + const event: SlugFallbackAuditEvent = { + ts: new Date().toISOString(), + slug, + source_path: sourcePath, + severity: 'info', + code: 'SLUG_FALLBACK_FRONTMATTER', + }; + const dir = resolveAuditDir(); + const file = path.join(dir, computeSlugFallbackAuditFilename()); + try { + fs.mkdirSync(dir, { recursive: true }); + fs.appendFileSync(file, JSON.stringify(event) + '\n', { encoding: 'utf8' }); + } catch (err) { + const msg = err instanceof Error ? err.message : String(err); + process.stderr.write(`[gbrain] slug-fallback audit write failed (${msg}); import continues\n`); + } +} + +/** + * Read recent (`days` window, default 7) slug-fallback events from the + * latest week's JSONL. Used by `gbrain doctor`'s slug_fallback_audit check. + * Missing file / corrupt rows are skipped silently — the audit trail is + * informational and shouldn't block doctor. + */ +export function readRecentSlugFallbacks(days = 7, now: Date = new Date()): SlugFallbackAuditEvent[] { + const dir = resolveAuditDir(); + const cutoff = now.getTime() - days * 86400000; + const out: SlugFallbackAuditEvent[] = []; + // Walk the current + previous ISO week so a 7-day window straddling + // Monday-midnight stays covered. + const filenames = [ + computeSlugFallbackAuditFilename(now), + computeSlugFallbackAuditFilename(new Date(now.getTime() - 7 * 86400000)), + ]; + for (const filename of filenames) { + const file = path.join(dir, filename); + let content: string; + try { + content = fs.readFileSync(file, 'utf8'); + } catch { + continue; + } + for (const line of content.split('\n')) { + if (line.length === 0) continue; + try { + const ev = JSON.parse(line) as SlugFallbackAuditEvent; + const ts = Date.parse(ev.ts); + if (Number.isFinite(ts) && ts >= cutoff) out.push(ev); + } catch { + // Corrupt row — skip. + } + } + } + return out; +} diff --git a/src/core/chunkers/recursive.ts b/src/core/chunkers/recursive.ts index 07cc59259..dada7d540 100644 --- a/src/core/chunkers/recursive.ts +++ b/src/core/chunkers/recursive.ts @@ -5,25 +5,40 @@ * 5-level delimiter hierarchy: * 1. Paragraphs (\n\n) * 2. Lines (\n) - * 3. Sentences (. ! ? followed by space or newline) - * 4. Clauses (; : , ) - * 5. Words (whitespace) + * 3. Sentences (. ! ? followed by space or newline; plus CJK 。！？) + * 4. Clauses (; : , ; plus CJK ；：，、) + * 5. Words (whitespace + CJK char-slice fallback) * * Config: 300-word chunks with 50-word sentence-aware overlap. + * v0.32.7: maxChars hard cap (default 6000) sliding-window safety belt + * guarantees no chunk overflows OpenAI's 8192-token embedding limit even + * on pathological CJK / whitespace-less text. + * * Lossless invariant: non-overlapping portions reassemble to original. */ +import { countCJKAwareWords, CJK_SENTENCE_DELIMITERS, CJK_CLAUSE_DELIMITERS } from '../cjk.ts'; + +/** + * Markdown chunker version. Folded into the per-page chunker_version column + * so post-upgrade reindex sweeps can find pages built with old chunkers and + * rebuild them on the new shape. Bump on any change that affects chunk + * boundaries (delimiters, word counting, maxChars cap). + */ +export const MARKDOWN_CHUNKER_VERSION = 2; + const DELIMITERS: string[][] = [ ['\n\n'], // L0: paragraphs ['\n'], // L1: lines - ['. ', '! ', '? ', '.\n', '!\n', '?\n'], // L2: sentences - ['; ', ': ', ', '], // L3: clauses - [], // L4: words (whitespace split) + ['. ', '! ', '? ', '.\n', '!\n', '?\n', ...CJK_SENTENCE_DELIMITERS], // L2: sentences + ['; ', ': ', ', ', ...CJK_CLAUSE_DELIMITERS], // L3: clauses + [], // L4: words (whitespace + CJK char-slice fallback) ]; export interface ChunkOptions { chunkSize?: number; // target words per chunk (default 300) chunkOverlap?: number; // overlap words (default 50) + maxChars?: number; // hard cap on any chunk's char length (default 6000) } export interface TextChunk { @@ -48,6 +63,7 @@ import { stripFactsFence } from '../facts-fence.ts'; export function chunkText(text: string, opts?: ChunkOptions): TextChunk[] { const chunkSize = opts?.chunkSize || 300; const chunkOverlap = opts?.chunkOverlap || 50; + const maxChars = opts?.maxChars || 6000; if (!text || text.trim().length === 0) return []; @@ -64,15 +80,47 @@ export function chunkText(text: string, opts?: ChunkOptions): TextChunk[] { const wordCount = countWords(stripped); if (wordCount <= chunkSize) { - return [{ text: stripped.trim(), index: 0 }]; + // Single-chunk path: still apply the maxChars cap. + const capped = capByChars(stripped.trim(), maxChars); + return capped.map((t, i) => ({ text: t, index: i })); } // Recursively split, then greedily merge to target size const pieces = recursiveSplit(stripped, 0, chunkSize); const merged = greedyMerge(pieces, chunkSize); const withOverlap = applyOverlap(merged, chunkOverlap); + // v0.32.7: hard char cap. Catches pathological CJK + whitespace-less text + // that the word-level pipeline can't bound (a single Chinese paragraph can + // exceed 8192 OpenAI embedding tokens at any word count). + const capped: string[] = []; + for (const chunk of withOverlap) { + capped.push(...capByChars(chunk.trim(), maxChars)); + } + return capped.map((t, i) => ({ text: t, index: i })); +} - return withOverlap.map((t, i) => ({ text: t.trim(), index: i })); +/** + * Hard-cap a chunk's char length via a sliding window. Returns the input + * unchanged when it's already ≤ maxChars. + * + * Overlap is min(500, maxChars/10) so successive windows preserve semantic + * continuity across the cut. + * + * v0.32.7. BMP-only safe (does not split astral surrogate pairs in practice + * because declared CJK ranges are all BMP; widening to astral Han support + * is a v0.33+ follow-up that requires Array.from-style codepoint iteration). + */ +function capByChars(text: string, maxChars: number): string[] { + if (text.length <= maxChars) return text.length > 0 ? [text] : []; + const overlap = Math.min(500, Math.floor(maxChars / 10)); + const stride = Math.max(1, maxChars - overlap); + const out: string[] = []; + for (let i = 0; i < text.length; i += stride) { + const slice = text.slice(i, i + maxChars).trim(); + if (slice.length > 0) out.push(slice); + if (i + maxChars >= text.length) break; + } + return out; } function recursiveSplit(text: string, level: number, target: number): string[] { @@ -149,10 +197,28 @@ function splitAtDelimiters(text: string, delimiters: string[]): string[] { /** * Fallback: split on whitespace boundaries to hit target word count. + * v0.32.7: when the input is whitespace-less or any single "word" exceeds + * the target (CJK paragraph, base64 blob, long URL), slice on character + * boundaries so we still bound chunk size and the chunker makes forward + * progress. The downstream maxChars cap tightens this further. */ function splitOnWhitespace(text: string, target: number): string[] { const words = text.match(/\S+\s*/g) || []; - if (words.length === 0) return []; + + // No whitespace tokens, OR a single token longer than `target` chars + // (greedy /\S+/g returns a CJK paragraph as one "word"). Slice by char. + const noUsefulWhitespace = + words.length === 0 || (words.length === 1 && words[0].length > target); + if (noUsefulWhitespace) { + if (text.trim().length === 0) return []; + const pieces: string[] = []; + const charsPerPiece = Math.max(1, target); + for (let i = 0; i < text.length; i += charsPerPiece) { + const slice = text.slice(i, i + charsPerPiece); + if (slice.trim().length > 0) pieces.push(slice); + } + return pieces; + } const pieces: string[] = []; for (let i = 0; i < words.length; i += target) { @@ -231,6 +297,18 @@ function extractTrailingContext(text: string, targetWords: number): string { return trailing; } +/** + * Word count, CJK-aware (v0.32.7). For Latin-dominant text this behaves + * exactly like the historical `text.match(/\S+/g).length`. When CJK char + * density exceeds CJK_DENSITY_THRESHOLD (30%), each non-whitespace char is + * counted as one "word" so the chunker actually splits CJK paragraphs + * (whitespace-tokenization counts a whole Chinese paragraph as 1 word, + * letting it overflow the OpenAI embedding token limit). + * + * Delegated to src/core/cjk.ts so the slugify whitelist, expansion + * detection, and PGLite keyword fallback all agree on what "CJK enough" + * means. + */ function countWords(text: string): number { - return (text.match(/\S+/g) || []).length; + return countCJKAwareWords(text); } diff --git a/src/core/cjk.ts b/src/core/cjk.ts new file mode 100644 index 000000000..d9c531876 --- /dev/null +++ b/src/core/cjk.ts @@ -0,0 +1,67 @@ +/** + * Shared CJK (Chinese / Japanese / Korean) detection and handling primitives. + * + * Replaces the inline copy in `src/core/search/expansion.ts:58` and provides + * a single source of truth for every downstream caller: slug grammar, chunker + * word-counting, chunker delimiters, PGLite keyword fallback. + * + * Scope: BMP-only Unicode ranges that cover ~99% of real CJK content: + * - Han (CJK Unified Ideographs): U+4E00–U+9FFF + * - Hiragana: U+3040–U+309F + * - Katakana: U+30A0–U+30FF + * - Hangul Syllables: U+AC00–U+D7AF + * + * Out of scope (v0.32.7): Han extensions A/B/C, halfwidth katakana, + * compatibility ideographs, compatibility Jamo, iteration marks (々/〇). + * Filed as v0.33+ follow-up. + */ + +export const CJK_SLUG_CHARS = '一-鿿぀-ゟ゠-ヿ가-힯'; + +export const CJK_RANGES_REGEX = new RegExp(`[${CJK_SLUG_CHARS}]`); + +export const CJK_SENTENCE_DELIMITERS = ['。', '！', '？']; // 。！？ +export const CJK_CLAUSE_DELIMITERS = ['；', '：', '，', '、']; // ；：，、 + +/** + * Density threshold for switching word-count strategy. Below this CJK char + * density, a doc is treated as Latin-mostly and stays whitespace-tokenized + * (so a 5000-word English doc with one Japanese term doesn't get char-counted + * and over-split). At or above, it's CJK-mostly. + */ +export const CJK_DENSITY_THRESHOLD = 0.30; + +export function hasCJK(s: string): boolean { + return CJK_RANGES_REGEX.test(s); +} + +/** + * CJK-aware "word" count. CJK languages aren't whitespace-tokenized, so a + * paragraph of Chinese collapses to 1 word under /\S+/g and downstream chunkers + * never split it (the 8192-token OpenAI embedding limit then rejects the chunk). + * + * Heuristic (per codex outside-voice C13): switch on CJK character density, + * not mere presence. Below CJK_DENSITY_THRESHOLD the doc is Latin-dominant + * and whitespace tokens are the right unit; at or above it's CJK-dominant + * and char count is the right unit. + */ +export function countCJKAwareWords(s: string): number { + if (s.length === 0) return 0; + const cjkMatches = s.match(new RegExp(`[${CJK_SLUG_CHARS}]`, 'g')); + const cjkCount = cjkMatches ? cjkMatches.length : 0; + const nonWhitespace = s.replace(/\s/g, '').length; + if (nonWhitespace === 0) return 0; + const density = cjkCount / nonWhitespace; + if (density >= CJK_DENSITY_THRESHOLD) { + return nonWhitespace; + } + return (s.match(/\S+/g) || []).length; +} + +/** + * LIKE-pattern escape for PGLite/Postgres `ILIKE ... ESCAPE '\'`. + * Must escape backslash FIRST so the introduced backslashes aren't double-escaped. + */ +export function escapeLikePattern(s: string): string { + return s.replace(/\\/g, '\\\\').replace(/%/g, '\\%').replace(/_/g, '\\_'); +} diff --git a/src/core/embedding-pricing.ts b/src/core/embedding-pricing.ts new file mode 100644 index 000000000..5292dbc98 --- /dev/null +++ b/src/core/embedding-pricing.ts @@ -0,0 +1,68 @@ +/** + * v0.32.7 CJK wave — embedding model pricing lookup table. + * + * Sibling to `anthropic-pricing.ts`. Used by `gbrain upgrade`'s post-upgrade + * cost-estimate prompt so users with large brains see a dollar figure + * before the chunker-version sweep re-embeds. + * + * Prices in USD per 1M tokens. Numbers as of 2026-05-11. Verify alongside + * the Anthropic-pricing refresh cycle; drift here produces estimates + * that mislead operators. + * + * Codex outside-voice C3 fold: non-OpenAI embedding providers (Voyage, + * Hunyuan, Dashscope, etc.) return UNKNOWN_PROVIDER from `lookupPrice` + * so the cost-estimate prompt can fall back to a "estimate unavailable + * for ; press Ctrl-C in 10s to abort" message rather than + * fabricate numbers. + */ + +export interface EmbeddingPricing { + /** USD per 1M tokens (embedding cost; embeddings have no separate output rate). */ + pricePerMTok: number; +} + +/** + * `provider:model` keyed pricing. The colon-separated key matches + * gateway model strings (e.g. 'openai:text-embedding-3-large'). + */ +export const EMBEDDING_PRICING: Record = { + // OpenAI (https://openai.com/api/pricing/, verified 2026-05-11) + 'openai:text-embedding-3-large': { pricePerMTok: 0.13 }, + 'openai:text-embedding-3-small': { pricePerMTok: 0.02 }, + // Legacy OpenAI ada (still common in older brains) + 'openai:text-embedding-ada-002': { pricePerMTok: 0.10 }, + // Voyage (https://www.voyageai.com/pricing — voyage-3-large default) + 'voyage:voyage-3-large': { pricePerMTok: 0.18 }, + 'voyage:voyage-3': { pricePerMTok: 0.06 }, +}; + +export type PriceLookupResult = + | { kind: 'known'; pricePerMTok: number; key: string } + | { kind: 'unknown'; provider: string; model: string }; + +/** + * Resolve a model string into a price-per-1M-tokens. Accepts both + * `provider:model` and bare `model` forms (bare assumes openai). + */ +export function lookupEmbeddingPrice(modelString: string): PriceLookupResult { + const [providerRaw, modelRaw] = modelString.includes(':') + ? modelString.split(':', 2) + : ['openai', modelString]; + const provider = providerRaw.trim().toLowerCase(); + const model = (modelRaw ?? '').trim(); + const key = `${provider}:${model}`; + const hit = EMBEDDING_PRICING[key]; + if (hit) return { kind: 'known', pricePerMTok: hit.pricePerMTok, key }; + return { kind: 'unknown', provider, model }; +} + +/** + * Estimate USD cost for embedding `charCount` characters. Uses + * 3.5 chars/token as the OpenAI tiktoken-shaped approximation for English; + * CJK-heavy brains will under-estimate by ~2x (one char ≈ one token), but + * we'd rather under-estimate than spook users with a 10x worst-case figure. + */ +export function estimateCostFromChars(charCount: number, pricePerMTok: number): number { + const tokens = Math.ceil(charCount / 3.5); + return (tokens / 1_000_000) * pricePerMTok; +} diff --git a/src/core/import-file.ts b/src/core/import-file.ts index 5a6f1165b..65e6d0b93 100644 --- a/src/core/import-file.ts +++ b/src/core/import-file.ts @@ -12,6 +12,8 @@ import { embedBatch, embedMultimodal } from './embedding.ts'; import { slugifyPath, slugifyCodePath, isCodeFilePath } from './sync.ts'; import type { ChunkInput, PageInput, PageType } from './types.ts'; import { computeEffectiveDate } from './effective-date.ts'; +import { MARKDOWN_CHUNKER_VERSION } from './chunkers/recursive.ts'; +import { logSlugFallback } from './audit-slug-fallback.ts'; /** * v0.20.0 Cathedral II Layer 8 D2 — markdown fence extraction helper. @@ -195,6 +197,23 @@ export async function importFromContent( * disk path; the put_page MCP op derives it from the slug tail. */ filename?: string; + /** + * v0.32.7 CJK wave: repo-relative path captured at import. Stored on + * `pages.source_path` so sync's delete/rename code can look up the + * page slug by path when the slug isn't derivable (frontmatter + * fallback). MCP `put_page` callers leave undefined (no file). + */ + sourcePath?: string; + /** + * v0.32.7 CJK wave (codex post-merge F1): bypass the + * `existing.content_hash === hash` short-circuit and ALWAYS re-chunk + + * re-embed. Used by `gbrain reindex --markdown` so a chunker version + * bump actually reaches unchanged-source pages. Without this, the + * sweep silently no-ops on every page whose markdown body hasn't + * been edited since the last import — defeating the whole purpose of + * the version bump. + */ + forceRechunk?: boolean; } = {}, ): Promise { // v0.18.0+ multi-source: when caller is syncing under a non-default source, @@ -240,7 +259,7 @@ export async function importFromContent( }; const existing = await engine.getPage(slug, sourceId ? { sourceId } : undefined); - if (existing?.content_hash === hash) { + if (existing?.content_hash === hash && !opts.forceRechunk) { return { slug, status: 'skipped', chunks: 0, parsedPage }; } @@ -310,6 +329,12 @@ export async function importFromContent( effective_date: effectiveDate, effective_date_source: effectiveDateSource, import_filename: filenameForChain, + // v0.32.7 CJK wave: stamp the chunker version so the post-upgrade + // reindex sweep can find pre-bump pages via `chunker_version < 2`. + // Also capture the repo-relative source path so sync's delete/rename + // code can resolve frontmatter-fallback slugs back to their files. + chunker_version: MARKDOWN_CHUNKER_VERSION, + source_path: opts.sourcePath ?? null, }, txOpts); // Tag reconciliation: remove stale, add current @@ -384,7 +409,7 @@ export async function importFromFile( engine: BrainEngine, filePath: string, relativePath: string, - opts: { noEmbed?: boolean; inferFrontmatter?: boolean; sourceId?: string } = {}, + opts: { noEmbed?: boolean; inferFrontmatter?: boolean; sourceId?: string; forceRechunk?: boolean } = {}, ): Promise { // Defense-in-depth: reject symlinks before reading content. const lstat = lstatSync(filePath); @@ -426,8 +451,37 @@ export async function importFromFile( // Enforce path-authoritative slug. parseMarkdown prefers frontmatter.slug over // the path-derived slug, so a mismatch here means the frontmatter is trying // to rewrite a page whose filesystem location says something different. + // + // parsed.slug is `frontmatter.slug || inferSlug(filePath)` where inferSlug + // falls back to slugifyPath(). So parsed.slug.length > 0 with empty + // expectedSlug = frontmatter provided one; both empty = no usable slug. const expectedSlug = slugifyPath(relativePath); - if (parsed.slug !== expectedSlug) { + let resolvedSlug = expectedSlug; + let usedFrontmatterFallback = false; + + if (expectedSlug === '') { + if (parsed.slug && parsed.slug.length > 0) { + // v0.32.7 CJK wave (PR #598 + codex C1/C6): path-derived slug is empty + // (emoji / Thai / Arabic / exotic-script filename). Frontmatter slug + // takes over. logSlugFallback fires below once we know the import + // isn't going to short-circuit. + resolvedSlug = parsed.slug; + usedFrontmatterFallback = true; + } else { + // No path slug, no frontmatter slug — friendlier error (D6=B). + return { + slug: '', + status: 'skipped', + chunks: 0, + error: + `Filename "${relativePath}" produces no usable slug. ` + + `Add a "slug:" to the frontmatter, or rename the file to use ` + + `ASCII / Chinese / Japanese / Korean characters.`, + }; + } + } else if (parsed.slug !== expectedSlug) { + // Anti-spoof preserved: path DOES derive a slug, but the frontmatter slug + // claims a different one. Reject. return { slug: expectedSlug, status: 'skipped', @@ -438,13 +492,23 @@ export async function importFromFile( }; } - // Pass the path-derived slug explicitly so that any future change to + // Emit the dual-channel audit entry AFTER we know we're not going to + // short-circuit, so we don't log noise for failed imports. + if (usedFrontmatterFallback) { + logSlugFallback(resolvedSlug, relativePath); + } + + // Pass the resolved slug explicitly so that any future change to // parseMarkdown's precedence rules cannot re-introduce this bug. // v0.29.1: thread the basename (without extension) for filename-date // precedence in computeEffectiveDate. e.g. `daily/2024-03-15.md` → // filename `2024-03-15`. const fileBasename = basename(relativePath, '.md'); - return importFromContent(engine, expectedSlug, content, { ...opts, filename: fileBasename }); + return importFromContent(engine, resolvedSlug, content, { + ...opts, + filename: fileBasename, + sourcePath: relativePath, + }); } /** diff --git a/src/core/migrate.ts b/src/core/migrate.ts index e0667bf41..c8d5d1185 100644 --- a/src/core/migrate.ts +++ b/src/core/migrate.ts @@ -2672,6 +2672,38 @@ export const MIGRATIONS: Migration[] = [ ON eval_contradictions_runs (ran_at DESC); `, }, + { + version: 54, + name: 'cjk_wave_pages_chunker_version_and_source_path', + // v0.32.7 CJK fix wave. Two new columns on `pages` so the post-upgrade + // reindex sweep can find markdown pages built by the old chunker AND so + // sync's delete/rename code can resolve frontmatter-fallback slugs by + // path (CJK files where path → slug is non-derivable). + // + // chunker_version: bumped to 2 in this release. New imports populate + // it; existing rows inherit DEFAULT 1. `gbrain reindex --markdown` + // walks `WHERE chunker_version < 2 AND page_kind = 'markdown'` + // and re-imports each, bumping the column. + // + // source_path: import-time repo-relative path. Lets sync's delete/ + // rename resolve fallback slugs (`小米.md` w/ frontmatter slug → + // non-path-derivable). NULL for pre-migration rows; populated on + // next import / reindex. + // + // Both columns engine-agnostic. Partial indexes scope to the rows + // we actually query (markdown-only chunker_version; non-NULL source_path). + idempotent: true, + sql: ` + ALTER TABLE pages ADD COLUMN IF NOT EXISTS chunker_version SMALLINT NOT NULL DEFAULT 1; + ALTER TABLE pages ADD COLUMN IF NOT EXISTS source_path TEXT; + + CREATE INDEX IF NOT EXISTS pages_chunker_version_idx + ON pages (chunker_version) WHERE page_kind = 'markdown'; + + CREATE INDEX IF NOT EXISTS pages_source_path_idx + ON pages (source_path) WHERE source_path IS NOT NULL; + `, + }, ]; export const LATEST_VERSION = MIGRATIONS.length > 0 diff --git a/src/core/operations.ts b/src/core/operations.ts index c4b76ac97..6696496f7 100644 --- a/src/core/operations.ts +++ b/src/core/operations.ts @@ -19,6 +19,7 @@ import { extractPageLinks, isAutoLinkEnabled, isAutoTimelineEnabled, parseTimeli import { isFactsBackstopEligible } from './facts/eligibility.ts'; import { stripTakesFence } from './takes-fence.ts'; import { stripFactsFence } from './facts-fence.ts'; +import { CJK_SLUG_CHARS } from './cjk.ts'; import * as db from './db.ts'; import { VERSION } from '../version.ts'; import { @@ -145,8 +146,11 @@ export function validatePageSlug(slug: string): void { if (slug.length > 255) { throw new OperationError('invalid_params', 'page_slug exceeds 255 characters'); } - if (!/^[a-z0-9][a-z0-9\-]*(\/[a-z0-9][a-z0-9\-]*)*$/i.test(slug)) { - throw new OperationError('invalid_params', `Invalid page_slug: ${slug} (allowed: alphanumeric, hyphens, forward-slash separated segments)`); + // v0.32.7: CJK ranges (Han / Hiragana / Katakana / Hangul Syllables) allowed + // in segments. ASCII shape rules (lead char, hyphen continuation) preserved. + const PAGE_SLUG_SEG = `[a-z0-9${CJK_SLUG_CHARS}][a-z0-9${CJK_SLUG_CHARS}\\-]*`; + if (!new RegExp(`^${PAGE_SLUG_SEG}(\\/${PAGE_SLUG_SEG})*$`, 'i').test(slug)) { + throw new OperationError('invalid_params', `Invalid page_slug: ${slug} (allowed: alphanumeric, CJK, hyphens, forward-slash separated segments)`); } } @@ -187,8 +191,11 @@ export function validateFilename(name: string): void { if (name.length > 255) { throw new OperationError('invalid_params', 'Filename exceeds 255 characters'); } - if (!/^[a-zA-Z0-9][a-zA-Z0-9._\-]*$/.test(name)) { - throw new OperationError('invalid_params', `Invalid filename: ${name} (allowed: alphanumeric, dot, underscore, hyphen — no leading dot/dash, no control chars or backslash)`); + // v0.32.7: CJK ranges (Han / Hiragana / Katakana / Hangul) allowed in filenames. + // Leading-dot / leading-dash rejection preserved. + const FILENAME_RE = new RegExp(`^[a-zA-Z0-9${CJK_SLUG_CHARS}][a-zA-Z0-9${CJK_SLUG_CHARS}._\\-]*$`); + if (!FILENAME_RE.test(name)) { + throw new OperationError('invalid_params', `Invalid filename: ${name} (allowed: alphanumeric, CJK, dot, underscore, hyphen — no leading dot/dash, no control chars or backslash)`); } } diff --git a/src/core/pglite-engine.ts b/src/core/pglite-engine.ts index edc436987..525ba2e0e 100644 --- a/src/core/pglite-engine.ts +++ b/src/core/pglite-engine.ts @@ -41,6 +41,7 @@ import { GBrainError, PAGE_SORT_SQL } from './types.ts'; import { computeAnomaliesFromBuckets } from './cycle/anomaly.ts'; import { resolveBoostMap, resolveHardExcludes } from './search/source-boost.ts'; import { buildSourceFactorCase, buildHardExcludeClause, buildVisibilityClause, buildRecencyComponentSql } from './search/sql-ranking.ts'; +import { hasCJK, escapeLikePattern } from './cjk.ts'; type PGLiteDB = PGlite; @@ -530,9 +531,12 @@ export class PGLiteEngine implements BrainEngine { : (page.effective_date ?? null); const effectiveDateSource = page.effective_date_source ?? null; const importFilename = page.import_filename ?? null; + // v0.32.7 CJK wave: chunker_version + source_path columns. + const chunkerVersion = page.chunker_version ?? null; + const sourcePath = page.source_path ?? null; const { rows } = await this.db.query( - `INSERT INTO pages (source_id, slug, type, page_kind, title, compiled_truth, timeline, frontmatter, content_hash, updated_at, effective_date, effective_date_source, import_filename) - VALUES ($1, $2, $3, $4, $5, $6, $7, $8::jsonb, $9, now(), $10::timestamptz, $11, $12) + `INSERT INTO pages (source_id, slug, type, page_kind, title, compiled_truth, timeline, frontmatter, content_hash, updated_at, effective_date, effective_date_source, import_filename, chunker_version, source_path) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8::jsonb, $9, now(), $10::timestamptz, $11, $12, COALESCE($13, 1), $14) ON CONFLICT (source_id, slug) DO UPDATE SET type = EXCLUDED.type, page_kind = EXCLUDED.page_kind, @@ -544,9 +548,11 @@ export class PGLiteEngine implements BrainEngine { updated_at = now(), effective_date = COALESCE(EXCLUDED.effective_date, pages.effective_date), effective_date_source = COALESCE(EXCLUDED.effective_date_source, pages.effective_date_source), - import_filename = COALESCE(EXCLUDED.import_filename, pages.import_filename) + import_filename = COALESCE(EXCLUDED.import_filename, pages.import_filename), + chunker_version = COALESCE(EXCLUDED.chunker_version, pages.chunker_version), + source_path = COALESCE(EXCLUDED.source_path, pages.source_path) RETURNING id, slug, type, title, compiled_truth, timeline, frontmatter, content_hash, created_at, updated_at, effective_date, effective_date_source, import_filename`, - [sourceId, slug, page.type, pageKind, page.title, page.compiled_truth, page.timeline || '', JSON.stringify(frontmatter), hash, effectiveDate, effectiveDateSource, importFilename] + [sourceId, slug, page.type, pageKind, page.title, page.compiled_truth, page.timeline || '', JSON.stringify(frontmatter), hash, effectiveDate, effectiveDateSource, importFilename, chunkerVersion, sourcePath] ); return rowToPage(rows[0] as Record); } @@ -722,6 +728,21 @@ export class PGLiteEngine implements BrainEngine { const hardExcludePrefixes = resolveHardExcludes(opts?.exclude_slug_prefixes, opts?.include_slug_prefixes); const hardExcludeClause = buildHardExcludeClause('p.slug', hardExcludePrefixes); + // v0.26.5: visibility filter (soft-deleted + archived-source). + const visibilityClause = buildVisibilityClause('p', 's'); + + // v0.32.7: CJK query branch. PGLite uses websearch_to_tsquery('english') + // which can't tokenize CJK; queries return empty. Switch to ILIKE on + // chunk_text with bigram-frequency-count ranking when the query contains + // CJK characters. ASCII path stays exactly the same below. + if (hasCJK(query)) { + return this._searchKeywordCJK(query, { + limit, offset, innerLimit, sourceFactorCase, + hardExcludeClause, visibilityClause, detailFilter, opts, + dedup: true, + }); + } + // v0.20.0 Cathedral II Layer 10 C1/C2: language + symbol-kind filters. const params: unknown[] = [query, innerLimit, limit, offset]; let extraFilter = ''; @@ -746,9 +767,6 @@ export class PGLiteEngine implements BrainEngine { extraFilter += ` AND COALESCE(p.effective_date, p.updated_at, p.created_at) < $${params.length}::timestamptz`; } - // v0.26.5: visibility filter (soft-deleted + archived-source). - const visibilityClause = buildVisibilityClause('p', 's'); - const { rows } = await this.db.query( `WITH ranked AS ( SELECT @@ -783,6 +801,130 @@ export class PGLiteEngine implements BrainEngine { return (rows as Record[]).map(rowToSearchResult); } + /** + * v0.32.7 CJK keyword fallback. PGLite's `websearch_to_tsquery('english')` + * can't tokenize CJK so the FTS path returns empty for Chinese / Japanese / + * Korean queries. This routes to an ILIKE substring scan with + * bigram-frequency-count ranking as a ts_rank substitute. + * + * Codex outside-voice C8 corrections in place: + * - Two distinct parameter bindings: $qLike (LIKE-escaped, for ILIKE) and + * $qRaw (un-escaped, for ranking arithmetic via position/replace). + * Escaped chars cannot be reused as ranking substrings. + * - Explicit `ESCAPE '\'` on the ILIKE clause. + * - Symmetric: no asymmetric whitespace strip (caller's query and + * chunk_text are compared as-stored). + * - Empty-query guard returns no results without binding SQL. + * + * Postgres engine is intentionally untouched (multi-tenant deployments + * can install pgroonga / zhparser when needed; out of scope here). + */ + private async _searchKeywordCJK( + query: string, + ctx: { + limit: number; + offset: number; + innerLimit: number; + sourceFactorCase: string; + hardExcludeClause: string; + visibilityClause: string; + detailFilter: string; + opts: SearchOpts | undefined; + dedup: boolean; + }, + ): Promise { + const { limit, offset, innerLimit, sourceFactorCase, hardExcludeClause, visibilityClause, detailFilter, opts, dedup } = ctx; + const qRaw = query; + if (qRaw.length === 0) return []; + const qLike = escapeLikePattern(qRaw); + + // $1 = qLike (escaped for ILIKE) + // $2 = qRaw (raw for position()/replace() ranking arithmetic) + // $3 = inner limit (dedup path) OR final limit (chunk-grain path) + // $4 = final limit (dedup path only) — see callers + // $5 = offset (dedup path) / $4 = offset (chunk-grain path) + const params: unknown[] = dedup + ? [qLike, qRaw, innerLimit, limit, offset] + : [qLike, qRaw, limit, offset]; + + let extraFilter = ''; + if (opts?.language) { + params.push(opts.language); + extraFilter += ` AND cc.language = $${params.length}`; + } + if (opts?.symbolKind) { + params.push(opts.symbolKind); + extraFilter += ` AND cc.symbol_type = $${params.length}`; + } + if (opts?.afterDate) { + params.push(opts.afterDate); + extraFilter += ` AND COALESCE(p.effective_date, p.updated_at, p.created_at) > $${params.length}::timestamptz`; + } + if (opts?.beforeDate) { + params.push(opts.beforeDate); + extraFilter += ` AND COALESCE(p.effective_date, p.updated_at, p.created_at) < $${params.length}::timestamptz`; + } + + // Bigram-frequency count: count occurrences of $qRaw in chunk_text via + // (length(chunk) - length(replace(chunk, q, ''))) / length(q). Acts as + // a ts_rank substitute. position()-tiebreaker so earlier-in-chunk hits + // outrank later ones at the same occurrence count. + const scoreExpr = ` + ((LENGTH(cc.chunk_text) - LENGTH(REPLACE(cc.chunk_text, $2, ''))) / NULLIF(LENGTH($2), 0)::real + + 1.0 / NULLIF(POSITION($2 IN cc.chunk_text), 0)::real) + * ${sourceFactorCase} + `; + + if (dedup) { + const { rows } = await this.db.query( + `WITH ranked AS ( + SELECT + p.slug, p.id as page_id, p.title, p.type, p.source_id, + cc.id as chunk_id, cc.chunk_index, cc.chunk_text, cc.chunk_source, + ${scoreExpr} AS score, + CASE WHEN p.updated_at < ( + SELECT MAX(te.created_at) FROM timeline_entries te WHERE te.page_id = p.id + ) THEN true ELSE false END AS stale + FROM content_chunks cc + JOIN pages p ON p.id = cc.page_id + JOIN sources s ON s.id = p.source_id + WHERE cc.chunk_text ILIKE '%' || $1 || '%' ESCAPE '\\' ${detailFilter}${extraFilter} ${hardExcludeClause} ${visibilityClause} + AND cc.modality = 'text' + ORDER BY score DESC + LIMIT $3 + ), + best_per_page AS ( + SELECT DISTINCT ON (slug) * + FROM ranked + ORDER BY slug, score DESC + ) + SELECT * FROM best_per_page + ORDER BY score DESC + LIMIT $4 OFFSET $5`, + params, + ); + return (rows as Record[]).map(rowToSearchResult); + } else { + const { rows } = await this.db.query( + `SELECT + p.slug, p.id as page_id, p.title, p.type, p.source_id, + cc.id as chunk_id, cc.chunk_index, cc.chunk_text, cc.chunk_source, + ${scoreExpr} AS score, + CASE WHEN p.updated_at < ( + SELECT MAX(te.created_at) FROM timeline_entries te WHERE te.page_id = p.id + ) THEN true ELSE false END AS stale + FROM content_chunks cc + JOIN pages p ON p.id = cc.page_id + JOIN sources s ON s.id = p.source_id + WHERE cc.chunk_text ILIKE '%' || $1 || '%' ESCAPE '\\' ${detailFilter}${extraFilter} ${hardExcludeClause} ${visibilityClause} + ORDER BY score DESC + LIMIT $3 OFFSET $4`, + params, + ); + return (rows as Record[]).map(rowToSearchResult); + } + } + /** * v0.20.0 Cathedral II Layer 3 (1b) chunk-grain keyword search. * @@ -810,6 +952,18 @@ export class PGLiteEngine implements BrainEngine { const sourceFactorCase = buildSourceFactorCase('p.slug', boostMap, opts?.detail); const hardExcludePrefixes = resolveHardExcludes(opts?.exclude_slug_prefixes, opts?.include_slug_prefixes); const hardExcludeClause = buildHardExcludeClause('p.slug', hardExcludePrefixes); + const visibilityClause = buildVisibilityClause('p', 's'); + + // v0.32.7: CJK branch (same as searchKeyword but without page-dedup). + if (hasCJK(query)) { + return this._searchKeywordCJK(query, { + limit, offset, + innerLimit: 0, // unused on chunk-grain (no inner CTE) + sourceFactorCase, + hardExcludeClause, visibilityClause, detailFilter, opts, + dedup: false, + }); + } const params: unknown[] = [query, limit, offset]; let extraFilter = ''; @@ -831,8 +985,7 @@ export class PGLiteEngine implements BrainEngine { extraFilter += ` AND COALESCE(p.effective_date, p.updated_at, p.created_at) < $${params.length}::timestamptz`; } - // v0.26.5: visibility filter for the chunk-grain anchor primitive. - const visibilityClause = buildVisibilityClause('p', 's'); + // visibilityClause already declared above (v0.32.7: hoisted so CJK branch can reuse). const { rows } = await this.db.query( `SELECT diff --git a/src/core/post-upgrade-reembed.ts b/src/core/post-upgrade-reembed.ts new file mode 100644 index 000000000..a6b4f35e2 --- /dev/null +++ b/src/core/post-upgrade-reembed.ts @@ -0,0 +1,156 @@ +/** + * v0.32.7 CJK wave — post-upgrade chunker-bump cost prompt. + * + * When `MARKDOWN_CHUNKER_VERSION` bumps, every markdown page needs a + * re-chunk + re-embed. Re-embed has a real OpenAI bill ($X) and wall-clock + * cost (Y min) proportional to the brain size. On a 1386-page brain that's + * pennies; on a 100K-page brain it's tens of dollars. Surprise OpenAI bills + * are how trust breaks. + * + * Per D3=B: print a stderr line with the real-data estimate before the + * sweep starts, give the operator 10 seconds to Ctrl-C, then proceed. + * + * TTY-only wait so non-TTY upgrades (CI, cron-driven, headless) don't hang. + * + * Codex C3 corrections in place: + * - Real SQL queries against `pages.chunker_version < N AND page_kind = 'markdown'` + * for both page count and char total. No phantom `markdown_body` column. + * - Pricing lookup through `src/core/embedding-pricing.ts` keyed on + * `provider:model` from the configured gateway, with a clear + * "estimate unavailable" message for unknown providers. + */ + +import type { BrainEngine } from './engine.ts'; +import { MARKDOWN_CHUNKER_VERSION } from './chunkers/recursive.ts'; +import { lookupEmbeddingPrice, estimateCostFromChars } from './embedding-pricing.ts'; + +export interface ReembedEstimate { + pendingCount: number; + pendingChars: number; + estimatedTokens: number; + estimatedCostUsd: number | null; + modelString: string; + pricingKnown: boolean; +} + +/** + * Compute the re-embed estimate using only what's actually on the `pages` + * table after migration v54 applied. Used by both the post-upgrade prompt + * and tests. + */ +export async function computeReembedEstimate( + engine: BrainEngine, + modelString: string, +): Promise { + const rows = await engine.executeRaw<{ pending_count: string | number; pending_chars: string | number | null }>( + `SELECT COUNT(*)::bigint AS pending_count, + COALESCE(SUM(LENGTH(compiled_truth)) + SUM(LENGTH(timeline)), 0)::bigint AS pending_chars + FROM pages + WHERE page_kind = 'markdown' + AND chunker_version < $1 + AND deleted_at IS NULL`, + [MARKDOWN_CHUNKER_VERSION], + ); + const pendingCount = Number(rows[0]?.pending_count ?? 0); + const pendingChars = Number(rows[0]?.pending_chars ?? 0); + const price = lookupEmbeddingPrice(modelString); + + if (price.kind === 'known') { + const estimatedCostUsd = estimateCostFromChars(pendingChars, price.pricePerMTok); + return { + pendingCount, + pendingChars, + estimatedTokens: Math.ceil(pendingChars / 3.5), + estimatedCostUsd, + modelString, + pricingKnown: true, + }; + } + return { + pendingCount, + pendingChars, + estimatedTokens: Math.ceil(pendingChars / 3.5), + estimatedCostUsd: null, + modelString, + pricingKnown: false, + }; +} + +/** + * Format the operator-facing stderr line. Pure function so tests can pin + * the exact wording. + */ +export function formatReembedPrompt(est: ReembedEstimate, graceSeconds: number): string { + if (est.pendingCount === 0) { + return `[chunker-bump] No pending markdown pages. Skipping re-embed.`; + } + const minEst = Math.max(1, Math.ceil(est.pendingCount / 60)); // ~60 pages/min wall-clock heuristic + if (est.pricingKnown && est.estimatedCostUsd !== null) { + const dollars = est.estimatedCostUsd.toFixed(2); + return `[chunker-bump] Will re-embed ~${est.pendingCount} markdown pages via ${est.modelString}, est. ~$${dollars}, ~${minEst}min. Press Ctrl-C within ${graceSeconds}s to abort.`; + } + return `[chunker-bump] Will re-embed ~${est.pendingCount} markdown pages via ${est.modelString}; pricing estimate unavailable for this provider. Press Ctrl-C within ${graceSeconds}s to abort.`; +} + +export interface PromptResult { + proceeded: boolean; + reason: 'no_pending' | 'bypassed_no_reembed' | 'tty_proceeded' | 'non_tty_proceeded'; + estimate: ReembedEstimate; +} + +/** + * Run the post-upgrade chunker-bump prompt + grace window. Returns whether + * the caller should proceed to invoke `gbrain reindex --markdown`. + * + * Env overrides (codex C3 + D3=B): + * - GBRAIN_NO_REEMBED=1 → bail out entirely (writes a doctor warning marker). + * - GBRAIN_REEMBED_GRACE_SECONDS=0 → skip wait (proceed immediately). + * - Non-TTY (CI / cron) → skip wait, proceed. + */ +export async function runPostUpgradeReembedPrompt( + engine: BrainEngine, + modelString: string, + opts: { + /** Override for tests: pretend stdin is/isn't a TTY. */ + isTTY?: boolean; + /** Override for tests: how long the wait window is. */ + graceSeconds?: number; + /** Override for tests: env-var bag. Defaults to process.env. */ + env?: Record; + /** Override for tests: where to write. Defaults to process.stderr. */ + write?: (line: string) => void; + } = {}, +): Promise { + const env = opts.env ?? process.env; + const writeFn = opts.write ?? ((line: string) => process.stderr.write(line + '\n')); + const estimate = await computeReembedEstimate(engine, modelString); + + if (estimate.pendingCount === 0) { + return { proceeded: false, reason: 'no_pending', estimate }; + } + + if (env.GBRAIN_NO_REEMBED === '1') { + writeFn(`[chunker-bump] GBRAIN_NO_REEMBED=1 set; skipping re-embed sweep. Pending: ${estimate.pendingCount} pages. Re-run \`gbrain reindex --markdown\` when ready.`); + return { proceeded: false, reason: 'bypassed_no_reembed', estimate }; + } + + const grace = typeof opts.graceSeconds === 'number' + ? opts.graceSeconds + : (() => { + const n = parseInt(env.GBRAIN_REEMBED_GRACE_SECONDS ?? '', 10); + return Number.isFinite(n) && n >= 0 ? n : 10; + })(); + + writeFn(formatReembedPrompt(estimate, grace)); + + const isTTY = typeof opts.isTTY === 'boolean' + ? opts.isTTY + : Boolean(process.stdin.isTTY); + + if (!isTTY || grace === 0) { + return { proceeded: true, reason: isTTY ? 'tty_proceeded' : 'non_tty_proceeded', estimate }; + } + + await new Promise(resolveSleep => setTimeout(resolveSleep, grace * 1000)); + return { proceeded: true, reason: 'tty_proceeded', estimate }; +} diff --git a/src/core/postgres-engine.ts b/src/core/postgres-engine.ts index 41bcf187c..1bfdf86da 100644 --- a/src/core/postgres-engine.ts +++ b/src/core/postgres-engine.ts @@ -591,9 +591,12 @@ export class PostgresEngine implements BrainEngine { const effectiveDate = page.effective_date ?? null; const effectiveDateSource = page.effective_date_source ?? null; const importFilename = page.import_filename ?? null; + // v0.32.7 CJK wave: chunker_version + source_path columns. + const chunkerVersion = page.chunker_version ?? null; + const sourcePath = page.source_path ?? null; const rows = await sql` - INSERT INTO pages (source_id, slug, type, page_kind, title, compiled_truth, timeline, frontmatter, content_hash, updated_at, effective_date, effective_date_source, import_filename) - VALUES (${sourceId}, ${slug}, ${page.type}, ${pageKind}, ${page.title}, ${page.compiled_truth}, ${page.timeline || ''}, ${sql.json(frontmatter as Parameters[0])}, ${hash}, now(), ${effectiveDate}, ${effectiveDateSource}, ${importFilename}) + INSERT INTO pages (source_id, slug, type, page_kind, title, compiled_truth, timeline, frontmatter, content_hash, updated_at, effective_date, effective_date_source, import_filename, chunker_version, source_path) + VALUES (${sourceId}, ${slug}, ${page.type}, ${pageKind}, ${page.title}, ${page.compiled_truth}, ${page.timeline || ''}, ${sql.json(frontmatter as Parameters[0])}, ${hash}, now(), ${effectiveDate}, ${effectiveDateSource}, ${importFilename}, COALESCE(${chunkerVersion}::smallint, 1), ${sourcePath}) ON CONFLICT (source_id, slug) DO UPDATE SET type = EXCLUDED.type, page_kind = EXCLUDED.page_kind, @@ -605,7 +608,9 @@ export class PostgresEngine implements BrainEngine { updated_at = now(), effective_date = COALESCE(EXCLUDED.effective_date, pages.effective_date), effective_date_source = COALESCE(EXCLUDED.effective_date_source, pages.effective_date_source), - import_filename = COALESCE(EXCLUDED.import_filename, pages.import_filename) + import_filename = COALESCE(EXCLUDED.import_filename, pages.import_filename), + chunker_version = COALESCE(EXCLUDED.chunker_version, pages.chunker_version), + source_path = COALESCE(EXCLUDED.source_path, pages.source_path) RETURNING id, slug, type, title, compiled_truth, timeline, frontmatter, content_hash, created_at, updated_at, effective_date, effective_date_source, import_filename `; return rowToPage(rows[0]); diff --git a/src/core/search/expansion.ts b/src/core/search/expansion.ts index c70eea1e6..ddcfad474 100644 --- a/src/core/search/expansion.ts +++ b/src/core/search/expansion.ts @@ -11,6 +11,7 @@ */ import { expand as gatewayExpand, isAvailable as gatewayIsAvailable } from '../ai/gateway.ts'; +import { countCJKAwareWords } from '../cjk.ts'; const MAX_QUERIES = 3; const MIN_WORDS = 3; @@ -54,10 +55,7 @@ export function sanitizeExpansionOutput(alternatives: unknown[]): string[] { } export async function expandQuery(query: string): Promise { - // CJK text is not space-delimited. - const hasCJK = /[\u4e00-\u9fff\u3040-\u309f\u30a0-\u30ff\uac00-\ud7af]/.test(query); - const wordCount = hasCJK ? query.replace(/\s/g, '').length : (query.match(/\S+/g) || []).length; - if (wordCount < MIN_WORDS) return [query]; + if (countCJKAwareWords(query) < MIN_WORDS) return [query]; // Skip LLM call entirely if gateway has no expansion provider configured. if (!gatewayIsAvailable('expansion')) return [query]; diff --git a/src/core/sync.ts b/src/core/sync.ts index c89c84b6b..89778243c 100644 --- a/src/core/sync.ts +++ b/src/core/sync.ts @@ -11,6 +11,8 @@ * pathToSlug() → convert file paths to page slugs */ +import { CJK_SLUG_CHARS } from './cjk.ts'; + export interface SyncManifest { added: string[]; modified: string[]; @@ -246,17 +248,23 @@ export function isSyncable(path: string, opts: SyncableOptions = {}): boolean { * Pattern is the inner character class only (no anchors); callers wrap it * in `^...$` or compose it with prefixes like `(?:people|companies)/...`. */ -export const SLUG_SEGMENT_PATTERN = /[a-z0-9._-]+/; +export const SLUG_SEGMENT_PATTERN = new RegExp(`[a-z0-9._\\-${CJK_SLUG_CHARS}]+`); /** * Slugify a single path segment: lowercase, strip special chars, spaces → hyphens. + * CJK ranges (Han / Hiragana / Katakana / Hangul Syllables) are preserved (v0.32.7). + * NFC re-normalize after the NFD-strip-accents pass so Hangul Jamo recomposes back + * into precomposed syllables that fall inside the whitelist. */ +const SLUGIFY_KEEP_RE = new RegExp(`[^a-z0-9.\\s_\\-${CJK_SLUG_CHARS}]`, 'g'); + export function slugifySegment(segment: string): string { return segment .normalize('NFD') // Decompose accented chars .replace(/[\u0300-\u036f]/g, '') // Strip accent marks + .normalize('NFC') // Recompose Hangul Jamo back to Syllables (v0.32.7) .toLowerCase() - .replace(/[^a-z0-9.\s_-]/g, '') // Keep alphanumeric, dots, spaces, underscores, hyphens + .replace(SLUGIFY_KEEP_RE, '') // Keep alnum, dots, spaces, _-, and CJK (v0.32.7) .replace(/[\s]+/g, '-') // Spaces → hyphens .replace(/-+/g, '-') // Collapse multiple hyphens .replace(/^-|-$/g, ''); // Strip leading/trailing hyphens diff --git a/src/core/types.ts b/src/core/types.ts index a2b90bf85..9b5477435 100644 --- a/src/core/types.ts +++ b/src/core/types.ts @@ -132,6 +132,21 @@ export interface PageInput { effective_date_source?: EffectiveDateSource | null; /** v0.29.1: basename without extension captured at import. */ import_filename?: string | null; + /** + * v0.32.7 CJK wave: bumped to MARKDOWN_CHUNKER_VERSION (2) on import so the + * post-upgrade `gbrain reindex --markdown` sweep can find pre-bump pages + * via `WHERE chunker_version < 2`. Defaults to 1 at the schema level when + * omitted (existing rows pre-migration inherit 1; new imports overwrite + * with the current version). + */ + chunker_version?: number | null; + /** + * v0.32.7 CJK wave: repo-relative import path. Lets sync's delete/rename + * paths resolve a frontmatter-fallback slug back to its filesystem source + * (CJK + emoji + exotic-script files whose path doesn't derive a slug). + * NULL on legacy / non-file callers (MCP `put_page`, fixture seeds). + */ + source_path?: string | null; } export interface PageFilters { diff --git a/test/audit-slug-fallback.serial.test.ts b/test/audit-slug-fallback.serial.test.ts new file mode 100644 index 000000000..15578abfa --- /dev/null +++ b/test/audit-slug-fallback.serial.test.ts @@ -0,0 +1,100 @@ +/** + * v0.32.7 CJK wave — slug-fallback audit JSONL. + * + * Direct unit coverage for `logSlugFallback` and `readRecentSlugFallbacks`. + * Uses GBRAIN_AUDIT_DIR override pointed at a tmpdir for hermeticity (the + * shared-audit-dir helper honors that env var; see shell-audit.ts). + */ + +import { describe, test, expect, beforeAll, afterAll, beforeEach, afterEach } from 'bun:test'; +import { mkdtempSync, rmSync, readdirSync, readFileSync, writeFileSync, mkdirSync } from 'fs'; +import { join } from 'path'; +import { tmpdir } from 'os'; +import { + computeSlugFallbackAuditFilename, + logSlugFallback, + readRecentSlugFallbacks, +} from '../src/core/audit-slug-fallback.ts'; + +let auditDir: string; +let savedEnv: string | undefined; + +beforeAll(() => { + auditDir = mkdtempSync(join(tmpdir(), 'gbrain-slug-fallback-audit-')); + savedEnv = process.env.GBRAIN_AUDIT_DIR; + process.env.GBRAIN_AUDIT_DIR = auditDir; +}); + +afterAll(() => { + if (savedEnv === undefined) delete process.env.GBRAIN_AUDIT_DIR; + else process.env.GBRAIN_AUDIT_DIR = savedEnv; + rmSync(auditDir, { recursive: true, force: true }); +}); + +afterEach(() => { + // Empty the audit dir between tests so file rotation doesn't leak state. + for (const f of readdirSync(auditDir)) rmSync(join(auditDir, f)); +}); + +describe('slug-fallback audit (v0.32.7)', () => { + test('computeSlugFallbackAuditFilename has stable ISO-week shape', () => { + const filename = computeSlugFallbackAuditFilename(new Date('2026-05-15T12:00:00Z')); + expect(filename).toMatch(/^slug-fallback-\d{4}-W\d{2}\.jsonl$/); + expect(filename).toContain('2026-W20'); + }); + + test('logSlugFallback writes one JSONL row + stderr line', () => { + logSlugFallback('projects/launch', '🚀.md'); + const files = readdirSync(auditDir); + expect(files.length).toBe(1); + expect(files[0]).toMatch(/^slug-fallback-\d{4}-W\d{2}\.jsonl$/); + const content = readFileSync(join(auditDir, files[0]), 'utf8'); + const lines = content.split('\n').filter(l => l.length > 0); + expect(lines.length).toBe(1); + const parsed = JSON.parse(lines[0]); + expect(parsed.slug).toBe('projects/launch'); + expect(parsed.source_path).toBe('🚀.md'); + expect(parsed.severity).toBe('info'); + expect(parsed.code).toBe('SLUG_FALLBACK_FRONTMATTER'); + expect(parsed.ts).toMatch(/^\d{4}-\d{2}-\d{2}T/); + }); + + test('readRecentSlugFallbacks returns events from the last 7 days', () => { + logSlugFallback('a/one', '🚀.md'); + logSlugFallback('b/two', '🌟.md'); + const events = readRecentSlugFallbacks(7); + expect(events.length).toBe(2); + expect(events.every(e => e.severity === 'info')).toBe(true); + }); + + test('readRecentSlugFallbacks skips corrupt rows silently', () => { + const filename = computeSlugFallbackAuditFilename(); + const file = join(auditDir, filename); + writeFileSync(file, [ + JSON.stringify({ ts: new Date().toISOString(), slug: 'a/good', source_path: 'a.md', severity: 'info', code: 'SLUG_FALLBACK_FRONTMATTER' }), + 'not-json', + '{"ts": "garbage", "slug": "b/bad"', + '', + ].join('\n')); + const events = readRecentSlugFallbacks(7); + expect(events.length).toBe(1); + expect(events[0].slug).toBe('a/good'); + }); + + test('readRecentSlugFallbacks returns empty when no file exists', () => { + const events = readRecentSlugFallbacks(7); + expect(events).toEqual([]); + }); + + test('readRecentSlugFallbacks honors the days window', () => { + // Write an event with a 10-day-old timestamp. + const oldTs = new Date(Date.now() - 10 * 86400000).toISOString(); + const filename = computeSlugFallbackAuditFilename(); + writeFileSync( + join(auditDir, filename), + JSON.stringify({ ts: oldTs, slug: 'stale', source_path: 'old.md', severity: 'info', code: 'SLUG_FALLBACK_FRONTMATTER' }) + '\n', + ); + expect(readRecentSlugFallbacks(7).length).toBe(0); + expect(readRecentSlugFallbacks(30).length).toBe(1); + }); +}); diff --git a/test/chunkers/recursive.test.ts b/test/chunkers/recursive.test.ts index ead1df8e8..87dafe8e9 100644 --- a/test/chunkers/recursive.test.ts +++ b/test/chunkers/recursive.test.ts @@ -133,3 +133,82 @@ describe('Recursive Text Chunker', () => { expect(chunks.length).toBeGreaterThan(1); }); }); + +describe('CJK chunking (v0.32.7)', () => { + test('MARKDOWN_CHUNKER_VERSION is 2', async () => { + const mod = await import('../../src/core/chunkers/recursive.ts'); + expect(mod.MARKDOWN_CHUNKER_VERSION).toBe(2); + }); + + test('long pure-Chinese paragraph splits into multiple chunks', () => { + // Pre-fix: 1000 Chinese chars counts as 1 word, never splits. + // Post-fix: density >= 30% → char-count → splits at chunkSize. + const text = '品牌圣经测试用例'.repeat(200); // 1600 CJK chars, no whitespace + const chunks = chunkText(text, { chunkSize: 100, chunkOverlap: 10 }); + expect(chunks.length).toBeGreaterThan(1); + }); + + test('Japanese with 。 sentence terminator splits at CJK delimiter', () => { + // Each sentence is small (10 chars). chunkSize 5 → must split. + // With CJK delimiter `。` in L2, the splitter finds sentence boundaries. + const text = '今日は晴れです。明日は雨です。明後日は曇りです。'.repeat(20); + const chunks = chunkText(text, { chunkSize: 50, chunkOverlap: 5 }); + expect(chunks.length).toBeGreaterThan(1); + // Verify chunks generally end near a sentence boundary + const someEndAtPunct = chunks.some(c => /[。！？]/.test(c.text.slice(-3))); + expect(someEndAtPunct).toBe(true); + }); + + test('Korean Hangul + spaces splits cleanly', () => { + // Mixed CJK density but with spaces — should still split. + const text = '한글 테스트 입니다 짧은 문장 여러개 '.repeat(50); + const chunks = chunkText(text, { chunkSize: 30 }); + expect(chunks.length).toBeGreaterThan(1); + }); + + test('mixed CJK + English still splits', () => { + const para = 'This is English text. 这是中文文本。 More English here. '; + const text = para.repeat(30); + const chunks = chunkText(text, { chunkSize: 20 }); + expect(chunks.length).toBeGreaterThan(1); + }); + + test('maxChars hard cap fires on whitespace-less CJK at chunkSize boundary', () => { + // 20K char pure-Chinese blob with no whitespace; chunkSize 10K (huge) + // is overridden by maxChars=6000 cap. + const text = '测试'.repeat(10000); // 20K chars + const chunks = chunkText(text, { chunkSize: 100000, chunkOverlap: 0, maxChars: 6000 }); + expect(chunks.length).toBeGreaterThan(1); + for (const c of chunks) { + expect(c.text.length).toBeLessThanOrEqual(6000); + } + }); + + test('maxChars sliding window preserves overlap for continuity', () => { + const text = 'A'.repeat(15000); // 15K of one char, no delimiters + const chunks = chunkText(text, { chunkSize: 100000, chunkOverlap: 0, maxChars: 6000 }); + expect(chunks.length).toBeGreaterThanOrEqual(3); + // Successive chunks should overlap by ~500 chars at the cap boundary + expect(chunks[0].text.length).toBeLessThanOrEqual(6000); + expect(chunks[1].text.length).toBeLessThanOrEqual(6000); + }); + + test('maxChars applies on single-short-chunk path too', () => { + // A short doc (under chunkSize words) but with one huge whitespace-less + // line that exceeds maxChars. The single-chunk fast path must still cap. + const text = 'a'.repeat(8000); // 1 "word" of 8000 chars + const chunks = chunkText(text, { chunkSize: 300, maxChars: 6000 }); + expect(chunks.length).toBeGreaterThanOrEqual(2); + for (const c of chunks) { + expect(c.text.length).toBeLessThanOrEqual(6000); + } + }); + + test('REGRESSION: pure English doc unchanged', () => { + const para = 'The quick brown fox jumps over the lazy dog. '.repeat(50); + const chunks = chunkText(para, { chunkSize: 50 }); + expect(chunks.length).toBeGreaterThan(0); + // Should have been chunked by word boundaries (English-dominant doc). + expect(chunks.every(c => /^[\x20-\x7e\s]+$/.test(c.text))).toBe(true); + }); +}); diff --git a/test/cjk.test.ts b/test/cjk.test.ts new file mode 100644 index 000000000..373127988 --- /dev/null +++ b/test/cjk.test.ts @@ -0,0 +1,130 @@ +import { describe, test, expect } from 'bun:test'; +import { + hasCJK, + countCJKAwareWords, + CJK_DENSITY_THRESHOLD, + CJK_RANGES_REGEX, + CJK_SLUG_CHARS, + CJK_SENTENCE_DELIMITERS, + CJK_CLAUSE_DELIMITERS, + escapeLikePattern, +} from '../src/core/cjk.ts'; + +describe('hasCJK', () => { + test('true on Han', () => { + expect(hasCJK('品牌圣经')).toBe(true); + }); + test('true on Hiragana', () => { + expect(hasCJK('ひらがな')).toBe(true); + }); + test('true on Katakana', () => { + expect(hasCJK('カタカナ')).toBe(true); + }); + test('true on Hangul Syllables', () => { + expect(hasCJK('한글')).toBe(true); + }); + test('false on ASCII', () => { + expect(hasCJK('hello world')).toBe(false); + }); + test('false on Latin-with-accents', () => { + expect(hasCJK('café résumé')).toBe(false); + }); + test('true on mixed CJK + ASCII', () => { + expect(hasCJK('hello 世界')).toBe(true); + }); + test('false on empty string', () => { + expect(hasCJK('')).toBe(false); + }); +}); + +describe('countCJKAwareWords (30% density threshold)', () => { + test('pure Chinese paragraph counts chars', () => { + // 6 Han characters, all CJK → char count + expect(countCJKAwareWords('品牌圣经测试用例')).toBe(8); + }); + + test('pure ASCII paragraph counts whitespace tokens', () => { + expect(countCJKAwareWords('this is a normal english sentence')).toBe(6); + }); + + test('CJK-dominant mixed switches to char count', () => { + // ~80% CJK by char count → char-count branch + const s = '品牌圣经品牌圣经 is the brand'; + expect(countCJKAwareWords(s)).toBe(s.replace(/\s/g, '').length); + }); + + test('English doc with one Japanese term stays whitespace-tokenized', () => { + // 1 CJK / ~50 non-whitespace chars = ~2%, well below 30% + const s = 'the user wrote a long english blog post about マンガ and other interests'; + // Should NOT char-count (would be ~60). Should whitespace-tokenize. + expect(countCJKAwareWords(s)).toBe((s.match(/\S+/g) || []).length); + }); + + test('exactly at 30% threshold uses CJK branch', () => { + // 3 CJK chars + 7 ASCII non-ws chars = 10 total; 3/10 = 0.30 → CJK + const s = '世界世 abcdefg'; + expect(countCJKAwareWords(s)).toBe(10); + }); + + test('just below threshold uses whitespace branch', () => { + // 2 CJK + 8 ASCII = 10 total; 2/10 = 0.20 < 0.30 → whitespace + const s = '世界 abcdefgh'; + expect(countCJKAwareWords(s)).toBe(2); // two whitespace-delimited tokens + }); + + test('empty string returns 0', () => { + expect(countCJKAwareWords('')).toBe(0); + }); + + test('whitespace-only returns 0', () => { + expect(countCJKAwareWords(' \n\t ')).toBe(0); + }); +}); + +describe('constants', () => { + test('CJK_DENSITY_THRESHOLD is 0.30', () => { + expect(CJK_DENSITY_THRESHOLD).toBe(0.30); + }); + + test('CJK_RANGES_REGEX matches all four scripts', () => { + expect(CJK_RANGES_REGEX.test('一')).toBe(true); + expect(CJK_RANGES_REGEX.test('あ')).toBe(true); + expect(CJK_RANGES_REGEX.test('カ')).toBe(true); + expect(CJK_RANGES_REGEX.test('한')).toBe(true); + expect(CJK_RANGES_REGEX.test('a')).toBe(false); + }); + + test('CJK_SLUG_CHARS can be embedded into a character class', () => { + const re = new RegExp(`^[a-z0-9${CJK_SLUG_CHARS}]+$`); + expect(re.test('品牌圣经')).toBe(true); + expect(re.test('hello')).toBe(true); + expect(re.test('hello-world')).toBe(false); // no hyphen in this test class + }); + + test('CJK_SENTENCE_DELIMITERS covers 。！？', () => { + expect(CJK_SENTENCE_DELIMITERS).toEqual(['。', '！', '？']); + }); + + test('CJK_CLAUSE_DELIMITERS covers ；：，、', () => { + expect(CJK_CLAUSE_DELIMITERS).toEqual(['；', '：', '，', '、']); + }); +}); + +describe('escapeLikePattern', () => { + test('escapes % and _', () => { + expect(escapeLikePattern('100% off_today')).toBe('100\\% off\\_today'); + }); + test('escapes backslash first', () => { + // 'a\b' → 'a\\b' (backslash doubled) + expect(escapeLikePattern('a\\b')).toBe('a\\\\b'); + }); + test('escapes all three meta-chars together', () => { + expect(escapeLikePattern('a\\%b_c')).toBe('a\\\\\\%b\\_c'); + }); + test('no-op on plain text', () => { + expect(escapeLikePattern('hello world')).toBe('hello world'); + }); + test('no-op on CJK', () => { + expect(escapeLikePattern('测试')).toBe('测试'); + }); +}); diff --git a/test/e2e/cjk-roundtrip.test.ts b/test/e2e/cjk-roundtrip.test.ts new file mode 100644 index 000000000..96b920b0c --- /dev/null +++ b/test/e2e/cjk-roundtrip.test.ts @@ -0,0 +1,119 @@ +/** + * v0.32.7 CJK wave — end-to-end CJK roundtrip on PGLite. + * + * Proves the complete pipeline delivers for CJK users: + * 1. Import a CJK-named markdown file (slugify CJK preservation). + * 2. Chunk the body (countCJKAwareWords + CJK delimiters + maxChars cap). + * 3. Search via the PGLite CJK keyword fallback (ILIKE + bigram count). + * 4. Assert the page is findable by a CJK substring. + * + * Vector path requires OPENAI_API_KEY; skipped gracefully when absent. + */ + +import { describe, test, expect, beforeAll, afterAll, beforeEach } from 'bun:test'; +import { PGLiteEngine } from '../../src/core/pglite-engine.ts'; +import { importFromContent } from '../../src/core/import-file.ts'; + +let engine: PGLiteEngine; + +beforeAll(async () => { + engine = new PGLiteEngine(); + await engine.connect({}); + await engine.initSchema(); +}); + +afterAll(async () => { + await engine.disconnect(); +}); + +beforeEach(async () => { + await (engine as any).db.exec('DELETE FROM content_chunks'); + await (engine as any).db.exec('DELETE FROM pages'); +}); + +describe('CJK roundtrip (v0.32.7)', () => { + test('Chinese page: import → chunk → search by 测试 substring', async () => { + const md = `--- +type: concept +title: Chinese essay +--- + +这是一个测试文档。测试内容很重要。我们再次测试一下系统。`; + + const result = await importFromContent(engine, 'originals/chinese-roundtrip', md, { noEmbed: true }); + expect(result.status).toBe('imported'); + expect(result.chunks).toBeGreaterThan(0); + + const hits = await engine.searchKeyword('测试'); + expect(hits.length).toBeGreaterThan(0); + expect(hits[0].slug).toBe('originals/chinese-roundtrip'); + }); + + test('Japanese page: import → chunk on 。 delimiter → search', async () => { + const md = `--- +type: concept +title: Japanese essay +--- + +今日は晴れです。明日は雨です。明後日は曇りです。`; + + const result = await importFromContent(engine, 'originals/japanese-roundtrip', md, { noEmbed: true }); + expect(result.status).toBe('imported'); + expect(result.chunks).toBeGreaterThan(0); + + const hits = await engine.searchKeyword('晴れ'); + expect(hits.length).toBeGreaterThan(0); + expect(hits[0].slug).toBe('originals/japanese-roundtrip'); + }); + + test('Korean Hangul page imports + searches cleanly', async () => { + const md = `--- +type: concept +title: Korean essay +--- + +한글 테스트 문서 입니다. 또 한번 한글 테스트.`; + + const result = await importFromContent(engine, 'originals/korean-roundtrip', md, { noEmbed: true }); + expect(result.status).toBe('imported'); + + const hits = await engine.searchKeyword('한글'); + expect(hits.length).toBeGreaterThan(0); + expect(hits[0].slug).toBe('originals/korean-roundtrip'); + }); + + test('REGRESSION: ASCII pipeline still routes via FTS path on the same brain', async () => { + const md = `--- +type: concept +title: English essay +--- + +NovaMind builds AI agents for enterprise automation. Real production deployments scale to thousands.`; + + await importFromContent(engine, 'originals/english-roundtrip', md, { noEmbed: true }); + const hits = await engine.searchKeyword('NovaMind'); + expect(hits.length).toBeGreaterThan(0); + expect(hits[0].slug).toBe('originals/english-roundtrip'); + }); + + test('CJK + ASCII mixed query lands on the CJK page (LIKE branch)', async () => { + await importFromContent(engine, 'originals/mixed-roundtrip', `--- +type: note +title: Mixed +--- + +The system uses 测试 framework for validation.`, { noEmbed: true }); + const hits = await engine.searchKeyword('测试'); + expect(hits.some(h => h.slug === 'originals/mixed-roundtrip')).toBe(true); + }); + + test('vector path skip-gracefully without OPENAI_API_KEY', () => { + if (!process.env.OPENAI_API_KEY) { + // Documented behavior — surface to the CI log so reviewers know the + // vector path didn't run. Test still passes. + // eslint-disable-next-line no-console + console.log('[skip] vector path — set OPENAI_API_KEY to exercise'); + } + expect(true).toBe(true); + }); +}); diff --git a/test/e2e/sync-cjk-git.test.ts b/test/e2e/sync-cjk-git.test.ts new file mode 100644 index 000000000..57ac520a3 --- /dev/null +++ b/test/e2e/sync-cjk-git.test.ts @@ -0,0 +1,95 @@ +/** + * v0.32.7 CJK wave — real-git E2E for core.quotepath=false fix. + * + * Hermetic: no DATABASE_URL, no API keys. Spawns real `git` in a tmpdir, + * commits a markdown file with a CJK filename, and asserts buildSyncManifest + * receives the UTF-8 path literal — not the octal-escaped form git emits by + * default. Unit tests pin the args array; this test proves real-CLI behavior. + */ + +import { describe, test, expect, beforeAll, afterAll } from 'bun:test'; +import { execFileSync } from 'child_process'; +import { mkdtempSync, writeFileSync, rmSync, mkdirSync } from 'fs'; +import { tmpdir } from 'os'; +import { join } from 'path'; +import { buildSyncManifest } from '../../src/core/sync.ts'; +import { buildGitInvocation } from '../../src/commands/sync.ts'; + +let repoPath: string; + +beforeAll(() => { + repoPath = mkdtempSync(join(tmpdir(), 'gbrain-cjk-git-')); + execFileSync('git', ['-C', repoPath, 'init', '--quiet']); + execFileSync('git', ['-C', repoPath, 'config', 'user.email', 'cjk-test@example.com']); + execFileSync('git', ['-C', repoPath, 'config', 'user.name', 'CJK Test']); + execFileSync('git', ['-C', repoPath, 'config', 'commit.gpgsign', 'false']); +}); + +afterAll(() => { + rmSync(repoPath, { recursive: true, force: true }); +}); + +function gitWithHelper(args: string[]): string { + // Mirrors the production helper at src/commands/sync.ts:git() + return execFileSync('git', buildGitInvocation(repoPath, args), { + encoding: 'utf-8', + timeout: 30000, + }).trim(); +} + +describe('real git emits UTF-8 paths with core.quotepath=false', () => { + test('Chinese filename round-trips through diff --name-status', () => { + // Create + commit a Chinese-named markdown file + const filename = '品牌圣经.md'; + writeFileSync(join(repoPath, filename), '# 测试\n\nbody\n'); + execFileSync('git', ['-C', repoPath, 'add', '--all']); + execFileSync('git', ['-C', repoPath, 'commit', '--quiet', '-m', 'add chinese file']); + + // Diff against the empty tree (the file is the only thing in the commit) + const emptyTree = execFileSync('git', ['-C', repoPath, 'hash-object', '-t', 'tree', '--stdin'], { + input: '', + encoding: 'utf-8', + }).trim(); + const diff = gitWithHelper(['diff', '--name-status', '-M', `${emptyTree}..HEAD`]); + + // The literal CJK chars should appear; the octal-escape form (\345\223\201) should NOT + expect(diff).toContain('品牌圣经.md'); + expect(diff).not.toContain('\\345'); + + const manifest = buildSyncManifest(diff); + expect(manifest.added).toContain('品牌圣经.md'); + }); + + test('Japanese filename with spaces (Apple Notes export pattern)', () => { + // Make sure we have a clean base commit before this case + const initialHead = gitWithHelper(['rev-parse', 'HEAD']); + + const filename = '2026-04-14 22_38 記録-個人智能体_原文.md'; + mkdirSync(join(repoPath, 'inbox'), { recursive: true }); + writeFileSync(join(repoPath, 'inbox', filename), '# meeting\n'); + execFileSync('git', ['-C', repoPath, 'add', '--all']); + execFileSync('git', ['-C', repoPath, 'commit', '--quiet', '-m', 'add jp file']); + + const diff = gitWithHelper(['diff', '--name-status', '-M', `${initialHead}..HEAD`]); + expect(diff).toContain('記録-個人智能体_原文.md'); + expect(diff).not.toMatch(/\\\d{3}/); // no octal-escape sequences anywhere + + const manifest = buildSyncManifest(diff); + expect(manifest.added.some(p => p.includes('記録-個人智能体_原文.md'))).toBe(true); + }); + + test('rename entry with CJK paths', () => { + const beforeRename = gitWithHelper(['rev-parse', 'HEAD']); + + // Rename the first file. -M in diff catches it. + execFileSync('git', ['-C', repoPath, 'mv', '品牌圣经.md', '销售论证文档.md']); + execFileSync('git', ['-C', repoPath, 'commit', '--quiet', '-m', 'rename chinese file']); + + const diff = gitWithHelper(['diff', '--name-status', '-M', `${beforeRename}..HEAD`]); + expect(diff).toContain('销售论证文档.md'); + expect(diff).not.toMatch(/\\\d{3}/); + + const manifest = buildSyncManifest(diff); + expect(manifest.renamed.some(r => r.to === '销售论证文档.md' && r.from === '品牌圣经.md')).toBe(true); + }); +}); diff --git a/test/import-file.test.ts b/test/import-file.test.ts index c2505f3d8..60c2d4556 100644 --- a/test/import-file.test.ts +++ b/test/import-file.test.ts @@ -1,8 +1,9 @@ import { describe, test, expect, beforeAll, afterAll } from 'bun:test'; -import { writeFileSync, mkdirSync, rmSync, symlinkSync } from 'fs'; +import { writeFileSync, mkdirSync, rmSync, symlinkSync, readdirSync, readFileSync, existsSync } from 'fs'; import { join } from 'path'; import { importFile, importFromContent } from '../src/core/import-file.ts'; import type { BrainEngine } from '../src/core/engine.ts'; +import { MARKDOWN_CHUNKER_VERSION } from '../src/core/chunkers/recursive.ts'; const TMP = join(import.meta.dir, '.tmp-import-test'); @@ -408,3 +409,99 @@ ${longText} } }); }); + +describe('importFile — CJK wave (v0.32.7)', () => { + test('REGRESSION: pure-CJK filename with NO frontmatter slug imports cleanly as CJK slug', async () => { + // After #115, slugifyPath('小米.md') = '小米' (CJK preserved). The + // anti-spoof rule is content with no frontmatter slug present. + const filePath = join(TMP, '小米.md'); + writeFileSync(filePath, `--- +type: company +title: Xiaomi +--- + +Body text. +`); + const engine = mockEngine(); + const result = await importFile(engine, filePath, '小米.md', { noEmbed: true }); + expect(result.status).toBe('imported'); + expect(result.slug).toBe('小米'); + const putCall = (engine as any)._calls.find((c: any) => c.method === 'putPage'); + expect(putCall.args[1].chunker_version).toBe(MARKDOWN_CHUNKER_VERSION); + expect(putCall.args[1].source_path).toBe('小米.md'); + }); + + test('empty-path-slug + frontmatter slug → fallback path fires (emoji filename)', async () => { + // 🚀.md slugifies empty even after #115 (emoji not in CJK ranges). + // Frontmatter slug must take over. logSlugFallback fires. + const filePath = join(TMP, '🚀.md'); + writeFileSync(filePath, `--- +type: project +title: Launch +slug: projects/launch +--- + +Lifting off. +`); + const engine = mockEngine(); + const result = await importFile(engine, filePath, '🚀.md', { noEmbed: true }); + expect(result.status).toBe('imported'); + expect(result.slug).toBe('projects/launch'); + const putCall = (engine as any)._calls.find((c: any) => c.method === 'putPage'); + expect(putCall.args[0]).toBe('projects/launch'); + expect(putCall.args[1].source_path).toBe('🚀.md'); + }); + + test('empty-path-slug + NO frontmatter slug → friendly D6=B error message', async () => { + // 🌟🚀 slugifies to '' (both emoji stripped, no remaining chars). + // No frontmatter slug to fall back on → friendly error. + const filePath = join(TMP, '🌟🚀.md'); + writeFileSync(filePath, `# Bare body without frontmatter slug + +just content. +`); + const engine = mockEngine(); + const result = await importFile(engine, filePath, '🌟🚀.md', { noEmbed: true }); + expect(result.status).toBe('skipped'); + expect(result.error).toContain('no usable slug'); + expect(result.error).toContain('ASCII / Chinese / Japanese / Korean'); + expect((engine as any)._calls.length).toBe(0); + }); + + test('REGRESSION: anti-spoof still rejects when path DOES derive a slug', async () => { + // notes/random.md derives slug `notes/random`. Frontmatter `slug: people/elon` + // is a mismatch and MUST still be rejected (the original PR #598 + C1 test + // fixture contradiction concern). + const filePath = join(TMP, 'antispoof-cjk-wave.md'); + writeFileSync(filePath, `--- +type: person +title: Elon +slug: people/elon +--- + +Hijack. +`); + const engine = mockEngine(); + const result = await importFile(engine, filePath, 'notes/antispoof-cjk-wave.md', { noEmbed: true }); + expect(result.status).toBe('skipped'); + expect(result.error).toContain('does not match'); + expect((engine as any)._calls.length).toBe(0); + }); + + test('chunker_version + source_path populated on every import', async () => { + const filePath = join(TMP, 'cjk-source-path.md'); + writeFileSync(filePath, `--- +type: concept +title: Has source path +--- + +Content. +`); + const engine = mockEngine(); + await importFile(engine, filePath, 'concepts/cjk-source-path.md', { noEmbed: true }); + const putCall = (engine as any)._calls.find((c: any) => c.method === 'putPage'); + expect(putCall).toBeTruthy(); + expect(putCall.args[1].chunker_version).toBe(MARKDOWN_CHUNKER_VERSION); + expect(putCall.args[1].source_path).toBe('concepts/cjk-source-path.md'); + }); +}); diff --git a/test/migrations-cjk-wave.test.ts b/test/migrations-cjk-wave.test.ts new file mode 100644 index 000000000..1c9de5e9d --- /dev/null +++ b/test/migrations-cjk-wave.test.ts @@ -0,0 +1,73 @@ +/** + * v0.32.7 CJK wave — migration v54 (cjk_wave_pages_chunker_version_and_source_path). + * + * Asserts the two new columns + partial indexes on `pages` exist after schema + * initialization on PGLite. Postgres parity is covered by test/e2e/schema-drift.test.ts. + */ + +import { describe, test, expect, beforeAll, afterAll } from 'bun:test'; +import { PGLiteEngine } from '../src/core/pglite-engine.ts'; + +let engine: PGLiteEngine; + +beforeAll(async () => { + engine = new PGLiteEngine(); + await engine.connect({}); + await engine.initSchema(); +}); + +afterAll(async () => { + await engine.disconnect(); +}); + +describe('migration v54: cjk_wave_pages_chunker_version_and_source_path', () => { + test('pages.chunker_version exists with default 1', async () => { + const rows = await engine.executeRaw<{ column_name: string; data_type: string; column_default: string | null; is_nullable: string }>( + `SELECT column_name, data_type, column_default, is_nullable + FROM information_schema.columns + WHERE table_name = 'pages' AND column_name = 'chunker_version'`, + ); + expect(rows.length).toBe(1); + expect(rows[0].is_nullable).toBe('NO'); + expect(rows[0].column_default).toContain('1'); + }); + + test('pages.source_path exists, nullable', async () => { + const rows = await engine.executeRaw<{ column_name: string; is_nullable: string }>( + `SELECT column_name, is_nullable + FROM information_schema.columns + WHERE table_name = 'pages' AND column_name = 'source_path'`, + ); + expect(rows.length).toBe(1); + expect(rows[0].is_nullable).toBe('YES'); + }); + + test('partial index pages_chunker_version_idx exists', async () => { + const rows = await engine.executeRaw<{ indexname: string }>( + `SELECT indexname FROM pg_indexes WHERE indexname = 'pages_chunker_version_idx'`, + ); + expect(rows.length).toBe(1); + }); + + test('partial index pages_source_path_idx exists', async () => { + const rows = await engine.executeRaw<{ indexname: string }>( + `SELECT indexname FROM pg_indexes WHERE indexname = 'pages_source_path_idx'`, + ); + expect(rows.length).toBe(1); + }); + + test('default-inherited rows show chunker_version=1', async () => { + // Insert a page WITHOUT specifying chunker_version; verify default fires. + await engine.executeRaw( + `INSERT INTO pages (slug, type, title, source_path) + VALUES ('test/cjk-migration', 'note', 'Test', 'test/cjk-migration.md') + ON CONFLICT DO NOTHING`, + ); + const rows = await engine.executeRaw<{ chunker_version: number; source_path: string | null }>( + `SELECT chunker_version, source_path FROM pages WHERE slug = 'test/cjk-migration'`, + ); + expect(rows.length).toBe(1); + expect(Number(rows[0].chunker_version)).toBe(1); + expect(rows[0].source_path).toBe('test/cjk-migration.md'); + }); +}); diff --git a/test/pglite-engine.test.ts b/test/pglite-engine.test.ts index 51913a389..0663fd8be 100644 --- a/test/pglite-engine.test.ts +++ b/test/pglite-engine.test.ts @@ -217,6 +217,118 @@ describe('PGLiteEngine: Search', () => { }); }); +// ───────────────────────────────────────────────────────────────── +// CJK keyword fallback (v0.32.7) +// ───────────────────────────────────────────────────────────────── +describe('PGLiteEngine: CJK keyword fallback (v0.32.7)', () => { + beforeEach(async () => { + await truncateAll(); + // Three pages with Chinese / Japanese / Korean content; the Chinese + // page contains the substring 测试 three times so bigram ranking + // can rank it above the others. + await engine.putPage('originals/chinese-essay', { + type: 'concept', title: 'Chinese essay', + compiled_truth: '测试 内容 测试 测试 多次', + }); + await engine.upsertChunks('originals/chinese-essay', [ + { chunk_index: 0, chunk_text: '测试 内容 测试 测试 多次', chunk_source: 'compiled_truth' }, + ]); + + await engine.putPage('originals/japanese-essay', { + type: 'concept', title: 'Japanese essay', + compiled_truth: '今日は晴れです。明日は雨です。', + }); + await engine.upsertChunks('originals/japanese-essay', [ + { chunk_index: 0, chunk_text: '今日は晴れです。明日は雨です。', chunk_source: 'compiled_truth' }, + ]); + + await engine.putPage('originals/korean-essay', { + type: 'concept', title: 'Korean essay', + compiled_truth: '한글 테스트 문서 입니다', + }); + await engine.upsertChunks('originals/korean-essay', [ + { chunk_index: 0, chunk_text: '한글 테스트 문서 입니다', chunk_source: 'compiled_truth' }, + ]); + + // Plus an English page so we can verify the ASCII path still works. + await engine.putPage('originals/english-essay', { + type: 'concept', title: 'English essay', + compiled_truth: 'NovaMind builds AI agents for enterprise automation.', + }); + await engine.upsertChunks('originals/english-essay', [ + { chunk_index: 0, chunk_text: 'NovaMind builds AI agents for enterprise', chunk_source: 'compiled_truth' }, + ]); + }); + + test('CJK query routes to LIKE branch and finds Chinese substring', async () => { + const results = await engine.searchKeyword('测试'); + expect(results.length).toBeGreaterThan(0); + expect(results[0].slug).toBe('originals/chinese-essay'); + }); + + test('CJK query finds Japanese substring', async () => { + const results = await engine.searchKeyword('晴れ'); + expect(results.length).toBeGreaterThan(0); + expect(results[0].slug).toBe('originals/japanese-essay'); + }); + + test('CJK query finds Korean Hangul substring', async () => { + const results = await engine.searchKeyword('한글'); + expect(results.length).toBeGreaterThan(0); + expect(results[0].slug).toBe('originals/korean-essay'); + }); + + test('bigram ranking: 3-hit page outranks 1-hit page', async () => { + // Add another Chinese page with only ONE occurrence of 测试. + await engine.putPage('originals/chinese-one-hit', { + type: 'concept', title: 'One-hit', + compiled_truth: '只有一个 测试 in this page', + }); + await engine.upsertChunks('originals/chinese-one-hit', [ + { chunk_index: 0, chunk_text: '只有一个 测试 in this page', chunk_source: 'compiled_truth' }, + ]); + + const results = await engine.searchKeyword('测试'); + // 3-occurrence page should rank ahead of 1-occurrence page. + const idxThreeHits = results.findIndex(r => r.slug === 'originals/chinese-essay'); + const idxOneHit = results.findIndex(r => r.slug === 'originals/chinese-one-hit'); + expect(idxThreeHits).toBeGreaterThanOrEqual(0); + expect(idxOneHit).toBeGreaterThanOrEqual(0); + expect(idxThreeHits).toBeLessThan(idxOneHit); + }); + + test('REGRESSION: ASCII query still uses FTS path and returns English hits', async () => { + const results = await engine.searchKeyword('NovaMind'); + expect(results.length).toBeGreaterThan(0); + expect(results[0].slug).toBe('originals/english-essay'); + }); + + test('REGRESSION: ASCII query does NOT match CJK pages', async () => { + // English query against a brain containing Chinese pages should not + // return Chinese hits (English tokenizer + Chinese text → no FTS match). + const results = await engine.searchKeyword('NovaMind'); + expect(results.every(r => !r.slug.includes('chinese') && !r.slug.includes('japanese') && !r.slug.includes('korean'))).toBe(true); + }); + + test('LIKE-meta-char escape: query with literal % does not wildcard-match', async () => { + // After our escape pass, ILIKE '%' || '\%' || '%' ESCAPE '\' looks + // for a literal `%` character — which our seeded CJK pages don't + // contain. So results should be empty (or at least not all 3 CJK pages). + const results = await engine.searchKeyword('100% 测试'); + // Either the literal "100% 测试" exists nowhere (expected empty), or + // only the exact-substring pages match. None of our seeded pages + // contain this exact string. + expect(results.length).toBe(0); + }); + + test('empty CJK query returns no results', async () => { + const results = await engine.searchKeyword(''); + // Empty query: our CJK branch detects hasCJK('') === false, so it falls + // to the ASCII FTS path which also returns nothing for empty. + expect(results).toEqual([]); + }); +}); + // ───────────────────────────────────────────────────────────────── // Chunks // ───────────────────────────────────────────────────────────────── diff --git a/test/reindex.test.ts b/test/reindex.test.ts new file mode 100644 index 000000000..9901f2044 --- /dev/null +++ b/test/reindex.test.ts @@ -0,0 +1,138 @@ +/** + * v0.32.7 CJK wave — reindex sweep tests. + * + * Drives `gbrain reindex --markdown` against an in-memory PGLite brain, + * verifies the chunker_version sweep updates rows below the current + * MARKDOWN_CHUNKER_VERSION and is idempotent on re-run. + */ + +import { describe, test, expect, beforeAll, afterAll, beforeEach } from 'bun:test'; +import { PGLiteEngine } from '../src/core/pglite-engine.ts'; +import { runReindex } from '../src/commands/reindex.ts'; +import { MARKDOWN_CHUNKER_VERSION } from '../src/core/chunkers/recursive.ts'; + +let engine: PGLiteEngine; + +beforeAll(async () => { + engine = new PGLiteEngine(); + await engine.connect({}); + await engine.initSchema(); +}); + +afterAll(async () => { + await engine.disconnect(); +}); + +beforeEach(async () => { + await (engine as any).db.exec('DELETE FROM content_chunks'); + await (engine as any).db.exec('DELETE FROM pages'); +}); + +async function seedLegacyPage(slug: string, body: string, sourcePath: string | null = null) { + // Force chunker_version=1 explicitly to simulate a pre-bump row. + await engine.executeRaw( + `INSERT INTO pages (slug, type, title, compiled_truth, page_kind, chunker_version, source_path) + VALUES ($1, 'note', $2, $3, 'markdown', 1, $4)`, + [slug, slug.split('/').pop() ?? slug, body, sourcePath], + ); +} + +describe('gbrain reindex --markdown (v0.32.7)', () => { + test('dry-run reports pending count and does not write', async () => { + await seedLegacyPage('note-a', 'body a'); + await seedLegacyPage('note-b', 'body b'); + + const result = await runReindex(engine, ['--markdown', '--dry-run']); + expect(result.dryRun).toBe(true); + expect(result.pending).toBe(2); + expect(result.reindexed).toBe(0); + + // chunker_version still 1 after dry-run + const rows = await engine.executeRaw<{ chunker_version: number }>( + `SELECT chunker_version FROM pages WHERE slug IN ('note-a', 'note-b') ORDER BY slug`, + ); + expect(rows.every(r => Number(r.chunker_version) === 1)).toBe(true); + }); + + test('actual sweep bumps chunker_version on each row', async () => { + await seedLegacyPage('note-c', 'content for c\n\nmore content'); + await seedLegacyPage('note-d', 'content for d'); + + const result = await runReindex(engine, ['--markdown', '--no-embed']); + expect(result.reindexed).toBe(2); + expect(result.failed).toBe(0); + + const rows = await engine.executeRaw<{ chunker_version: number }>( + `SELECT chunker_version FROM pages WHERE slug IN ('note-c', 'note-d')`, + ); + expect(rows.every(r => Number(r.chunker_version) === MARKDOWN_CHUNKER_VERSION)).toBe(true); + }); + + test('idempotent: re-run on a fully-updated brain reports nothing to do', async () => { + await seedLegacyPage('note-e', 'body e'); + await runReindex(engine, ['--markdown', '--no-embed']); + const second = await runReindex(engine, ['--markdown', '--no-embed']); + expect(second.pending).toBe(0); + expect(second.reindexed).toBe(0); + }); + + test('--limit caps the work done in one invocation', async () => { + for (let i = 0; i < 5; i++) await seedLegacyPage(`note-lim-${i}`, `body ${i}`); + const result = await runReindex(engine, ['--markdown', '--no-embed', '--limit', '2']); + expect(result.reindexed).toBe(2); + + const remaining = await engine.executeRaw<{ count: string | number }>( + `SELECT COUNT(*)::bigint AS count + FROM pages + WHERE page_kind = 'markdown' AND chunker_version < $1`, + [MARKDOWN_CHUNKER_VERSION], + ); + expect(Number(remaining[0].count)).toBe(3); + }); + + test('REGRESSION: forceRechunk bypasses content_hash short-circuit (codex F1)', async () => { + // The bug: importFromContent skips pages whose content_hash matches even + // when the chunker version is stale. The fix: reindex passes + // forceRechunk: true so the bumped chunker actually applies. + // + // We can't easily verify chunk_text changed (CJK delimiters are additive + // for English text), but we can verify chunker_version was bumped on the + // row even though compiled_truth + content_hash are unchanged from the + // import. + await seedLegacyPage('regression-force-rechunk', 'unchanged body text'); + + // First reindex pass — content_hash gets stamped to match the body. + await runReindex(engine, ['--markdown', '--no-embed']); + + // Mock a "stale chunker" state: reset chunker_version to 1 WITHOUT + // changing compiled_truth. A non-forceRechunk import would now skip. + await engine.executeRaw( + `UPDATE pages SET chunker_version = 1 WHERE slug = 'regression-force-rechunk'`, + ); + + // Second reindex pass — must bump chunker_version DESPITE content_hash + // matching the stored value. + const result = await runReindex(engine, ['--markdown', '--no-embed']); + expect(result.reindexed).toBe(1); + + const rows = await engine.executeRaw<{ chunker_version: number }>( + `SELECT chunker_version FROM pages WHERE slug = 'regression-force-rechunk'`, + ); + expect(Number(rows[0].chunker_version)).toBe(MARKDOWN_CHUNKER_VERSION); + }); + + test('skips pages already at current chunker_version', async () => { + // Pre-bump page (chunker_version = 1) + await seedLegacyPage('note-up', 'pending body'); + // Already-bumped page (chunker_version = current) + await engine.executeRaw( + `INSERT INTO pages (slug, type, title, compiled_truth, page_kind, chunker_version) + VALUES ('note-current', 'note', 'note-current', 'current body', 'markdown', $1)`, + [MARKDOWN_CHUNKER_VERSION], + ); + + const result = await runReindex(engine, ['--markdown', '--no-embed']); + expect(result.pending).toBe(1); + expect(result.reindexed).toBe(1); + }); +}); diff --git a/test/slug-validation.test.ts b/test/slug-validation.test.ts index b9fa9afb5..7100d2b2f 100644 --- a/test/slug-validation.test.ts +++ b/test/slug-validation.test.ts @@ -1,5 +1,5 @@ import { describe, test, expect } from 'bun:test'; -import { slugifySegment, slugifyPath } from '../src/core/sync.ts'; +import { slugifySegment, slugifyPath, SLUG_SEGMENT_PATTERN } from '../src/core/sync.ts'; // Test the validateSlug behavior via the engine // We can't import validateSlug directly (it's private), so we test through putPage mock behavior @@ -183,3 +183,86 @@ describe('validateSlug (widened for any filename chars)', () => { expect(validateSlug('notes/..')).toBe(false); }); }); + +describe('CJK slug preservation (v0.32.7)', () => { + test('Han characters preserved (Chinese)', () => { + expect(slugifySegment('品牌圣经')).toBe('品牌圣经'); + expect(slugifySegment('销售论证文档')).toBe('销售论证文档'); + }); + + test('Hiragana preserved', () => { + expect(slugifySegment('ひらがなテスト')).toBe('ひらがなテスト'); + }); + + test('Katakana preserved (full-width)', () => { + expect(slugifySegment('カタカナテスト')).toBe('カタカナテスト'); + }); + + test('Hangul Syllables preserved (Korean)', () => { + expect(slugifySegment('한글테스트')).toBe('한글테스트'); + }); + + test('NFC re-composition for Hangul', () => { + // NFD decomposes Hangul Syllables into conjoining Jamo (U+1100 block). + // Without normalize('NFC') after the accent strip, the result would + // collapse to empty because Jamo sits outside the Syllables range. + const decomposed = '한글테스트'.normalize('NFD'); + expect(slugifySegment(decomposed)).toBe('한글테스트'); + }); + + test('mixed CJK + ASCII: lowercase ASCII, preserve CJK', () => { + expect(slugifySegment('ICP-理想客户画像')).toBe('icp-理想客户画像'); + }); + + test('collision regression: different CJK names produce different slugs', () => { + expect(slugifySegment('品牌圣经')).not.toBe(slugifySegment('销售论证文档')); + }); + + test('slugifyPath preserves pure-CJK files', () => { + expect(slugifyPath('inbox/品牌圣经.md')).toBe('inbox/品牌圣经'); + }); + + test('slugifyPath collision regression at path level', () => { + expect(slugifyPath('inbox/品牌圣经.md')).not.toBe(slugifyPath('inbox/销售论证文档.md')); + }); + + test('CJK directory names preserved', () => { + expect(slugifyPath('档案/2024-记录.md')).toBe('档案/2024-记录'); + }); + + test('REGRESSION: café still slugifies to cafe (NFD-strip-accents chain preserved)', () => { + // Iron rule: the NFC re-normalize must not break existing Latin-with-accent + // behavior. café (Latin) decomposes to 'cafe' + combining acute under NFD, + // strip-combining drops the acute, NFC recomposes 'cafe', then lowercase. + expect(slugifySegment('café')).toBe('cafe'); + }); + + test('REGRESSION: existing English slugs unchanged', () => { + expect(slugifySegment('hello world')).toBe('hello-world'); + expect(slugifySegment('notes (march 2024)')).toBe('notes-march-2024'); + }); +}); + +describe('SLUG_SEGMENT_PATTERN (v0.32.7)', () => { + test('matches pure-CJK slug segments', () => { + expect(SLUG_SEGMENT_PATTERN.test('品牌圣经')).toBe(true); + expect(SLUG_SEGMENT_PATTERN.test('한글')).toBe(true); + }); + + test('matches existing ASCII slug shapes', () => { + expect(SLUG_SEGMENT_PATTERN.test('hello-world')).toBe(true); + expect(SLUG_SEGMENT_PATTERN.test('companies/acme.io')).toBe(true); + expect(SLUG_SEGMENT_PATTERN.test('people/foo_bar')).toBe(true); + }); + + test('matches mixed CJK + ASCII', () => { + expect(SLUG_SEGMENT_PATTERN.test('icp-理想客户画像')).toBe(true); + }); + + test('REGRESSION: rejects non-CJK Unicode (Vietnamese)', () => { + // Scope is CJK only; Vietnamese with combining diacritics stays rejected + // until we widen to Unicode property escapes in v0.33+. + const result = 'người-dùng'.match(new RegExp(`^${SLUG_SEGMENT_PATTERN.source}$`)); + expect(result).toBeNull(); + }); +}); diff --git a/test/sync.test.ts b/test/sync.test.ts index 8106b304d..a7d0d67cc 100644 --- a/test/sync.test.ts +++ b/test/sync.test.ts @@ -1,5 +1,6 @@ import { describe, test, expect, beforeAll, afterAll, beforeEach, afterEach } from 'bun:test'; import { buildSyncManifest, isSyncable, pathToSlug } from '../src/core/sync.ts'; +import { buildGitInvocation } from '../src/commands/sync.ts'; import { mkdtempSync, writeFileSync, rmSync, mkdirSync } from 'fs'; import { join } from 'path'; import { execSync } from 'child_process'; @@ -465,3 +466,97 @@ describe('sync regression — #132 nested transaction deadlock', () => { } }); }); + +describe('resolveSlugByPathOrSourcePath (CJK wave v0.32.7, codex F4)', () => { + let pgEngine: PGLiteEngine; + + beforeAll(async () => { + pgEngine = new PGLiteEngine(); + await pgEngine.connect({}); + await pgEngine.initSchema(); + }); + + afterAll(async () => { + await pgEngine.disconnect(); + }); + + beforeEach(async () => { + await (pgEngine as any).db.exec('DELETE FROM content_chunks'); + await (pgEngine as any).db.exec('DELETE FROM pages'); + }); + + test('returns stored slug when source_path matches a row', async () => { + const { resolveSlugByPathOrSourcePath } = await import('../src/commands/sync.ts'); + // Seed a frontmatter-fallback page: slug doesn't derive from path (emoji) + await pgEngine.executeRaw( + `INSERT INTO pages (slug, type, title, compiled_truth, page_kind, source_path) + VALUES ('projects/launch', 'project', 'Launch', 'body', 'markdown', '🚀.md')`, + ); + const slug = await resolveSlugByPathOrSourcePath(pgEngine, '🚀.md'); + expect(slug).toBe('projects/launch'); + }); + + test('falls back to resolveSlugForPath when no source_path matches', async () => { + const { resolveSlugByPathOrSourcePath } = await import('../src/commands/sync.ts'); + // No row seeded — fallback returns the path-derived slug. + const slug = await resolveSlugByPathOrSourcePath(pgEngine, 'concepts/hello-world.md'); + expect(slug).toBe('concepts/hello-world'); + }); + + test('scoped by source_id when provided', async () => { + const { resolveSlugByPathOrSourcePath } = await import('../src/commands/sync.ts'); + // Same source_path under TWO sources — without source_id scope we'd + // get either at random. With source_id we get the right one. + await pgEngine.executeRaw( + `INSERT INTO sources (id, name) VALUES ('source-a', 'A') ON CONFLICT DO NOTHING`, + ); + await pgEngine.executeRaw( + `INSERT INTO sources (id, name) VALUES ('source-b', 'B') ON CONFLICT DO NOTHING`, + ); + await pgEngine.executeRaw( + `INSERT INTO pages (source_id, slug, type, title, compiled_truth, page_kind, source_path) + VALUES ('source-a', 'slug-a/page', 'note', 'A', 'a', 'markdown', '🚀.md')`, + ); + await pgEngine.executeRaw( + `INSERT INTO pages (source_id, slug, type, title, compiled_truth, page_kind, source_path) + VALUES ('source-b', 'slug-b/page', 'note', 'B', 'b', 'markdown', '🚀.md')`, + ); + expect(await resolveSlugByPathOrSourcePath(pgEngine, '🚀.md', 'source-a')).toBe('slug-a/page'); + expect(await resolveSlugByPathOrSourcePath(pgEngine, '🚀.md', 'source-b')).toBe('slug-b/page'); + }); +}); + +describe('git() helper invocation order (CJK wave v0.32.7)', () => { + // The git CLI requires `-c key=val` to appear BEFORE the subcommand, + // and `-C path` BEFORE the subcommand too. Pin the emit order so a future + // refactor can't silently put `-c` after the subcommand and break CJK + // path emission. + + test('core.quotepath=false is always emitted first', () => { + const argv = buildGitInvocation('/repo', ['diff', '--name-status']); + expect(argv).toEqual([ + '-c', 'core.quotepath=false', + '-C', '/repo', + 'diff', '--name-status', + ]); + }); + + test('extra configs append AFTER quotepath, BEFORE -C and subcommand', () => { + const argv = buildGitInvocation('/repo', ['diff'], ['foo=bar', 'baz=qux']); + expect(argv).toEqual([ + '-c', 'core.quotepath=false', + '-c', 'foo=bar', + '-c', 'baz=qux', + '-C', '/repo', + 'diff', + ]); + }); + + test('empty args produces a valid invocation', () => { + const argv = buildGitInvocation('/repo', []); + expect(argv).toEqual([ + '-c', 'core.quotepath=false', + '-C', '/repo', + ]); + }); +}); diff --git a/test/upgrade-reembed-prompt.test.ts b/test/upgrade-reembed-prompt.test.ts new file mode 100644 index 000000000..d03077a5e --- /dev/null +++ b/test/upgrade-reembed-prompt.test.ts @@ -0,0 +1,167 @@ +/** + * v0.32.7 CJK wave — post-upgrade chunker-bump cost prompt tests. + * + * Asserts the prompt fires with real-data estimates, honors the non-TTY + * skip-wait + GBRAIN_NO_REEMBED env overrides, and falls back to an + * "estimate unavailable" message for unknown embedding providers. + */ + +import { describe, test, expect, beforeAll, afterAll, beforeEach } from 'bun:test'; +import { PGLiteEngine } from '../src/core/pglite-engine.ts'; +import { + computeReembedEstimate, + formatReembedPrompt, + runPostUpgradeReembedPrompt, +} from '../src/core/post-upgrade-reembed.ts'; +import { MARKDOWN_CHUNKER_VERSION } from '../src/core/chunkers/recursive.ts'; + +let engine: PGLiteEngine; + +beforeAll(async () => { + engine = new PGLiteEngine(); + await engine.connect({}); + await engine.initSchema(); +}); + +afterAll(async () => { + await engine.disconnect(); +}); + +beforeEach(async () => { + await (engine as any).db.exec('DELETE FROM content_chunks'); + await (engine as any).db.exec('DELETE FROM pages'); +}); + +async function seedPage(slug: string, body: string, version = 1) { + await engine.executeRaw( + `INSERT INTO pages (slug, type, title, compiled_truth, timeline, page_kind, chunker_version) + VALUES ($1, 'note', $2, $3, '', 'markdown', $4)`, + [slug, slug, body, version], + ); +} + +describe('computeReembedEstimate (v0.32.7)', () => { + test('returns real SQL counts + chars', async () => { + await seedPage('a', 'x'.repeat(1000)); + await seedPage('b', 'y'.repeat(2000)); + const est = await computeReembedEstimate(engine, 'openai:text-embedding-3-large'); + expect(est.pendingCount).toBe(2); + expect(est.pendingChars).toBe(3000); + expect(est.pricingKnown).toBe(true); + expect(est.estimatedCostUsd).toBeGreaterThan(0); + }); + + test('already-bumped pages excluded', async () => { + await seedPage('a', 'old body', 1); + await seedPage('b', 'new body', MARKDOWN_CHUNKER_VERSION); + const est = await computeReembedEstimate(engine, 'openai:text-embedding-3-large'); + expect(est.pendingCount).toBe(1); + }); + + test('unknown provider → pricingKnown=false, estimatedCostUsd=null', async () => { + await seedPage('a', 'body'); + const est = await computeReembedEstimate(engine, 'hunyuan:hunyuan-embedding-v1'); + expect(est.pricingKnown).toBe(false); + expect(est.estimatedCostUsd).toBeNull(); + }); +}); + +describe('formatReembedPrompt (v0.32.7)', () => { + test('known provider includes dollar figure', () => { + const line = formatReembedPrompt( + { pendingCount: 100, pendingChars: 100000, estimatedTokens: 28571, estimatedCostUsd: 0.034, modelString: 'openai:text-embedding-3-large', pricingKnown: true }, + 10, + ); + expect(line).toContain('100 markdown pages'); + expect(line).toContain('openai:text-embedding-3-large'); + expect(line).toContain('$0.03'); + expect(line).toContain('Ctrl-C within 10s'); + }); + + test('unknown provider says "estimate unavailable"', () => { + const line = formatReembedPrompt( + { pendingCount: 50, pendingChars: 50000, estimatedTokens: 14286, estimatedCostUsd: null, modelString: 'hunyuan:hunyuan-embedding-v1', pricingKnown: false }, + 10, + ); + expect(line).toContain('estimate unavailable'); + expect(line).toContain('hunyuan:hunyuan-embedding-v1'); + }); + + test('no pending → "Skipping re-embed"', () => { + const line = formatReembedPrompt( + { pendingCount: 0, pendingChars: 0, estimatedTokens: 0, estimatedCostUsd: 0, modelString: 'openai:text-embedding-3-large', pricingKnown: true }, + 10, + ); + expect(line).toContain('No pending markdown pages'); + }); +}); + +describe('runPostUpgradeReembedPrompt (v0.32.7)', () => { + test('no pending → does NOT prompt, returns proceeded=false', async () => { + await seedPage('a', 'body', MARKDOWN_CHUNKER_VERSION); + const writes: string[] = []; + const result = await runPostUpgradeReembedPrompt(engine, 'openai:text-embedding-3-large', { + isTTY: false, + env: {}, + write: (l) => writes.push(l), + }); + expect(result.proceeded).toBe(false); + expect(result.reason).toBe('no_pending'); + expect(writes.length).toBe(0); + }); + + test('non-TTY proceeds without wait', async () => { + await seedPage('a', 'body'); + const writes: string[] = []; + const result = await runPostUpgradeReembedPrompt(engine, 'openai:text-embedding-3-large', { + isTTY: false, + env: {}, + write: (l) => writes.push(l), + graceSeconds: 99, // would block forever if respected + }); + expect(result.proceeded).toBe(true); + expect(result.reason).toBe('non_tty_proceeded'); + expect(writes.length).toBe(1); + expect(writes[0]).toContain('Ctrl-C'); + }); + + test('GBRAIN_NO_REEMBED=1 bails out with doctor-warning marker', async () => { + await seedPage('a', 'body'); + const writes: string[] = []; + const result = await runPostUpgradeReembedPrompt(engine, 'openai:text-embedding-3-large', { + isTTY: true, + env: { GBRAIN_NO_REEMBED: '1' }, + write: (l) => writes.push(l), + graceSeconds: 99, + }); + expect(result.proceeded).toBe(false); + expect(result.reason).toBe('bypassed_no_reembed'); + expect(writes.some(w => w.includes('GBRAIN_NO_REEMBED=1'))).toBe(true); + }); + + test('GBRAIN_REEMBED_GRACE_SECONDS=0 skips wait on TTY', async () => { + await seedPage('a', 'body'); + const writes: string[] = []; + const t0 = Date.now(); + const result = await runPostUpgradeReembedPrompt(engine, 'openai:text-embedding-3-large', { + isTTY: true, + env: { GBRAIN_REEMBED_GRACE_SECONDS: '0' }, + write: (l) => writes.push(l), + }); + expect(result.proceeded).toBe(true); + expect(result.reason).toBe('tty_proceeded'); + expect(Date.now() - t0).toBeLessThan(1000); // didn't actually wait + }); + + test('unknown provider still prompts + proceeds (degrades to "estimate unavailable")', async () => { + await seedPage('a', 'body'); + const writes: string[] = []; + const result = await runPostUpgradeReembedPrompt(engine, 'hunyuan:hunyuan-embedding-v1', { + isTTY: false, + env: {}, + write: (l) => writes.push(l), + }); + expect(result.proceeded).toBe(true); + expect(writes[0]).toContain('estimate unavailable'); + }); +});