admin/gbrain
1
0
Fork 0
mirror of https://github.com/garrytan/gbrain.git synced 2026-07-28 14:59:47 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/master' into reland/2145

Browse Source
This commit is contained in:
Garry Tan
2026-07-24 10:57:30 -07:00
parent 142c4e9c05 1f319e6d5a
commit ed2a8a9d75
133 changed files with 5964 additions and 904 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/workflows/actionlint.yml
+1 -1
View File
@@ -28,5 +28,5 @@ jobs:
runs-on: ubuntu-latest
timeout-minutes: 5
steps:
- uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
- uses: rhysd/actionlint@393031adb9afb225ee52ae2ccd7a5af5525e03e8 # v1.7.11
.github/workflows/e2e.yml
+3 -3
View File
@@ -45,7 +45,7 @@ jobs:
--health-timeout 5s
--health-retries 5
steps:
- uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
- uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2
with:
bun-version: 1.3.13
@@ -82,7 +82,7 @@ jobs:
--health-timeout 5s
--health-retries 5
steps:
- uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
- uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2
with:
bun-version: 1.3.13
@@ -116,7 +116,7 @@ jobs:
--health-timeout 5s
--health-retries 5
steps:
- uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
- uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2
with:
bun-version: 1.3.13
.github/workflows/heavy-tests.yml
+1 -1
View File
@@ -55,7 +55,7 @@ jobs:
--health-timeout 5s
--health-retries 5
steps:
- uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
- uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2
with:
bun-version: 1.3.13
.github/workflows/release.yml
+2 -2
View File
@@ -24,7 +24,7 @@ jobs:
id-token: write # for attest-build-provenance (Sigstore OIDC)
attestations: write # for attest-build-provenance
steps:
- uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
- uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2
with:
bun-version: 1.3.13
@@ -49,7 +49,7 @@ jobs:
with:
path: artifacts
- name: Create release
uses: softprops/action-gh-release@153bb8e04406b158c6c84fc1615b65b24149a1fe # v2
uses: softprops/action-gh-release@3bb12739c298aeb8a4eeaf626c5b8d85266b0e65 # v2
with:
files: |
artifacts/gbrain-darwin-arm64/gbrain-darwin-arm64
.github/workflows/semgrep.yml
+1 -1
View File
@@ -26,7 +26,7 @@ jobs:
container:
image: semgrep/semgrep:1.170.0@sha256:c98f8829eea377274ee4b10656458b078b88232469b2ff913f091c2317347c9d
steps:
- uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
# Non-blocking initially (continue-on-error): the first runs establish a
# baseline without failing unrelated PRs. Graduation path: once the
# baseline findings are triaged (fixed or `# nosemgrep`'d), remove
.github/workflows/test.yml
+7 -7
View File
@@ -43,7 +43,7 @@ jobs:
hit: ${{ steps.lookup.outputs.cache-hit }}
hash: ${{ steps.compute.outputs.hash }}
steps:
- uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
- name: Compute content hash
id: compute
run: |
@@ -84,7 +84,7 @@ jobs:
runs-on: ubuntu-latest
timeout-minutes: 10
steps:
- uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
with:
fetch-depth: 0
- uses: gitleaks/gitleaks-action@dcedce43c6f43de0b836d1fe38946645c9c638dc # v2
@@ -103,7 +103,7 @@ jobs:
runs-on: ubuntu-latest
timeout-minutes: 12
steps:
- uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
- uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2
with:
bun-version: 1.3.13
@@ -124,7 +124,7 @@ jobs:
runs-on: ubuntu-latest
timeout-minutes: 15
steps:
- uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
- uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2
with:
bun-version: 1.3.13
@@ -149,7 +149,7 @@ jobs:
runs-on: ubuntu-latest
timeout-minutes: 12
steps:
- uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
- uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2
with:
bun-version: 1.3.13
@@ -172,7 +172,7 @@ jobs:
runs-on: ubuntu-latest
timeout-minutes: 12
steps:
- uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
- uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2
with:
bun-version: 1.3.13
@@ -216,7 +216,7 @@ jobs:
matrix:
shard: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]
steps:
- uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
- uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2
with:
bun-version: 1.3.13
.gitignore
+5
View File
@@ -35,6 +35,11 @@ export/
# .context/test-shards/. Workspace-local by design — never committed.
.context/
# Local agent instruction overrides (CLAUDE.local.md / AGENTS.local.md) — personal,
# per-clone, loaded after the committed CLAUDE.md/AGENTS.md. Never committed.
CLAUDE.local.md
AGENTS.local.md
# Tier 3 PGLite snapshot fixture (built on demand by build:pglite-snapshot)
test/fixtures/pglite-snapshot.tar
test/fixtures/pglite-snapshot.version
CHANGELOG.md
+138
View File
@@ -2,6 +2,144 @@
All notable changes to GBrain will be documented in this file.
## [0.42.65.0] - 2026-07-23
**A large maintenance release: 93 verified fixes and small features merged since v0.42.64.0, most of them community contributions.**
If you use gbrain day to day, this release makes the boring parts trustworthy. Importing and syncing notes is safer: a failed pull no longer pretends everything is up to date, imported pages are read back after writing to confirm they landed, and a page with real content can no longer be silently overwritten by an empty one. Search answers get better inputs: the think command now picks excerpts that actually match your question, and results respect your federated source settings. Background enrichment (the "dream" cycle) wastes less money and retries properly when an AI provider is down. Spending caps now fail closed, so a billing hiccup can never turn into an uncapped spend. And `gbrain doctor` is quieter, with several false alarms removed and real problems (like an embedding backlog with no worker running) now flagged.
More AI providers work out of the box, including OpenRouter prompt caching, MiniMax and Zhipu GLM recipes, Ollama Matryoshka embedding dimensions, and llama-server batch limits.
## To take advantage of v0.42.65.0
`gbrain upgrade` should do this automatically. No new schema migrations ship in this release.
1. **Upgrade and verify:**
```bash
gbrain upgrade
gbrain doctor
gbrain stats
```
2. **If `gbrain doctor` reports new findings after upgrading,** that is the quieter, more accurate check set working as intended. Each finding names its fix.
3. **If any step fails,** please file an issue at https://github.com/garrytan/gbrain/issues with the output of `gbrain doctor` and `~/.gbrain/upgrade-errors.jsonl` if it exists.
### Itemized changes
#### Security
- MCP source scoping for remote callers got a hardening pass, so agent-facing connections stay confined to the sources they were granted. (#2881, contributed by @spinsirr)
- Paid MCP spend accounting is now atomic and fails closed, and resolver spend is recorded before a cap error is raised, so caps cannot be raced past or undercounted. (#3203, #3204, contributed by @caterpillarC15)
- The OAuth token endpoint rate limit on the HTTP server is now configurable via env for deployments behind shared IPs. (#3114, contributed by @time-attack)
- `WWW-Authenticate` responses now carry `resource_metadata` per the MCP spec and RFC 9728, so conforming clients can discover the auth server. (#1410, contributed by @rayers)
#### Search, retrieval, and think
- `think` selects query-relevant excerpts instead of generic ones. (#3197, contributed by @Y0lan)
- Unqualified local CLI `search`/`query` now honors `sources.config.federated` read visibility. (#2561, #3141, contributed by @time-attack)
- Email citation metadata is projected into search results. (#2873, contributed by @amtagrwl)
- The `think` Gaps section renders once instead of twice. (#1662, contributed by @howwohmm)
- Fuzzy entity lookup threads the caller's source scope and skips soft-deleted entities. (#1508, contributed by @tim404x)
- `code-def` surfaces method, constructor, field, and struct definitions, not just top-level symbols. (#1628, contributed by @rayers)
- Briefing pages are excluded from their own Brain Pulse salience. (#1202, contributed by @rwbaker)
- Reranker calls with missing auth are classified as configuration errors before falling back. (#2059, #3139, contributed by @time-attack)
#### Import, sync, and ingestion
- A failed git pull with zero imports reports `partial (pull_failed)` instead of `up_to_date`. (#3068, #3253, contributed by @Masashi-Ono0611)
- Imports run a post-write read-back verification with a durable ingest-log record. (#2869, contributed by @Andredsouza1984)
- `put` refuses to overwrite a non-empty page with empty content. (#2708, contributed by @symmetric-matthew)
- `putPage` restores soft-deleted rows instead of colliding with them. (#2779, contributed by @RerankerGuo)
- Mixed-case slugs are normalized before chunk upsert, ending duplicate-chunk churn. (#430, #3143, contributed by @time-attack)
- Imports fall back to the body H1 for the title when frontmatter lacks `title:`. (#2446, #3072, contributed by @time-attack)
- YAML comments inside the frontmatter fence are no longer treated as markdown headings. (#3225, #3247, contributed by @Masashi-Ono0611)
- Write-through guards case-insensitive filesystem collisions before the atomic write. (#2831, #3119, contributed by @time-attack)
- Path-qualified wikilinks outside the known directory pattern resolve on the DB/put_page path. (#2866, contributed by @paul-0320)
- CJK slugs are supported in the slug registry and dream-cycle summary slugs. (#782, #738, #3083, contributed by @time-attack)
- Three ingest/sync/serve singleton fixes: page-type round-trip, deleted-slug embed noise, and a stateless width guard. (#3140, contributed by @time-attack)
- Sync honors the `embedding_disabled` sentinel as an implicit `--no-embed`. (#2879, contributed by @gawievanblerk)
- Verified sync head sentinels are cleared correctly. (#2734, contributed by @symmetric-matthew)
- Resumed syncs report the pinned commit they actually landed on. (#3202, contributed by @caterpillarC15)
- The expected `discover_git_root` probe failure stays off stderr. (#3232, contributed by @Masashi-Ono0611)
- `extract --stale` runs the real resolver so basename resolution reaches stale pages, and clears pre-version-bump pages. (#2576, #2717, contributed by @paul-0320; #1791, contributed by @Nazim22)
- Oversized code chunks are capped so they stay embeddable, and code-chunk metadata survives re-embeds. (#1675, contributed by @lubosxyz; #769, #1232, contributed by @rayers)
#### Background cycle, dream, and facts
- Path-derived dream sources are stamped, and the engine closes cleanly on autopilot shutdown. (#3178, contributed by @time-attack)
- All-provider-failed atom drains propagate so durable jobs retry instead of silently dropping work. (#3218, #3248, contributed by @Masashi-Ono0611)
- Atom extraction raises `maxTokens` and case-normalizes `atom_type` for Gemini models. (#3211, contributed by @alexey-metaengage)
- The conversation extractor gates anonymous-speaker self-attribution instead of guessing. (#3228, contributed by @asenkovskiy)
- Incremental dream extraction stamps its watermark so re-runs stop reprocessing. (#2636, #3115, contributed by @time-attack)
- `dream --dry-run --json` keeps stdout clean of embed summaries. (#394, #3109, contributed by @time-attack)
- Synthesized dream pages require a self-contained opening summary. (#2770, contributed by @Masashi-Ono0611)
- PGLite inline synth subagent drains complete, and `lint` gains `--exclude`. (#2699, #2649, #3162, contributed by @time-attack)
- Live context reads the documented "P1 Today" heading form with plain checkbox tasks, matching the daily-task-manager skill's output format. (#2186, #3124, contributed by @time-attack)
- Queued AI jobs refresh gateway config at execution time instead of using a stale snapshot. (#2125, contributed by @maxpetrusenkoagent)
- `brainstorm`/`propose_takes` honor configured models: cost preview uses the configured model, the judge reads its config key, provider probes are skipped when unneeded, and page projection is narrowed. (#3120, contributed by @time-attack)
- Backlog hardening wave: x-to-brain health check, propose_takes deadlines, capture title truncation, extract_atoms backlog handling, and pooler direct-URL routing. (#3165, contributed by @time-attack)
- `skillopt` emits `proposed.md` in no-mutate mode. (#2635, #3182, contributed by @time-attack)
- Nightly quality probe enable path and conversation-parser probe are wired up. (#2629, #2630, #3094, contributed by @time-attack)
#### Doctor, health, and maintenance
- New safe maintenance automation with a shared orphan-exclusion policy, so routine cleanup runs without risking linked content. (#3015, #3023, contributed by @time-attack)
- `orphan_ratio` excludes the chronicle volume under `life/events/`. (#2264, #3214, contributed by @asenkovskiy)
- `brain_score` orphan/timeline components use the orphans-audit linkable scope. (#3155, contributed by @time-attack)
- Entity timeline coverage is measured separately from whole-brain density. (#2761, contributed by @TurgutKural)
- Doctor flags embed backfills queued with no worker running. (#2696, contributed by @javieraldape)
- Two doctor false-positive/timeout fixes: the drift walk skips `node_modules`, and the bare-tweet check skips inline code and cited lines. (#1772, contributed by @sonlndv)
- A dead `llm_fallback_enabled` recommendation is dropped from conversation format coverage. (#1903, contributed by @ElliotDrel)
- Skill triggers with CRLF line endings parse on Windows. (#1149, contributed by @samporter-31)
- Onboard check names are registered in doctor categories, ending unknown-check warnings, and onboard-check remediations survive the `--apply --auto` path. (#3075, #3097, contributed by @time-attack)
- Dead slug prefixes are counted by slug. (#2697, contributed by @RerankerGuo)
- The backlinks worker defaults to check, not fix, and `check-backlinks` honors its positional directory argument. (#1853, contributed by @choomz; #3076, contributed by @time-attack)
- Calibration resolves the owner holder via config, defaulting to `self`. (#3077, contributed by @time-attack)
- Memory throttling on Linux reads `/proc/meminfo` MemAvailable. (#556, contributed by @chengzehsu)
#### AI providers and gateway
- OpenRouter gets family-scoped prompt caching, and query expansion works on chat-capable openai-compat recipes. (#3152, contributed by @time-attack)
- MiniMax recipe: embedding wire-shape compat fetch plus a chat touchpoint. (#1977, #3089, contributed by @time-attack)
- The Zhipu recipe gains a chat touchpoint so GLM subagents work. (#1157, #3084, contributed by @time-attack)
- Tier-configured models reach the recipe allowlist, Anthropic model lists are refreshed, tier resolutions are registered, and probe labels are honest. (#2800, contributed by @p3ob7o)
- Provider base URL config merges from the DB. (#1676, contributed by @TheLordArgus)
- The gateway falls back to the pooler when the derived direct host is unreachable. (#1641, #3088, contributed by @time-attack)
- Config-plane `voyage_api_key` folds into `VOYAGE_API_KEY` like the other hosted keys. (#3236, contributed by @Masashi-Ono0611)
- The `zeroentropyai:zerank-2` reranker has a pricing entry so the budget tracker can meter it. (#3223, #3233, contributed by @Masashi-Ono0611)
- llama-server embedding batches are capped at its 32-input request limit. (#1281, contributed by @mmekkaoui)
- Matryoshka dimensions thread through for Qwen3-Embedding on Ollama. (#1072, contributed by @mgandal)
- `init` seeds AI options from env on cold install, and `whoami` reports the stdio transport. (#3091, contributed by @time-attack)
- The `models` dispatch subcommand reads its first argument correctly. (#1428, contributed by @BenjaminDSmithy)
- Synopsis generation tail-truncates document text for small-model chat handlers. (#1427, contributed by @BenjaminDSmithy)
- The contradiction judge token cap is raised for thinking models. (#3210, contributed by @alexey-metaengage)
#### Schema, migrations, and storage engines
- Engine migration counts and surfaces per-page copy failures instead of silently advancing. (#3241, contributed by @Masashi-Ono0611)
- Invalid `CONCURRENTLY`-build index remnants are dropped without a DO block. (#3191, contributed by @Masashi-Ono0611)
- Unsupported large-dimension HNSW indexes are skipped instead of failing schema setup. (#1734, #3080, contributed by @time-attack)
- The v0.32.2 migration dirty-check scopes to targeted sources and surfaces failed phase detail. (#3093, contributed by @time-attack)
- Schema packs merge the full `extends` chain and `borrow_from` into the resolved manifest. (#1749, #3181, contributed by @time-attack)
- The schema-pack stats catch-all is narrowed so masked errors surface instead of fake zero-page counts. (#2466, #3133, contributed by @time-attack)
- Bundled schema-pack inspection reports the pack actually shipped in the binary, and minion subagent auth resolves through config. (#3110, contributed by @time-attack)
- PGLite `putPage` guards against zero-row RETURNING. (#1649, contributed by @alexhawkins)
#### MCP server and CLI surface
- `list_pages` rows include `source_id`. (#3209, contributed by @alexey-metaengage)
- Running CLI commands while `gbrain serve` (MCP) holds the brain now notifies about the conflict instead of failing confusingly. (#3243, contributed by @fdefitte)
- The OpenClaw plugin manifest entry is declared so the plugin loads. (#2551, #3185, contributed by @time-attack)
#### For contributors
- CI scanner roots are normalized on macOS. (#3198, contributed by @caterpillarC15)
- CI shard timeout raised to 22 minutes plus a delta-assert reporter leak test. (#3231, contributed by @time-attack)
- E2E suite hardening: flaky tests, no-op assertions, and cross-test coupling removed. (#1704, contributed by @auroracapital)
- `mechanical.test.ts` isolates `$HOME` so the E2E suite stops clobbering user config. (#434, contributed by @lloydarmbrust)
- The lint code-fence-wrap detector and fixer regex now agree. (#1597, contributed by @chungty)
- README project links for OpenClaw and Hermes are corrected. (#1961, #3179, contributed by @time-attack)
- A completed TODOS entry is dropped. (#3229, contributed by @Masashi-Ono0611)
## [0.42.64.0] - 2026-07-20
### Fixed
SECURITY.md
+12
View File
@@ -135,6 +135,18 @@ the PGLite schema. Local agents continue to use stdio (`gbrain serve`).
Running `--http` against a PGLite-backed install fails fast with a clear
error message at startup.
### Docker network isolation (self-hosted Postgres)
OAuth and source scoping enforce isolation on the `serve --http` path only.
Raw Postgres reachability bypasses both: a container that shares Docker's
default `bridge` network with the brain's Postgres can open a direct DB
session without any token and read every source. Put the brain's Postgres on
a user-defined Docker network with nothing untrusted on it, publish its port
loopback-only (if at all), and never put `DATABASE_URL` or a Postgres
password in untrusted agent containers — those should reach the brain
exclusively via OAuth against `serve --http`. Full operator checklist:
[docs/mcp/DEPLOY.md — Co-located Docker workloads](docs/mcp/DEPLOY.md#co-located-docker-workloads-self-hosted-postgres).
### CORS
Default-deny: no `Access-Control-Allow-Origin` header is sent unless an
VERSION
+1 -1
View File
@@ -1 +1 @@
0.42.64.0
0.42.65.0
bun.lock
+10 -5
View File
@@ -51,8 +51,9 @@
"@electric-sql/pglite",
],
"overrides": {
"@hono/node-server": "^1.19.13",
"fast-uri": "^3.1.2",
"@hono/node-server": "^2.0.5",
"body-parser": "^2.3.0",
"fast-uri": "^3.1.4",
"fast-xml-builder": "^1.1.7",
"fast-xml-parser": "^5.7.0",
"form-data": "^4.0.6",
@@ -162,7 +163,7 @@
"@electric-sql/pglite": ["@electric-sql/pglite@0.4.3", "", {}, "sha512-ichuWTgtd4mOM1G4SpyGJa5trT03lWbMypDV0fUXUCXg5hiHqVAz/bZyV68NqmkLB7WcYmj1RMJVSp8HV/v/ZQ=="],
"@hono/node-server": ["@hono/node-server@1.19.14", "", { "peerDependencies": { "hono": "^4" } }, "sha512-GwtvgtXxnWsucXvbQXkRgqksiH2Qed37H9xHZocE5sA3N8O8O8/8FA3uclQXxXVzc9XBZuEOMK7+r02FmSpHtw=="],
"@hono/node-server": ["@hono/node-server@2.0.11", "", { "peerDependencies": { "hono": "^4" } }, "sha512-bjD221KPLoJTWUwso1J6fGKiTXEUFedG/s0visavY4zakFPkeGURMRNly+FhBHs7T8Dz4qHaZIMX9ZoJHSJtKA=="],
"@jsquash/avif": ["@jsquash/avif@2.1.1", "", { "dependencies": { "wasm-feature-detect": "^1.2.11" } }, "sha512-LMRxd0fMgfCLtobDh0/sFYJMMiRJTNYSEEWvRDKXlAeZ08t3gI5V+1thIT0XjXJ+SVG7Zug9B0XPyx0Ti5VRNA=="],
@@ -326,7 +327,7 @@
"asynckit": ["asynckit@0.4.0", "", {}, "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q=="],
"body-parser": ["body-parser@2.2.2", "", { "dependencies": { "bytes": "^3.1.2", "content-type": "^1.0.5", "debug": "^4.4.3", "http-errors": "^2.0.0", "iconv-lite": "^0.7.0", "on-finished": "^2.4.1", "qs": "^6.14.1", "raw-body": "^3.0.1", "type-is": "^2.0.1" } }, "sha512-oP5VkATKlNwcgvxi0vM0p/D3n2C3EReYVX+DNYs5TjZFn/oQt2j+4sVJtSMr18pdRr8wjTcBl6LoV+FUwzPmNA=="],
"body-parser": ["body-parser@2.3.0", "", { "dependencies": { "bytes": "^3.1.2", "content-type": "^2.0.0", "debug": "^4.4.3", "http-errors": "^2.0.1", "iconv-lite": "^0.7.2", "on-finished": "^2.4.1", "qs": "^6.15.2", "raw-body": "^3.0.2", "type-is": "^2.1.0" } }, "sha512-2cGmJupaNgg+QUwVLAucDuWuoMZ6EX9iHDRswZ5lsNYEmwPaRknMPCLZz07yTzVq/83p4o/wzbDZbBrTvGGTIw=="],
"bowser": ["bowser@2.14.1", "", {}, "sha512-tzPjzCxygAKWFOJP011oxFHs57HzIhOEracIgAePE4pqB3LikALKnSzUyU4MGs9/iCEUuHlAJTjTc5M+u7YEGg=="],
@@ -400,7 +401,7 @@
"fast-deep-equal": ["fast-deep-equal@3.1.3", "", {}, "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q=="],
"fast-uri": ["fast-uri@3.1.3", "", {}, "sha512-i70LwGWUduXqzicKXWshooq+sWL1K3WUU5rKZNG/0i3a1OSoX3HqhH5WbWwTmqWfor4urUakGPiRQcleRZTwOg=="],
"fast-uri": ["fast-uri@3.1.4", "", {}, "sha512-8JnbkQ4juDyvYs4mgFGQqg4yCYtFDtUtmp2QIQq11ZZe5CFQ5wcqm1rqDgAh/QdMySuBnPzMUiJUNZG5N/AiQw=="],
"fast-xml-builder": ["fast-xml-builder@1.3.0", "", { "dependencies": { "path-expression-matcher": "^1.6.2", "xml-naming": "^0.3.0" } }, "sha512-F74cZEdCvuw9P41GAC3rod4X04jjWGM1JPEv/GWSqFTWLsdyMSBMBMlm9Hk3GLBgLBbdBNY8yee0pQh2RBVESQ=="],
@@ -614,6 +615,10 @@
"@types/bun/bun-types": ["bun-types@1.3.11", "", { "dependencies": { "@types/node": "*" } }, "sha512-1KGPpoxQWl9f6wcZh57LvrPIInQMn2TQ7jsgxqpRzg+l0QPOFvJVH7HmvHo/AiPgwXy+/Thf6Ov3EdVn1vOabg=="],
"body-parser/content-type": ["content-type@2.0.0", "", {}, "sha512-j/O/d7GcZCyNl7/hwZAb606rzqkyvaDctLmckbxLzHvFBzTJHuGEdodATcP3yIRoDrLHkIATJuvzbFlp/ki2cQ=="],
"body-parser/type-is": ["type-is@2.1.0", "", { "dependencies": { "content-type": "^2.0.0", "media-typer": "^1.1.0", "mime-types": "^3.0.0" } }, "sha512-faYHw0anBbc/kWF3zFTEnxSFOAGUX9GFbOBthvDdLsIlEoWOFOtS0zgCiQYwIskL9iGXZL3kAXD8OoZ4GmMATA=="],
"es-set-tostringtag/hasown": ["hasown@2.0.2", "", { "dependencies": { "function-bind": "^1.1.2" } }, "sha512-0hJU9SCPvmMzIBdZFqNPXWa6dqh7WdH0cII9y+CyS8rG3nL48Bclra9HmKhVVUHyPWNH5Y7xDwAB7bfgSjkUMQ=="],
"eventsource/eventsource-parser": ["eventsource-parser@3.0.6", "", {}, "sha512-Vo1ab+QXPzZ4tCa8SwIHJFaSzy4R6SHf7BY79rFBDf0idraZWAkYrDjDj8uWaSm3S2TK+hJ7/t1CEmZ7jXw+pg=="],
docs/architecture/KEY_FILES.md
+1 -1
View File
File diff suppressed because one or more lines are too long
docs/guides/minions-deployment.md
-30
View File
@@ -93,29 +93,6 @@ usually want both.
| **Linux VM with systemd** | Two-layer recommended: systemd supervises `gbrain jobs supervisor`, which in turn supervises `gbrain jobs work`. Buys you automatic restart on reboot (systemd) plus fast crash recovery (supervisor). See [systemd](#systemd). |
| **Dev laptop / macOS** | `gbrain jobs supervisor` in a terminal. Ctrl-C stops it. No system-level setup needed. |
### Running alongside autopilot (production deployment shape)
`gbrain autopilot` and `gbrain jobs supervisor` overlap: by default,
autopilot spawns its own managed worker for the jobs it dispatches. If you
run BOTH a default autopilot and a supervisor on the same brain, you get two
worker lanes claiming from the same queue — double concurrency, double memory,
and two processes competing for the same job locks. Pick one shape:
| Shape | When | How |
|---|---|---|
| **Split (recommended for production)** | You already run (or want) a platform-supervised worker: systemd, Fly, container. | `gbrain autopilot --no-worker` as the dispatcher + `gbrain jobs supervisor` as the single worker lane. Autopilot submits jobs and probes for peer-worker liveness (it warns loudly after a few cycles if nothing is claiming); the supervisor owns execution, crash recovery, and drain. |
| **All-in-one** | Single machine, nothing else runs workers. | `gbrain autopilot` alone (it manages its own worker child). Do NOT also start a supervisor. |
Never run a default (worker-spawning) autopilot and a supervisor
side-by-side. If `gbrain jobs stats` shows more active workers than you
expect, this footgun is the first thing to check.
`gbrain autopilot --install` currently installs the all-in-one shape; for
the split shape, install the supervisor via systemd/your platform (below)
and run autopilot with `--no-worker`. See also
[queue-operations-runbook.md](queue-operations-runbook.md) for the
`--no-worker` liveness probe.
### Variables used in this guide
Substitute these once before copy-pasting any snippet.
@@ -325,13 +302,6 @@ silently. The stall detector then dead-letters the job after
(since v0.13.1). These write onto the job row at submit time — which is
what `handleStalled()` reads — so per-job tuning is the real knob today.
**Tune per-worker.** `--lock-duration MS` on `gbrain jobs work` /
`gbrain jobs supervisor` (env: `GBRAIN_LOCK_DURATION`, flag wins; minimum
1000) raises the 30 s stall-lock window itself. The wall-clock dead-letter
cap for a running job is `lock-duration × max_stalled`, so with the
defaults (30000 × 5 ≈ 180 s wall-clock sweep window) long jobs on flaky
connections benefit from raising either side.
### DO NOT pass `maxStalledCount` to `MinionWorker`
It's a no-op. The stall detector reads the row's `max_stalled` column
docs/mcp/DEPLOY.md
+37
View File
@@ -258,6 +258,43 @@ the user owns the machine.
See [ALTERNATIVES.md](ALTERNATIVES.md) for a comparison of ngrok, Tailscale
Funnel, and cloud hosts (Fly.io, Railway).
### Co-located Docker workloads (self-hosted Postgres)
OAuth scopes and source scoping guard the `gbrain serve --http` path. They do
NOT guard raw Postgres. If the brain's Postgres runs as a container on the same
Docker host as other workloads (agent runtimes, n8n, staging fixtures), any
container sharing Docker's default `bridge` network can open a direct DB
session — no OAuth token required — and read every source. That silently
recreates a privileged path underneath the isolation you configured at the MCP
layer.
Network-zone the host so untrusted containers can never reach Postgres:
```
Docker host
├── gbrain-net ← ONLY the brain's Postgres (+ gbrain serve, if containerized)
├── agent-<id>-net ← each untrusted agent runtime, isolated
└── default bridge ← no secret-bearing databases
```
Operator checklist:
```text
[ ] Postgres is on a user-defined Docker network, not the default bridge
(or nothing else runs on that bridge)
[ ] If Postgres publishes a host port at all, it binds loopback only
(`-p 127.0.0.1:5432:5432`, never `0.0.0.0`)
[ ] Untrusted agent containers have no DATABASE_URL or Postgres password
[ ] Untrusted agents reach the brain via OAuth/Bearer against serve --http only
(host loopback via host.docker.internal / host gateway — never gbrain-net)
[ ] OAuth clients are least-privilege: scoped --source / --federated-read,
pre-minted short-lived tokens preferred over long-lived client secrets
[ ] Isolation verified: a team-scoped client cannot read internal-only sources
```
Optional defense-in-depth: a dedicated Postgres role (or RLS) limited to the
allowed `source_id`s, so even a leaked connection string can't read everything.
## Troubleshooting
**"missing_auth" error**
docs/tutorials/company-brain.md
+4
View File
@@ -484,6 +484,10 @@ Returns a per-source dashboard: when each source last synced, how many pages, ho
The admin dashboard at `https://brain.acme-co.com/admin` shows live request volume, registered OAuth clients, recent activity, and brain stats. Use the admin bootstrap token from Part 4 to log in the first time, then register additional admin users from inside the dashboard.
### If agents run as containers on the same Docker host
OAuth source scoping only guards the HTTP MCP path. If the brain's Postgres and your teammates' agent runtimes are containers on the same Docker host, make sure the agents can't reach Postgres directly over Docker's default bridge network — a direct DB session skips OAuth entirely. Put Postgres on its own user-defined network, publish it loopback-only if at all, and never hand agent containers a `DATABASE_URL`. The copy-paste operator checklist lives in [docs/mcp/DEPLOY.md — Co-located Docker workloads](../mcp/DEPLOY.md#co-located-docker-workloads-self-hosted-postgres).
---
## Part 13: Cost and speed expectations
llms-full.txt
+37
View File
@@ -3905,6 +3905,43 @@ the user owns the machine.
See [ALTERNATIVES.md](ALTERNATIVES.md) for a comparison of ngrok, Tailscale
Funnel, and cloud hosts (Fly.io, Railway).
### Co-located Docker workloads (self-hosted Postgres)
OAuth scopes and source scoping guard the `gbrain serve --http` path. They do
NOT guard raw Postgres. If the brain's Postgres runs as a container on the same
Docker host as other workloads (agent runtimes, n8n, staging fixtures), any
container sharing Docker's default `bridge` network can open a direct DB
session — no OAuth token required — and read every source. That silently
recreates a privileged path underneath the isolation you configured at the MCP
layer.
Network-zone the host so untrusted containers can never reach Postgres:
```
Docker host
├── gbrain-net ← ONLY the brain's Postgres (+ gbrain serve, if containerized)
├── agent-<id>-net ← each untrusted agent runtime, isolated
└── default bridge ← no secret-bearing databases
```
Operator checklist:
```text
[ ] Postgres is on a user-defined Docker network, not the default bridge
(or nothing else runs on that bridge)
[ ] If Postgres publishes a host port at all, it binds loopback only
(`-p 127.0.0.1:5432:5432`, never `0.0.0.0`)
[ ] Untrusted agent containers have no DATABASE_URL or Postgres password
[ ] Untrusted agents reach the brain via OAuth/Bearer against serve --http only
(host loopback via host.docker.internal / host gateway — never gbrain-net)
[ ] OAuth clients are least-privilege: scoped --source / --federated-read,
pre-minted short-lived tokens preferred over long-lived client secrets
[ ] Isolation verified: a team-scoped client cannot read internal-only sources
```
Optional defense-in-depth: a dedicated Postgres role (or RLS) limited to the
allowed `source_id`s, so even a leaked connection string can't read everything.
## Troubleshooting
**"missing_auth" error**
package.json
+4 -3
View File
@@ -144,10 +144,11 @@
"bun": ">=1.3.10"
},
"license": "MIT",
"version": "0.42.64.0",
"version": "0.42.65.0",
"overrides": {
"@hono/node-server": "^1.19.13",
"fast-uri": "^3.1.2",
"@hono/node-server": "^2.0.5",
"fast-uri": "^3.1.4",
"body-parser": "^2.3.0",
"fast-xml-builder": "^1.1.7",
"fast-xml-parser": "^5.7.0",
"form-data": "^4.0.6",
scripts/run-unit-parallel.sh
+12 -1
View File
@@ -133,6 +133,7 @@ for i in $(seq 1 "$N"); do
env SHARD="$i/$N" \
bash scripts/run-unit-shard.sh --max-concurrency="$INTRA_CONC" \
> "$SHARD_LOG" 2>&1
rc=$?
else
env SHARD="$i/$N" \
bash scripts/run-unit-shard.sh --max-concurrency="$INTRA_CONC" \
@@ -142,10 +143,20 @@ for i in $(seq 1 "$N"); do
sleep 5 && kill -KILL "$pid" 2>/dev/null ) &
cap_pid=$!
wait "$pid" 2>/dev/null
# Capture the shard's exit code from ITS `wait`, before any watchdog
# teardown runs. The teardown commands below overwrite $? — the killed
# watchdog reports 143 — which used to get stamped into every shard's
# sentinel on machines with no gtimeout/timeout: every run "failed"
# with rc=143 summaries even when all tests passed.
rc=$?
# Reap the watchdog's `sleep` child too (pkill -P), then the watchdog.
# Killing only the subshell leaves the sleep orphaned until
# $SHARD_TIMEOUT elapses — same quirk the heartbeat cleanup below works
# around; CI's orphan-process sweep flags those.
pkill -P "$cap_pid" 2>/dev/null
kill "$cap_pid" 2>/dev/null
wait "$cap_pid" 2>/dev/null
fi
rc=$?
echo "$rc" > "$LOG_DIR/shard-$i.exit"
[ "$rc" = "124" ] && echo "WEDGED" > "$LOG_DIR/shard-$i.wedged"
) &
scripts/run-verify-parallel.sh
+12 -1
View File
@@ -126,6 +126,7 @@ for c in "${CHECKS[@]}"; do
(
if [ -n "$TIMEOUT_BIN" ]; then
"$TIMEOUT_BIN" "${TIMEOUT}s" bun run "$c" > "$LOG_FILE" 2>&1
rc=$?
else
bun run "$c" > "$LOG_FILE" 2>&1 &
pid=$!
@@ -133,10 +134,20 @@ for c in "${CHECKS[@]}"; do
sleep 5 && kill -KILL "$pid" 2>/dev/null ) &
cap_pid=$!
wait "$pid" 2>/dev/null
# Capture the check's exit code from ITS `wait`, before any watchdog
# teardown runs. The teardown commands below overwrite $? — the killed
# watchdog reports 143 — which used to get stamped into every sentinel
# on machines with no gtimeout/timeout: verify reported pass=0
# fail=<all> while every per-check log said OK.
rc=$?
# Reap the watchdog's `sleep` child too (pkill -P), then the watchdog.
# Killing only the subshell leaves the sleep orphaned until $TIMEOUT
# elapses — same quirk the heartbeat cleanup in run-unit-parallel.sh
# works around; CI's orphan-process sweep flags those.
pkill -P "$cap_pid" 2>/dev/null
kill "$cap_pid" 2>/dev/null
wait "$cap_pid" 2>/dev/null
fi
rc=$?
echo "$rc" > "$EXIT_FILE"
) &
PIDS+=($!)
src/cli.ts
+64
View File
@@ -24,6 +24,7 @@ import type { GBrainConfig } from './core/config.ts';
import type { AIGatewayConfig } from './core/ai/types.ts';
import type { BrainEngine } from './core/engine.ts';
import { operations, OperationError } from './core/operations.ts';
import { resolveSourceIdEngineFree } from './core/source-resolver.ts';
import { formatVolunteeredPage } from './core/context/volunteer.ts';
import type { Operation, OperationContext } from './core/operations.ts';
import { shouldForceExitAfterMain, finishCliTeardown, flushThenExit, currentExitCode, setCliExitVerdict } from './core/cli-force-exit.ts';
@@ -384,6 +385,15 @@ async function main() {
if (op.localOnly) {
refuseThinClient(command, cfgPre!.remote_mcp!.mcp_url);
}
// #2098: the local path resolves --source / GBRAIN_SOURCE / .gbrain-source
// inside makeContext (ctx.sourceId), which this route never reaches — so
// scope must be mapped onto the op's source_id wire param before the call.
try {
applyThinClientSourceScope(op, params);
} catch (e: unknown) {
console.error(e instanceof Error ? e.message : String(e));
process.exit(1);
}
await runThinClientRouted(op, params, cfgPre!, cliOpts);
return;
}
@@ -804,6 +814,60 @@ export function parseOpArgs(op: Operation, args: string[]): Record<string, unkno
return params;
}
/**
* #2098: thin-client source scoping. Locally, --source / GBRAIN_SOURCE /
* .gbrain-source resolve to ctx.sourceId in makeContext; the thin-client
* route short-circuits before that, so `gbrain query --source X` against a
* remote brain silently searched unscoped. This runs the engine-free tiers
* (flag → env → dotfile; the DB-backed tiers can't run without an engine —
* the server's grant scoping covers the rest) and maps the result onto the
* op's `source_id` wire param.
*
* Ops that declare their OWN `source` param (facts add, etc.) are left
* untouched — their --source is an op param, not scope. An explicit --source
* on an op with no source_id wire param throws (loud beats silent drop);
* ambient env/dotfile scope with nowhere to send it is ignored, matching the
* pre-fix behavior for non-scopeable ops. Exported for tests.
*/
// Ops whose `source_id` wire param is NOT read-scope semantics: get_skill's
// source_id flips the lookup from host catalog to brain-resident-pack
// (getResidentSkillDetail). Ambient env/dotfile scope must never leak into
// these; an explicit --source-id still passes through untouched above.
const NON_SCOPE_SOURCE_ID_OPS = new Set(['get_skill']);
export function applyThinClientSourceScope(
op: Operation,
params: Record<string, unknown>,
cwd?: string,
): void {
if ('source' in op.params) return; // the op owns --source; not a scope flag
const explicit = typeof params.source === 'string' && params.source.length > 0
? (params.source as string)
: null;
delete params.source; // never a wire param on these ops — don't leak it
// Explicit per-call scope already on the wire wins over ambient tiers.
if (params.source_id !== undefined || params.all_sources === true) {
if (explicit) {
throw new Error('Pass either --source or --source-id/--all-sources, not both.');
}
return;
}
const resolved = resolveSourceIdEngineFree(explicit, cwd);
if (!resolved) return;
if (!('source_id' in op.params) || NON_SCOPE_SOURCE_ID_OPS.has(op.name)) {
if (explicit) {
const hint = NON_SCOPE_SOURCE_ID_OPS.has(op.name)
? `(its source_id parameter is not a scope filter; pass --source-id explicitly if you mean it)`
: `(the remote op has no source_id parameter; the server scopes it to your grant)`;
throw new Error(
`gbrain ${op.cliHints?.name || op.name} does not accept --source on a thin-client install ${hint}.`,
);
}
return; // ambient env/dotfile scope with nowhere to send it
}
params.source_id = resolved;
}
async function makeContext(engine: BrainEngine, params: Record<string, unknown>): Promise<OperationContext> {
// v0.31.8 (D11): resolve sourceId via the canonical 6-tier chain. Honors
// --source / GBRAIN_SOURCE / .gbrain-source / path-match / brain default /
src/commands/apply-migrations.ts
+7 -6
View File
@@ -133,14 +133,15 @@ function indexCompleted(entries: CompletedMigrationEntry[]): CompletedIndex {
* Returns the resolved status for a migration based on its entries.
*
* Semantics (Bug 3 — keep "complete wins" safety):
* - If any entry is `complete`, the version is complete. Terminal state.
* - Otherwise, if the latest entry is `retry`, the version is pending
* (user requested a fresh attempt).
* - If the latest entry is `retry`, the version is pending. This is the
* explicit escape hatch written by `--force-retry`, and it overrides an
* earlier `complete` entry without hand-editing the ledger.
* - Otherwise, if any entry is `complete`, the version is complete.
* - Otherwise, if any entry is `partial`, the version is partial.
* - Otherwise, pending.
*
* `complete` never regresses. A later accidental `partial` append cannot
* undo a completed migration.
* `complete` never regresses accidentally. A later `partial` append cannot
* undo a completed migration; only a trailing, explicit `retry` marker can.
*/
function statusForVersion(
version: string,
@@ -148,9 +149,9 @@ function statusForVersion(
): 'complete' | 'partial' | 'pending' | 'wedged' {
const entries = idx.byVersion.get(version) ?? [];
if (entries.length === 0) return 'pending';
if (entries.some(e => e.status === 'complete')) return 'complete';
const latest = entries[entries.length - 1];
if (latest.status === 'retry') return 'pending';
if (entries.some(e => e.status === 'complete')) return 'complete';
// Bug 3 attempt cap — count consecutive partials from the end (stopping
// at any 'retry' or 'complete'). If we hit MAX_CONSECUTIVE_PARTIALS,
// the migration is wedged and needs explicit --force-retry to try again.
src/commands/auth.ts
+60
View File
@@ -515,6 +515,60 @@ async function registerClient(name: string, args: string[]) {
}
}
/**
* v0.42.x (#1914): rescope an existing OAuth client's write source and/or
* federated read scope. This is the operator surface the DCR registration
* comment promised ("rescope via the CLI later") — DCR clients land with
* source_id='default' / federated_read=['default'] and must not self-widen,
* so widening happens here (trusted local CLI) or via the requireAdmin
* /admin/api/rescope-client endpoint.
*/
async function rescopeClient(clientId: string, args: string[]) {
const usage = 'Usage: auth rescope-client <client_id> [--source SOURCE] [--federated-read SRC1,SRC2,...]';
if (!clientId) {
console.error(usage);
process.exit(1);
}
let sourceId: string | undefined;
let federatedRead: string[] | undefined;
for (let i = 0; i < args.length; i += 2) {
const flag = args[i];
const value = args[i + 1];
if (value === undefined || value.startsWith('--')) {
console.error(`Error: ${flag} requires a value`);
console.error(usage);
process.exit(1);
}
if (flag === '--source') sourceId = value;
else if (flag === '--federated-read') {
federatedRead = value.split(',').map(s => s.trim()).filter(Boolean);
} else {
console.error(`Error: Unknown flag: ${flag}`);
console.error(usage);
process.exit(1);
}
}
if (sourceId === undefined && federatedRead === undefined) {
console.error('Error: pass --source and/or --federated-read');
console.error(usage);
process.exit(1);
}
try {
await withConfiguredSql(async (sql) => {
const { GBrainOAuthProvider } = await import('../core/oauth-provider.ts');
const provider = new GBrainOAuthProvider({ sql });
const result = await provider.rescopeClient(clientId, { sourceId, federatedRead });
console.log(`OAuth client rescoped: "${result.clientName}" (${result.clientId})\n`);
console.log(` Write source: ${result.sourceId}`);
console.log(` Federated reads: ${result.federatedRead.join(', ') || '<none>'}`);
console.log('\nTakes effect on the client\'s next request (existing tokens included).');
});
} catch (e: any) {
console.error('Error:', e.message);
process.exit(1);
}
}
/**
* Entry point for the `gbrain auth` CLI subcommand. Also reused by the
* direct-script path (see bottom of file) so `bun run src/commands/auth.ts`
@@ -556,6 +610,7 @@ export async function runAuth(args: string[]): Promise<void> {
return;
}
case 'register-client': await registerClient(rest[0], rest.slice(1)); return;
case 'rescope-client': await rescopeClient(rest[0], rest.slice(1)); return;
case 'revoke-client': await revokeClient(rest[0]); return;
case 'test': {
const tokenIdx = rest.indexOf('--token');
@@ -593,6 +648,11 @@ Usage:
--bound-slug-prefixes <prefix1,prefix2> Bind submit_agent writes to slug prefixes
--bound-max-concurrent <n> Bound submit_agent concurrency (default: 1)
--budget-usd-per-day <usd> Bound submit_agent daily spend cap
gbrain auth rescope-client <client_id> [options] Change an existing client's source scope (e.g. a DCR
client stuck on the 'default' source). Only the flags
you pass change; the other axis is left as-is.
--source <id> New write source
--federated-read <id1,id2,...> New read-scope source list
gbrain auth revoke-client <client_id> Hard-delete an OAuth 2.1 client (cascades to tokens + codes)
gbrain auth test <url> --token <token> Smoke-test a remote MCP server
`);
src/commands/autopilot.ts
+5 -33
View File
@@ -361,7 +361,7 @@ export async function runAutopilot(engine: BrainEngine, args: string[]) {
if (args.includes('--help') || args.includes('-h')) {
console.log(
'Usage: gbrain autopilot [--repo <path>] [--interval N] [--json] [--no-worker]\n' +
' gbrain autopilot --install [--repo <path>] [--interval N]\n' +
' gbrain autopilot --install [--repo <path>]\n' +
' gbrain autopilot --uninstall\n' +
' gbrain autopilot --status [--json]\n\n' +
'Self-maintaining brain daemon. Runs the full maintenance cycle\n' +
@@ -385,12 +385,7 @@ export async function runAutopilot(engine: BrainEngine, args: string[]) {
}
const repoPath = parseArg(args, '--repo') || await engine.getConfig('sync.repo_path');
// Flag → persisted config (written by --install --interval, #2794) → 300s.
const intervalRaw = parseArg(args, '--interval')
|| await engine.getConfig('autopilot.interval')
|| '300';
const intervalParsed = parseInt(intervalRaw, 10);
const baseInterval = Number.isFinite(intervalParsed) && intervalParsed >= 1 ? intervalParsed : 300;
const baseInterval = parseInt(parseArg(args, '--interval') || '300', 10);
const jsonMode = args.includes('--json');
const forceInline = args.includes('--inline');
const noWorker = !shouldSpawnAutopilotWorker(args);
@@ -1303,8 +1298,7 @@ function detectOpenClaw(): { detected: boolean; bootstrapCandidates: string[] }
return { detected: signal, bootstrapCandidates: existing };
}
// Exported for tests (issue #2794: the wrapper must carry --interval).
export function writeWrapperScript(repoPath: string, intervalSeconds?: number): string {
function writeWrapperScript(repoPath: string): string {
const home = process.env.HOME || '';
const gbrainDir = join(home, '.gbrain');
mkdirSync(gbrainDir, { recursive: true });
@@ -1324,7 +1318,7 @@ export function writeWrapperScript(repoPath: string, intervalSeconds?: number):
# OPENAI/ANTHROPIC keys exported in zshenv reach autopilot.
[ -f ~/.zshenv ] && source ~/.zshenv 2>/dev/null
source ~/.zshrc 2>/dev/null || source ~/.bashrc 2>/dev/null || true
exec '${safeGbrainPath}' autopilot --repo '${safeRepoPath}'${intervalSeconds !== undefined ? ` --interval '${intervalSeconds}'` : ''}
exec '${safeGbrainPath}' autopilot --repo '${safeRepoPath}'
`;
writeFileSync(wrapperPath, wrapper, { mode: 0o755 });
return wrapperPath;
@@ -1343,29 +1337,7 @@ async function installDaemon(engine: BrainEngine, args: string[]) {
const injectBootstrap = args.includes('--inject-bootstrap');
const noInject = args.includes('--no-inject');
// issue #2794: --install used to silently drop --interval — the wrapper
// always exec'd bare `autopilot --repo ...` (default 300s). Parse it here,
// persist to config so a later flag-less --install regenerates the wrapper
// with the same tuning, and thread it into the exec line.
const intervalRaw = parseArg(args, '--interval');
let intervalSeconds: number | undefined;
if (intervalRaw !== undefined) {
const parsed = Number(intervalRaw);
if (!Number.isInteger(parsed) || parsed < 1) {
console.error(`Error: --interval must be a positive integer (seconds), got "${intervalRaw}"`);
process.exit(2);
}
intervalSeconds = parsed;
await engine.setConfig('autopilot.interval', String(parsed));
} else {
const persisted = await engine.getConfig('autopilot.interval');
if (persisted) {
const parsed = Number(persisted);
if (Number.isInteger(parsed) && parsed >= 1) intervalSeconds = parsed;
}
}
const wrapperPath = writeWrapperScript(repoPath, intervalSeconds);
const wrapperPath = writeWrapperScript(repoPath);
const home = process.env.HOME || '';
switch (target) {
src/commands/doctor.ts
+113 -8
View File
@@ -529,6 +529,61 @@ export async function childTableOrphansCheck(engine: BrainEngine): Promise<Check
};
}
/**
* Raw-source persistence guarantee (#1978, warn-only v1).
*
* Invariant: every synthesized/derived page (dream_generated:true frontmatter
* or type:synthesis) must either carry a raw trace or declare an explicit
* exemption. Accepted traces:
* - frontmatter key `raw_trace` / `raw_source` / `source_uri`
* - an attached `raw_data` row
* - `synthesis_evidence` rows (think-op citations)
* - explicit `raw_trace_exempt: true` (reason in `raw_trace_exempt_reason`)
*
* v1 is deliberately warn-only — no write path is blocked. Escalation to
* fail-closed enforcement in the synthesis/import write paths is the v2
* follow-up once real brains run clean.
*
* Pure helper (engine.executeRaw only) for parity with
* childTableOrphansCheck so tests can target it directly.
*/
export async function rawProvenanceCheck(engine: BrainEngine): Promise<Check> {
const where = `
p.deleted_at IS NULL
AND (COALESCE(p.frontmatter->>'dream_generated', '') = 'true' OR p.type = 'synthesis')
AND NOT (COALESCE(p.frontmatter, '{}'::jsonb) ?| ARRAY['raw_trace', 'raw_source', 'source_uri', 'raw_trace_exempt'])
AND NOT EXISTS (SELECT 1 FROM raw_data rd WHERE rd.page_id = p.id)
AND NOT EXISTS (SELECT 1 FROM synthesis_evidence se WHERE se.synthesis_page_id = p.id)`;
try {
const rows = await engine.executeRaw<{ n: string | number }>(
`SELECT COUNT(*)::int AS n FROM pages p WHERE ${where}`,
);
const n = Number(rows[0]?.n ?? 0);
if (n === 0) {
return {
name: 'raw_provenance',
status: 'ok',
message: 'All synthesized pages carry a raw trace or explicit exemption',
};
}
const sample = await engine.executeRaw<{ slug: string }>(
`SELECT p.slug FROM pages p WHERE ${where} ORDER BY p.slug LIMIT 5`,
);
const slugs = sample.map(r => r.slug).join(', ');
return {
name: 'raw_provenance',
status: 'warn',
message:
`${n} synthesized page(s) lack a raw trace (no raw_trace/raw_source/source_uri frontmatter, ` +
`raw_data row, or synthesis evidence) and carry no raw_trace_exempt marker. e.g. ${slugs}. ` +
`Fix: stamp raw_source (path/URI of the source material) or raw_trace_exempt: true + ` +
`raw_trace_exempt_reason in frontmatter. Warn-only (#1978).`,
};
} catch {
return { name: 'raw_provenance', status: 'warn', message: 'Could not check raw provenance (older schema?)' };
}
}
export async function doctorReportRemote(engine: BrainEngine): Promise<DoctorReport> {
const checks: Check[] = [];
@@ -6290,6 +6345,12 @@ export async function buildChecks(
progress.heartbeat('child_table_orphans');
checks.push(await childTableOrphansCheck(engine));
// 10d. Raw-source persistence guarantee (#1978, warn-only v1).
// Every synthesized/derived page must carry a raw trace or an explicit
// exemption. Warn-only in v1 — surfaces violations, blocks nothing.
progress.heartbeat('raw_provenance');
checks.push(await rawProvenanceCheck(engine));
// v0.33: whoknows_health — fixture presence + row count. The eval
// gate itself runs via `gbrain eval whoknows`; this check is the
// "did you do the assignment?" signal.
@@ -7863,27 +7924,71 @@ export async function runRemediationPlan(
return;
}
// Human output
console.log(`Brain score: ${plan.brain_score_current}/100 → target ${targetScore}`);
for (const line of renderRemediationPlanLines(plan, targetScore)) {
console.log(line);
}
}
/**
* Human-render the remediation plan into a sequence of console lines.
* Exported for unit-test access — `runRemediationPlan` consumes it
* verbatim and only adds the JSON-mode short-circuit.
*
* Gating the "at target" line on `brain_score_current >= targetScore`
* is load-bearing: when the plan is empty AND the target is unreachable,
* the prior shape printed both "Target unreachable: …" and "Brain is at
* target" back-to-back, which contradicted itself and hid the real next
* step (manual prereq config to lift `max_reachable_score`).
*/
export function renderRemediationPlanLines(
plan: RemediationPlanShape,
targetScore: number,
): string[] {
const lines: string[] = [];
lines.push(`Brain score: ${plan.brain_score_current}/100 → target ${targetScore}`);
if (plan.target_unreachable) {
console.log(`Target unreachable: max with autonomous remediation is ${plan.max_reachable_score}/100.`);
lines.push(`Target unreachable: max with autonomous remediation is ${plan.max_reachable_score}/100.`);
}
if (plan.plan.length === 0) {
console.log('No remediations needed. Brain is at target.');
if (plan.brain_score_current >= targetScore) {
lines.push('No remediations needed. Brain is at target.');
}
// When brain_score < targetScore and plan is empty, the unreachable
// line (if applicable) is the user-facing explanation; the blocked-
// checks block below surfaces the manual gap. Don't follow with a
// misleading "at target" claim.
} else {
console.log(`Plan: ${plan.plan.length} step(s), est ${plan.est_total_seconds}s, est $${plan.est_total_usd_cost.toFixed(2)}`);
lines.push(`Plan: ${plan.plan.length} step(s), est ${plan.est_total_seconds}s, est $${plan.est_total_usd_cost.toFixed(2)}`);
for (const step of plan.plan) {
const protectedMark = step.protected ? ' [PROTECTED]' : '';
const costMark = step.est_usd_cost ? ` ($${step.est_usd_cost.toFixed(2)})` : '';
console.log(` ${step.step}. [${step.severity}] ${step.job}${protectedMark} — ${step.rationale}${costMark}`);
lines.push(` ${step.step}. [${step.severity}] ${step.job}${protectedMark} — ${step.rationale}${costMark}`);
}
}
if (plan.blocked.length > 0) {
console.log(`\nBlocked checks (prereq missing):`);
lines.push(`\nBlocked checks (prereq missing):`);
for (const b of plan.blocked) {
console.log(` - ${b.check}: ${b.reason}`);
lines.push(` - ${b.check}: ${b.reason}`);
}
}
return lines;
}
interface RemediationPlanShape {
brain_score_current: number;
target_unreachable: boolean;
max_reachable_score: number;
plan: Array<{
step: number;
severity: string;
job: string;
protected?: boolean;
est_usd_cost?: number;
rationale: string;
}>;
est_total_seconds: number;
est_total_usd_cost: number;
blocked: Array<{ check: string; reason: string }>;
}
/**
src/commands/dream.ts
+2 -2
View File
@@ -86,7 +86,7 @@ interface DreamArgs {
* `--phase <name>`; bare `--once` is a usage error (there'd be no single
* phase to target). Applies only to phases with a config `.enabled` gate
* (patterns, synthesize, conversation_facts_backfill, enrich_thin,
* skillopt) — a no-op for phases that always run when named directly.
* skillopt, drift) — a no-op for phases that always run when named directly.
*/
once: boolean;
}
@@ -367,7 +367,7 @@ Options:
unlike toggling the flag on/off around the run, a
crash mid-invocation can't leave it stuck. Applies to
patterns, synthesize, conversation_facts_backfill,
enrich_thin, skillopt; no-op on phases with no such
enrich_thin, skillopt, drift; no-op on phases with no such
gate. Requires an EXPLICIT --phase <name> — a phase
implied by --input or --drain does not count (bare
--once, or --once with --input/--drain and no
src/commands/extract.ts
+22 -3
View File
@@ -433,7 +433,10 @@ export async function extractLinksFromFile(
async resolve(name: string, dirHint?: string | string[]): Promise<string | null> {
if (!name) return null;
const trimmed = name.trim();
if (/^[a-z][a-z0-9-]*\/[a-z0-9][a-z0-9-]*$/.test(trimmed) && allSlugs.has(trimmed)) {
// Same broadened slug-shape as makeResolver step 1: accepts
// digit-leading folders (`90-people/nicolai`) and nested paths.
// Exact Set membership guards it — no false positives.
if (/\//.test(trimmed) && /^[a-z0-9][a-z0-9/_-]*$/.test(trimmed) && allSlugs.has(trimmed)) {
return trimmed;
}
const hints = Array.isArray(dirHint) ? dirHint : (dirHint ? [dirHint] : []);
@@ -582,6 +585,17 @@ export interface ExtractOpts {
* before (single-'default'-source brains unaffected).
*/
sourceId?: string;
/**
* v0.42 — also extract frontmatter links on the incremental (slugs) path.
* `extractForSlugs` extracts BODY links only by default; set this true to also
* parse each changed page's frontmatter so `sources:`/`related:` edges stay fresh
* when YAML is edited externally and synced in. Applied PER changed page, so the
* incremental walk stays bounded (no switch to a full DB scan). Only honored on
* the incremental path (`slugs` defined); the full-walk path already covers
* frontmatter via its own dispatch. Gated upstream by the config key
* `autopilot.incremental_extract_include_frontmatter` (default off).
*/
includeFrontmatter?: boolean;
}
/**
@@ -620,7 +634,7 @@ export async function runExtractCore(engine: BrainEngine, opts: ExtractOpts): Pr
// Nothing changed — skip entirely.
return result;
}
const r = await extractForSlugs(engine, opts.dir, opts.slugs, opts.mode, dryRun, jsonMode, workers, opts.signal, opts.sourceId);
const r = await extractForSlugs(engine, opts.dir, opts.slugs, opts.mode, dryRun, jsonMode, workers, opts.signal, opts.sourceId, opts.includeFrontmatter);
result.links_created = r.links_created;
result.timeline_entries_created = r.timeline_created;
result.pages_processed = r.pages;
@@ -1011,6 +1025,11 @@ async function extractForSlugs(
signal?: AbortSignal,
// #1747/#1503: stamp resolved brain source id on batch rows (see ExtractOpts.sourceId).
sourceId?: string,
// v0.42: when true, also extract frontmatter links per changed page so
// externally-edited YAML (`sources:`/`related:`) stays fresh on the cycle.
// Default false preserves the body-only incremental behavior. Gated upstream
// by `autopilot.incremental_extract_include_frontmatter`.
includeFrontmatter: boolean = false,
): Promise<{ links_created: number; timeline_created: number; pages: number }> {
// Build the full slug set for link resolution (fast: just readdir, no file reads)
const allFiles = walkMarkdownFiles(brainDir);
@@ -1089,7 +1108,7 @@ async function extractForSlugs(
const content = readFileSync(fullPath, 'utf-8');
if (doLinks) {
const links = await extractLinksFromFile(content, relPath, allSlugs, { globalBasename });
const links = await extractLinksFromFile(content, relPath, allSlugs, { globalBasename, includeFrontmatter });
for (const link of links) {
if (dryRun) {
if (!jsonMode) console.log(` ${link.from_slug} → ${link.to_slug} (${link.link_type})`);
src/commands/init.ts
+65 -21
View File
@@ -26,6 +26,8 @@ export async function runInit(args: string[]) {
return;
}
validateInitFlags(args);
const isSupabase = args.includes('--supabase');
const isPGLite = args.includes('--pglite');
const isMcpOnly = args.includes('--mcp-only');
@@ -151,6 +153,65 @@ export async function runInit(args: string[]) {
return initPostgres({ databaseUrl, jsonOutput, apiKey, aiOpts, schemaPack, skipEmbedCheck });
}
const INIT_BOOLEAN_FLAGS = new Set([
'--pglite',
'--supabase',
'--mcp-only',
'--force',
'--non-interactive',
'--migrate-only',
'--json',
'--no-embedding',
'--skip-embed-check',
]);
const INIT_VALUE_FLAGS = new Set([
'--url',
'--key',
'--path',
'--schema-pack',
'--embedding-model',
'--model',
'--embedding-dimensions',
'--expansion-model',
'--chat-model',
'--mcp-url',
'--issuer-url',
'--oauth-client-id',
'--oauth-client-secret',
]);
function validateInitFlags(args: string[]) {
for (let i = 0; i < args.length; i++) {
const arg = args[i];
if (!arg.startsWith('-')) continue;
if (INIT_BOOLEAN_FLAGS.has(arg)) continue;
if (INIT_VALUE_FLAGS.has(arg)) {
if (i + 1 >= args.length || args[i + 1].startsWith('-')) {
failInitFlag(`gbrain init: ${arg} requires a value`, args.includes('--json'));
}
i += 1;
continue;
}
if (arg.startsWith('--')) {
failInitFlag(`gbrain init: unknown flag ${arg}`, args.includes('--json'));
}
}
}
function failInitFlag(message: string, jsonOutput: boolean): never {
if (jsonOutput) {
console.log(JSON.stringify({ status: 'error', reason: 'invalid_flag', message }));
} else {
console.error(message);
console.error('Run `gbrain init --help` for supported flags.');
}
process.exit(1);
}
interface ResolveAIOptionsArgs {
verbose: string | null; // --embedding-model
shorthand: string | null; // --model
@@ -249,13 +310,6 @@ async function resolveAIOptions(opts: ResolveAIOptionsArgs): Promise<ResolvedAIO
// --- Tier 1+2: explicit flags ---------------------------------------------
// #2301: an explicit embedding flag on THIS invocation overrides the
// persisted deferred-setup sentinel above. Without this, a stale
// `embedding_disabled: true` in config.json made every re-init defer
// embedding — including `gbrain init --embedding-model ...`, the exact
// recovery path the deferred-setup message tells users to take.
if (verbose || shorthand) delete out.noEmbedding;
if (verbose) {
out.embedding_model = verbose;
} else if (shorthand) {
@@ -442,7 +496,7 @@ function printNoEmbeddingProviderHint(typos: Array<{ userSet: string; suggested:
console.error(' gbrain init --pglite --embedding-model openai:text-embedding-3-large');
console.error('');
console.error('Or defer setup: gbrain init --pglite --no-embedding');
console.error(' (you can configure later with `gbrain init --force --embedding-model <provider>:<model>`)');
console.error(' (you can configure later with `gbrain config set embedding_model <id>`)');
// D13: surface near-miss env vars (e.g. OPENAPI_API_KEY → OPENAI_API_KEY).
if (typos.length > 0) {
console.error('');
@@ -840,7 +894,7 @@ async function initPGLite(opts: {
let resolvedModel: string | undefined;
if (opts.aiOpts?.noEmbedding) {
// D9 deferred-setup mode: skip preflight, no model/dim resolved.
console.log(` --no-embedding: deferred setup — run \`gbrain init --force --embedding-model <provider>:<model>\` before import`);
console.log(` --no-embedding: deferred setup — configure with \`gbrain config set embedding_model <id>\` before import`);
} else if (opts.aiOpts?.embedding_model) {
const { resolveSchemaEmbeddingDim } = await import('../core/embedding-dim-check.ts');
const pre = resolveSchemaEmbeddingDim({
@@ -979,12 +1033,6 @@ async function initPGLite(opts: {
// unless explicitly overridden by --schema-pack on re-init.
...(opts.schemaPack ? { schema_pack: opts.schemaPack } : {}),
};
// #2301: a resolved embedding model supersedes any stale deferred-setup
// sentinel carried over via ...existingFile — otherwise the sentinel
// re-defers embedding on every future init/embed forever.
if (!opts.aiOpts?.noEmbedding && resolvedModel && resolvedDim) {
delete config.embedding_disabled;
}
// PR1: new installs publish their skill catalog over MCP by default
// (existing config wins on re-init, so a prior opt-out is preserved).
config.mcp = { publish_skills: true, ...(config.mcp ?? {}) };
@@ -1069,7 +1117,7 @@ async function initPostgres(opts: {
let resolvedDim: number | undefined;
let resolvedModel: string | undefined;
if (opts.aiOpts?.noEmbedding) {
console.log(` --no-embedding: deferred setup — run \`gbrain init --force --embedding-model <provider>:<model>\` before import`);
console.log(` --no-embedding: deferred setup — configure with \`gbrain config set embedding_model <id>\` before import`);
} else if (opts.aiOpts?.embedding_model) {
const { resolveSchemaEmbeddingDim } = await import('../core/embedding-dim-check.ts');
const pre = resolveSchemaEmbeddingDim({
@@ -1233,10 +1281,6 @@ async function initPostgres(opts: {
// v0.42 (T17): same schema_pack default as PGLite path.
...(opts.schemaPack ? { schema_pack: opts.schemaPack } : {}),
};
// #2301: same stale-sentinel drop as the PGLite path above.
if (!opts.aiOpts?.noEmbedding && resolvedModel && resolvedDim) {
delete config.embedding_disabled;
}
// PR1: new installs publish their skill catalog over MCP by default
// (existing config wins on re-init, so a prior opt-out is preserved).
config.mcp = { publish_skills: true, ...(config.mcp ?? {}) };
@@ -1508,7 +1552,7 @@ export function reportModStatus(): void {
console.log(' cd ~/.claude/skills/gstack && ./setup');
}
console.log('Resolver: skills/RESOLVER.md');
console.log('Soul audit: run `gbrain soul-audit` to customize agent identity');
console.log('Soul audit: ask your agent to "run a soul audit" to customize its identity (see skills/soul-audit)');
// Retrieval Reflex (#1981): the deterministic pointer layer is ON by default
// (no action needed). The policy skill is installed into the HOST repo on
// request — we PRINT the command rather than silently mutating the host repo.
src/commands/integrations.ts
-83
View File
@@ -707,14 +707,6 @@ async function cmdDoctor(args: string[]): Promise<void> {
}
}
// Cross-cutting check (PR #1185, @ethanbeard): dead jobs in the minions
// queue. Shell jobs submitted by collectors die silently from the
// per-integration perspective — the collector's drain sees the submit
// succeed and stays green while the worker job dead-letters on the queue
// side, so a whole pipeline can be broken for days before anyone notices.
const deadJobsCheck = checkDeadJobs();
if (deadJobsCheck) results.push(deadJobsCheck);
if (jsonMode) {
const fails = results.filter(r => r.status !== 'ok');
console.log(JSON.stringify({
@@ -734,85 +726,10 @@ async function cmdDoctor(args: string[]): Promise<void> {
}
}
// Surface cross-cutting [queue] checks below the per-integration sections.
const queueChecks = results.filter(r => r.integration === '[queue]');
if (queueChecks.length > 0) {
console.log(` [queue]: ${queueChecks.every(c => c.status === 'ok') ? 'OK' : 'ISSUES'}`);
for (const c of queueChecks) {
const icon = c.status === 'ok' ? ' ✓' : c.status === 'timeout' ? ' ⏱' : ' ✗';
console.log(`${icon} ${c.output}`);
}
}
const totalFails = results.filter(r => r.status !== 'ok').length;
console.log(`\n OVERALL: ${totalFails === 0 ? 'All checks passed' : `${totalFails} issue(s) found`}`);
}
/**
* Pure aggregation for the dead-jobs queue check (PR #1185 rework): given
* parsed `jobs list --status dead --json` output, count jobs whose
* finished_at falls inside the last 24 hours — parity with the main
* `gbrain doctor` queue checks, so one ancient dead job can't flag
* `[queue]: ISSUES` forever — grouped by job name, biggest first.
* Exported for unit tests.
*/
export function summarizeDeadJobs(
jobs: Array<{ name?: unknown; finished_at?: unknown }>,
now: Date = new Date(),
): { total: number; breakdown: string } {
const cutoff = now.getTime() - 24 * 60 * 60 * 1000;
const byName: Record<string, number> = {};
let total = 0;
for (const job of jobs) {
if (typeof job.finished_at !== 'string' && !(job.finished_at instanceof Date)) continue;
const finished = new Date(job.finished_at as string | Date);
if (!Number.isFinite(finished.getTime()) || finished.getTime() < cutoff) continue;
const name = typeof job.name === 'string' ? job.name : 'unknown';
byName[name] = (byName[name] ?? 0) + 1;
total++;
}
const breakdown = Object.entries(byName)
.sort((a, b) => b[1] - a[1])
.map(([n, c]) => `${n}:${c}`)
.join(', ');
return { total, breakdown };
}
/**
* Dead minion jobs in the last 24h. Returns a CheckResult only when dead
* jobs exist (silent when healthy, per doctor's ISSUES-only model).
*
* Shells out to `gbrain jobs list --json` instead of opening a DB
* connection — preserves this file's standalone-CLI design. The JSON
* surface exists for exactly this: the human table's column widths shift
* with long job names, so screen-scraping it is not an option.
*/
function checkDeadJobs(): CheckResult | null {
try {
// --limit 500 caps memory while comfortably covering a day of failures;
// getJobs orders by created_at DESC so the most recent dead jobs come first.
const out = execSync('gbrain jobs list --status dead --limit 500 --json', {
encoding: 'utf8',
timeout: 10_000,
stdio: ['ignore', 'pipe', 'ignore'],
});
const jobs: unknown = JSON.parse(out);
if (!Array.isArray(jobs)) return null;
const { total, breakdown } = summarizeDeadJobs(jobs);
if (total === 0) return null;
return {
integration: '[queue]',
check: 'dead_jobs',
status: 'fail',
output: `${total} dead job(s) in the last 24h (${breakdown}). Inspect: gbrain jobs list --status dead; details: gbrain jobs get <id>`,
};
} catch {
// DB unreachable / gbrain not on PATH / non-JSON output — skip silently.
// Doctor still surfaces the per-recipe checks; this one is best-effort.
return null;
}
}
function cmdStats(args: string[]): void {
const jsonMode = args.includes('--json');
const recipes = loadAllRecipes();
src/commands/jobs.ts
+5 -61
View File
@@ -122,27 +122,6 @@ export function parseNiceFlag(args: string[], env: NodeJS.ProcessEnv = process.e
}
}
/** Parse `--lock-duration MS` (then `GBRAIN_LOCK_DURATION` env; flag wins).
* Tunes the worker's stall-lock window — the wall-clock dead-letter cap is
* lockDuration × max_stalled, so the default 30s lock caps long jobs at
* ~180s with the schema defaults (issue #1014). Returns undefined when
* absent (worker default 30000ms applies). Throws on non-integer values or
* values < 1000 — sub-1000 is almost always a seconds-vs-ms unit-confusion
* typo (parity with --health-interval). Exported for unit tests; CLI call
* sites wrap with console.error + process.exit(1). */
export function parseLockDurationFlag(args: string[], env: NodeJS.ProcessEnv = process.env): number | undefined {
const raw = parseFlag(args, '--lock-duration') ?? env.GBRAIN_LOCK_DURATION;
if (raw === undefined || raw === '') return undefined;
const parsed = Number(raw);
if (!Number.isInteger(parsed) || parsed < 1000) {
throw new Error(
`--lock-duration must be an integer >= 1000 (milliseconds), got "${raw}". ` +
`The flag takes milliseconds; for a 60-second lock pass 60000.`,
);
}
return parsed;
}
export function resolveWorkerConcurrency(args: string[], env: NodeJS.ProcessEnv = process.env): number {
const raw = parseFlag(args, '--concurrency') ?? env.GBRAIN_WORKER_CONCURRENCY ?? '1';
const parsed = parseInt(raw, 10);
@@ -225,7 +204,7 @@ USAGE
[--idempotency-key K] [--queue Q] [--dry-run]
[--redact-secrets] (shell only; scrubs inherit
values from stdout/stderr)
gbrain jobs list [--status S] [--queue Q] [--limit N] [--json]
gbrain jobs list [--status S] [--queue Q] [--limit N]
gbrain jobs get <id>
gbrain jobs cancel <id>
gbrain jobs retry <id>
@@ -234,17 +213,12 @@ USAGE
gbrain jobs stats
gbrain jobs smoke
gbrain jobs work [--queue Q] [--concurrency N] [--max-rss MB]
[--health-interval MS] [--nice N] [--lock-duration MS]
[--health-interval MS] [--nice N]
gbrain jobs supervisor [start] [--detach] [--json]
[--concurrency N] [--queue Q] [--pid-file PATH]
[--max-crashes N] [--health-interval N]
[--allow-shell-jobs] [--cli-path PATH]
[--max-rss MB] [--nice N] [--lock-duration MS]
--lock-duration MS Worker stall-lock window in milliseconds (default
30000). The wall-clock dead-letter cap for a running job is
lock-duration × max_stalled, so raise this for long-running
handlers. Minimum 1000. Env: GBRAIN_LOCK_DURATION (flag wins).
[--max-rss MB] [--nice N]
--nice N OS scheduling priority, -20 (highest) to 19 (nicest). Lowers CPU
priority without cutting concurrency — full throughput when the
@@ -510,7 +484,6 @@ HANDLER TYPES (built in)
const status = parseFlag(args, '--status') as MinionJobStatus | undefined;
const queueName = parseFlag(args, '--queue');
const limit = parseInt(parseFlag(args, '--limit') ?? '20', 10);
const jsonMode = hasFlag(args, '--json');
// v0.32: thin-client routing. The `list_jobs` MCP op is admin-scoped
// but not localOnly, so a thin-client install with admin access can
@@ -530,14 +503,6 @@ HANDLER TYPES (built in)
jobs = await queue.getJobs({ status, queue: queueName, limit });
}
// --json: machine-readable output (one JSON array on stdout) so
// downstream tooling (e.g. `gbrain integrations doctor`) never
// screen-scrapes the human table, whose column widths shift.
if (jsonMode) {
console.log(JSON.stringify(jobs));
return;
}
if (jobs.length === 0) {
console.log('No jobs found.');
return;
@@ -980,16 +945,6 @@ HANDLER TYPES (built in)
healthCheckInterval = parsed;
}
// --lock-duration MS (issue #1014): tune the stall-lock window (and so
// the lockDuration × max_stalled wall-clock dead-letter cap).
let lockDuration: number | undefined;
try {
lockDuration = parseLockDurationFlag(args);
} catch (e) {
console.error(`Error: ${e instanceof Error ? e.message : String(e)}`);
process.exit(1);
}
// --nice N (issue #1815): renice this worker process so background work
// yields CPU to foreground tasks without sacrificing concurrency. Applied
// at the CLI layer (worker.ts stays embeddable). Niceness inherits to the
@@ -1011,7 +966,6 @@ HANDLER TYPES (built in)
const worker = new MinionWorker(engine, {
queue: queueName, concurrency, maxRssMb, healthCheckInterval,
...(lockDuration !== undefined ? { lockDuration } : {}),
});
await registerBuiltinHandlers(worker, engine);
@@ -1052,8 +1006,7 @@ HANDLER TYPES (built in)
: `, health-check: ${Math.round(healthCheckInterval / 1000)}s`)
: '';
const niceNote = niceResult ? `, nice: ${formatNice(niceResult.effective ?? niceVal!)}` : '';
const lockNote = lockDuration !== undefined ? `, lock: ${lockDuration}ms` : '';
console.log(`Minion worker started (queue: ${queueName}, concurrency: ${concurrency}${watchdogNote}${healthNote}${niceNote}${lockNote})`);
console.log(`Minion worker started (queue: ${queueName}, concurrency: ${concurrency}${watchdogNote}${healthNote}${niceNote})`);
console.log(`Registered handlers: ${worker.registeredNames.join(', ')}`);
// Register in the live worker registry (issue #1815) so jobs stats / doctor
@@ -1305,15 +1258,6 @@ HANDLER TYPES (built in)
}
healthInterval = parsed;
}
// --lock-duration (supervisor): validated here (fail-fast even for
// --detach), passed down to the spawned worker via buildWorkerArgs.
let supLockDuration: number | undefined;
try {
supLockDuration = parseLockDurationFlag(args);
} catch (e) {
console.error(`Error: ${e instanceof Error ? e.message : String(e)}`);
process.exit(1);
}
const allowShellJobs = hasFlag(args, '--allow-shell-jobs') ||
!!process.env.GBRAIN_ALLOW_SHELL_JOBS;
const detach = hasFlag(args, '--detach');
@@ -1381,7 +1325,6 @@ HANDLER TYPES (built in)
allowShellJobs,
json: jsonMode,
maxRssMb,
...(supLockDuration !== undefined ? { lockDuration: supLockDuration } : {}),
...(supNice !== undefined ? { nice_requested: supNice } : {}),
...(supNiceResult?.effective != null ? { nice_effective: supNiceResult.effective } : {}),
...(supNiceResult?.error ? { nice_error: supNiceResult.error } : {}),
@@ -1942,6 +1885,7 @@ export async function registerBuiltinHandlers(
signal: job.signal,
deadlineAtMs: job.deadlineAtMs, // #2781: phases budget sub-work from remaining time
phases,
forceGlobalOrphans: true,
yieldBetweenPhases: async () => { await new Promise<void>((r) => setImmediate(r)); },
});
src/commands/serve-http.ts
+37 -2
View File
@@ -1567,6 +1567,38 @@ export async function runServeHttp(engine: BrainEngine, options: ServeHttpOption
}
});
// v0.42.x (#1914): rescope an OAuth client's write source / federated read
// scope. Admin-gated on purpose — DCR clients must never self-widen their
// scope (fail-closed trust); only the operator rescopes, here or via
// `gbrain auth rescope-client`. Source ids are validated by the canonical
// validator inside rescopeClient.
app.post('/admin/api/rescope-client', requireAdmin, express.json(), async (req: Request, res: Response) => {
try {
const { clientId, sourceId, federatedRead } = req.body ?? {};
if (!clientId || typeof clientId !== 'string') {
res.status(400).json({ error: 'clientId required' });
return;
}
if (federatedRead !== undefined &&
!(Array.isArray(federatedRead) && federatedRead.every((s: unknown) => typeof s === 'string'))) {
res.status(400).json({ error: 'federatedRead must be an array of source id strings' });
return;
}
if (sourceId !== undefined && typeof sourceId !== 'string') {
res.status(400).json({ error: 'sourceId must be a string' });
return;
}
const result = await oauthProvider.rescopeClient(clientId, { sourceId, federatedRead });
res.json(result);
} catch (e) {
const message = e instanceof Error ? e.message : 'Rescope failed';
const status = /No OAuth client found/.test(message) ? 404
: /Invalid source_id|requires --source|cannot be empty|does not exist/.test(message) ? 400
: 500;
res.status(status).json({ error: message });
}
});
// Revoke OAuth client
app.post('/admin/api/revoke-client', requireAdmin, express.json(), async (req: Request, res: Response) => {
try {
@@ -2217,8 +2249,10 @@ export async function runServeHttp(engine: BrainEngine, options: ServeHttpOption
// Other event types (ping, pull_request, etc.) return 202 'ignored'
// so GitHub doesn't retry.
// D15.5: HMAC compare uses the shared safeHexEqual helper.
// D18: submits 'sync' job with auto_embed_backfill=true and priority -10
// (above autopilot's 0).
// D18: submits 'sync' job with extraction + auto_embed_backfill enabled and
// priority -10 (above autopilot's 0). This opts normal incremental pushes
// into sync's inline extraction while pagesAffected still identifies the
// changed pages. The sync core can still defer large (>100) changes.
// ---------------------------------------------------------------------------
const githubWebhookLimiter = rateLimit({
windowMs: 60_000,
@@ -2338,6 +2372,7 @@ export async function runServeHttp(engine: BrainEngine, options: ServeHttpOption
'sync',
{
sourceId: source.id,
noExtract: false,
auto_embed_backfill: true,
embed_reason: 'webhook',
},
src/commands/sync.ts
+4
View File
@@ -1423,6 +1423,7 @@ See also:
{
sourceId: sourceIdArg,
repoPath: source.local_path,
noExtract: false,
auto_embed_backfill: true,
embed_reason: 'sync_trigger',
},
@@ -3384,6 +3385,9 @@ async function performSyncInner(engine: BrainEngine, opts: SyncOpts): Promise<Sy
// Log ingest
await engine.logIngest({
// #3242 (attribution sub-bug): credit the sync to the source it wrote
// to, not the shared 'default' bucket.
...(opts.sourceId ? { source_id: opts.sourceId } : {}),
source_type: 'git_sync',
source_ref: `${repoPath} @ ${headCommit.slice(0, 8)}`,
pages_updated: pagesAffected,
src/commands/takes.ts
+1 -1
View File
@@ -292,7 +292,7 @@ async function cmdSupersede(engine: BrainEngine, args: string[], sourceId?: stri
const pageId = await getPageId(engine, slug, sourceId);
// Read existing row to inherit kind/holder unless overridden
const existing = await engine.listTakes({ page_id: pageId, active: false, limit: 500 });
const existing = await engine.listTakes({ page_id: pageId, active: true, limit: 500 });
const target = existing.find(t => t.row_num === rowNum);
if (!target) {
console.error(`Row #${rowNum} not found on ${slug}.`);
src/core/ai/anthropic-key.ts
+13 -3
View File
@@ -18,12 +18,22 @@
import { loadConfig } from '../config.ts';
export function hasAnthropicKey(): boolean {
if (process.env.ANTHROPIC_API_KEY) return true;
return resolveAnthropicKey() !== undefined;
}
/**
* Resolve the actual key value: env first, then the gbrain config file.
* Callers constructing an Anthropic client directly (e.g. the legacy
* subagent path) must pass this as `apiKey` — a bare `new Anthropic()`
* only sees env, so launchd/MCP workers with config-stored keys fail.
*/
export function resolveAnthropicKey(): string | undefined {
if (process.env.ANTHROPIC_API_KEY) return process.env.ANTHROPIC_API_KEY;
try {
const cfg = loadConfig();
if (cfg?.anthropic_api_key) return true;
if (cfg?.anthropic_api_key) return cfg.anthropic_api_key;
} catch {
// loadConfig may throw on first-run installs; treat as no key available.
}
return false;
return undefined;
}
src/core/ai/dims.ts
+45 -3
View File
@@ -90,6 +90,30 @@ export function isValidOpenAITextEmbedding3Dim(modelId: string, dims: number): b
return Number.isInteger(dims) && dims >= 1 && dims <= max;
}
// Perplexity hosted embeddings (#1046): Matryoshka-style flexible dims,
// any integer from 128 up to the model's native size. `dimensions` is the
// native wire field (no translation needed); output encoding divergence
// (base64 int8) is handled by perplexityCompatFetch in gateway.ts.
const PERPLEXITY_EMBEDDING_MAX_DIMS: Record<string, number> = {
'pplx-embed-v1-0.6b': 1024,
'pplx-embed-v1-4b': 2560,
};
export const PERPLEXITY_MIN_DIMS = 128;
export function isPerplexityEmbeddingModel(modelId: string): boolean {
return modelId in PERPLEXITY_EMBEDDING_MAX_DIMS;
}
export function maxPerplexityEmbeddingDim(modelId: string): number | undefined {
return PERPLEXITY_EMBEDDING_MAX_DIMS[modelId];
}
export function isValidPerplexityDim(modelId: string, dims: number): boolean {
const max = PERPLEXITY_EMBEDDING_MAX_DIMS[modelId];
if (max === undefined) return false;
return Number.isInteger(dims) && dims >= PERPLEXITY_MIN_DIMS && dims <= max;
}
// NVIDIA NIM hosted embedding models use asymmetric input_type values. Most
// emit fixed natural dimensions, but llama-nemotron-embed-1b-v2 accepts
// Matryoshka-style dimension overrides (e.g. matching an existing 1280d
@@ -226,6 +250,23 @@ export function dimsProviderOptions(
},
};
}
// Perplexity pplx-embed-v1-* — flexible dims via the native
// `dimensions` field. Fail-loud when the configured dim is outside
// the model's range (same rationale as the Voyage/ZE guards: the
// upstream HTTP 400 misroutes as a transient network error).
// Symmetric retrieval — inputType is never emitted.
if (isPerplexityEmbeddingModel(modelId)) {
if (!isValidPerplexityDim(modelId, dims)) {
const max = maxPerplexityEmbeddingDim(modelId)!;
throw new AIConfigError(
`Perplexity model "${modelId}" supports embedding_dimensions in ` +
`${PERPLEXITY_MIN_DIMS}..${max}, got ${dims}.`,
`Set \`embedding_dimensions\` to a value between ${PERPLEXITY_MIN_DIMS} and ${max} ` +
`in your gbrain config.`,
);
}
return { openaiCompatible: { dimensions: dims } };
}
// NVIDIA NIM hosted embeddings are OpenAI-compatible but require
// asymmetric input_type. Use passage for indexing/document-side vectors
// and query for search-side vectors. Only llama-nemotron-embed-1b-v2
@@ -244,9 +285,10 @@ export function dimsProviderOptions(
// configured for a smaller width (e.g. 1536) hard-fail at first embed.
// Azure/OpenAI-compat embeddings are symmetric — inputType ignored.
// v0.36.0.0 (D13): same range validation as native-openai path.
if (modelId.startsWith('text-embedding-3')) {
if (isOpenAITextEmbedding3Model(modelId) && !isValidOpenAITextEmbedding3Dim(modelId, dims)) {
const max = maxOpenAITextEmbedding3Dim(modelId)!;
const bareModelId = modelId.includes('/') ? modelId.split('/').pop()! : modelId;
if (bareModelId.startsWith('text-embedding-3')) {
if (isOpenAITextEmbedding3Model(bareModelId) && !isValidOpenAITextEmbedding3Dim(bareModelId, dims)) {
const max = maxOpenAITextEmbedding3Dim(bareModelId)!;
throw new AIConfigError(
`OpenAI model "${modelId}" supports embedding_dimensions in 1..${max}, got ${dims}.`,
`Set \`embedding_dimensions\` to a value between 1 and ${max} ` +
src/core/ai/gateway.ts
+133
View File
@@ -267,6 +267,18 @@ export class ZeroEntropyResponseTooLargeError extends Error {
}
}
/** Perplexity twin of the Voyage/ZE OOM caps (#1046). Int8 components are
* 1 byte each, so a real response (512 texts × 2560 dims) is ~1.3 MB —
* anything near this cap is unambiguously not legitimate. */
const MAX_PERPLEXITY_RESPONSE_BYTES = 256 * 1024 * 1024;
export class PerplexityResponseTooLargeError extends Error {
constructor(message: string) {
super(message);
this.name = 'PerplexityResponseTooLargeError';
}
}
// ---- Unified auth resolution (D12=A) ----
//
// Pre-v0.32, openai-compatible auth was duplicated across instantiateEmbedding,
@@ -1298,6 +1310,103 @@ const openAICompatAsymmetricFetch = (async (input: RequestInfo | URL, init?: Req
return fetch(typeof input === 'string' ? input : input.toString(), baseInit);
}) as unknown as typeof fetch;
/**
* Perplexity compatibility shim (#1046). Perplexity's `/v1/embeddings`
* endpoint is OpenAI-shaped but diverges on two points that break the AI
* SDK's openai-compatible adapter:
* - `encoding_format` only accepts 'base64_int8' (default) or
* 'base64_binary'; the SDK sends 'float', which Perplexity rejects.
* Force 'base64_int8' on the wire.
* - The response `embedding` is a base64 string encoding SIGNED INT8
* components (natively quantized output). The SDK schema expects
* `number[]` — decode Int8Array → number[] here. Cosine similarity is
* scale-invariant, so the raw int8 components rank correctly.
* `dimensions` is Perplexity's native field name — no translation needed
* (dims.ts emits it directly). Layer 1/Layer 2 OOM caps mirror the Voyage
* pattern.
*
* Exported for tests (behavioral coverage of the int8 decode); not part of
* the public gateway API.
*/
export const perplexityCompatFetch = (async (input: RequestInfo | URL, init?: RequestInit) => {
// OUTBOUND: force the encoding Perplexity actually accepts.
if (init?.body && typeof init.body === 'string') {
try {
const parsed = JSON.parse(init.body);
if (parsed && typeof parsed === 'object' && parsed.encoding_format !== 'base64_int8') {
parsed.encoding_format = 'base64_int8';
// Drop Content-Length so fetch recomputes from the new body.
const headers = new Headers(init.headers ?? {});
headers.delete('content-length');
init = { ...init, body: JSON.stringify(parsed), headers };
}
} catch {
// Body wasn't JSON — pass through untouched.
}
}
const resp = await fetch(input as any, init);
if (!resp.ok) return resp;
const ct = resp.headers.get('content-type') ?? '';
if (!ct.toLowerCase().includes('application/json')) return resp;
// Layer 1: Content-Length pre-check BEFORE the body is parsed.
const contentLengthHeader = resp.headers.get('content-length');
if (contentLengthHeader) {
const len = parseInt(contentLengthHeader, 10);
if (Number.isFinite(len) && len > MAX_PERPLEXITY_RESPONSE_BYTES) {
throw new PerplexityResponseTooLargeError(
`Perplexity response Content-Length=${len} exceeds ${MAX_PERPLEXITY_RESPONSE_BYTES} bytes — ` +
`likely compromised endpoint or misconfiguration`,
);
}
}
// INBOUND: decode base64 int8 embeddings to number[] so the SDK's Zod
// schema validates.
try {
const json: any = await resp.clone().json();
if (!json || typeof json !== 'object') return resp;
let modified = false;
if (Array.isArray(json.data)) {
for (const item of json.data) {
if (item && typeof item.embedding === 'string') {
// Layer 2: per-embedding cap for chunked responses that skipped
// Layer 1. base64 → bytes is the canonical 0.75 ratio.
const estDecoded = Math.ceil(item.embedding.length * 0.75);
if (estDecoded > MAX_PERPLEXITY_RESPONSE_BYTES) {
throw new PerplexityResponseTooLargeError(
`Perplexity embedding base64 exceeds ${MAX_PERPLEXITY_RESPONSE_BYTES} bytes ` +
`(estimated ${estDecoded} bytes from ${item.embedding.length} base64 chars)`,
);
}
// base64_int8: one signed int8 per component.
const bytes = Buffer.from(item.embedding, 'base64');
item.embedding = Array.from(new Int8Array(bytes.buffer, bytes.byteOffset, bytes.byteLength));
modified = true;
}
}
}
if (json.usage && typeof json.usage === 'object' && json.usage.prompt_tokens === undefined) {
json.usage.prompt_tokens = typeof json.usage.total_tokens === 'number'
? json.usage.total_tokens
: 0;
modified = true;
}
if (!modified) return resp;
return new Response(JSON.stringify(json), {
status: resp.status,
statusText: resp.statusText,
headers: resp.headers,
});
} catch (err) {
// OOM-cap throws MUST propagate; anything else falls back to the
// original response (same contract as voyageCompatFetch).
if (err instanceof PerplexityResponseTooLargeError) throw err;
return resp;
}
}) as unknown as typeof fetch;
async function resolveEmbeddingProvider(modelStr: string): Promise<{ model: any; recipe: Recipe; modelId: string }> {
const { parsed, recipe } = resolveRecipe(modelStr);
assertTouchpoint(recipe, 'embedding', parsed.modelId, getExtendedModelsForProvider(parsed.providerId));
@@ -1342,6 +1451,10 @@ function instantiateEmbedding(recipe: Recipe, modelId: string, cfg: AIGatewayCon
throw new AIConfigError(
`Anthropic has no embedding model. Use openai or google for embeddings.`,
);
case 'claude-cli':
throw new AIConfigError(
`claude-cli has no embedding model. Use openai or google for embeddings.`,
);
case 'openai-compatible': {
// D12=A: unified auth via Recipe.resolveAuth (or default).
const auth = applyResolveAuth(recipe, cfg, 'embedding');
@@ -1366,6 +1479,8 @@ function instantiateEmbedding(recipe: Recipe, modelId: string, cfg: AIGatewayCon
? zeroEntropyCompatFetch
: recipe.id === 'nvidia'
? nvidiaCompatFetch
: recipe.id === 'perplexity'
? perplexityCompatFetch
: openAICompatAsymmetricFetch);
const client = createOpenAICompatible({
name: recipe.id,
@@ -2284,6 +2399,15 @@ function instantiateExpansion(recipe: Recipe, modelId: string, cfg: AIGatewayCon
const baseURL = resolveNativeBaseUrl('anthropic', cfg);
return createAnthropic({ apiKey, ...(baseURL ? { baseURL } : {}) }).languageModel(modelId);
}
case 'claude-cli': {
// The CLI handles its own auth (OAuth session); spawn the subprocess
// directly via the same LanguageModelV2 implementation chat uses. There
// is no separate expansion path because claude-cli does not declare a
// separate expansion touchpoint — but routing here keeps the switch
// exhaustive and lets a future expansion touchpoint use the same code.
const { ClaudeCliLanguageModel } = require('./providers/claude-cli-language-model.ts');
return new ClaudeCliLanguageModel(modelId);
}
case 'openai-compatible': {
// D12=A: unified auth via Recipe.resolveAuth (or default).
const auth = applyResolveAuth(recipe, cfg, 'expansion');
@@ -2783,6 +2907,15 @@ function instantiateChat(recipe: Recipe, modelId: string, cfg: AIGatewayConfig):
const baseURL = resolveNativeBaseUrl('anthropic', cfg);
return createAnthropic({ apiKey, ...(baseURL ? { baseURL } : {}) }).languageModel(modelId);
}
case 'claude-cli': {
// The CLI handles its own auth (OAuth session managed by `claude`
// login). Subprocess-based LanguageModelV2 dispatches via the recipe
// path so per-call routing works: `claude-cli:claude-sonnet-4-6` lands
// here, while sibling `litellm:gpt-5.4` continues through the
// openai-compatible path below. No env-var switch, no global flag.
const { ClaudeCliLanguageModel } = require('./providers/claude-cli-language-model.ts');
return new ClaudeCliLanguageModel(modelId);
}
case 'openai-compatible': {
// D12=A: unified auth via Recipe.resolveAuth (or default).
const auth = applyResolveAuth(recipe, cfg, 'chat');
src/core/ai/providers/claude-cli-language-model.ts
+444
View File
@@ -0,0 +1,444 @@
/**
* ai-sdk LanguageModelV2 implementation that dispatches via the `claude --print`
* CLI subprocess. Used by the `claude-cli` recipe to route gateway.toolLoop /
* gateway.chat calls through Claude Code's OAuth session instead of the
* Anthropic SDK + ANTHROPIC_API_KEY.
*
* Per-call routing is the contract: the gateway resolves the model string
* to this recipe based on the `claude-cli:` prefix, instantiates one of
* these objects per modelId, and dispatches doGenerate. Sibling subagent
* jobs with `litellm:gpt-5.4` continue routing through litellm-proxy in
* the same worker; no env-var switch, no global state.
*
* Tool use is supported via system-prompt-instructed JSON emission:
* The recipe injects a fenced instruction block into the system prompt
* that teaches the model the `<use_tools>[{id,name,input}, ...]</use_tools>`
* emission format. The adapter parses those blocks back into ai-sdk
* `tool-call` content parts. Parallel tool calls (multiple entries in
* the JSON array) round-trip cleanly — this is the case that breaks
* on the codex-proxy / litellm GPT-5.x bridge today.
*
* Context isolation:
* The subprocess is spawned from a dedicated tmpdir so claude-cli's
* CLAUDE.md auto-discovery has no local files to find. `--system-prompt`
* replaces the default system prompt; `--disable-slash-commands` skips
* skill resolution. User-level ~/.claude/CLAUDE.md still loads because
* the only way to skip it is `--bare`, which forces ANTHROPIC_API_KEY
* auth and defeats the whole point of this provider. The ~42k cached
* tokens from user-level instructions are accepted as a cost-trivial
* trade-off on the subscription path.
*
* doStream is not yet implemented; the model declares no streaming. Callers
* (gateway.toolLoop primarily) use doGenerate.
*/
import { spawn } from 'node:child_process';
import { mkdirSync } from 'node:fs';
import { tmpdir } from 'node:os';
import { join } from 'node:path';
import type {
LanguageModelV2,
LanguageModelV2CallOptions,
LanguageModelV2Content,
LanguageModelV2FunctionTool,
LanguageModelV2Prompt,
LanguageModelV2Message,
LanguageModelV2ProviderDefinedTool,
} from '@ai-sdk/provider';
function claudeBin(): string {
return process.env.GBRAIN_CLAUDE_CLI_BIN ?? 'claude';
}
const CLAUDE_CWD = join(tmpdir(), `gbrain-claude-cli-cwd-${process.pid}`);
let cwdEnsured = false;
function ensureCleanCwd(): string {
if (!cwdEnsured) {
mkdirSync(CLAUDE_CWD, { recursive: true });
cwdEnsured = true;
}
return CLAUDE_CWD;
}
/** Parsed shape of `claude --print --output-format json`. */
interface ClaudeJsonResult {
type: 'result';
subtype: 'success' | string;
is_error: boolean;
result: string;
stop_reason: string | null;
session_id: string;
num_turns: number;
usage?: {
input_tokens?: number;
output_tokens?: number;
cache_read_input_tokens?: number;
cache_creation_input_tokens?: number;
};
}
/**
* Build the system-prompt addendum that teaches the model the
* `<use_tools>...</use_tools>` emission format. Returns the empty string
* when no tools are registered for this turn so the model gets a normal
* text-completion prompt without protocol noise.
*/
function buildToolUseInstructions(
tools: ReadonlyArray<LanguageModelV2FunctionTool | LanguageModelV2ProviderDefinedTool> | undefined,
): string {
if (!tools || tools.length === 0) return '';
const functionTools = tools.filter((t): t is LanguageModelV2FunctionTool => t.type === 'function');
if (functionTools.length === 0) return '';
const toolSpecs = functionTools.map(t => ({
name: t.name,
description: t.description ?? '',
input_schema: t.inputSchema ?? { type: 'object', properties: {} },
}));
return [
'',
'## Tool Use Protocol',
'',
'You have access to these tools:',
'',
'```json',
JSON.stringify(toolSpecs, null, 2),
'```',
'',
'To call one or more tools in this turn, emit EXACTLY ONE block of this form, ' +
'with no other text outside the block on its own lines:',
'',
'<use_tools>',
'[',
' {"id": "<unique tool call id, like toolu_01ABC>", "name": "<tool name>", "input": <input object matching the tool\'s input_schema>}',
']',
'</use_tools>',
'',
'Multiple tool calls go in the array. Tool results are returned to you on the ' +
'next turn as [tool_result <text>] entries. You may then call more tools or emit a final response.',
'',
'When you are ready to give a final answer instead of calling tools, respond with prose text only — ' +
'do not include a <use_tools> block in that case.',
'',
].join('\n');
}
/**
* Render the ai-sdk message array into a single text prompt for `claude --print`
* stdin. System messages are extracted up-front and concatenated into the
* `--system-prompt` flag value. Tool calls and tool results are rendered as
* placeholders so the model sees the conversation in a coherent shape even
* though the adapter does not natively round-trip tool calls through claude-cli.
*/
function renderPrompt(prompt: LanguageModelV2Prompt): { systemText: string; userPrompt: string } {
const systemParts: string[] = [];
const convo: string[] = [];
for (const msg of prompt as ReadonlyArray<LanguageModelV2Message>) {
if (msg.role === 'system') {
systemParts.push(msg.content);
continue;
}
if (msg.role === 'user') {
const text = msg.content
.map(p => {
if (p.type === 'text') return p.text;
// File parts get a stub — multimodal is not supported via subprocess yet.
if (p.type === 'file') return `[file ${p.mediaType ?? 'unknown'}]`;
return '';
})
.filter(s => s.length > 0)
.join('\n');
if (text) convo.push(`User: ${text}`);
continue;
}
if (msg.role === 'assistant') {
const rendered = msg.content
.map(p => {
if (p.type === 'text') return p.text;
if (p.type === 'reasoning') return ''; // dropped on replay
if (p.type === 'tool-call') {
return `[tool_use ${p.toolName}(${p.input})]`;
}
if (p.type === 'tool-result') {
const out = typeof p.output === 'string' ? p.output : JSON.stringify(p.output);
return `[tool_result ${out}]`;
}
return '';
})
.filter(s => s.length > 0)
.join('\n');
if (rendered) convo.push(`Assistant: ${rendered}`);
continue;
}
if (msg.role === 'tool') {
const rendered = msg.content
.map(p => {
const out = typeof p.output === 'string' ? p.output : JSON.stringify(p.output);
return `[tool_result ${out}]`;
})
.join('\n');
if (rendered) convo.push(`User: ${rendered}`);
continue;
}
}
return { systemText: systemParts.join('\n'), userPrompt: convo.join('\n\n') };
}
/**
* Spawn `claude --print` with the contamination-suppression flags and return
* the parsed `--output-format json` envelope. Aborts propagate to SIGTERM on
* the child.
*/
function runClaude(
systemPrompt: string,
userPrompt: string,
model: string,
signal?: AbortSignal,
): Promise<ClaudeJsonResult> {
return new Promise((resolve, reject) => {
const args = [
'--print',
'--output-format', 'json',
'--model', model,
'--disable-slash-commands',
// Agent isolation: this subprocess must behave like a raw LLM, not a
// full Claude Code agent. `--tools ""` disables every built-in tool
// (Bash/Read/WebSearch/...); `--strict-mcp-config` ignores all user-level
// MCP servers (without it, each call would boot the user's MCP servers —
// including gbrain's own MCP → recursion + PGLite single-writer lock
// contention). Verified against claude CLI 2.1.145 --help.
'--tools', '',
'--strict-mcp-config',
];
if (systemPrompt) {
args.push('--system-prompt', systemPrompt);
}
// Env scrub: guarantee the CLI authenticates via its own OAuth session
// (subscription), never via an inherited API key. Without this, an
// ANTHROPIC_API_KEY in gbrain's env (the exact setup this recipe is meant
// to replace) silently flips billing to per-token API usage.
const env = { ...process.env };
delete env.ANTHROPIC_API_KEY;
delete env.ANTHROPIC_AUTH_TOKEN;
delete env.ANTHROPIC_BASE_URL;
const child = spawn(claudeBin(), args, {
stdio: ['pipe', 'pipe', 'pipe'],
cwd: ensureCleanCwd(),
env,
});
let stdout = '';
let stderr = '';
child.stdout.on('data', chunk => { stdout += String(chunk); });
child.stderr.on('data', chunk => { stderr += String(chunk); });
const onAbort = () => {
child.kill('SIGTERM');
reject(new Error('claude-cli adapter aborted'));
};
if (signal) {
if (signal.aborted) {
onAbort();
return;
}
signal.addEventListener('abort', onAbort, { once: true });
}
child.on('error', err => {
if (signal) signal.removeEventListener('abort', onAbort);
reject(new Error(`claude-cli spawn failed: ${err instanceof Error ? err.message : String(err)}`));
});
child.on('close', code => {
if (signal) signal.removeEventListener('abort', onAbort);
if (code !== 0) {
reject(new Error(`claude-cli exited ${code}: ${stderr.trim() || stdout.trim()}`));
return;
}
try {
let parsed = JSON.parse(stdout) as unknown;
// Compat: when the user has `"verbose": true` in ~/.claude/settings.json,
// `--print --output-format json` emits an ARRAY of events
// ([{type:"system",subtype:"init",...}, ..., {type:"result",...}])
// instead of the bare result object. There is no CLI flag to force it
// off (no --no-verbose; --settings '{}' merges, does not replace), so
// tolerate both shapes and pick the result event. Verified on CLI 2.1.145.
if (Array.isArray(parsed)) {
const resultEvent = parsed.find(
(ev): ev is ClaudeJsonResult =>
!!ev && typeof ev === 'object' && (ev as { type?: unknown }).type === 'result',
);
if (!resultEvent) {
reject(new Error(`claude-cli JSON event array had no "result" event\n--- raw ---\n${stdout.slice(0, 500)}`));
return;
}
parsed = resultEvent;
}
const envelope = parsed as ClaudeJsonResult;
if (envelope.is_error) {
reject(new Error(`claude-cli reported error: ${envelope.result || envelope.subtype}`));
return;
}
resolve(envelope);
} catch (e) {
reject(new Error(`claude-cli output not JSON: ${e instanceof Error ? e.message : String(e)}\n--- raw ---\n${stdout.slice(0, 500)}`));
}
});
// stdin error handler: if the binary does not exist (ENOENT) or the child
// dies before draining stdin, write/end can emit an unhandled 'error'
// (EPIPE) that would crash the worker. The spawn-level 'error' / non-zero
// 'close' handlers above already surface the real failure, so the stdin
// error itself is safe to swallow.
child.stdin.on('error', () => { /* surfaced via child 'error'/'close' */ });
try {
child.stdin.write(userPrompt);
child.stdin.end();
} catch (e) {
if (signal) signal.removeEventListener('abort', onAbort);
reject(new Error(`claude-cli stdin write failed (is the claude binary installed?): ${e instanceof Error ? e.message : String(e)}`));
}
});
}
interface ParsedToolCall {
id: string;
name: string;
/** Stringified JSON, matching the ai-sdk LanguageModelV2ToolCall.input contract. */
input: string;
}
/**
* Locate and parse the `<use_tools>...</use_tools>` block in the assistant's
* raw text response. Returns the parsed tool calls plus whatever prose
* surrounded the block. Returns an empty `toolCalls` array when no block is
* present, malformed, or unterminated — the caller then treats the full
* raw text as a final text response.
*/
function extractToolCalls(raw: string): {
toolCalls: ParsedToolCall[];
beforeText: string;
afterText: string;
} {
const openTag = '<use_tools>';
const closeTag = '</use_tools>';
const openIdx = raw.indexOf(openTag);
if (openIdx === -1) {
return { toolCalls: [], beforeText: raw.trim(), afterText: '' };
}
const closeIdx = raw.indexOf(closeTag, openIdx + openTag.length);
if (closeIdx === -1) {
// Unterminated block — recover gracefully.
return { toolCalls: [], beforeText: raw.trim(), afterText: '' };
}
const beforeText = raw.slice(0, openIdx).trim();
const afterText = raw.slice(closeIdx + closeTag.length).trim();
let inner = raw.slice(openIdx + openTag.length, closeIdx).trim();
if (inner.startsWith('```')) {
inner = inner.replace(/^```(?:json|JSON)?\s*\n?/, '').replace(/\n?```$/, '').trim();
}
let parsed: unknown;
try {
parsed = JSON.parse(inner);
} catch {
return { toolCalls: [], beforeText: raw.trim(), afterText: '' };
}
if (!Array.isArray(parsed)) {
return { toolCalls: [], beforeText: raw.trim(), afterText: '' };
}
const toolCalls: ParsedToolCall[] = [];
for (const entry of parsed) {
if (!entry || typeof entry !== 'object') continue;
const e = entry as Record<string, unknown>;
const name = typeof e.name === 'string' ? e.name : null;
if (!name) continue;
const id = typeof e.id === 'string' && e.id.length > 0
? e.id
: `toolu_claude_cli_${Math.random().toString(36).slice(2, 12)}`;
const inputJson = JSON.stringify(e.input ?? {});
toolCalls.push({ id, name, input: inputJson });
}
return { toolCalls, beforeText, afterText };
}
/**
* Strip provider prefixes (`anthropic:`, `litellm:`, `claude-cli:`) that the
* underlying CLI does not understand. The gateway hands us a bare model id
* via `recipe.aliases` resolution, but defensive normalization here keeps
* direct LanguageModelV2 construction (in tests, for example) ergonomic.
*/
function normalizeModel(model: string): string {
const idx = model.indexOf(':');
return idx >= 0 ? model.slice(idx + 1) : model;
}
export class ClaudeCliLanguageModel implements LanguageModelV2 {
readonly specificationVersion = 'v2' as const;
readonly provider = 'claude-cli';
readonly modelId: string;
readonly supportedUrls = {};
constructor(modelId: string) {
this.modelId = normalizeModel(modelId);
}
async doGenerate(options: LanguageModelV2CallOptions): Promise<{
content: LanguageModelV2Content[];
finishReason: 'stop' | 'length' | 'content-filter' | 'tool-calls' | 'error' | 'other' | 'unknown';
usage: { inputTokens: number | undefined; outputTokens: number | undefined; totalTokens: number | undefined };
warnings: never[];
}> {
const { systemText, userPrompt } = renderPrompt(options.prompt);
const toolInstructions = buildToolUseInstructions(options.tools);
const systemPrompt = [systemText, toolInstructions].filter(s => s.length > 0).join('\n');
const result = await runClaude(systemPrompt, userPrompt, this.modelId, options.abortSignal);
const { toolCalls, beforeText, afterText } = extractToolCalls(result.result);
const content: LanguageModelV2Content[] = [];
if (beforeText) content.push({ type: 'text', text: beforeText });
for (const call of toolCalls) {
content.push({
type: 'tool-call',
toolCallId: call.id,
toolName: call.name,
input: call.input,
});
}
if (afterText) content.push({ type: 'text', text: afterText });
if (content.length === 0) {
// Empty response — still hand the caller a well-formed content array.
content.push({ type: 'text', text: result.result ?? '' });
}
const finishReason = toolCalls.length > 0 ? 'tool-calls' as const : 'stop' as const;
const inputTokens = result.usage?.input_tokens;
const outputTokens = result.usage?.output_tokens;
const totalTokens = (inputTokens ?? 0) + (outputTokens ?? 0);
return {
content,
finishReason,
usage: {
inputTokens,
outputTokens,
totalTokens: inputTokens !== undefined && outputTokens !== undefined ? totalTokens : undefined,
},
warnings: [],
};
}
async doStream(): Promise<never> {
throw new Error(
'claude-cli LanguageModel does not support streaming. Use doGenerate or set ' +
'the model on a non-streaming chat surface (gateway.toolLoop is non-streaming).',
);
}
}
src/core/ai/recipes/claude-cli.ts
+71
View File
@@ -0,0 +1,71 @@
import type { Recipe } from '../types.ts';
/**
* Claude via the local `claude` CLI binary, using its built-in OAuth session
* (Claude Code / Claude Max subscription). No ANTHROPIC_API_KEY needed — the
* CLI manages its own auth state and the gateway dispatches via subprocess.
*
* Solves the #334 case where Max subscribers want Minions subagent dispatch
* to run against their existing subscription instead of paying per-token API
* charges. The recipe sits alongside the existing `anthropic` recipe so users
* pick per call: `anthropic:claude-sonnet-4-6` (API key + per-token billing)
* vs `claude-cli:claude-sonnet-4-6` (OAuth subscription, no API key).
*
* Chat-only. Claude has no first-party embedding model; users wanting an
* Anthropic chat path with embeddings still combine this with openai/google/
* voyage for embedding the way the existing `anthropic` recipe documents.
*
* Auth: `auth_env.required: []` because the CLI handles auth itself. The
* `claude` binary on PATH (or `GBRAIN_CLAUDE_CLI_BIN`) IS the auth surface;
* there is nothing for the gateway to forward.
*
* Setup expectation: `claude` CLI installed and logged in (Claude Code
* onboarding does this), or `GBRAIN_CLAUDE_CLI_BIN` pointing at the binary.
*/
export const claudeCli: Recipe = {
id: 'claude-cli',
name: 'Claude (via CLI)',
tier: 'native',
implementation: 'claude-cli',
// The CLI owns auth; no env vars are required from the gateway side.
auth_env: {
required: [],
},
touchpoints: {
// No embedding or expansion touchpoints — chat-only.
chat: {
models: [
'claude-opus-4-7',
'claude-sonnet-4-6',
'claude-haiku-4-5-20251001',
],
supports_tools: true,
supports_subagent_loop: true,
// The CLI handles caching internally and does not surface it via the
// standard cache_control control plane. From the gateway's POV the
// model does not support prompt caching.
supports_prompt_cache: false,
max_context_tokens: 200000,
// Cost figures match the underlying Claude API tier, but the actual
// bill is borne by the subscription. We report them for the budget
// ledger's per-call accounting; operators on flat-rate subscriptions
// can treat the numbers as nominal.
cost_per_1m_input_usd: 3.0,
cost_per_1m_output_usd: 15.0,
price_last_verified: '2026-06-17',
},
},
// Friendly aliases mirror the `anthropic` recipe so config strings stay
// portable: switching `anthropic:claude-sonnet-4-6` to `claude-cli:claude-sonnet-4-6`
// is a one-token edit. Reverse aliases rewrite legacy IDs back to canonical.
aliases: {
'claude-haiku-4-5': 'claude-haiku-4-5-20251001',
'claude-sonnet-4-6-20250929': 'claude-sonnet-4-6',
'sonnet': 'claude-sonnet-4-6',
'haiku': 'claude-haiku-4-5-20251001',
'opus': 'claude-opus-4-7',
},
setup_hint:
'Install Claude Code (`claude` CLI) and run `claude` once to log in. ' +
'Set GBRAIN_CLAUDE_CLI_BIN if the binary is not on PATH.',
};
src/core/ai/recipes/dashscope-rerank.ts
+61
View File
@@ -0,0 +1,61 @@
import type { Recipe } from '../types.ts';
/**
* Alibaba DashScope (灵积) reranker. DashScope's OpenAI-compatible surface
* splits by capability: embeddings live under `/compatible-mode/v1` (see the
* sibling `dashscope` recipe) while rerank lives under `/compatible-api/v1`
* with a PLURAL leaf — `POST {base}/reranks`. Wire shape matches ZeroEntropy:
* request `{model, query, documents, top_n?}`, response
* `{results: [{index, relevance_score}]}` — so it rides gateway.rerank()'s
* native path with only the recipe-pluggable `path` override (v0.40.6.1).
*
* This is a SEPARATE recipe rather than a reranker touchpoint on `dashscope`
* because the two capabilities need different base URLs (`compatible-mode`
* vs `compatible-api`) and `provider_base_urls` is keyed by recipe id — one
* recipe can't point embeddings and rerank at different prefixes. Same
* topology precedent as llama-server vs llama-server-reranker.
*
* Live-verified against the China endpoint (2026-07): `/reranks` with
* `qwen3-rerank` → 200 `results[].relevance_score`; `/rerank` (singular)
* → 404; `gte-rerank-v2` → 404 "Unsupported model for OpenAI compatibility
* mode" (native-API only, so it is deliberately NOT listed here).
*
* Note: the international endpoint requires a region-aware DASHSCOPE_API_KEY.
* China-region users point at https://dashscope.aliyuncs.com/compatible-api/v1
* via `provider_base_urls['dashscope-rerank']`, mirroring the embedding
* recipe's convention.
*/
export const dashscopeRerank: Recipe = {
id: 'dashscope-rerank',
name: 'Alibaba DashScope (灵积, reranker)',
tier: 'openai-compat',
implementation: 'openai-compatible',
base_url_default: 'https://dashscope-intl.aliyuncs.com/compatible-api/v1',
auth_env: {
required: ['DASHSCOPE_API_KEY'],
setup_url: 'https://help.aliyun.com/zh/model-studio/getting-started/',
},
touchpoints: {
reranker: {
// Only the model verified live on the OpenAI-compat /reranks surface.
// gte-rerank-v2 exists on DashScope's native API but the compat path
// rejects it ("Unsupported model for OpenAI compatibility mode").
models: ['qwen3-rerank'],
default_model: 'qwen3-rerank',
// Mirror ZE's defensive per-request ceiling; gateway.rerank()
// pre-flights body size and fails open.
max_payload_bytes: 5_000_000,
// PLURAL leaf under compatible-api — the whole reason this recipe
// exists. `${base_url}${path}` → `…/compatible-api/v1/reranks`.
path: '/reranks',
// Hosted API: no local warmup, but cross-region latency can exceed
// the 5s gateway default (same rationale as llama-server-reranker).
default_timeout_ms: 30_000,
},
},
setup_hint:
'Get an API key at https://help.aliyun.com/zh/model-studio/getting-started/, then ' +
'`export DASHSCOPE_API_KEY=...` and `gbrain config set search.reranker.model ' +
'dashscope-rerank:qwen3-rerank`. China-region accounts: `gbrain config set ' +
'provider_base_urls.dashscope-rerank https://dashscope.aliyuncs.com/compatible-api/v1`.',
};
src/core/ai/recipes/index.ts
+6
View File
@@ -9,6 +9,7 @@ import type { Recipe } from '../types.ts';
import { openai } from './openai.ts';
import { google } from './google.ts';
import { anthropic } from './anthropic.ts';
import { claudeCli } from './claude-cli.ts';
import { ollama } from './ollama.ts';
import { openrouter } from './openrouter.ts';
import { voyage } from './voyage.ts';
@@ -19,6 +20,7 @@ import { together } from './together.ts';
import { llamaServer } from './llama-server.ts';
import { minimax } from './minimax.ts';
import { dashscope } from './dashscope.ts';
import { dashscopeRerank } from './dashscope-rerank.ts';
import { zhipu } from './zhipu.ts';
import { azureOpenAI } from './azure-openai.ts';
import { zeroentropyai } from './zeroentropyai.ts';
@@ -26,11 +28,13 @@ import { llamaServerReranker } from './llama-server-reranker.ts';
import { moonshot } from './moonshot.ts';
import { mistral } from './mistral.ts';
import { nvidia } from './nvidia.ts';
import { perplexity } from './perplexity.ts';
const ALL: Recipe[] = [
openai,
google,
anthropic,
claudeCli,
ollama,
openrouter,
voyage,
@@ -42,12 +46,14 @@ const ALL: Recipe[] = [
llamaServerReranker,
minimax,
dashscope,
dashscopeRerank,
zhipu,
azureOpenAI,
zeroentropyai,
moonshot,
mistral,
nvidia,
perplexity,
];
/** Map from `provider:id` key to recipe. */
src/core/ai/recipes/perplexity.ts
+54
View File
@@ -0,0 +1,54 @@
import type { Recipe } from '../types.ts';
/**
* Perplexity's hosted embeddings API (#1046). OpenAI-shaped at
* `POST {base}/embeddings` but diverges on the wire:
* - `encoding_format` only accepts 'base64_int8' (default) or
* 'base64_binary' — the AI SDK's 'float' default is rejected.
* - The response `embedding` is a base64 string encoding SIGNED INT8
* components (natively quantized output), not a float array.
* Both divergences are handled by perplexityCompatFetch in gateway.ts
* (force 'base64_int8' outbound; decode Int8Array → number[] inbound).
* Cosine similarity is scale-invariant, so the raw int8 components store
* and rank correctly as floats.
*
* Models (per docs.perplexity.ai/api-reference/embeddings-post, 2026-07):
* - pplx-embed-v1-0.6b: dims 128..1024 (default 1024)
* - pplx-embed-v1-4b: dims 128..2560 (default 2560)
* The flexible-dim range validation lives in src/core/ai/dims.ts
* (PERPLEXITY_EMBEDDING_MAX_DIMS). default_dims is pinned at 1024 so both
* models work out of the box on a plain vector(N) column; users who want
* the 4b model's full 2560 width set `embedding_dimensions: 2560` and the
* existing halfvec path (dims > 2000) covers storage + ANN.
*
* Auth is PERPLEXITY_API_KEY only — deliberately NO OPENAI_API_KEY
* fallback (a Perplexity brain must never silently bill/route through
* OpenAI). If your key lives in PPLX_API_KEY, re-export it.
*/
export const perplexity: Recipe = {
id: 'perplexity',
name: 'Perplexity',
tier: 'openai-compat',
implementation: 'openai-compatible',
base_url_default: 'https://api.perplexity.ai/v1',
auth_env: {
required: ['PERPLEXITY_API_KEY'],
setup_url: 'https://www.perplexity.ai/settings/api',
},
touchpoints: {
embedding: {
models: ['pplx-embed-v1-0.6b', 'pplx-embed-v1-4b'],
default_dims: 1024,
cost_per_1m_tokens_usd: 0.03, // pplx-embed-v1-4b; 0.6b is $0.004/M
price_last_verified: '2026-07-21',
// Perplexity enforces 120K combined tokens (and 512 texts) per
// request. Same pre-split posture as Voyage: assume a dense
// tokenizer (1 char ≈ 1 token) at 0.5 utilization; the gateway's
// recursive halving is the runtime safety net.
max_batch_tokens: 120_000,
chars_per_token: 1,
safety_factor: 0.5,
},
},
setup_hint: 'Get an API key at https://www.perplexity.ai/settings/api, then `export PERPLEXITY_API_KEY=...` (re-export PPLX_API_KEY if that is where your key lives).',
};
src/core/ai/types.ts
+2 -1
View File
@@ -22,7 +22,8 @@ export type Implementation =
| 'native-openai'
| 'native-google'
| 'native-anthropic'
| 'openai-compatible';
| 'openai-compatible'
| 'claude-cli';
export interface EmbeddingTouchpoint {
models: string[];
src/core/brainstorm/orchestrator.ts
+40 -5
View File
@@ -486,9 +486,44 @@ const DEFAULT_PARALLELISM = 4;
* src/core/errors.ts (the v0.19.0 envelope every new agent-facing
* surface uses) rather than introducing a new BrainstormError class.
*/
/** File-config slice the orchestrator reads (see loadConfig in core/config.ts). */
export interface BrainstormRunConfig {
embedding_model?: string;
chat_model?: string;
emotional_weight?: { user_holder?: string };
}
/**
* Model used for the cost preview + hard cost ceiling. Mirrors what the
* gateway will actually run: explicit --model override, else the configured
* chat_model (gateway default), else the hardcoded gateway fallback. Before
* this resolved through config, a non-Sonnet chat_model got its preview
* priced against the wrong model. (Takeover of PR #1855 by @starm2010.)
*/
export function resolveBrainstormChatModel(
config: { chat_model?: string },
modelOverride?: string,
): string {
return modelOverride ?? config.chat_model ?? 'anthropic:claude-sonnet-4-6';
}
/**
* Judge-phase model precedence: --judge-model flag, else the
* `models.brainstorm.judge` config key, else undefined (falls back to
* `modelOverride` then the gateway default at the runJudge callsite).
*/
export async function resolveBrainstormJudgeModel(
engine: BrainEngine,
judgeModelFlag?: string,
): Promise<string | undefined> {
if (judgeModelFlag) return judgeModelFlag;
const configured = await engine.getConfig('models.brainstorm.judge');
return configured ?? undefined;
}
export async function runBrainstorm(
engine: BrainEngine,
config: { embedding_model?: string; emotional_weight?: { user_holder?: string } },
config: BrainstormRunConfig,
opts: BrainstormOptions
): Promise<BrainstormResult> {
// v0.39.3.0 (Phase 5, CV11+T4): outer try/catch around the orchestrator
@@ -510,7 +545,7 @@ export async function runBrainstorm(
async function runBrainstormImpl(
engine: BrainEngine,
config: { embedding_model?: string; emotional_weight?: { user_holder?: string } },
config: BrainstormRunConfig,
opts: BrainstormOptions,
): Promise<BrainstormResult> {
// v0.39.0.0 T10: install a gateway-layer BudgetTracker scope around the
@@ -530,7 +565,7 @@ async function runBrainstormImpl(
async function _runBrainstormInner(
engine: BrainEngine,
config: { embedding_model?: string; emotional_weight?: { user_holder?: string } },
config: BrainstormRunConfig,
opts: BrainstormOptions,
): Promise<BrainstormResult> {
const profile = opts.profile ?? BRAINSTORM_PROFILE;
@@ -539,7 +574,7 @@ async function _runBrainstormInner(
const embedFn = opts.embedQueryFn ?? embedQuery;
// ---- Phase 0: cost preview + TTY grace ----
const modelStr = opts.modelOverride ?? 'anthropic:claude-sonnet-4-6';
const modelStr = resolveBrainstormChatModel(config, opts.modelOverride);
const { aborted, estimate } = await previewCostAndWait({
profile,
model: modelStr,
@@ -848,7 +883,7 @@ async function _runBrainstormInner(
far_slug: i.far_slug,
}));
const judgeResult = await runJudge(profile.judge_config, judgeInput, {
modelOverride: opts.judgeModel ?? opts.modelOverride,
modelOverride: (await resolveBrainstormJudgeModel(engine, opts.judgeModel)) ?? opts.modelOverride,
chatFn: opts.chatFn,
activeBiasTags: activeBiasTags ?? undefined,
abortSignal: opts.abortSignal,
src/core/budget/budget-tracker.ts
+7
View File
@@ -32,6 +32,7 @@ import { mkdirSync, appendFileSync } from 'node:fs';
import { dirname } from 'node:path';
import { gbrainPath } from '../config.ts';
import { ANTHROPIC_PRICING, type ModelPricing } from '../anthropic-pricing.ts';
import { canonicalLookup } from '../model-pricing.ts';
import { EMBEDDING_PRICING, lookupEmbeddingPrice } from '../embedding-pricing.ts';
import { splitProviderModelId } from '../model-id.ts';
import { isoWeekFilename, resolveAuditDir } from '../audit-week-file.ts';
@@ -213,6 +214,12 @@ function lookupPricing(modelId: string, kind: BudgetKind): ModelPricing | null {
if (kind === 'rerank' && providerId && FREE_LOCAL_RERANK_PROVIDERS.has(providerId)) {
return { input: 0, output: 0 };
}
// Fall back to the full canonical pricing table so non-Anthropic chat
// models with a known price (openai:*, google:*, deepseek:*) resolve under
// --max-cost instead of TX2 no_pricing hard-failing at $0. ANTHROPIC_PRICING
// above is only the bare-keyed Claude view.
const canon = canonicalLookup(modelId);
if (canon) return canon;
return null;
}
src/core/chunkers/code.ts
+19 -1
View File
@@ -1235,7 +1235,25 @@ export function estimateTokens(text: string): number {
tiktokenInitialized = true;
}
if (tiktokenEncoder) {
return tiktokenEncoder.encode(text).length;
try {
return tiktokenEncoder.encode(text).length;
} catch {
// Code legitimately contains tiktoken special-token strings (e.g. CLIP/GPT
// tokenizers embed the literal "<|endoftext|>"). The default encode() uses
// disallowed_special='all' and THROWS on those, crashing reindex-code on
// valid source files. For a token COUNT we don't need special-token
// semantics: re-encode treating them as ordinary text (never throws),
// heuristic only if even that fails.
try {
return (
tiktokenEncoder as unknown as {
encode: (s: string, allowed: string[], disallowed: string[]) => Uint32Array;
}
).encode(text, [], []).length;
} catch {
return Math.max(1, Math.ceil(text.length / 4));
}
}
}
return Math.max(1, Math.ceil(text.length / 4));
}
src/core/config.ts
+13
View File
@@ -146,6 +146,18 @@ export interface GBrainConfig {
/** Daily spend cap (USD); bounds drains/day = floor(cap / ~$0.30). Default 2.0. */
max_usd_per_day?: number;
};
/**
* v0.42 — keep frontmatter links fresh on the incremental cycle. The cycle's
* extract phase re-extracts only the slugs a sync changed, but `extractForSlugs`
* extracts BODY links only — frontmatter (`sources:`/`related:` etc.) link edges
* silently drift stale when a page's YAML is edited externally and synced in.
* Set true to also extract frontmatter links per changed page each cycle, keeping
* externally-edited YAML edges fresh without a full rescan. Default false
* (preserves current behavior). Read via the file/env/DB plane in the cycle's
* extract dispatch. Disable/enable with
* `gbrain config set autopilot.incremental_extract_include_frontmatter <bool>`.
*/
incremental_extract_include_frontmatter?: boolean;
};
eval?: {
/** false disables capture entirely. Defaults to true. */
@@ -962,6 +974,7 @@ export const KNOWN_CONFIG_KEYS: readonly string[] = [
'models.subagent',
'models.expansion',
'models.chat',
'models.brainstorm.judge',
'models.eval.longmemeval',
'facts.extraction_model',
// #2113: output-token cap for the per-turn facts extractor (default 4000).
src/core/contextual-retrieval-service.ts
+164 -79
View File
@@ -45,7 +45,6 @@ import { embedBatch } from './embedding.ts';
import { resolveContextualRetrievalMode } from './contextual-retrieval-resolver.ts';
import {
buildContextualPrefix,
extractFirstTwoSentences,
modeRequiresHaiku,
modeRequiresWrapper,
sanitizeTitle,
@@ -57,10 +56,8 @@ import {
SYNOPSIS_DOC_MAX_CHARS,
type GeneratePerChunkSynopsisResult,
} from './page-summary.ts';
import {
logSynopsisFailure,
type SynopsisFailureKind,
} from './audit-synopsis.ts';
import type { SynopsisFailureKind } from './audit-synopsis.ts';
import { runSlidingPool } from './worker-pool.ts';
import type { BrainEngine } from './engine.ts';
import type { ChunkInput, CRMode, Page } from './types.ts';
import type { SourceRow } from './sources-ops.ts';
@@ -73,6 +70,24 @@ import type { SourceRow } from './sources-ops.ts';
* corpus_generation hash.
*/
export const TITLE_WRAPPER_VERSION = 1;
const DEFAULT_HAIKU_MODEL = 'anthropic:claude-haiku-4-5-20251001';
export const DEFAULT_CONTEXTUAL_CHUNK_CONCURRENCY = 4;
export const MAX_CONTEXTUAL_CHUNK_CONCURRENCY = 16;
export function resolveContextualChunkConcurrency(
env: Record<string, string | undefined> = process.env,
): number {
const raw = env.GBRAIN_CONTEXTUAL_CHUNK_CONCURRENCY;
if (raw === undefined || raw.trim() === '') return DEFAULT_CONTEXTUAL_CHUNK_CONCURRENCY;
const n = Number(raw);
if (!Number.isFinite(n)) return DEFAULT_CONTEXTUAL_CHUNK_CONCURRENCY;
return clampContextualChunkConcurrency(n);
}
function clampContextualChunkConcurrency(n: number): number {
if (!Number.isFinite(n)) return DEFAULT_CONTEXTUAL_CHUNK_CONCURRENCY;
return Math.max(1, Math.min(MAX_CONTEXTUAL_CHUNK_CONCURRENCY, Math.trunc(n)));
}
/**
* Embedding model placeholder. The actual model name lands here from
@@ -208,12 +223,16 @@ export interface ReembedPageArgs {
* src/core/minions/rate-leases.ts here; inline callers (import-file,
* reindex command) pass undefined and rely on gateway-level retry.
*/
acquireSynopsisLease?: () => Promise<void>;
releaseSynopsisLease?: () => Promise<void>;
acquireSynopsisLease?: () => Promise<unknown>;
releaseSynopsisLease?: (lease?: unknown) => Promise<void>;
/**
* Intra-page per-chunk synopsis concurrency. 1 preserves the legacy
* sequential loop exactly; higher values only parallelize Haiku synopsis
* calls. Embedding remains one batch after all synopses succeed.
*/
chunkConcurrency?: number;
}
const DEFAULT_HAIKU_MODEL = 'anthropic:claude-haiku-4-5-20251001';
/**
* Re-embed one page through the active CR mode. Implements the D26 P0-2
* two-phase build pattern.
@@ -432,82 +451,41 @@ async function tryBuildPhase1(opts: {
}
// per_chunk_synopsis path. Read source text via fallback chain,
// generate synopsis per chunk sequentially within this page (D10),
// generate synopsis per chunk through a bounded sliding pool, then
// batch embed at the end (D27 P2-2).
const sourceText = readSourceTextWithFallback(page, chunks);
const wrappedTexts: string[] = [];
const wrappedTexts: string[] = new Array(chunks.length);
const chunkConcurrency = clampContextualChunkConcurrency(
args.chunkConcurrency ?? resolveContextualChunkConcurrency(),
);
for (let i = 0; i < chunks.length; i++) {
const c = chunks[i];
// Code chunks always bypass the wrapper (D20-T4) — pass through.
if (c.chunk_source === 'fenced_code') {
wrappedTexts.push(c.chunk_text);
continue;
}
// Acquire rate-lease per chunk (D26 P0-3). Inline callers pass no
// hooks; only the Minion handler wires through rate-leases.ts.
if (args.acquireSynopsisLease) {
await args.acquireSynopsisLease();
}
let synopsisResult: GeneratePerChunkSynopsisResult;
try {
synopsisResult = await generatePerChunkSynopsis({
documentText: sourceText,
chunkText: c.chunk_text,
pageTitle: page.title,
pageSlug: args.pageSlug,
sourceId: args.sourceId,
chunkIndex: c.chunk_index,
model: haikuModel,
abortSignal: args.abortSignal,
const poolResult = await runSlidingPool({
items: chunks,
workers: chunkConcurrency,
signal: args.abortSignal,
onError: 'abort',
failureLabel: (c) => String(c.chunk_index),
onItem: async (c, i) => {
wrappedTexts[i] = await buildWrappedChunkText({
chunk: c,
sourceText,
safeTitle,
page,
args,
haikuModel,
});
} finally {
if (args.releaseSynopsisLease) {
try {
await args.releaseSynopsisLease();
} catch {
// Lease release failure shouldn't abort the page; surfacing it
// would race with the synopsis result. Audit-only.
}
}
}
},
});
if (synopsisResult.kind === 'success') {
const prefix = buildContextualPrefix(safeTitle, synopsisResult.synopsis);
wrappedTexts.push(
wrapChunkForEmbedding(c.chunk_text, prefix, c.chunk_source),
);
continue;
if (poolResult.failures.length > 0) {
const failure = [...poolResult.failures].sort((a, b) => a.idx - b.idx)[0].error;
if (failure instanceof ChunkSynopsisPhase1Error) {
return failure.result;
}
// Failure classification per D27 P1-2:
// refusal | empty | malformed → page-level fall-back to title-only
// auth_failure → permanent (won't fix with retry)
// rate_limit | timeout | network | provider_5xx → transient
// source_missing → walked into fallback already; would be 'malformed'
// from generatePerChunkSynopsis if we ever propagated it here
if (
synopsisResult.kind === 'refusal' ||
synopsisResult.kind === 'empty' ||
synopsisResult.kind === 'malformed'
) {
return { kind: 'page_level_fallback_requested', cause: synopsisResult.kind };
}
if (synopsisResult.kind === 'auth_failure') {
return {
kind: 'permanent',
cause: synopsisResult.kind,
detail: synopsisResult.detail ?? 'auth failure',
};
}
return {
kind: 'transient',
cause: synopsisResult.kind,
detail: synopsisResult.detail ?? 'transient',
};
throw failure;
}
if (poolResult.aborted || args.abortSignal?.aborted) {
return { kind: 'transient', cause: 'timeout', detail: 'aborted' };
}
// All chunks synthesized successfully. Single batch embed (D27 P2-2).
@@ -528,6 +506,113 @@ async function tryBuildPhase1(opts: {
}
}
class ChunkSynopsisPhase1Error extends Error {
constructor(readonly result: Exclude<Phase1Result, Phase1Success>) {
super(`chunk synopsis failed: ${result.kind}`);
this.name = 'ChunkSynopsisPhase1Error';
}
}
async function buildWrappedChunkText(opts: {
chunk: ChunkInput;
sourceText: string;
safeTitle: string;
page: Page;
args: ReembedPageArgs;
haikuModel: string;
}): Promise<string> {
const { chunk: c, sourceText, safeTitle, page, args, haikuModel } = opts;
// Code chunks always bypass the wrapper (D20-T4) — pass through.
if (c.chunk_source === 'fenced_code') {
return c.chunk_text;
}
// Acquire rate-lease per chunk (D26 P0-3). Inline callers pass no
// hooks; only the Minion handler wires through rate-leases.ts.
let lease: unknown;
let leaseAcquired = false;
let synopsisResult: GeneratePerChunkSynopsisResult;
try {
if (args.acquireSynopsisLease) {
try {
lease = await args.acquireSynopsisLease();
} catch (err) {
if (args.abortSignal?.aborted || isAbortError(err)) {
throw new ChunkSynopsisPhase1Error({
kind: 'transient',
cause: 'timeout',
detail: 'aborted',
});
}
throw err;
}
leaseAcquired = true;
}
synopsisResult = await generatePerChunkSynopsis({
documentText: sourceText,
chunkText: c.chunk_text,
pageTitle: page.title,
pageSlug: args.pageSlug,
sourceId: args.sourceId,
chunkIndex: c.chunk_index,
model: haikuModel,
abortSignal: args.abortSignal,
});
} finally {
if (leaseAcquired && args.releaseSynopsisLease) {
try {
await args.releaseSynopsisLease(lease);
} catch {
// Lease release failure shouldn't abort the page; surfacing it
// would race with the synopsis result. Audit-only.
}
}
}
if (synopsisResult.kind === 'success') {
const prefix = buildContextualPrefix(safeTitle, synopsisResult.synopsis);
return wrapChunkForEmbedding(c.chunk_text, prefix, c.chunk_source);
}
// Failure classification per D27 P1-2:
// refusal | empty | malformed → page-level fall-back to title-only
// auth_failure → permanent (won't fix with retry)
// rate_limit | timeout | network | provider_5xx → transient
// source_missing → walked into fallback already; would be 'malformed'
// from generatePerChunkSynopsis if we ever propagated it here
if (
synopsisResult.kind === 'refusal' ||
synopsisResult.kind === 'empty' ||
synopsisResult.kind === 'malformed'
) {
throw new ChunkSynopsisPhase1Error({
kind: 'page_level_fallback_requested',
cause: synopsisResult.kind,
});
}
if (synopsisResult.kind === 'auth_failure') {
throw new ChunkSynopsisPhase1Error({
kind: 'permanent',
cause: synopsisResult.kind,
detail: synopsisResult.detail ?? 'auth failure',
});
}
throw new ChunkSynopsisPhase1Error({
kind: 'transient',
cause: synopsisResult.kind,
detail: synopsisResult.detail ?? 'transient',
});
}
function isAbortError(err: unknown): boolean {
return (
typeof err === 'object' &&
err !== null &&
(err as { name?: unknown }).name === 'AbortError'
);
}
/**
* Source-text fallback chain per D11:
* 1. read page.source_path from disk (truest "document")
src/core/cycle.ts
+96 -11
View File
@@ -66,6 +66,10 @@ export type CyclePhase =
// - calibration_profile: aggregates the resolved subset into 2-4
// narrative pattern statements + active bias tags. Voice-gated.
| 'propose_takes' | 'grade_takes' | 'calibration_profile'
// #2653 — drift detection (default OFF; dream.drift.enabled). LLM-judges
// soft-band takes against recent timeline evidence; report-only in v1
// (writes reports/drift-<date>; auto_update mutates nothing).
| 'drift'
| 'embed' | 'orphans' | 'purge'
// v0.39 T12: schema-suggest passive trigger (D3 + D4 plan-eng-review).
// Wraps runSuggest() — same library the CLI verb + EIIRP call.
@@ -151,6 +155,10 @@ export const ALL_PHASES: CyclePhase[] = [
'propose_takes',
'grade_takes',
'calibration_profile',
// #2653 — drift detection. Default OFF (dream.drift.enabled). Runs AFTER
// the calibration trio (fresh take resolutions) and BEFORE embed so the
// drift report page gets embedded same-cycle. Report-only in v1.
'drift',
// v0.41.11.0 — opt-in conversation-facts backfill. Default OFF; reads
// cycle.conversation_facts_backfill.enabled gate inside the wrapper.
// Ordered AFTER calibration_profile (matches the runCycle dispatch
@@ -195,8 +203,10 @@ export const ALL_PHASES: CyclePhase[] = [
* - `source`: safe to parallelize per source. Sync reads/writes the
* one source's rows; extract walks changed slugs.
* - `global`: must serialize across the brain. Embed walks all stale
* chunks; orphans/purge sweep brain-wide; grade_takes + calibration
* aggregate across sources; resolve_symbol_edges walks every chunk.
* chunks; purge sweeps brain-wide; orphans can report a single
* resolved source but still belongs in the serialized global lane;
* grade_takes + calibration aggregate across sources;
* resolve_symbol_edges walks every chunk.
* - `mixed`: per-phase decomposition needed before parallelizing.
* Synthesize reads the brain-global transcripts dir but writes to
* per-source slugs (via subagent allowlist). Patterns reads
@@ -221,6 +231,9 @@ export const PHASE_SCOPE: Record<CyclePhase, PhaseScope> = {
propose_takes: 'source',
grade_takes: 'global',
calibration_profile: 'global',
// #2653 — drift walks takes brain-wide (same posture as grade_takes) and
// writes one brain-global report page.
drift: 'global',
embed: 'global',
orphans: 'global',
purge: 'global',
@@ -289,6 +302,8 @@ const NEEDS_LOCK_PHASES: ReadonlySet<CyclePhase> = new Set([
'propose_takes',
'grade_takes',
'calibration_profile',
// #2653 — writes the reports/drift-<date> page.
'drift',
// v0.41 T9 — extract_atoms writes atom-typed pages via put_page;
// synthesize_concepts writes concept-typed pages + tier updates. Both
// mutate DB state and need the lock.
@@ -453,6 +468,12 @@ export interface CycleOpts {
* loop bug (codex finding #3).
*/
synthBypassDreamGuard?: boolean;
/**
* Force the orphans phase to scan brain-wide even when `brainDir` resolves to
* a source. Used by autopilot global maintenance, whose phase set is
* intentionally brain-wide.
*/
forceGlobalOrphans?: boolean;
/**
* AbortSignal from the Minions worker (v0.22.1, #403). When aborted
* (timeout, cancel, lock-loss), runCycle bails between phases and
@@ -469,12 +490,14 @@ export interface CycleOpts {
* + every existing caller).
*
* **Note for follow-up waves:** this only scopes the LOCK. Several
* cycle phases (`embed`, `orphans`, `purge`, `resolve_symbol_edges`,
* `grade_takes`, `calibration_profile`) still operate brain-wide
* regardless of sourceId — see the `PHASE_SCOPE` taxonomy. Per-source
* cycle locks let two cycles RUN, but the global-scoped phases
* inside each will still touch the same rows. Genuine per-source
* fan-out requires the deferred TODOs in the plan.
* cycle phases (`embed`, `purge`, `resolve_symbol_edges`, `grade_takes`,
* `calibration_profile`) still operate brain-wide regardless of sourceId
* — see the `PHASE_SCOPE` taxonomy. `orphans` uses the resolved source
* for its candidate set when one exists, but it remains in the serialized
* global lane for autopilot scheduling. Per-source cycle locks let two
* cycles RUN, but the global-scoped phases inside each will still touch
* the same rows. Genuine per-source fan-out requires the deferred TODOs
* in the plan.
*
* Validated via `assertValidSourceId` in `cycleLockIdFor` (defense-in-depth).
*/
@@ -1013,6 +1036,21 @@ async function runPhaseExtract(
): Promise<PhaseResult> {
try {
const { runExtractCore } = await import('../commands/extract.ts');
const { loadConfig } = await import('./config.ts');
// Default off: the incremental cycle extracts body links only unless the
// operator opts in to keeping externally-edited frontmatter links fresh too.
// Both planes, file wins (env > file > DB precedence, per loadConfigWithEngine):
// `gbrain config set autopilot.incremental_extract_include_frontmatter true`
// writes the DB plane (engine.setConfig), so a file-plane-only read here
// would make the documented enable command a silent no-op (#2120 class).
const fileVal = loadConfig()?.autopilot?.incremental_extract_include_frontmatter;
let includeFrontmatter = fileVal === true;
if (fileVal === undefined) {
try {
includeFrontmatter =
(await engine.getConfig('autopilot.incremental_extract_include_frontmatter')) === 'true';
} catch { /* config table unreadable → default off */ }
}
// Extract is read-mostly against the filesystem + write to links table.
// Honor dryRun by skipping with a 'skipped' entry: extract doesn't have
// a clean dry-run mode today and runCycle should be honest about it.
@@ -1033,6 +1071,7 @@ async function runPhaseExtract(
slugs: changedSlugs, // undefined = full walk (first run / manual)
signal,
sourceId,
includeFrontmatter, // honored on the incremental (slugs) path only
});
const linksCreated = result?.links_created ?? 0;
const timelineCreated = result?.timeline_entries_created ?? 0;
@@ -1410,10 +1449,10 @@ async function runPhasePurge(engine: BrainEngine, dryRun: boolean): Promise<Phas
* to avoid a static import (purge phase is only loaded in the autopilot path). */
const SOFT_DELETE_TTL_HOURS_FOR_PURGE = 72;
async function runPhaseOrphans(engine: BrainEngine): Promise<PhaseResult> {
async function runPhaseOrphans(engine: BrainEngine, sourceId?: string): Promise<PhaseResult> {
try {
const { findOrphans } = await import('../commands/orphans.ts');
const result = await findOrphans(engine);
const result = await findOrphans(engine, sourceId !== undefined ? { sourceId } : {});
const count = result.total_orphans;
// Orphans are a code-smell signal, not a fatal condition. The
// original `count > 20` cutoff was tuned for small dev brains; on
@@ -1432,8 +1471,10 @@ async function runPhaseOrphans(engine: BrainEngine): Promise<PhaseResult> {
summary: `${count} orphan page(s) out of ${result.total_pages} total`,
details: {
total_orphans: count,
total_linkable: result.total_linkable,
total_pages: result.total_pages,
excluded: result.excluded,
...(sourceId !== undefined ? { source_id: sourceId } : {}),
},
};
} catch (e) {
@@ -1497,6 +1538,7 @@ export async function runCycle(
const cycleSourceId: string | undefined = engine
? (opts.sourceId ?? (await resolveSourceForDir(engine, brainDir)))
: opts.sourceId;
const orphansSourceId = opts.forceGlobalOrphans ? undefined : cycleSourceId;
const progress = createProgress(cliOptsToProgressOptions(getCliOptions()));
@@ -2135,6 +2177,49 @@ export async function runCycle(
}
}
// ── #2653: drift detection ──────────────────────────────────
// Default OFF (dream.drift.enabled). LLM-judges soft-band takes
// against recent timeline evidence; report-only in v1 — writes one
// reports/drift-<date> page, mutates no takes regardless of
// dream.drift.auto_update.
if (phases.includes('drift')) {
checkAborted(opts.signal);
if (!engine) {
phaseResults.push({
phase: 'drift',
status: 'skipped',
duration_ms: 0,
summary: 'no database connected',
details: { reason: 'no_database' },
});
} else {
progress.start('cycle.drift');
const { runPhaseDrift } = await import('./cycle/drift.ts');
const { result, duration_ms } = await timePhase(async (): Promise<PhaseResult> => {
const r = await runPhaseDrift(engine, {
dryRun,
brainDir: brainDir ?? undefined,
forceEnabled: opts.onceForPhase === 'drift',
});
const status: PhaseStatus =
r.status === 'complete' ? 'ok' :
r.status === 'partial' ? 'warn' :
r.status === 'failed' ? 'fail' : 'skipped';
return {
phase: 'drift',
status,
duration_ms: 0,
summary: r.detail,
details: { ...(r.totals ?? {}) },
};
});
result.duration_ms = duration_ms;
phaseResults.push(result);
progress.finish();
}
await safeYield(opts.yieldBetweenPhases);
}
// ── v0.41.11.0: conversation_facts_backfill ─────────────────
// Opt-in (default OFF). Walks long-form conversation/meeting/slack/
// email pages, segments by 30-min gap, runs facts extractor with a
@@ -2268,7 +2353,7 @@ export async function runCycle(
});
} else {
progress.start('cycle.orphans');
const { result, duration_ms } = await timePhase(() => runPhaseOrphans(engine));
const { result, duration_ms } = await timePhase(() => runPhaseOrphans(engine, orphansSourceId));
result.duration_ms = duration_ms;
phaseResults.push(result);
progress.finish();
src/core/cycle/drift.ts
+250 -35
View File
@@ -1,18 +1,25 @@
/**
* v0.28: drift dream phase.
* Drift dream phase (#2653 — wired v0.42.x; scaffold shipped v0.28).
*
* Detects takes where the underlying evidence has shifted since the take
* was made. v0.28 ships the SCAFFOLD: the phase iterates active takes,
* runs a lightweight check against recent timeline entries on the same
* page, and writes a drift-report-<date>.md if any takes look stale.
* Detects takes whose underlying evidence has shifted since the take was
* made. Two stages:
* 1. Cheap SQL heuristic (findDriftCandidates): soft-band takes
* (weight 0.3–0.85, active, unresolved) on pages with fresh
* timeline_entries evidence inside the lookback window.
* 2. LLM judge: each candidate's claim is compared against the recent
* timeline evidence; the judge returns {drifted, confidence,
* reasoning, suggested_weight}. BudgetMeter-gated.
*
* The full LLM-driven drift detection (compare each take's claim to recent
* page evidence and propose a weight adjustment) is the v0.29 follow-up.
* v0.28 lays the phase orchestration so the contract is stable.
* Output is REPORT-ONLY (v1 conservative posture): judged candidates land
* on a `reports/drift-<date>` page. `dream.drift.auto_update` mutates
* NOTHING in v1 — it is recorded in the report so operators can see the
* flag state, and a future wave may wire weight adjustment behind it.
*
* Default-disabled. Operator opts in:
* gbrain config set dream.drift.enabled true
* gbrain config set dream.drift.lookback_days 30
* gbrain config set dream.drift.max_per_cycle 20
* gbrain config set dream.drift.budget 1.0
*/
import type { BrainEngine } from '../engine.ts';
@@ -25,6 +32,10 @@ export interface DriftPhaseOpts {
dryRun: boolean;
/** Override the audit ledger path (tests). */
auditPath?: string;
/** issue #2860 --once: bypass the dream.drift.enabled gate for this run only. */
forceEnabled?: boolean;
/** Inject the judge model call (tests). Defaults to gateway chat. */
judge?: DriftJudgeFn;
}
export interface DriftConfig {
@@ -32,6 +43,7 @@ export interface DriftConfig {
lookbackDays: number;
budgetUsd: number;
autoUpdate: boolean;
maxPerCycle: number;
}
async function loadDriftConfig(engine: BrainEngine): Promise<DriftConfig> {
@@ -39,16 +51,19 @@ async function loadDriftConfig(engine: BrainEngine): Promise<DriftConfig> {
const lookbackStr = await engine.getConfig('dream.drift.lookback_days');
const budgetStr = await engine.getConfig('dream.drift.budget');
const autoStr = await engine.getConfig('dream.drift.auto_update');
const maxPerStr = await engine.getConfig('dream.drift.max_per_cycle');
return {
enabled: enabledStr === 'true',
lookbackDays: lookbackStr ? Math.max(1, parseInt(lookbackStr, 10) || 30) : 30,
budgetUsd: budgetStr ? Math.max(0, parseFloat(budgetStr) || 1.0) : 1.0,
autoUpdate: autoStr === 'true',
maxPerCycle: maxPerStr ? Math.max(1, parseInt(maxPerStr, 10) || 20) : 20,
};
}
interface DriftCandidate {
export interface DriftCandidate {
takeId: number;
pageId: number;
pageSlug: string;
rowNum: number;
claim: string;
@@ -57,24 +72,121 @@ interface DriftCandidate {
recentEvidenceCount: number;
}
/** Judge verdict: has the claim drifted relative to the evidence? */
export interface DriftVerdict {
drifted: boolean;
confidence: number;
reasoning: string;
/** Judge's suggested new weight (advisory only — v1 never applies it). */
suggested_weight?: number;
}
/** Judge function signature — injected for tests. */
export type DriftJudgeFn = (input: {
candidate: DriftCandidate;
evidence: string;
modelHint?: string;
}) => Promise<DriftVerdict>;
export const DRIFT_JUDGE_PROMPT = `You are auditing a knowledge-base "take" (a weighted claim) for drift:
has newer evidence shifted the ground under this claim since it was made?
Output ONLY one JSON object with these fields:
- drifted (boolean) — true when the evidence meaningfully contradicts,
supersedes, or reframes the claim; false when it is
consistent or merely adjacent.
- confidence (number in [0,1]) — your confidence in the drifted verdict.
- reasoning (string, <=300 chars) — what in the evidence drove the verdict.
- suggested_weight (number in [0,1], optional) — where the take's weight should
move if drifted. Advisory only.
If the evidence is sparse or unrelated to the claim, return drifted=false with
low confidence.
TAKE:
Claim: {CLAIM}
Weight: {WEIGHT}
Page: {PAGE}
RECENT EVIDENCE (timeline entries on the same page):
{EVIDENCE_BLOCK}
`;
/**
* Parse the judge model's JSON output. Tolerant of fence wrapping and
* leading prose; returns null on unrecoverable parse failure.
*/
export function parseDriftOutput(raw: string): DriftVerdict | null {
if (!raw || raw.trim().length === 0) return null;
let text = raw.trim();
const fenced = text.match(/^```(?:json)?\s*\n?([\s\S]*?)\n?```$/);
if (fenced) text = (fenced[1] ?? '').trim();
const firstObj = text.indexOf('{');
if (firstObj === -1) return null;
let parsed: unknown;
try {
parsed = JSON.parse(text.slice(firstObj));
} catch {
return null;
}
if (typeof parsed !== 'object' || parsed === null) return null;
const r = parsed as Record<string, unknown>;
if (typeof r.drifted !== 'boolean') return null;
const confRaw = typeof r.confidence === 'number' ? r.confidence : Number.parseFloat(String(r.confidence ?? ''));
if (!Number.isFinite(confRaw)) return null;
const verdict: DriftVerdict = {
drifted: r.drifted,
confidence: Math.max(0, Math.min(1, confRaw)),
reasoning: typeof r.reasoning === 'string' ? r.reasoning.slice(0, 300) : '',
};
const sw = typeof r.suggested_weight === 'number' ? r.suggested_weight : NaN;
if (Number.isFinite(sw)) verdict.suggested_weight = Math.max(0, Math.min(1, sw));
return verdict;
}
/** Production judge — calls gateway.chat with the DRIFT_JUDGE_PROMPT. */
export async function defaultDriftJudge(input: {
candidate: DriftCandidate;
evidence: string;
modelHint?: string;
}): Promise<DriftVerdict> {
const { chat } = await import('../ai/gateway.ts');
const prompt = DRIFT_JUDGE_PROMPT
.replace('{CLAIM}', input.candidate.claim)
.replace('{WEIGHT}', String(input.candidate.weight))
.replace('{PAGE}', input.candidate.pageSlug)
.replace('{EVIDENCE_BLOCK}', input.evidence);
const result = await chat({
messages: [{ role: 'user', content: prompt }],
...(input.modelHint ? { model: input.modelHint } : {}),
maxTokens: 400,
});
const parsed = parseDriftOutput(result.text);
if (!parsed) {
// Failed parse — conservative no-drift at zero confidence so the row
// still surfaces in the report instead of disappearing silently.
return { drifted: false, confidence: 0, reasoning: 'judge_output_parse_failed' };
}
return parsed;
}
/**
* Cheap pre-LLM heuristic: takes that have substantial recent timeline
* evidence on the same page MAY have drifted. Surface them; the v0.29
* LLM judge will decide if the weight should move.
* evidence on the same page MAY have drifted. Surface them; the LLM judge
* decides.
*/
async function findDriftCandidates(
engine: BrainEngine,
lookbackDays: number,
): Promise<DriftCandidate[]> {
const cutoffMs = Date.now() - lookbackDays * 86_400_000;
const cutoffIso = new Date(cutoffMs).toISOString().slice(0, 10);
const cutoffIso = lookbackCutoffIso(lookbackDays);
// Only consider takes with weight in the "soft" middle band (0.3..0.85)
// — facts (1.0) don't drift, very-low hunches (<0.3) aren't actionable yet.
const rows = await engine.executeRaw<{
take_id: number; page_slug: string; row_num: number;
take_id: number; page_id: number; page_slug: string; row_num: number;
claim: string; weight: number; recent_evidence: number;
}>(`
SELECT t.id AS take_id, p.slug AS page_slug, t.row_num,
SELECT t.id AS take_id, p.id AS page_id, p.slug AS page_slug, t.row_num,
t.claim, t.weight,
(SELECT count(*)::int FROM timeline_entries te
WHERE te.page_id = p.id
@@ -92,6 +204,7 @@ async function findDriftCandidates(
.filter(r => Number(r.recent_evidence) >= 1)
.map(r => ({
takeId: Number(r.take_id),
pageId: Number(r.page_id),
pageSlug: String(r.page_slug),
rowNum: Number(r.row_num),
claim: String(r.claim),
@@ -100,6 +213,58 @@ async function findDriftCandidates(
}));
}
function lookbackCutoffIso(lookbackDays: number): string {
return new Date(Date.now() - lookbackDays * 86_400_000).toISOString().slice(0, 10);
}
/** Format the candidate page's recent timeline entries as judge evidence. */
async function loadEvidence(
engine: BrainEngine,
pageId: number,
cutoffIso: string,
): Promise<string> {
const rows = await engine.executeRaw<{ date: string; source: string; summary: string }>(
`SELECT date::text AS date, source, summary FROM timeline_entries
WHERE page_id = $1 AND date >= $2::date
ORDER BY date DESC LIMIT 12`,
[pageId, cutoffIso],
);
if (rows.length === 0) return '(no timeline entries in window)';
return rows.map(r => `- ${r.date} [${r.source}] ${r.summary}`).join('\n');
}
interface JudgedCandidate {
candidate: DriftCandidate;
verdict: DriftVerdict;
}
function buildReportBody(
judged: JudgedCandidate[],
cfg: DriftConfig,
modelId: string,
): string {
const drifted = judged.filter(j => j.verdict.drifted);
const lines: string[] = [
`Drift check over active soft-band takes (weight 0.3–0.85) with timeline`,
`evidence from the last ${cfg.lookbackDays} day(s). Judge model: ${modelId}.`,
``,
`**Report-only:** no takes were modified. \`dream.drift.auto_update\` is`,
`${cfg.autoUpdate ? 'set but ignored in v1 (report-only posture)' : 'off'}; review and adjust weights manually.`,
``,
`${drifted.length} of ${judged.length} judged take(s) look drifted.`,
``,
];
for (const { candidate: c, verdict: v } of judged) {
lines.push(`## ${v.drifted ? 'DRIFTED' : 'stable'} — ${c.pageSlug} (take #${c.rowNum})`);
lines.push(`- Claim: ${c.claim}`);
lines.push(`- Weight: ${c.weight}${v.suggested_weight !== undefined ? ` → suggested ${v.suggested_weight}` : ''}`);
lines.push(`- Confidence: ${v.confidence.toFixed(2)}`);
if (v.reasoning) lines.push(`- Reasoning: ${v.reasoning}`);
lines.push('');
}
return lines.join('\n');
}
function skipped(_reason: string, detail: string): DreamPhaseResult {
return { name: 'drift', status: 'skipped', detail, duration_ms: 0 };
}
@@ -110,7 +275,7 @@ export async function runPhaseDrift(
): Promise<DreamPhaseResult> {
const start = Date.now();
const config = await loadDriftConfig(engine);
if (!config.enabled) {
if (!config.enabled && !opts.forceEnabled) {
return skipped('not_configured', 'dream.drift.enabled is false');
}
@@ -125,9 +290,16 @@ export async function runPhaseDrift(
};
}
// Resolve model for the (future v0.29) LLM judge. For v0.28 we just
// surface the candidates — the meter call is a no-op when we don't actually
// submit, but resolveModel sets the right pricing key when v0.29 ships.
if (opts.dryRun) {
return {
name: 'drift',
status: 'skipped',
detail: `dry-run: ${Math.min(candidates.length, config.maxPerCycle)} of ${candidates.length} candidates would be evaluated`,
totals: { candidates: candidates.length },
duration_ms: Date.now() - start,
};
}
const modelId = await resolveModel(engine, {
configKey: 'models.drift',
deprecatedConfigKey: 'dream.drift.model',
@@ -139,27 +311,70 @@ export async function runPhaseDrift(
phase: 'drift',
auditPath: opts.auditPath,
});
const judge = opts.judge ?? defaultDriftJudge;
const cutoffIso = lookbackCutoffIso(config.lookbackDays);
// v0.28 scaffold: write a candidate report. v0.29 wires LLM-driven weight
// adjustment through autoUpdate. modelId + meter are wired now so the
// ledger captures the gate state even when we don't submit.
void modelId; void meter;
if (opts.dryRun) {
return {
name: 'drift',
status: 'skipped',
detail: `dry-run: ${candidates.length} candidates would be evaluated`,
totals: { candidates: candidates.length },
duration_ms: Date.now() - start,
};
const judged: JudgedCandidate[] = [];
let budgetExhausted = false;
let failed = 0;
for (const candidate of candidates.slice(0, config.maxPerCycle)) {
const check = meter.check({
modelId,
estimatedInputTokens: 1500,
maxOutputTokens: 400,
label: `drift:${candidate.pageSlug}#${candidate.rowNum}`,
});
if (!check.allowed) {
budgetExhausted = true;
break;
}
const evidence = await loadEvidence(engine, candidate.pageId, cutoffIso);
try {
const verdict = await judge({ candidate, evidence, modelHint: modelId });
judged.push({ candidate, verdict });
} catch (e) {
failed += 1;
process.stderr.write(`[drift] judge failed on take ${candidate.takeId}: ${(e as Error).message}\n`);
}
}
const driftedCount = judged.filter(j => j.verdict.drifted).length;
let reportSlug: string | undefined;
if (judged.length > 0) {
const date = new Date().toISOString().slice(0, 10);
reportSlug = `reports/drift-${date}`;
// Report-only v1: the report page is the ONLY write this phase makes.
// Lands in the default source (brain-global artifact, same-day re-runs
// upsert the same slug).
await engine.putPage(reportSlug, {
type: 'report',
title: `Drift report ${date}`,
compiled_truth: buildReportBody(judged, config, modelId),
});
}
const detail =
`judged ${judged.length}/${candidates.length} candidates: ${driftedCount} drifted` +
(reportSlug ? ` → ${reportSlug}` : '') +
(budgetExhausted ? ' (budget exhausted)' : '') +
(failed > 0 ? ` (${failed} judge failure(s))` : '') +
`. Cumulative cost: $${meter.totalSpent.toFixed(4)} / $${config.budgetUsd.toFixed(2)}` +
`. Report-only: auto_update=${config.autoUpdate} mutates nothing in v1.`;
return {
name: 'drift',
status: 'complete',
detail: `surfaced ${candidates.length} drift candidates (LLM judge: v0.29 follow-up). autoUpdate=${config.autoUpdate}`,
totals: { candidates: candidates.length },
status: judged.length > 0
? (budgetExhausted || failed > 0 ? 'partial' : 'complete')
// Zero judged: budget capped before any judge ran → partial (capped,
// not broken); otherwise every judge call failed → failed.
: (budgetExhausted ? 'partial' : 'failed'),
detail,
totals: {
candidates: candidates.length,
judged: judged.length,
drifted: driftedCount,
failed,
},
duration_ms: Date.now() - start,
};
}
src/core/cycle/extract-atoms.ts
+26 -3
View File
@@ -171,10 +171,20 @@ interface ExtractedAtom {
body: string;
source_quote?: string;
lesson?: string;
/**
* 1-3 kebab-case topic labels for concept clustering. Consumed by
* synthesize_concepts (groups atoms by `frontmatter.concepts`; only
* labels shared by >=2 atoms materialize a concept page, so the prompt
* biases reuse-over-coinage). #2123.
*/
concepts?: string[];
virality_score?: number;
emotional_register?: string;
}
/** kebab-case validator for concept labels ("captive-portal", "channel-pricing"). */
const CONCEPT_LABEL_RE = /^[a-z0-9]+(-[a-z0-9]+)*$/;
const EXTRACT_PROMPT = `You extract atomic content nuggets from a transcript.
An atom is a single-source, self-contained idea that could become a tweet,
@@ -185,12 +195,17 @@ quote, or short essay angle. Each atom must:
Output a JSON array of atoms (1-3 per transcript, never more than 3).
Each atom: {title (≤80 chars), atom_type, body (2-4 sentences),
source_quote (verbatim ≤200 chars), lesson (one sentence), virality_score
(0-100), emotional_register (one of: shocking, inspiring, funny, sobering,
practical, controversial)}.
source_quote (verbatim ≤200 chars), lesson (one sentence), concepts
(1-3 topic labels), virality_score (0-100), emotional_register (one of:
shocking, inspiring, funny, sobering, practical, controversial)}.
atom_type MUST be one of: ${ATOM_TYPES.join(', ')}.
concepts are kebab-case English TOPIC labels used to cluster atoms into
concept pages (e.g. "captive-portal", "channel-pricing-strategy") — never
entity or brand names. Use the same label for the same topic across atoms;
prefer a label you already used over coining a near-synonym.
Output ONLY the JSON array, no prose.`;
interface DiscoveredPage {
@@ -622,6 +637,7 @@ export async function runPhaseExtractAtoms(
source_hash: item.contentHash.slice(0, 16),
...(atom.source_quote && { source_quote: atom.source_quote }),
...(atom.lesson && { lesson: atom.lesson }),
...(atom.concepts && atom.concepts.length > 0 && { concepts: atom.concepts }),
...(atom.virality_score !== undefined && { virality_score: atom.virality_score }),
...(atom.emotional_register && { emotional_register: atom.emotional_register }),
extracted_at: new Date().toISOString(),
@@ -758,6 +774,13 @@ export function parseAtomsResponse(raw: string): ExtractedAtom[] {
body,
source_quote: typeof obj.source_quote === 'string' ? obj.source_quote.slice(0, 500) : undefined,
lesson: typeof obj.lesson === 'string' ? obj.lesson : undefined,
concepts: (() => {
if (!Array.isArray(obj.concepts)) return undefined;
const labels = obj.concepts
.filter((c): c is string => typeof c === 'string' && CONCEPT_LABEL_RE.test(c))
.slice(0, 3);
return labels.length > 0 ? labels : undefined;
})(),
virality_score:
typeof obj.virality_score === 'number' &&
obj.virality_score >= 0 &&
src/core/cycle/extract-facts.ts
+53 -17
View File
@@ -23,14 +23,24 @@
* page coordinate only; legacy NULL-source_markdown_slug rows survive
* because deleteFactsForPage targets source_markdown_slug = slug only.
*
* Empty-fence guard (Codex R2-#7): the phase refuses to do its
* destructive reconciliation pass when legacy rows (row_num IS NULL,
* entity_slug IS NOT NULL) still exist in the brain — they're the
* v0.31 hot-memory facts pending the v0_32_2 backfill. Status returns
* `warn` with a hint to run `gbrain apply-migrations --yes`. Without
* the guard, an interrupted upgrade where v0_32_2 hasn't run could
* leave the cycle silently misreporting "0 facts on people/alice"
* while legacy rows linger in the DB.
* Empty-fence guard (Codex R2-#7; #2484): the phase refuses to do its
* destructive reconciliation pass when genuinely-backfillable legacy
* rows still exist — `row_num IS NULL` (never fenced) AND `entity_slug`
* resolves to a live page in this source (so the v0_32_2 migration's
* Phase B could fence them). Status returns `warn` with a hint to run
* `gbrain apply-migrations --yes`. Without the guard, an interrupted
* upgrade where v0_32_2 hasn't run could leave the cycle silently
* misreporting "0 facts on people/alice" while legacy rows linger.
*
* The live-page requirement (#2484) is load-bearing: the inline facts
* writer keeps producing `row_num IS NULL, entity_slug IS NOT NULL`
* rows AFTER the migration completes, whenever a resolved slug has no
* fenceable page (slugify-floor / stub-guard-blocked unprefixed slugs).
* Those are structurally unfenceable — no page to fence onto, and the
* ledger-complete migration won't re-run — so they must NOT gate, or
* the phase jams forever (~16/day observed). Requiring a backing page
* keeps genuine pre-v0.32.2 rows (whose entity page exists) gating
* while excluding the inline-writer's permanent-unfenceable rows.
*/
import type { BrainEngine } from '../engine.ts';
@@ -163,22 +173,48 @@ export async function runExtractFacts(
phantomsMorePending: false,
};
// ── Empty-fence guard (Codex R2-#7) ────────────────────────────
// Pre-check: if any legacy fact rows exist (row_num NULL but
// entity_slug NOT NULL), refuse to run the destructive
// reconciliation pass. The v0_32_2 orchestrator must complete
// first.
// ── Empty-fence guard (Codex R2-#7; #2484) ─────────────────────
// Pre-check: if any genuinely-backfillable legacy fact rows exist,
// refuse to run the destructive reconciliation pass — the v0_32_2
// orchestrator must fence them first.
//
// A row is a real backfill candidate only when `row_num IS NULL`
// (never fenced) AND its `entity_slug` resolves to a LIVE page in
// this source (the migration's Phase B only fences rows whose
// entity_slug maps to a writable page). #2484: the original
// predicate was just `row_num IS NULL AND entity_slug IS NOT NULL`,
// which ALSO matched structurally-unfenceable hot-memory rows the
// inline writer keeps producing post-migration: the legacy DB-only
// fallback (backstop.ts) writes `entity_slug` (a resolved slug, e.g.
// a slugify-floor or stub-guard-blocked unprefixed slug like
// `people-jane-doe`) with `row_num` NULL whenever the slug has no
// fenceable page. Those rows can never satisfy the migration's exit
// condition (no page to fence onto, and `apply-migrations` is a
// ledger-complete no-op for them), so they jammed the phase forever
// — ~16/day, mislabeled "v0.31 pending backfill." We now require a
// live backing page, which both genuine pre-v0.32.2 rows (their
// entity page exists) satisfy and inline-writer unfenceable rows do
// not.
const legacy = await engine.executeRaw<{ n: string }>(
`SELECT COUNT(*) AS n FROM facts WHERE row_num IS NULL AND entity_slug IS NOT NULL`,
`SELECT COUNT(*) AS n
FROM facts f
WHERE f.row_num IS NULL
AND f.entity_slug IS NOT NULL
AND EXISTS (
SELECT 1 FROM pages p
WHERE p.source_id = f.source_id
AND p.slug = f.entity_slug
AND p.deleted_at IS NULL
)`,
);
const legacyCount = parseInt(legacy[0]?.n ?? '0', 10);
result.legacyRowsPending = legacyCount;
if (legacyCount > 0) {
result.guardTriggered = true;
result.warnings.push(
`extract_facts: ${legacyCount} legacy v0.31 fact rows pending fence backfill. ` +
`Run \`gbrain apply-migrations --yes\` to complete v0_32_2 before this phase ` +
`can safely reconcile fence → DB.`,
`extract_facts: ${legacyCount} legacy v0.31 fact rows (entity page present, not yet ` +
`fenced) pending fence backfill. Run \`gbrain apply-migrations --yes\` to complete ` +
`v0_32_2 before this phase can safely reconcile fence → DB.`,
);
return result;
}
src/core/cycle/propose-takes.ts
+99 -17
View File
@@ -39,11 +39,11 @@
import { randomUUID, createHash } from 'node:crypto';
import { BaseCyclePhase, type ScopedReadOpts, type BasePhaseOpts } from './base-phase.ts';
import { chat as gatewayChat, getChatModel } from '../ai/gateway.ts';
import { chat as gatewayChat, getChatModel, probeChatModel } from '../ai/gateway.ts';
import { normalizeModelId } from '../model-id.ts';
import { writeReceipt } from '../extract/receipt-writer.ts';
import { upsertExtractRollup } from '../extract/rollup-writer.ts';
import { GBrainError } from '../types.ts';
import type { Page, PageFilters } from '../types.ts';
import type { OperationContext } from '../operations.ts';
import type { BrainEngine } from '../engine.ts';
import type { PhaseStatus, CyclePhase } from '../cycle.ts';
@@ -160,6 +160,48 @@ export interface ProposeTakesResult {
warnings: string[];
}
/** Narrow projection of `pages` — the only columns this phase reads. */
interface ProposeTakesPageRow {
slug: string;
source_id: string;
compiled_truth: string | null;
}
/**
* Load proposal candidates with a narrow projection instead of
* `engine.listPages` (`SELECT p.*`). The phase only reads slug, source_id
* and compiled_truth — skipping timeline/frontmatter/title keeps large
* toasted columns out of the hot path. Scope precedence mirrors
* `sourceScopeOpts`: federated array (`sourceIds`) beats scalar
* (`sourceId`); ordering matches `PAGE_SORT_SQL.updated_desc` with an id
* tiebreak for determinism. (Takeover of PR #1979's projection by
* @shawnduggan.)
*/
async function listCandidatePages(
engine: BrainEngine,
scope: ScopedReadOpts,
limit: number,
): Promise<ProposeTakesPageRow[]> {
const where = ['deleted_at IS NULL'];
const params: unknown[] = [];
if (scope.sourceIds && scope.sourceIds.length > 0) {
params.push(scope.sourceIds);
where.push(`source_id = ANY($${params.length}::text[])`);
} else if (scope.sourceId) {
params.push(scope.sourceId);
where.push(`source_id = $${params.length}`);
}
params.push(limit);
return engine.executeRaw<ProposeTakesPageRow>(
`SELECT slug, source_id, compiled_truth
FROM pages
WHERE ${where.join(' AND ')}
ORDER BY updated_at DESC, id DESC
LIMIT $${params.length}`,
params,
);
}
/**
* Compute the content_hash key for the idempotency cache. SHA-256 of the
* page body suffices — page slug + prompt_version are separate columns in
@@ -257,6 +299,8 @@ export async function defaultExtractor(
export function parseExtractorOutput(raw: string): ProposedTake[] {
if (!raw || raw.trim().length === 0) return [];
let text = raw.trim();
// Strip <think>...</think> reasoning tags (MiniMax-M3, DeepSeek-R1, etc.).
text = text.replace(/<think>[\s\S]*?<\/think>/g, '').trim();
// Strip markdown code fence wrapper.
const fenced = text.match(/^```(?:json)?\s*\n?([\s\S]*?)\n?```$/);
if (fenced) text = (fenced[1] ?? '').trim();
@@ -269,7 +313,21 @@ export function parseExtractorOutput(raw: string): ProposedTake[] {
try {
parsed = JSON.parse(text.slice(start));
} catch {
return [];
// Fallback: truncate at last ] or } to handle trailing noise (e.g. leftover
// markdown fences after <think> stripping). Try array-closing first.
const sliced = text.slice(start);
const lastArr = sliced.lastIndexOf(']');
const lastObj = sliced.lastIndexOf('}');
const end = Math.max(lastArr, lastObj);
if (end > 0) {
try {
parsed = JSON.parse(sliced.slice(0, end + 1));
} catch {
return [];
}
} else {
return [];
}
}
const arr = Array.isArray(parsed) ? parsed : [parsed];
const out: ProposedTake[] = [];
@@ -330,6 +388,34 @@ class ProposeTakesPhase extends BaseCyclePhase {
const phaseStartMs = Date.now();
const proposalRunId = `propose-${new Date().toISOString().slice(0, 19).replace(/[-:T]/g, '')}-${randomUUID().slice(0, 8)}`;
const modelId = opts.model ?? getChatModel();
// With the default (gateway) extractor, skip cheaply when the resolved
// model's provider can't run — same probe semantics as patterns.ts /
// think/index.ts: unknown provider/model or Anthropic-without-key skips;
// other providers' auth surfaces lazily at chat() time. An injected
// extractor bypasses the gateway, so it is never gated. (Takeover of
// PR #1979's intent by @shawnduggan.)
if (!opts.extractor) {
const probe = probeChatModel(normalizeModelId(modelId));
if (!probe.ok) {
return {
summary: `propose_takes skipped: ${probe.detail}`,
details: {
reason: 'no_provider',
model: modelId,
pages_scanned: 0,
cache_hits: 0,
cache_misses: 0,
proposals_inserted: 0,
budget_exhausted: false,
warnings: [],
},
status: 'skipped',
};
}
}
const result: ProposeTakesResult = {
pages_scanned: 0,
cache_hits: 0,
@@ -340,19 +426,12 @@ class ProposeTakesPhase extends BaseCyclePhase {
};
// Load pages eligible for proposal. Source-scoped per BaseCyclePhase.
const pageFilters: PageFilters = {
...scope,
limit: pageLimit,
sort: 'updated_desc',
};
const pages: Page[] = await engine.listPages(pageFilters);
const pages = await listCandidatePages(engine, scope, pageLimit);
if (opts.reporter) {
opts.reporter.start('propose_takes.pages' as never, pages.length);
}
const modelId = opts.model ?? getChatModel();
for (const page of pages) {
// Phase deadline check. Break (not throw) so the phase returns a
// partial result with deadline_hit:true; work already banked stays.
@@ -421,16 +500,18 @@ class ProposeTakesPhase extends BaseCyclePhase {
continue;
}
// Write proposals to take_proposals. Each row is a separate INSERT
// because the composite idempotency key is on the per-page tuple — a
// bulk UPSERT would collapse a same-page-multi-claim run into one row.
// Write proposals to take_proposals. #2138: the idempotency key is
// per-CLAIM — take_proposals_idempotency_idx folds md5(claim_text) into
// the per-page tuple (migration v125), so a multi-claim page keeps every
// claim. RETURNING id prevents a repeated claim from inflating the count.
for (const p of proposals) {
await engine.executeRaw(
const inserted = await engine.executeRaw<{ id: number }>(
`INSERT INTO take_proposals
(source_id, page_slug, content_hash, prompt_version, proposal_run_id,
claim_text, kind, holder, weight, domain, dedup_against_fence_rows, model_id)
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12)
ON CONFLICT (source_id, page_slug, content_hash, prompt_version) DO NOTHING`,
ON CONFLICT (source_id, page_slug, content_hash, prompt_version, md5(claim_text)) DO NOTHING
RETURNING id`,
[
sourceId,
page.slug,
@@ -446,7 +527,7 @@ class ProposeTakesPhase extends BaseCyclePhase {
modelId,
],
);
result.proposals_inserted += 1;
result.proposals_inserted += inserted.length;
}
}
@@ -509,4 +590,5 @@ export const __testing = {
contentHash,
hasCompleteFence,
extractExistingTakesForDedup,
listCandidatePages,
};
src/core/cycle/synthesize.ts
+36 -9
View File
@@ -554,6 +554,8 @@ export async function runPhaseSynthesize(
const childIds: number[] = [];
/** Map child job_id → chunk metadata for D6 orchestrator-side slug rewrite. */
const chunkInfo = new Map<number, { idx: number; hash6: string }>();
/** #1978: map child job_id → source transcript path so written pages get a raw_source stamp. */
const jobRawSource = new Map<number, string>();
/** Skip reasons for the cycle report (D5 cap hits, D8 legacy-key skips). */
const skipReports: Array<{ filePath: string; reason: string }> = [];
@@ -638,6 +640,7 @@ export async function runPhaseSynthesize(
{ allowProtectedSubmit: true },
);
childIds.push(child.id);
jobRawSource.set(child.id, t.filePath);
if (isChunked) {
chunkInfo.set(child.id, { idx: i, hash6 });
}
@@ -682,7 +685,7 @@ export async function runPhaseSynthesize(
// (source, slug) row. #1586: refs are stamped with the cycle's resolved
// source (children write there via SubagentHandlerData.source_id).
const cycleSourceId = opts.sourceId ?? 'default';
const writtenRefs = await collectChildPutPageSlugs(engine, childIds, chunkInfo, cycleSourceId);
const writtenRefs = await collectChildPutPageSlugs(engine, childIds, chunkInfo, cycleSourceId, jobRawSource);
const summaryDate = opts.date ?? today();
@@ -1234,7 +1237,8 @@ async function collectChildPutPageSlugs(
childIds: number[],
chunkInfo: Map<number, { idx: number; hash6: string }>,
sourceId = 'default',
): Promise<Array<{ slug: string; source_id: string }>> {
jobRawSource?: Map<number, string>,
): Promise<Array<{ slug: string; source_id: string; raw_source?: string }>> {
if (childIds.length === 0) return [];
// Raw fetch — NO SELECT DISTINCT. Preserves per-child slug duplicates so
// the orchestrator sees what each child wrote. COALESCE handles both
@@ -1256,13 +1260,21 @@ async function collectChildPutPageSlugs(
AND status = 'complete'`,
[childIds],
);
const rewritten = new Set<string>();
// #1978: slug → source transcript path (first writer wins) so the
// provenance stamp can record WHERE the synthesized content came from.
const rewritten = new Map<string, string | undefined>();
for (const r of rows) {
if (typeof r.slug !== 'string' || r.slug.length === 0) continue;
const ci = chunkInfo.get(r.job_id);
rewritten.add(ci ? rewriteChunkedSlug(r.slug, ci.hash6, ci.idx) : r.slug);
const slug = ci ? rewriteChunkedSlug(r.slug, ci.hash6, ci.idx) : r.slug;
if (!rewritten.has(slug) || rewritten.get(slug) === undefined) {
rewritten.set(slug, jobRawSource?.get(r.job_id));
}
}
return Array.from(rewritten).sort().map(slug => ({ slug, source_id: sourceId }));
return Array.from(rewritten.keys()).sort().map(slug => {
const raw_source = rewritten.get(slug);
return { slug, source_id: sourceId, ...(raw_source ? { raw_source } : {}) };
});
}
/**
@@ -1308,12 +1320,12 @@ async function hasLegacySingleChunkCompletion(
*/
async function stampDreamProvenance(
engine: BrainEngine,
refs: Array<{ slug: string; source_id: string }>,
refs: Array<{ slug: string; source_id: string; raw_source?: string }>,
cycleDate: string,
): Promise<void> {
if (refs.length === 0) return;
const { executeRawJsonb } = await import('../sql-query.ts');
for (const { slug, source_id } of refs) {
for (const { slug, source_id, raw_source } of refs) {
try {
await executeRawJsonb(
engine,
@@ -1321,7 +1333,14 @@ async function stampDreamProvenance(
SET frontmatter = COALESCE(frontmatter, '{}'::jsonb) || $3::jsonb
WHERE slug = $1 AND source_id = $2`,
[slug, source_id],
[{ dream_generated: true, dream_cycle_date: cycleDate }],
// #1978 raw-source persistence: record the transcript path the
// synthesis was derived from, so `gbrain doctor` (raw_provenance
// check) can verify every generated page carries a raw trace.
[{
dream_generated: true,
dream_cycle_date: cycleDate,
...(raw_source ? { raw_source } : {}),
}],
);
} catch (e) {
const msg = e instanceof Error ? e.message : String(e);
@@ -1423,7 +1442,15 @@ async function writeSummaryPage(
// parseMarkdown below round-trips it into the DB-stored frontmatter, so the
// marker survives any later reverse-render of the summary page.
const fullMarkdown = serializeMarkdown(
{ dream_generated: true, dream_cycle_date: summaryDate } as Record<string, unknown>,
{
dream_generated: true,
dream_cycle_date: summaryDate,
// #1978: deterministic index page — no source document of its own;
// raw traces live on the listed pages. Explicit exemption keeps the
// doctor raw_provenance check quiet.
raw_trace_exempt: true,
raw_trace_exempt_reason: 'deterministic dream-cycle index; raw traces live on listed pages',
} as Record<string, unknown>,
body,
'',
{ type: 'note' as string, title: `Dream cycle ${summaryDate}`, tags: ['dream-cycle'] },
src/core/doctor-categories.ts
+1
View File
@@ -99,6 +99,7 @@ export const BRAIN_CHECK_NAMES: ReadonlySet<string> = new Set([
'orphan_ratio',
'oversized_pages',
'quarantined_pages',
'raw_provenance',
'flagged_pages',
'salience_health',
'scraper_junk_pages',
src/core/embedding-dim-check.ts
+16 -2
View File
@@ -32,6 +32,10 @@ import {
nvidiaEmbeddingDim,
nvidiaEmbeddingDimOptions,
supportsNvidiaEmbeddingDimension,
isPerplexityEmbeddingModel,
isValidPerplexityDim,
maxPerplexityEmbeddingDim,
PERPLEXITY_MIN_DIMS,
} from './ai/dims.ts';
/**
@@ -71,8 +75,9 @@ export function assertEmbeddingEnabled(cfg: { embedding_disabled?: boolean } | n
throw new EmbeddingDisabledError(
'This brain was initialized with `--no-embedding` (deferred setup).\n' +
'Configure an embedding provider before running embed / import:\n' +
' gbrain init --force --embedding-model <provider>:<model> # re-init to size schema\n' +
'(`gbrain config set embedding_model` is refused — schema-sizing fields are set at init.)\n',
' gbrain config set embedding_model <provider>:<model>\n' +
' gbrain config set embedding_dimensions <N>\n' +
' gbrain init --force --embedding-model <provider>:<model> # re-init to size schema\n',
);
}
}
@@ -461,6 +466,15 @@ function isCustomDimValidForProvider(
`(allowed: ${ZEROENTROPY_VALID_DIMS.join(', ')}).`,
};
}
if (recipe.id === 'perplexity' && isPerplexityEmbeddingModel(modelId)) {
if (isValidPerplexityDim(modelId, requestedDims)) return { valid: true, error: '' };
return {
valid: false,
error:
`Perplexity ${modelId} accepts dimensions ${PERPLEXITY_MIN_DIMS}..${maxPerplexityEmbeddingDim(modelId)}, ` +
`got ${requestedDims}.`,
};
}
if (recipe.id === 'openai' && isOpenAITextEmbedding3Model(modelId)) {
if (isValidOpenAITextEmbedding3Dim(modelId, requestedDims)) return { valid: true, error: '' };
const maxDim = maxOpenAITextEmbedding3Dim(modelId);
src/core/embedding-pricing.ts
+3
View File
@@ -44,6 +44,9 @@ export const EMBEDDING_PRICING: Record<string, EmbeddingPricing> = {
// Mistral (https://mistral.ai/pricing/api/, verified 2026-07-19)
'mistral:mistral-embed': { pricePerMTok: 0.10 },
'mistral:mistral-embed-2312': { pricePerMTok: 0.10 },
// Perplexity (https://docs.perplexity.ai/getting-started/pricing, verified 2026-07-21)
'perplexity:pplx-embed-v1-0.6b': { pricePerMTok: 0.004 },
'perplexity:pplx-embed-v1-4b': { pricePerMTok: 0.03 },
};
export type PriceLookupResult =
src/core/extract/receipt-writer.ts
+5
View File
@@ -157,6 +157,11 @@ function buildReceiptFrontmatter(input: ExtractReceiptInput): Record<string, unk
const fm: Record<string, unknown> = {
type: 'extract_receipt',
dream_generated: true,
// #1978: receipts record an operation, not a source document — the
// run_id/round fields ARE the provenance. Explicit exemption keeps the
// doctor raw_provenance check quiet.
raw_trace_exempt: true,
raw_trace_exempt_reason: 'operation receipt; provenance is run_id + round',
kind: input.kind,
source_id: input.source_id,
run_id: input.run_id,
src/core/link-extraction.ts
+36 -3
View File
@@ -960,8 +960,17 @@ export function makeResolver(
const hints = Array.isArray(dirHint) ? dirHint : (dirHint ? [dirHint] : []);
// Step 1: already a slug? (dir/name shape, lowercase, hyphenated)
if (/^[a-z][a-z0-9-]*\/[a-z0-9][a-z0-9-]*$/.test(trimmed)) {
// Step 1: already a slug? Try an exact page lookup for any slug-shaped
// value (contains '/', slug charset). Broadened beyond the original
// single-segment lowercase-leading form (`^[a-z][a-z0-9-]*\/[a-z0-9]...`)
// to also accept digit-leading folders (`90-people/nicolai`,
// `01-trading/...`) and nested paths (`a/b/c`) — common in PARA-numbered
// vaults. This is an EXACT getPage match only — no fuzzy — so it never
// produces a false positive; a non-existent slug just falls through to
// the steps below. Fixes frontmatter `related: [[dir/slug]]` values
// (unwrapped by unwrapWikilink) that name a real page the strict regex
// could not reach and whose full-path fuzzy score is below threshold.
if (/\//.test(trimmed) && /^[a-z0-9][a-z0-9/_-]*$/.test(trimmed)) {
const page = await engine.getPage(trimmed);
if (page) {
cache.set(cacheKey, trimmed);
@@ -1025,6 +1034,25 @@ export function makeResolver(
// ─── Frontmatter extractor ──────────────────────────────────────
/**
* Unwrap an Obsidian `[[wikilink]]` frontmatter value to its bare link
* target so the resolver (which expects bare titles / dir slugs) can match
* it. Mainstream Obsidian authors frontmatter links as `related: ["[[Page]]"]`;
* without this, the resolver treats the brackets as part of the value and a
* `[[90-people/nicolai]]` is normalized into `90peoplenicolai`, so it never
* resolves. Strips a trailing `|alias`, `#heading`, or `^block` suffix — the
* link target only. The regex is anchored to a wholly-wrapped value
* (`^\s*\[\[…\]\]\s*$`), so bare titles and any value not fully wrapped pass
* through unchanged and existing behavior is preserved exactly.
*/
export function unwrapWikilink(value: string): string {
const match = /^\s*\[\[(.+?)\]\]\s*$/.exec(value);
if (!match) return value;
// Take the link target: drop |alias, then #heading / ^block suffixes.
const target = match[1].split('|')[0].split('#')[0].split('^')[0];
return target.trim();
}
export interface UnresolvedFrontmatterRef {
/** The frontmatter field name. */
field: string;
@@ -1082,7 +1110,12 @@ export async function extractFrontmatterLinks(
}
if (!name) continue; // skip numbers, nulls, malformed objects
const resolved = await resolver.resolve(name, mapping.dirHint);
// Accept Obsidian `[[wikilink]]` values in frontmatter link fields by
// unwrapping to the bare target before resolution. Bare titles pass
// through unchanged; the original `name` is preserved for the
// unresolved report and edge context.
const linkTarget = unwrapWikilink(name);
const resolved = await resolver.resolve(linkTarget, mapping.dirHint);
if (!resolved) {
unresolved.push({ field, name });
continue;
src/core/migrate.ts
+14
View File
@@ -5710,6 +5710,20 @@ export const MIGRATIONS: Migration[] = [
`);
},
},
{
version: 125,
name: 'take_proposals_per_claim_idempotency',
// #2138: the original idempotency key was per page, so each INSERT after
// the first claim silently conflicted. md5(claim_text) makes it per claim
// without adding/backfilling a column. The new key is strictly finer than
// the old key and therefore preserves all existing rows.
idempotent: true,
sql: `
DROP INDEX IF EXISTS take_proposals_idempotency_idx;
CREATE UNIQUE INDEX IF NOT EXISTS take_proposals_idempotency_idx
ON take_proposals (source_id, page_slug, content_hash, prompt_version, md5(claim_text));
`,
},
];
export const LATEST_VERSION = MIGRATIONS.length > 0
src/core/minions/handler-timeouts.ts
+5
View File
@@ -24,6 +24,7 @@
*/
const THIRTY_MIN_MS = 30 * 60 * 1000;
const SIXTY_MIN_MS = 60 * 60 * 1000;
const TEN_MIN_MS = 10 * 60 * 1000;
/**
@@ -42,6 +43,10 @@ export const HANDLER_DEFAULT_TIMEOUT_MS: Readonly<Record<string, number>> = {
// few writes. Generous 10-min budget (vs the tight null-default) covers a
// slow gateway without the 30-min loop budget.
chronicle_extract: TEN_MIN_MS,
// Per-page contextual reindex jobs process chunks sequentially with one
// rate-leased LLM synopsis call per chunk; large transcript pages need more
// than the standard 30-min long-job budget.
contextual_reindex_per_chunk: SIXTY_MIN_MS,
};
/**
src/core/minions/handlers/contextual-reindex-per-chunk.ts
+31 -10
View File
@@ -40,6 +40,7 @@ import { UnrecoverableError } from '../types.ts';
import type { BrainEngine } from '../../engine.ts';
import {
reembedPageWithContextualRetrieval,
resolveContextualChunkConcurrency,
type ReembedPageResult,
} from '../../contextual-retrieval-service.ts';
import {
@@ -132,7 +133,7 @@ export function makeContextualReindexHandler(opts: MakeContextualReindexHandlerO
// call inside the service acquires/releases a lease against the
// shared key across all worker processes.
const maxConcurrent = resolveMaxConcurrent();
let currentLeaseId: number | null = null;
const chunkConcurrency = resolveContextualChunkConcurrency();
const result: ReembedPageResult = await reembedPageWithContextualRetrieval({
engine,
@@ -141,32 +142,32 @@ export function makeContextualReindexHandler(opts: MakeContextualReindexHandlerO
globalMode,
killSwitchDisabled,
abortSignal: ctx.signal,
chunkConcurrency,
acquireSynopsisLease: async () => {
// Poll-acquire with brief backoff. The service's per-chunk loop
// is sequential within a page; this guards against the cross-
// worker pile-up.
// is bounded within a page; this guards against the cross-worker
// pile-up and remains the global rate governor.
let attempts = 0;
const maxAttempts = 60; // ~1 min max wait per chunk before giving up
while (attempts < maxAttempts) {
if (ctx.signal.aborted) throw abortError();
const res = await acquireLease(engine, RATE_LEASE_KEY, ctx.id, maxConcurrent, {
ttlMs: 60_000,
});
if (res.acquired && res.leaseId != null) {
currentLeaseId = res.leaseId;
return;
return res.leaseId;
}
attempts++;
await new Promise((r) => setTimeout(r, 1000));
await sleepWithAbort(1000, ctx.signal);
}
throw new Error(
`Failed to acquire ${RATE_LEASE_KEY} lease after ${maxAttempts} attempts; ` +
`Haiku rate limit pile-up too deep.`,
);
},
releaseSynopsisLease: async () => {
if (currentLeaseId != null) {
await releaseLease(engine, currentLeaseId);
currentLeaseId = null;
releaseSynopsisLease: async (lease) => {
if (typeof lease === 'number') {
await releaseLease(engine, lease);
}
},
});
@@ -218,6 +219,26 @@ async function tryLoadPageAcrossSources(
return null;
}
function sleepWithAbort(ms: number, signal: AbortSignal): Promise<void> {
return new Promise((resolve, reject) => {
if (signal.aborted) {
reject(abortError());
return;
}
const timer = setTimeout(resolve, ms);
signal.addEventListener('abort', () => {
clearTimeout(timer);
reject(abortError());
}, { once: true });
});
}
function abortError(): Error {
const err = new Error('aborted');
err.name = 'AbortError';
return err;
}
function classifyResult(
pageSlug: string,
result: ReembedPageResult,
src/core/minions/handlers/subagent.ts
+5 -1
View File
@@ -48,6 +48,7 @@ import {
logSubagentHeartbeat,
} from './subagent-audit.ts';
import { resolveModel, isAnthropicProvider, TIER_DEFAULTS } from '../../model-config.ts';
import { resolveAnthropicKey } from '../../ai/anthropic-key.ts';
import { buildSystemPrompt, DEFAULT_SUBAGENT_SYSTEM } from '../system-prompt.ts';
import { toolLoop as gatewayToolLoop } from '../../ai/gateway.ts';
import type { ChatToolDef, ChatMessage, ChatBlock, ChatResult, ToolHandler } from '../../ai/gateway.ts';
@@ -186,7 +187,10 @@ export function makeSubagentHandler(deps: SubagentDeps) {
// lives at sdk.messages.create. Assigning sdk.messages directly gets the
// right object; JS method-call semantics preserve `this` at the call
// site (subagent.ts invokes client.create(...) with client === sdk.messages).
const makeAnthropic = deps.makeAnthropic ?? (() => new Anthropic());
// Resolve the key env-first, then config (anthropic_api_key) — a bare
// new Anthropic() only reads env, so launchd/MCP workers whose key lives
// in the gbrain config file would fail auth (#2048).
const makeAnthropic = deps.makeAnthropic ?? (() => new Anthropic({ apiKey: resolveAnthropicKey() }));
const client: MessagesClient = deps.client ?? makeAnthropic().messages;
const config = deps.config ?? loadConfig() ?? ({ engine: 'postgres' } as GBrainConfig);
const rateLeaseKey = deps.rateLeaseKey ?? DEFAULT_RATE_KEY;
src/core/minions/queue.ts
+16 -1
View File
@@ -133,12 +133,27 @@ export class MinionQueue {
// 1. Idempotency fast path — if a row already exists for this key, return it
// without doing any other work. The unique partial index guarantees
// no second row can be inserted with the same non-null key.
//
// Dead/cancelled jobs represent permanently-failed work whose
// idempotency slot must be freed so a fresh attempt can be inserted.
// We NULL the key (preserving the row for audit) and fall through
// to the INSERT path below.
if (opts?.idempotency_key) {
const existing = await tx.executeRaw<Record<string, unknown>>(
`SELECT * FROM minion_jobs WHERE idempotency_key = $1`,
[opts.idempotency_key]
);
if (existing.length > 0) return rowToMinionJob(existing[0]);
if (existing.length > 0) {
const existingJob = rowToMinionJob(existing[0]);
if (existingJob.status === 'dead' || existingJob.status === 'cancelled') {
await tx.executeRaw(
`UPDATE minion_jobs SET idempotency_key = NULL WHERE id = $1`,
[existingJob.id]
);
} else {
return existingJob;
}
}
}
// 1b. Submission-time backpressure for high-frequency named jobs.
src/core/minions/supervisor.ts
+1 -8
View File
@@ -87,10 +87,6 @@ export interface SupervisorOpts {
* resolveDefaultMaxRssMb() (issue #1678) instead of a flat default.
* Set to 0 to spawn the worker without a watchdog. */
maxRssMb: number;
/** Worker stall-lock window in ms (issue #1014), passed to the spawned
* worker as `--lock-duration N`. The wall-clock dead-letter cap is
* lockDuration × max_stalled. Undefined → worker default (30000ms). */
lockDuration?: number;
/** Niceness (issue #1815) the operator requested via `--nice` / `GBRAIN_NICE`,
* or undefined to inherit. When set, the worker is spawned with `--nice N` so
* it re-applies the value (the supervisor itself is reniced by the CLI layer,
@@ -176,7 +172,7 @@ const DEFAULTS: Omit<SupervisorOpts, 'cliPath'> = {
* niceness also inherits to the worker's own children automatically.
*/
export function buildWorkerArgs(
opts: Pick<SupervisorOpts, 'concurrency' | 'queue' | 'maxRssMb' | 'nice_requested' | 'lockDuration'>,
opts: Pick<SupervisorOpts, 'concurrency' | 'queue' | 'maxRssMb' | 'nice_requested'>,
): string[] {
const args = [
'jobs', 'work',
@@ -189,9 +185,6 @@ export function buildWorkerArgs(
if (opts.nice_requested !== undefined) {
args.push('--nice', String(opts.nice_requested));
}
if (opts.lockDuration !== undefined) {
args.push('--lock-duration', String(opts.lockDuration));
}
return args;
}
src/core/oauth-provider.ts
+61
View File
@@ -24,6 +24,7 @@ import type { OAuthRegisteredClientsStore } from '@modelcontextprotocol/sdk/serv
import type { AuthInfo as SdkAuthInfo } from '@modelcontextprotocol/sdk/server/auth/types.js';
import { InvalidTokenError, InvalidClientMetadataError } from '@modelcontextprotocol/sdk/server/auth/errors.js';
import { hashToken, generateToken, isUndefinedColumnError } from './utils.ts';
import { assertValidSourceId } from './source-id.ts';
import { hasScope, assertAllowedScopes, parseScopeString, InvalidScopeError } from './scope.ts';
import type { AuthInfo as CoreAuthInfo } from './operations.ts';
import { parseLegacyTokenScope } from './legacy-token-scope.ts';
@@ -1006,6 +1007,66 @@ export class GBrainOAuthProvider implements OAuthServerProvider {
return { clientId, clientSecret };
}
/**
* v0.42.x (#1914): admin-gated rescope for an existing OAuth client.
*
* DCR clients self-register with source_id='default' +
* federated_read=['default'] and MUST NOT be able to widen their own
* scope (fail-closed trust). This is the trusted-operator surface that
* changes it afterward: `gbrain auth rescope-client` (local CLI) and
* POST /admin/api/rescope-client (requireAdmin) both route here.
*
* Omitted fields are left untouched (COALESCE). Takes effect on the
* client's NEXT request even for already-issued tokens, because
* verifyAccessToken re-reads oauth_clients on every verification.
*/
async rescopeClient(
clientId: string,
opts: { sourceId?: string; federatedRead?: string[] },
): Promise<{ clientId: string; clientName: string; sourceId: string; federatedRead: string[] }> {
const { sourceId, federatedRead } = opts;
if (sourceId === undefined && federatedRead === undefined) {
throw new Error('rescope-client requires --source and/or --federated-read');
}
if (sourceId !== undefined) assertValidSourceId(sourceId);
if (federatedRead !== undefined) {
if (federatedRead.length === 0) {
throw new Error('--federated-read cannot be empty (pass at least one source id)');
}
for (const s of federatedRead) assertValidSourceId(s);
}
let rows: Record<string, unknown>[];
try {
rows = await this.sql`
UPDATE oauth_clients
SET source_id = COALESCE(${sourceId ?? null}::text, source_id),
federated_read = COALESCE(${federatedRead ? pgArray(federatedRead) : null}::text[], federated_read)
WHERE client_id = ${clientId}
RETURNING client_id, client_name, source_id, federated_read
`;
} catch (err) {
if (isUndefinedColumnError(err, 'source_id') || isUndefinedColumnError(err, 'federated_read')) {
throw new Error('rescope-client requires an up-to-date OAuth schema; run `gbrain apply-migrations --yes` and retry.');
}
// FK oauth_clients.source_id → sources(id): translate the raw 23503
// into an actionable message.
if ((err as { code?: string })?.code === '23503') {
throw new Error(`Source "${sourceId}" does not exist. Create it first: gbrain sources add ${sourceId} ...`);
}
throw err;
}
if (rows.length === 0) {
throw new Error(`No OAuth client found with id "${clientId}"`);
}
const row = rows[0];
return {
clientId: row.client_id as string,
clientName: (row.client_name as string | null) ?? '',
sourceId: (row.source_id as string | null) ?? 'default',
federatedRead: Array.isArray(row.federated_read) ? (row.federated_read as string[]) : [],
};
}
// -------------------------------------------------------------------------
// Internal: Issue access + optional refresh tokens
// -------------------------------------------------------------------------
src/core/operations.ts
+48 -30
View File
@@ -433,20 +433,25 @@ export interface OperationContext {
*/
sourceId: string;
/**
* #2561 — federated read scope for UNQUALIFIED local CLI reads.
* #2561 / #3242 — federated read scope for UNQUALIFIED reads.
*
* Set ONLY by the local CLI's context builder (src/cli.ts makeContext), and
* only when the source resolved via a non-explicit tier (local_path /
* brain_default / sole_non_default / seed_default — NOT --source, NOT
* GBRAIN_SOURCE, NOT a .gbrain-source dotfile). Contains the resolved
* source first, then every other `config.federated = true` source, so an
* unqualified `gbrain search "X"` spans federated sources as
* docs/guides/multi-source-brains.md promises.
* Set ONLY by trusted server-side context builders — never from caller
* params — and only when the caller carries no explicit source scope:
* - local CLI (src/cli.ts makeContext) when the source resolved via a
* non-explicit tier (local_path / brain_default / sole_non_default /
* seed_default — NOT --source, NOT GBRAIN_SOURCE, NOT a dotfile);
* - stdio MCP (src/mcp/server.ts) when GBRAIN_SOURCE is unset;
* - HTTP MCP (src/mcp/http-transport.ts) for legacy bearer tokens with
* NO operator-set `permissions.source_id` grant (the historical
* 'default' floor). Tokens WITH an explicit grant never widen.
*
* Consumed exclusively by `federatedSearchScope` and ONLY when
* `ctx.remote === false` — a remote caller's scope stays governed by
* `ctx.auth.allowedSources` / scalar `ctx.sourceId` (source-isolation
* invariant, fail-closed).
* Contains the resolved source first, then every other
* `config.federated = true` source, so an unqualified read/search spans
* federated sources as docs/guides/multi-source-brains.md promises.
*
* Consumed exclusively by `federatedSearchScope`. Fail-closed remains:
* a grant (`ctx.auth.allowedSources`) or a per-call `source_id` always
* wins, and a context without this field never widens.
*/
localFederatedSourceIds?: string[];
}
@@ -565,25 +570,26 @@ export function resolveRequestedScope(
}
/**
* #2561 — source scope for the search-shaped read ops (`search`, `query`).
* #2561 / #3242 — source scope for the page-visibility read ops (`search`,
* `query`, `get_page`, `list_pages`, `resolve_slugs`).
*
* Delegates to `resolveRequestedScope` (the single trust+grant resolver), then
* widens an UNQUALIFIED trusted-local scalar scope to the CLI-computed
* federated set (`ctx.localFederatedSourceIds`, resolved source first). This is
* what makes `sources add --federated` mean something for local search: a
* federated source participates in unqualified `gbrain search "X"` results.
* widens an UNQUALIFIED scalar scope to the transport-computed federated set
* (`ctx.localFederatedSourceIds`, resolved source first). This is what makes
* `sources add --federated` mean something: a federated source participates in
* unqualified reads (#3242 — pages ingested into a `federated: true` source
* were invisible to get_page/search/list_pages while resolve_slugs leaked them).
*
* The expansion NEVER applies when:
* - the caller is not strictly trusted-local (`ctx.remote !== false`) —
* remote scope stays grant-governed (fail-closed source isolation);
* - a per-call `source_id` was passed (explicit wins, including `__all__`);
* - the resolver already produced a federated array (OAuth grant);
* - the CLI resolved the source from an explicit signal (--source / env /
* dotfile) — makeContext leaves `localFederatedSourceIds` unset then.
* - the resolver already produced a federated array (OAuth grant governs);
* - the transport didn't populate `localFederatedSourceIds` (see that
* field's doc: it is only set for callers with NO explicit source scope,
* and never from caller-controlled params — so trust stays fail-closed).
*
* Deliberately NOT inside `sourceScopeOpts`: code-intel ops collapse a
* multi-element scope to an error (`resolveCodeIntelScope`), and non-search
* reads (get_page, get_links, …) keep their long-standing scalar behavior.
* multi-element scope to an error (`resolveCodeIntelScope`), and the remaining
* scalar reads (get_links, get_chunks, …) keep their long-standing behavior.
*/
export function federatedSearchScope(
ctx: OperationContext,
@@ -591,7 +597,6 @@ export function federatedSearchScope(
): { sourceId?: string; sourceIds?: string[] } {
const scope = resolveRequestedScope(ctx, sourceIdParam);
if (
ctx.remote === false &&
sourceIdParam === undefined &&
scope.sourceId !== undefined &&
scope.sourceIds === undefined &&
@@ -748,7 +753,9 @@ const get_page: Operation = {
// with a federated `allowedSources` grant (and no single ctx.sourceId) got
// an UNSCOPED exact lookup — a cross-source read of any page by slug. getPage
// now honors sourceIds[] (both engines), so the same scope closes both paths.
const sourceOpts = sourceScopeOpts(ctx);
// #3242: federatedSearchScope (not bare sourceScopeOpts) so an unqualified
// read sees pages in `federated: true` sources, matching search/query.
const sourceOpts = federatedSearchScope(ctx);
const fuzzyScope = sourceOpts;
let page = await ctx.engine.getPage(slug, { includeDeleted, ...sourceOpts });
@@ -1503,7 +1510,9 @@ const list_pages: Operation = {
// enumerate src-B pages. Pre-fix, ctx.sourceId / ctx.auth?.allowedSources
// were ignored at this op handler and the engine returned every source's
// pages indiscriminately.
const scope = sourceScopeOpts(ctx);
// #3242: federatedSearchScope so unqualified listing spans federated
// sources (same visibility set as search / get_page). Grants still win.
const scope = federatedSearchScope(ctx);
const pages = await ctx.engine.listPages({
type: p.type as any,
tag: p.tag as string,
@@ -2737,7 +2746,11 @@ const resolve_slugs: Operation = {
partial: { type: 'string', required: true },
},
handler: async (ctx, p) => {
return ctx.engine.resolveSlugs(p.partial as string);
// #3242: was fully UNSCOPED — the one read that leaked every source's
// slugs to any caller (the reporter's "resolve_slugs sees them but
// get_page doesn't" matrix). Route through the same visibility set as
// get_page/search: grant > federated set > scalar source.
return ctx.engine.resolveSlugs(p.partial as string, federatedSearchScope(ctx));
},
scope: 'read',
};
@@ -3823,7 +3836,7 @@ const whoami: Operation = {
name: 'whoami',
description:
'Introspect the calling identity. Returns one of three transport shapes: ' +
'{transport: "oauth", client_id, client_name, scopes, expires_at}, ' +
'{transport: "oauth", client_id, client_name, scopes, expires_at, source_id, federated_read}, ' +
'{transport: "legacy", token_name, scopes, expires_at: null}, or ' +
'{transport: "local", scopes: []}, or {transport: "stdio", scopes: []} ' +
'for the auth-less stdio MCP pipe. Throws unknown_transport when the ' +
@@ -3865,6 +3878,10 @@ const whoami: Operation = {
client_name: ctx.auth.clientName ?? ctx.auth.clientId,
scopes: ctx.auth.scopes,
expires_at: ctx.auth.expiresAt ?? null,
// Read-only self-introspection of the token's source grants —
// widens nothing; absent grants serialize fail-closed (null / []).
source_id: ctx.auth.sourceId ?? null,
federated_read: ctx.auth.allowedSources ?? [],
};
}
return {
@@ -4719,7 +4736,8 @@ const list_schema_packs: Operation = {
const { existsSync, readdirSync } = await import('node:fs');
const { join } = await import('node:path');
const { gbrainPath } = await import('./config.ts');
const bundled = ['gbrain-base', 'gbrain-recommended'];
const { BUNDLED_PACK_NAMES } = await import('./schema-pack/bundled.ts');
const bundled = [...BUNDLED_PACK_NAMES];
const installedDir = gbrainPath('schema-packs');
const installed: string[] = [];
if (existsSync(installedDir)) {
src/core/pglite-engine.ts
+20 -5
View File
@@ -2312,6 +2312,18 @@ export class PGLiteEngine implements BrainEngine {
const params: unknown[] = [];
let paramIdx = 1;
// Provenance fallback for chunks without an explicit `model`: resolve the
// gateway's runtime model, not the compile-time DEFAULT_EMBEDDING_MODEL.
// See postgres-engine.ts _upsertChunksOnce for the full rationale — pglite
// mirrors it for parity.
let resolvedModel: string = DEFAULT_EMBEDDING_MODEL;
try {
const gw = await import('./ai/gateway.ts');
resolvedModel = gw.getEmbeddingModel() || resolvedModel;
} catch {
// Gateway unconfigured (unit tests / pre-connect): keep the default.
}
for (const chunk of chunks) {
const embeddingStr = chunk.embedding
? '[' + Array.from(chunk.embedding).join(',') + ']'
@@ -2344,7 +2356,7 @@ export class PGLiteEngine implements BrainEngine {
if (embeddingImageStr) params.push(embeddingImageStr);
params.push(
pageId, chunk.chunk_index, chunk.chunk_text, chunk.chunk_source,
chunk.model || DEFAULT_EMBEDDING_MODEL, chunk.token_count || null,
chunk.model || resolvedModel, chunk.token_count || null,
chunk.language || null, chunk.symbol_name || null, chunk.symbol_type || null,
chunk.start_line ?? null, chunk.end_line ?? null,
parentPath, chunk.doc_comment || null, chunk.symbol_name_qualified || null,
@@ -4835,11 +4847,11 @@ export class PGLiteEngine implements BrainEngine {
const { rows } = await this.db.query(
`SELECT t.id AS take_id, t.page_id, p.slug AS page_slug, t.row_num,
t.claim, t.kind, t.holder, t.weight,
similarity(t.claim, $1)::real AS score
word_similarity($1, t.claim)::real AS score
FROM takes t
JOIN pages p ON p.id = t.page_id
WHERE t.active
AND t.claim % $1
AND $1 <% t.claim
AND ($2::text[] IS NULL OR t.holder = ANY($2::text[]))
AND ($4::text[] IS NULL OR p.source_id = ANY($4::text[]))
AND ($5::text IS NULL OR p.source_id = $5::text)
@@ -5261,7 +5273,7 @@ export class PGLiteEngine implements BrainEngine {
// dashboard, v0.10.3 metrics give entity-page-level granularity.
const { rows: [h] } = await this.db.query(`
WITH entity_pages AS (
SELECT id, slug FROM pages WHERE type IN ('person', 'company')
SELECT id, slug FROM pages WHERE type IN ('entity', 'person', 'company')
)
SELECT
(SELECT count(*) FROM pages) as page_count,
@@ -5289,7 +5301,7 @@ export class PGLiteEngine implements BrainEngine {
SELECT p.slug,
(SELECT count(*) FROM links l WHERE l.from_page_id = p.id OR l.to_page_id = p.id)::int as link_count
FROM pages p
WHERE p.type IN ('person', 'company')
WHERE p.type IN ('entity', 'person', 'company')
ORDER BY link_count DESC
LIMIT 5
`);
@@ -6007,6 +6019,9 @@ export class PGLiteEngine implements BrainEngine {
);
}
// Exclude dream/synthesize-generated pages (parity with postgres-engine).
where.push(`(p.frontmatter ->> 'dream_generated') IS DISTINCT FROM 'true'`);
const orderKey = ENRICH_ORDER_SQL[opts.order] ? opts.order : 'inbound-links';
const orderBy = ENRICH_ORDER_SQL[orderKey];
src/core/pglite-lock.ts
+31 -16
View File
@@ -124,6 +124,35 @@ function isProcessAlive(pid: number): boolean {
}
}
function formatLockTimestamp(value: unknown): string {
return typeof value === 'number' && Number.isFinite(value)
? new Date(value).toISOString()
: 'unknown time';
}
function pgliteLockTimeoutError(lockDir: string): Error {
const lockPath = join(lockDir, LOCK_FILE);
try {
const lockData = JSON.parse(readFileSync(lockPath, 'utf-8'));
const pid = String(lockData.pid ?? 'unknown');
const command = String(lockData.command ?? 'unknown');
const serveHint = command.includes('gbrain serve')
? ' The holder looks like `gbrain serve`, so this is probably serve↔sync contention from an MCP/HTTP server; stop that server/client and rerun the command.'
: '';
return new Error(
`GBrain: Timed out waiting for PGLite data-dir lock. Process ${pid} has held it since ${formatLockTimestamp(lockData.acquired_at)} (command: ${command}). ` +
`Lock directory: ${lockDir}. If that process is dead, remove the lock directory and try again. ` +
`This is a PGLite data-dir lock, not the \`gbrain-sync:*\` advisory lock; \`gbrain sync --break-lock\` will not clear a live PGLite holder.` +
serveHint,
);
} catch {
return new Error(
`GBrain: Timed out waiting for PGLite lock. Remove ${lockDir} and try again.`
);
}
}
/**
* Attempt to acquire an exclusive lock on the PGLite data directory.
* Returns { acquired: true } if the lock was obtained, { acquired: false } otherwise.
@@ -206,28 +235,14 @@ export async function acquireLock(dataDir: string | undefined, opts?: { timeoutM
// mkdir failed — someone else grabbed it between our check and mkdir
// This is fine, we'll retry
if (Date.now() - startTime >= timeoutMs) {
// Timeout — report which process holds the lock
const lockPath = join(lockDir, LOCK_FILE);
try {
const lockData = JSON.parse(readFileSync(lockPath, 'utf-8'));
throw new Error(
`GBrain: Timed out waiting for PGLite lock. Process ${lockData.pid} has held it since ${new Date(lockData.acquired_at).toISOString()} (command: ${lockData.command}). ` +
`If that process is dead, remove ${lockDir} and try again.`
);
} catch (readErr) {
if (readErr instanceof Error && readErr.message.startsWith('GBrain')) throw readErr;
throw new Error(
`GBrain: Timed out waiting for PGLite lock. Remove ${lockDir} and try again.`
);
}
throw pgliteLockTimeoutError(lockDir);
}
// Brief wait before retry
await new Promise(r => setTimeout(r, 500));
}
}
// Should not reach here, but just in case
throw new Error(`GBrain: Timed out waiting for PGLite lock.`);
throw pgliteLockTimeoutError(lockDir);
}
/**
src/core/pglite-schema.ts
+1 -1
View File
@@ -777,7 +777,7 @@ CREATE TABLE IF NOT EXISTS take_proposals (
predicted_brier_bucket_n INTEGER
);
CREATE UNIQUE INDEX IF NOT EXISTS take_proposals_idempotency_idx
ON take_proposals (source_id, page_slug, content_hash, prompt_version);
ON take_proposals (source_id, page_slug, content_hash, prompt_version, md5(claim_text));
CREATE INDEX IF NOT EXISTS take_proposals_pending_idx
ON take_proposals (source_id, status, proposed_at DESC)
WHERE status = 'pending';
src/core/postgres-engine.ts
+29 -5
View File
@@ -2438,6 +2438,23 @@ export class PostgresEngine implements BrainEngine {
const params: unknown[] = [];
let paramIdx = 1;
// Provenance fallback for chunks that don't carry an explicit `model`:
// resolve the model the gateway ACTUALLY uses at runtime, not the
// compile-time DEFAULT_EMBEDDING_MODEL constant. Callers like `embed`
// build ChunkInputs without a `model` field (src/commands/embed.ts), so
// the old `chunk.model || DEFAULT_EMBEDDING_MODEL` fallback stamped the
// hardcoded default (e.g. zeroentropyai:zembed-1) onto rows whose vectors
// were produced by a different, config-resolved model — corrupting the
// provenance that signature-drift staleness + dim-migration logic trust.
// Mirrors the resolve-then-fallback pattern used for schema sizing above.
let resolvedModel: string = DEFAULT_EMBEDDING_MODEL;
try {
const gw = await import('./ai/gateway.ts');
resolvedModel = gw.getEmbeddingModel() || resolvedModel;
} catch {
// Gateway unconfigured (unit tests / pre-connect): keep the default.
}
for (const chunk of chunks) {
const embeddingStr = chunk.embedding
? '[' + Array.from(chunk.embedding).join(',') + ']'
@@ -2467,7 +2484,7 @@ export class PostgresEngine implements BrainEngine {
if (embeddingImageStr) params.push(embeddingImageStr);
params.push(
pageId, chunk.chunk_index, chunk.chunk_text, chunk.chunk_source,
chunk.model || DEFAULT_EMBEDDING_MODEL, chunk.token_count || null,
chunk.model || resolvedModel, chunk.token_count || null,
chunk.language || null, chunk.symbol_name || null, chunk.symbol_type || null,
chunk.start_line ?? null, chunk.end_line ?? null,
parentPath, chunk.doc_comment || null, chunk.symbol_name_qualified || null,
@@ -4962,11 +4979,11 @@ export class PostgresEngine implements BrainEngine {
const rows = await sql`
SELECT t.id AS take_id, t.page_id, p.slug AS page_slug, t.row_num,
t.claim, t.kind, t.holder, t.weight,
similarity(t.claim, ${query})::real AS score
word_similarity(${query}, t.claim)::real AS score
FROM takes t
JOIN pages p ON p.id = t.page_id
WHERE t.active
AND t.claim % ${query}
AND ${query} <% t.claim
AND (
${opts.takesHoldersAllowList ?? null}::text[] IS NULL
OR t.holder = ANY(${opts.takesHoldersAllowList ?? null}::text[])
@@ -5364,7 +5381,7 @@ export class PostgresEngine implements BrainEngine {
// dashboard health.
const [h] = await sql`
WITH entity_pages AS (
SELECT id, slug FROM pages WHERE type IN ('person', 'company')
SELECT id, slug FROM pages WHERE type IN ('entity', 'person', 'company')
)
SELECT
(SELECT count(*) FROM pages) as page_count,
@@ -5389,7 +5406,7 @@ export class PostgresEngine implements BrainEngine {
SELECT p.slug,
(SELECT count(*) FROM links l WHERE l.from_page_id = p.id OR l.to_page_id = p.id)::int as link_count
FROM pages p
WHERE p.type IN ('person', 'company')
WHERE p.type IN ('entity', 'person', 'company')
ORDER BY link_count DESC
LIMIT 5
`;
@@ -6293,6 +6310,12 @@ export class PostgresEngine implements BrainEngine {
)`
: sql``;
// Exclude dream/synthesize-generated pages (reflections, originals, cycle
// logs carrying frontmatter dream_generated:true). enrich develops ENTITY
// stubs; running it on a generated essay/log creates circular self-citation
// and drops the H1. IS DISTINCT FROM 'true' keeps NULL/'false' rows.
const dreamCondition = sql`AND (p.frontmatter ->> 'dream_generated') IS DISTINCT FROM 'true'`;
// Whitelisted ORDER BY (no injection — enum maps to a literal fragment).
const orderKey = ENRICH_ORDER_SQL[opts.order] ? opts.order : 'inbound-links';
const orderBy = sql.unsafe(ENRICH_ORDER_SQL[orderKey]);
@@ -6316,6 +6339,7 @@ export class PostgresEngine implements BrainEngine {
AND (char_length(p.compiled_truth) + char_length(COALESCE(p.timeline, ''))) < ${threshold}
${sourceCondition}
${recencyCondition}
${dreamCondition}
ORDER BY ${orderBy}
LIMIT ${limit}
`;
src/core/schema-embedded.ts
+3 -4
View File
@@ -1273,9 +1273,8 @@ CREATE INDEX IF NOT EXISTS calibration_profiles_published_idx
ON calibration_profiles (source_id, published, holder)
WHERE published = true;
-- take_proposals: propose_takes phase queue. Idempotency cache via the
-- composite unique index (source_id, page_slug, content_hash, prompt_version)
-- mirrors v0.23 dream_verdicts. proposal_run_id supports --rollback by run.
-- take_proposals: per-claim idempotency via source/page/content/prompt plus
-- md5(claim_text). The old per-page key silently dropped claim #2+ (#2138).
CREATE TABLE IF NOT EXISTS take_proposals (
id BIGSERIAL PRIMARY KEY,
source_id TEXT NOT NULL REFERENCES sources(id) ON DELETE CASCADE,
@@ -1301,7 +1300,7 @@ CREATE TABLE IF NOT EXISTS take_proposals (
predicted_brier_bucket_n INTEGER
);
CREATE UNIQUE INDEX IF NOT EXISTS take_proposals_idempotency_idx
ON take_proposals (source_id, page_slug, content_hash, prompt_version);
ON take_proposals (source_id, page_slug, content_hash, prompt_version, md5(claim_text));
CREATE INDEX IF NOT EXISTS take_proposals_pending_idx
ON take_proposals (source_id, status, proposed_at DESC)
WHERE status = 'pending';
src/core/schema-pack/bundled.ts
+24
View File
@@ -0,0 +1,24 @@
// Bundled schema-pack registry — single source of truth for the packs that
// ship in src/core/schema-pack/base/. Keep every bundled-pack consumer
// (CLI/MCP inspection, active-pack loading, mutation guards, upgrade
// discovery) on this one list so they cannot drift.
//
// v0.39 T8 — gbrain-base + gbrain-recommended.
// v0.41 T4 — lens packs: creator, investor, engineer, everything (meta-pack).
// v0.42 type-unification — gbrain-base-v2, the 15-type canonical successor.
export const BUNDLED_PACK_NAMES = [
'gbrain-base',
'gbrain-recommended',
'gbrain-creator',
'gbrain-investor',
'gbrain-engineer',
'gbrain-everything',
'gbrain-base-v2',
] as const;
export type BundledPackName = typeof BUNDLED_PACK_NAMES[number];
export function isBundledPackName(name: string): name is BundledPackName {
return (BUNDLED_PACK_NAMES as readonly string[]).includes(name);
}
src/core/schema-pack/load-active.ts
+2 -22
View File
@@ -37,6 +37,7 @@ import {
type ResolutionInput,
type ResolutionResult,
} from './registry.ts';
import { isBundledPackName } from './bundled.ts';
/**
* Inputs the caller (operations.ts handler / engine query path) provides.
@@ -92,28 +93,7 @@ export function _resetPackLocatorForTests(): void {
* throwing UnknownPackError with a paste-ready install hint.
*/
function defaultPackLocator(name: string): string | null {
// v0.39 T8 — bundled packs registry. gbrain-base + gbrain-recommended
// ship in src/core/schema-pack/base/. Add a new entry here to bundle
// additional canonical packs.
//
// v0.41 T4 — lens packs join the bundle: creator (atoms + concepts +
// extract_atoms/synthesize_concepts phases), investor (theses + bet
// resolution + 3 calibration domains), engineer (gstack-learnings bridge
// + 3 calibration domains), everything (meta-pack stacking all three
// via extends + borrow_from). Each ships as a real YAML at base/<name>.yaml.
const BUNDLED: ReadonlyArray<string> = [
'gbrain-base',
'gbrain-recommended',
'gbrain-creator',
'gbrain-investor',
'gbrain-engineer',
'gbrain-everything',
// v0.42 type-unification: 15-type canonical successor to gbrain-base.
// Ships as install default (Lane E T17) + via gbrain onboard pack
// upgrade flow (the unify-types Minion handler).
'gbrain-base-v2',
];
if (BUNDLED.includes(name)) {
if (isBundledPackName(name)) {
// Resolve bundled YAML relative to this source file. Works in both
// direct-bun execution and bun --compile binaries.
const here = dirname(fileURLToPath(import.meta.url));
src/core/schema-pack/loader.ts
+31
View File
@@ -159,6 +159,29 @@ export function parseYamlMini(content: string): unknown {
return parseMapping(baseIndent);
}
function parseBlockScalar(parentIndent: number, folded: boolean): string {
const contentIndent = parentIndent + 2;
const out: string[] = [];
while (i < lines.length) {
const raw = lines[i];
// Inside a block scalar everything is literal content — '#' is NOT a
// comment here, so use the raw line (no stripComment / isBlank).
if (raw.trim() === '') {
out.push('');
i++;
continue;
}
const indent = indentOf(raw);
if (indent <= parentIndent) break;
out.push(raw.slice(Math.min(contentIndent, indent)));
i++;
}
if (folded) {
return out.join(' ').replace(/\s+$/u, '');
}
return out.join('\n').replace(/\n+$/u, '');
}
function parseSequence(baseIndent: number): unknown[] {
const result: unknown[] = [];
while (i < lines.length) {
@@ -227,6 +250,10 @@ export function parseYamlMini(content: string): unknown {
i++;
if (rest2 === '') {
map[key2] = parseBlock(nextIndent + 2);
} else if (rest2 === '|' || rest2 === '|-' || rest2 === '|+') {
map[key2] = parseBlockScalar(nextIndent, false);
} else if (rest2 === '>' || rest2 === '>-' || rest2 === '>+') {
map[key2] = parseBlockScalar(nextIndent, true);
} else {
map[key2] = parseScalar(rest2);
}
@@ -257,6 +284,10 @@ export function parseYamlMini(content: string): unknown {
i++;
if (rest === '') {
result[key] = parseBlock(indent + 2);
} else if (rest === '|' || rest === '|-' || rest === '|+') {
result[key] = parseBlockScalar(indent, false);
} else if (rest === '>' || rest === '>-' || rest === '>+') {
result[key] = parseBlockScalar(indent, true);
} else {
result[key] = parseScalar(rest);
}
src/core/schema-pack/mutate.ts
+2 -1
View File
@@ -65,6 +65,7 @@ import { invalidateQueryCache } from './query-cache-invalidator.ts';
import { logMutationFailure, logMutationSuccess, type MutationActor, type MutationOp } from './mutate-audit.ts';
import { runFilePlaneLintRules } from './lint-rules.ts';
import { withPackLock, type PackLockOpts } from './pack-lock.ts';
import { BUNDLED_PACK_NAMES as BUNDLED_PACK_NAME_LIST } from './bundled.ts';
import type { BrainEngine } from '../engine.ts';
export type PackFileFormat = 'json' | 'yaml';
@@ -93,7 +94,7 @@ export class SchemaPackMutationError extends Error {
}
}
export const BUNDLED_PACK_NAMES = new Set(['gbrain-base', 'gbrain-recommended', 'gbrain-base-v2']);
export const BUNDLED_PACK_NAMES = new Set<string>(BUNDLED_PACK_NAME_LIST);
export interface MutateResult {
/** Pack name that was mutated. */
src/core/source-resolver.ts
+27
View File
@@ -160,6 +160,33 @@ export async function resolveSourceId(
return 'default';
}
/**
* Engine-free tiers (1-3) of the resolution chain: explicit flag →
* GBRAIN_SOURCE env → .gbrain-source dotfile walk. Used by the thin-client
* CLI path (#2098), which has no local engine to run tiers 4-6 or
* assertSourceExists against — the remote server enforces existence + grant.
* Returns null when no engine-free tier fires.
*/
export function resolveSourceIdEngineFree(
explicit: string | null | undefined,
cwd: string = process.cwd(),
): string | null {
if (explicit) {
if (!SOURCE_ID_RE.test(explicit)) {
throw new Error(`Invalid --source value "${explicit}". Must match [a-z0-9-]{1,32}.`);
}
return explicit;
}
const env = process.env.GBRAIN_SOURCE;
if (env && env.length > 0) {
if (!SOURCE_ID_RE.test(env)) {
throw new Error(`Invalid GBRAIN_SOURCE value "${env}". Must match [a-z0-9-]{1,32}.`);
}
return env;
}
return readDotfileWalk(cwd);
}
/**
* Returns the id of the SINGLE registered non-default source with a
* local_path, when exactly one such row exists. Returns null when:
src/core/storage/supabase.ts
+8 -1
View File
@@ -195,7 +195,14 @@ export class SupabaseStorage implements StorageBackend {
throw new Error(`Supabase signed URL failed: ${res.status} ${body}`);
}
const result = await res.json() as { signedURL: string };
return `${this.projectUrl}${result.signedURL}`;
// Supabase returns `signedURL` relative to the Storage API root, e.g.
// "/object/sign/<bucket>/<path>?token=...". Prepend projectUrl + "/storage/v1"
// (not just projectUrl) or the link 404s. Tolerate an already-absolute URL or a
// value that already carries the /storage/v1 prefix.
const signed = result.signedURL;
if (/^https?:\/\//.test(signed)) return signed;
if (signed.startsWith('/storage/v1')) return `${this.projectUrl}${signed}`;
return `${this.projectUrl}/storage/v1${signed.startsWith('/') ? '' : '/'}${signed}`;
}
async getUrl(path: string): Promise<string> {
src/core/write-through.ts
+5 -1
View File
@@ -174,7 +174,11 @@ export async function writePageThrough(
}
}
mkdirSync(dirname(filePath), { recursive: true });
// On Bun + Windows, mkdirSync(dir, { recursive: true }) can still throw
// EEXIST when the directory already exists (POSIX no-ops it). That aborts
// the put_page / enrich / capture write-through whenever the prefix dir
// already exists, silently leaving the DB and the .md file plane out of sync.
if (!existsSync(dir)) mkdirSync(dir, { recursive: true });
// Atomic write: unique temp sibling + rename. Unique name (pid + random)
// so two concurrent saves to the same target can't clobber each other's
src/mcp/dispatch.ts
+8
View File
@@ -54,6 +54,13 @@ export interface DispatchOpts {
* resolves it from the per-token allow-list (eE3).
*/
sourceId?: string;
/**
* #3242: federated read set for callers with NO explicit source scope
* (stdio without GBRAIN_SOURCE; legacy HTTP tokens without an operator-set
* `permissions.source_id` grant). Transport-computed, never derived from
* caller params. See OperationContext.localFederatedSourceIds.
*/
localFederatedSourceIds?: string[];
/**
* v0.31 (eD3): hook called by the dispatcher AFTER op.handler succeeds
* to compute `_meta.brain_hot_memory` for the response. Wrapped in its
@@ -216,6 +223,7 @@ export function buildOperationContext(
// CLI / HTTP / stdio transports SHOULD pass an explicit sourceId via opts;
// this fallback covers code paths that historically passed undefined.
sourceId: opts.sourceId ?? 'default',
...(opts.localFederatedSourceIds ? { localFederatedSourceIds: opts.localFederatedSourceIds } : {}),
auth: opts.auth,
};
}
src/mcp/http-transport.ts
+22
View File
@@ -84,6 +84,14 @@ interface AuthResult {
* Bounded to the stored grant — never widened to "all".
*/
auth?: AuthInfo;
/**
* #3242: true when the token row carries an operator-set
* `permissions.source_id` (string OR array — even a malformed one, which
* fails closed to 'default' without widening). false = the historical
* no-grant 'default' floor; ONLY that case gets the federated read set
* (config.federated sources) threaded as localFederatedSourceIds.
*/
hasSourceGrant?: boolean;
}
/* Legacy token source-scope parsing lives in core/legacy-token-scope.ts and is
@@ -229,6 +237,9 @@ export async function startHttpTransport(opts: HttpTransportOptions) {
// source unless the token carries an explicit grant (#1336 above).
sourceId,
auth,
// #3242: distinguish "operator granted a scope" from "historical
// no-grant floor" — only the latter widens to federated sources.
hasSourceGrant: perms?.source_id != null,
};
} catch {
return { ok: false };
@@ -379,10 +390,21 @@ export async function startHttpTransport(opts: HttpTransportOptions) {
// takes_search / query (when it returns takes) can server-side filter.
// v0.34.1 (#861): thread source-isolation scope. Legacy access_tokens
// path defaults to 'default' per AuthResult.sourceId above.
// #3242: a token with NO operator-set source grant reads across the
// federated set (config.federated sources), not just the scalar
// 'default' floor. Granted tokens (hasSourceGrant) never widen.
let localFederated: string[] | undefined;
if (auth.hasSourceGrant === false && auth.sourceId) {
try {
const { localFederatedSourceIds } = await import('../core/source-resolver.ts');
localFederated = await localFederatedSourceIds(engine, auth.sourceId, 'seed_default');
} catch { /* scalar scope stands */ }
}
const result = await dispatchToolCall(engine, toolName, args, {
remote: true,
takesHoldersAllowList: auth.takesHoldersAllowList,
sourceId: auth.sourceId,
...(localFederated ? { localFederatedSourceIds: localFederated } : {}),
// #1336: thread the token's federated_read grant so read ops scope
// to the operator-granted sources via sourceScopeOpts.
auth: auth.auth,
src/mcp/server.ts
+15
View File
@@ -35,6 +35,20 @@ export async function startMcpServer(engine: BrainEngine) {
// shape and cast through `any` (the SDK accepts it via the ServerResult union).
server.setRequestHandler(CallToolRequestSchema, async (request: any): Promise<any> => {
const { name, arguments: params } = request.params;
// #3242: when the operator didn't pin a source via GBRAIN_SOURCE, stdio
// reads span every `config.federated = true` source (same visibility set
// as unqualified local CLI reads). GBRAIN_SOURCE set = explicit scope,
// no widening. Best-effort: a resolver failure keeps the scalar scope.
// ponytail: one tiny SELECT per tool call; cache it if it ever shows up.
let localFederated: string[] | undefined;
try {
const { localFederatedSourceIds } = await import('../core/source-resolver.ts');
localFederated = await localFederatedSourceIds(
engine,
process.env.GBRAIN_SOURCE || 'default',
process.env.GBRAIN_SOURCE ? 'env' : 'seed_default',
);
} catch { /* scalar scope stands */ }
// v0.28: stdio MCP has no per-token auth (local pipe). Default the
// takes-holder allow-list to ['world'] so agent-facing callers don't
// see private hunches via takes_list / takes_search / query. Operators
@@ -51,6 +65,7 @@ export async function startMcpServer(engine: BrainEngine) {
// Operators who want a different source on stdio MCP should set
// GBRAIN_SOURCE in the env or use --source via `gbrain call`.
sourceId: process.env.GBRAIN_SOURCE || 'default',
...(localFederated ? { localFederatedSourceIds: localFederated } : {}),
// v0.31 (eD3): _meta.brain_hot_memory injection so Claude Desktop /
// Code see the brain's relevant hot memory automatically alongside
// every tool-call response. Best-effort; absorbs errors.
src/schema.sql
+3 -4
View File
@@ -1269,9 +1269,8 @@ CREATE INDEX IF NOT EXISTS calibration_profiles_published_idx
ON calibration_profiles (source_id, published, holder)
WHERE published = true;
-- take_proposals: propose_takes phase queue. Idempotency cache via the
-- composite unique index (source_id, page_slug, content_hash, prompt_version)
-- mirrors v0.23 dream_verdicts. proposal_run_id supports --rollback by run.
-- take_proposals: per-claim idempotency via source/page/content/prompt plus
-- md5(claim_text). The old per-page key silently dropped claim #2+ (#2138).
CREATE TABLE IF NOT EXISTS take_proposals (
id BIGSERIAL PRIMARY KEY,
source_id TEXT NOT NULL REFERENCES sources(id) ON DELETE CASCADE,
@@ -1297,7 +1296,7 @@ CREATE TABLE IF NOT EXISTS take_proposals (
predicted_brier_bucket_n INTEGER
);
CREATE UNIQUE INDEX IF NOT EXISTS take_proposals_idempotency_idx
ON take_proposals (source_id, page_slug, content_hash, prompt_version);
ON take_proposals (source_id, page_slug, content_hash, prompt_version, md5(claim_text));
CREATE INDEX IF NOT EXISTS take_proposals_pending_idx
ON take_proposals (source_id, status, proposed_at DESC)
WHERE status = 'pending';
test/ai/anthropic-key.test.ts
+33 -1
View File
@@ -10,7 +10,7 @@ import { mkdtempSync, mkdirSync, writeFileSync, rmSync } from 'node:fs';
import { tmpdir } from 'node:os';
import { join } from 'node:path';
import { withEnv } from '../helpers/with-env.ts';
import { hasAnthropicKey } from '../../src/core/ai/anthropic-key.ts';
import { hasAnthropicKey, resolveAnthropicKey } from '../../src/core/ai/anthropic-key.ts';
const tmpDirs: string[] = [];
function freshHome(withConfig?: Record<string, unknown>): string {
@@ -62,3 +62,35 @@ describe('hasAnthropicKey', () => {
);
});
});
describe('resolveAnthropicKey (#2048 — subagent config-key auth)', () => {
test('env wins over config', async () => {
const home = freshHome({ anthropic_api_key: 'sk-from-config' });
await withEnv(
{ ANTHROPIC_API_KEY: 'sk-from-env', GBRAIN_HOME: home, DATABASE_URL: undefined, GBRAIN_DATABASE_URL: undefined },
async () => {
expect(resolveAnthropicKey()).toBe('sk-from-env');
},
);
});
test('config key returned when env unset', async () => {
const home = freshHome({ anthropic_api_key: 'sk-from-config' });
await withEnv(
{ ANTHROPIC_API_KEY: undefined, GBRAIN_HOME: home, DATABASE_URL: undefined, GBRAIN_DATABASE_URL: undefined },
async () => {
expect(resolveAnthropicKey()).toBe('sk-from-config');
},
);
});
test('neither → undefined', async () => {
const home = freshHome();
await withEnv(
{ ANTHROPIC_API_KEY: undefined, GBRAIN_HOME: home, DATABASE_URL: undefined, GBRAIN_DATABASE_URL: undefined },
async () => {
expect(resolveAnthropicKey()).toBeUndefined();
},
);
});
});
test/ai/dims-openai.test.ts
+27
View File
@@ -134,3 +134,30 @@ describe('dimsProviderOptions — OpenAI on openai-compatible adapter (Azure cas
expect(JSON.stringify(opts)).not.toContain('input_type');
});
});
describe('dimsProviderOptions — prefixed model IDs (OpenRouter / proxy providers)', () => {
test('openai/text-embedding-3-large at 1536d returns dimensions=1536', () => {
const opts = dimsProviderOptions('openai-compatible', 'openai/text-embedding-3-large', 1536);
expect(opts).toEqual({ openaiCompatible: { dimensions: 1536 } });
});
test('openai/text-embedding-3-small at 768d returns dimensions=768', () => {
const opts = dimsProviderOptions('openai-compatible', 'openai/text-embedding-3-small', 768);
expect(opts).toEqual({ openaiCompatible: { dimensions: 768 } });
});
test('openai/text-embedding-3-large at 5000d throws AIConfigError', () => {
expect(() => dimsProviderOptions('openai-compatible', 'openai/text-embedding-3-large', 5000))
.toThrow(AIConfigError);
});
test('error message preserves full prefixed model ID for clarity', () => {
try {
dimsProviderOptions('openai-compatible', 'openai/text-embedding-3-large', 5000);
throw new Error('should have thrown');
} catch (err) {
expect(err).toBeInstanceOf(AIConfigError);
expect((err as Error).message).toContain('openai/text-embedding-3-large');
}
});
});
test/ai/recipe-dashscope-rerank.test.ts
+79
View File
@@ -0,0 +1,79 @@
/**
* dashscope-rerank recipe smoke.
*
* Sibling of recipe-llama-server-reranker.test.ts. Pins the recipe shape so:
* - id + tier + implementation + base_url stay byte-stable
* - reranker touchpoint declares the PLURAL `/reranks` leaf (the whole
* reason this recipe exists — DashScope's compatible-api surface 404s
* on singular `/rerank`) + `default_timeout_ms`
* - only live-verified models are listed (gte-rerank-v2 is native-API only
* and rejected by the OpenAI-compat surface)
*/
import { describe, expect, test } from 'bun:test';
import { getRecipe } from '../../src/core/ai/recipes/index.ts';
import { defaultResolveAuth } from '../../src/core/ai/gateway.ts';
import { AIConfigError } from '../../src/core/ai/errors.ts';
describe('recipe: dashscope-rerank', () => {
test('registered with expected shape', () => {
const r = getRecipe('dashscope-rerank');
expect(r).toBeDefined();
expect(r!.id).toBe('dashscope-rerank');
expect(r!.tier).toBe('openai-compat');
expect(r!.implementation).toBe('openai-compatible');
expect(r!.base_url_default).toBe(
'https://dashscope-intl.aliyuncs.com/compatible-api/v1',
);
expect(r!.auth_env?.required).toEqual(['DASHSCOPE_API_KEY']);
});
test('declares reranker touchpoint with PLURAL /reranks path + timeout', () => {
const r = getRecipe('dashscope-rerank')!;
const tp = r.touchpoints.reranker;
expect(tp).toBeDefined();
expect(tp!.path).toBe('/reranks');
expect(tp!.default_timeout_ms).toBe(30_000);
expect(tp!.max_payload_bytes).toBe(5_000_000);
});
test('base_url + path concatenation produces /v1/reranks, NOT /v1/v1/…', () => {
const r = getRecipe('dashscope-rerank')!;
const combined =
r.base_url_default!.replace(/\/$/, '') + (r.touchpoints.reranker!.path ?? '/models/rerank');
expect(combined).toBe('https://dashscope-intl.aliyuncs.com/compatible-api/v1/reranks');
expect(combined).not.toContain('/v1/v1/');
expect(combined.endsWith('/reranks')).toBe(true);
});
test('lists only the live-verified compat-surface model', () => {
const r = getRecipe('dashscope-rerank')!;
const tp = r.touchpoints.reranker!;
expect(tp.models).toEqual(['qwen3-rerank']);
expect(tp.default_model).toBe('qwen3-rerank');
// gte-rerank-v2 is native-API only; the compat surface rejects it.
expect(tp.models).not.toContain('gte-rerank-v2');
});
test('default auth: DASHSCOPE_API_KEY set → Bearer token', () => {
const r = getRecipe('dashscope-rerank')!;
const auth = defaultResolveAuth(
r,
{ DASHSCOPE_API_KEY: 'sk-dashscope-fake' },
'reranker',
);
expect(auth.headerName).toBe('Authorization');
expect(auth.token).toBe('Bearer sk-dashscope-fake');
});
test('default auth: missing DASHSCOPE_API_KEY → AIConfigError', () => {
const r = getRecipe('dashscope-rerank')!;
expect(() => defaultResolveAuth(r, {}, 'reranker')).toThrow(AIConfigError);
});
test('does not perturb the sibling dashscope embedding recipe', () => {
const emb = getRecipe('dashscope')!;
expect(emb.base_url_default).toBe('https://dashscope-intl.aliyuncs.com/compatible-mode/v1');
expect(emb.touchpoints.reranker).toBeUndefined();
});
});
test/ai/recipe-perplexity.test.ts
+142
View File
@@ -0,0 +1,142 @@
/**
* #1046 — Perplexity hosted embeddings (pplx-embed-v1-*).
*
* Covers the three seams the recipe touches:
* - recipe registration + auth (PERPLEXITY_API_KEY only, never OPENAI_API_KEY)
* - flexible-dim validation (128..native max) in dims.ts + the init
* preflight (resolveSchemaEmbeddingDim), incl. the >2000-dim 4b case
* - perplexityCompatFetch: forces encoding_format=base64_int8 outbound and
* decodes the base64 int8 embedding payload to number[] inbound
*/
import { afterEach, describe, expect, test } from 'bun:test';
import {
dimsProviderOptions,
isPerplexityEmbeddingModel,
isValidPerplexityDim,
maxPerplexityEmbeddingDim,
} from '../../src/core/ai/dims.ts';
import { getRecipe, RECIPES } from '../../src/core/ai/recipes/index.ts';
import { perplexity } from '../../src/core/ai/recipes/perplexity.ts';
import { defaultResolveAuth, perplexityCompatFetch } from '../../src/core/ai/gateway.ts';
import { AIConfigError } from '../../src/core/ai/errors.ts';
import { resolveSchemaEmbeddingDim } from '../../src/core/embedding-dim-check.ts';
import { lookupEmbeddingPrice } from '../../src/core/embedding-pricing.ts';
describe('recipe: perplexity', () => {
test('registered as an OpenAI-compatible embedding provider', () => {
expect(RECIPES.has('perplexity')).toBe(true);
expect(getRecipe('perplexity')).toBe(perplexity);
expect(perplexity.tier).toBe('openai-compat');
expect(perplexity.implementation).toBe('openai-compatible');
expect(perplexity.base_url_default).toBe('https://api.perplexity.ai/v1');
const e = perplexity.touchpoints.embedding!;
expect(e.models).toEqual(['pplx-embed-v1-0.6b', 'pplx-embed-v1-4b']);
expect(e.default_dims).toBe(1024);
expect(e.max_batch_tokens).toBe(120_000);
});
test('auth is PERPLEXITY_API_KEY bearer — no OPENAI_API_KEY fallback', () => {
expect(perplexity.resolveAuth).toBeUndefined();
expect(perplexity.auth_env?.required).toEqual(['PERPLEXITY_API_KEY']);
expect(defaultResolveAuth(perplexity, { PERPLEXITY_API_KEY: 'fake-pplx' }, 'embedding')).toEqual({
headerName: 'Authorization',
token: 'Bearer fake-pplx',
});
// An OPENAI_API_KEY in the env must NOT satisfy Perplexity auth.
expect(() => defaultResolveAuth(perplexity, { OPENAI_API_KEY: 'sk-test' }, 'embedding')).toThrow(AIConfigError);
});
test('dims: 128..native-max range per model', () => {
expect(isPerplexityEmbeddingModel('pplx-embed-v1-4b')).toBe(true);
expect(maxPerplexityEmbeddingDim('pplx-embed-v1-4b')).toBe(2560);
expect(maxPerplexityEmbeddingDim('pplx-embed-v1-0.6b')).toBe(1024);
expect(isValidPerplexityDim('pplx-embed-v1-4b', 2560)).toBe(true);
expect(isValidPerplexityDim('pplx-embed-v1-4b', 128)).toBe(true);
expect(isValidPerplexityDim('pplx-embed-v1-4b', 64)).toBe(false);
expect(isValidPerplexityDim('pplx-embed-v1-0.6b', 2560)).toBe(false);
});
test('dimsProviderOptions emits native `dimensions`, fails loud out of range', () => {
expect(dimsProviderOptions('openai-compatible', 'pplx-embed-v1-4b', 2560)).toEqual({
openaiCompatible: { dimensions: 2560 },
});
// Symmetric provider — inputType never emitted.
expect(dimsProviderOptions('openai-compatible', 'pplx-embed-v1-4b', 1024, 'query')).toEqual({
openaiCompatible: { dimensions: 1024 },
});
expect(() => dimsProviderOptions('openai-compatible', 'pplx-embed-v1-0.6b', 2560)).toThrow(AIConfigError);
});
test('init preflight accepts the 4b model at its native 2560 dims (halfvec territory)', () => {
const res = resolveSchemaEmbeddingDim({
embedding_model: 'perplexity:pplx-embed-v1-4b',
embedding_dimensions: 2560,
});
expect(res).toEqual({
ok: true,
dim: 2560,
model: 'perplexity:pplx-embed-v1-4b',
provider: 'perplexity',
recipeDefault: 1024,
});
const bad = resolveSchemaEmbeddingDim({
embedding_model: 'perplexity:pplx-embed-v1-4b',
embedding_dimensions: 4096,
});
expect(bad.ok).toBe(false);
});
test('embedding pricing table knows both models', () => {
expect(lookupEmbeddingPrice('perplexity:pplx-embed-v1-4b')).toMatchObject({ kind: 'known', pricePerMTok: 0.03 });
expect(lookupEmbeddingPrice('perplexity:pplx-embed-v1-0.6b')).toMatchObject({ kind: 'known', pricePerMTok: 0.004 });
});
});
describe('perplexityCompatFetch — int8 wire shim', () => {
const realFetch = globalThis.fetch;
afterEach(() => {
globalThis.fetch = realFetch;
});
test('forces encoding_format=base64_int8 outbound and decodes int8 base64 inbound', async () => {
const int8 = new Int8Array([3, -7, 127, -128]);
const b64 = Buffer.from(int8.buffer).toString('base64');
let sentBody: any;
globalThis.fetch = (async (_input: any, init?: RequestInit) => {
sentBody = JSON.parse(init!.body as string);
return new Response(
JSON.stringify({
object: 'list',
model: 'pplx-embed-v1-4b',
data: [{ object: 'embedding', index: 0, embedding: b64 }],
usage: { prompt_tokens: 4, total_tokens: 4 },
}),
{ status: 200, headers: { 'content-type': 'application/json' } },
);
}) as any;
const resp = await (perplexityCompatFetch as any)('https://api.perplexity.ai/v1/embeddings', {
method: 'POST',
headers: { 'content-type': 'application/json' },
// The AI SDK sends encoding_format:'float' — Perplexity rejects it.
body: JSON.stringify({ model: 'pplx-embed-v1-4b', input: ['hi'], encoding_format: 'float', dimensions: 4 }),
});
expect(sentBody.encoding_format).toBe('base64_int8');
expect(sentBody.dimensions).toBe(4); // native field, untouched
const json = await resp.json();
expect(json.data[0].embedding).toEqual([3, -7, 127, -128]);
expect(json.usage.prompt_tokens).toBe(4);
});
test('non-JSON and error responses pass through untouched', async () => {
globalThis.fetch = (async () =>
new Response('nope', { status: 401, headers: { 'content-type': 'text/plain' } })) as any;
const resp = await (perplexityCompatFetch as any)('https://api.perplexity.ai/v1/embeddings', {
method: 'POST',
body: JSON.stringify({ model: 'pplx-embed-v1-4b', input: ['hi'] }),
});
expect(resp.status).toBe(401);
expect(await resp.text()).toBe('nope');
});
});
test/apply-migrations.test.ts
+35
View File
@@ -167,6 +167,41 @@ describe('buildPlan — diff against completed + installed VERSION', () => {
});
});
describe('force-retry escape hatch', () => {
test("complete then retry-latest → pending and buildPlan lists the version as pending", () => {
const idx = indexCompleted([
{ version: '0.11.0', status: 'complete' },
{ version: '0.11.0', status: 'retry' },
]);
expect(statusForVersion('0.11.0', idx)).toBe('pending');
const plan = buildPlan(idx, '0.11.1', '0.11.0');
expect(plan.pending.map(m => m.version)).toEqual(['0.11.0']);
expect(plan.applied).toEqual([]);
expect(plan.partial).toEqual([]);
expect(plan.wedged).toEqual([]);
});
test('complete then stray partial without retry → still complete', () => {
const idx = indexCompleted([
{ version: '0.11.0', status: 'complete' },
{ version: '0.11.0', status: 'partial' },
]);
expect(statusForVersion('0.11.0', idx)).toBe('complete');
});
test('retry followed by a newer complete → complete', () => {
const idx = indexCompleted([
{ version: '0.11.0', status: 'complete' },
{ version: '0.11.0', status: 'retry' },
{ version: '0.11.0', status: 'complete' },
]);
expect(statusForVersion('0.11.0', idx)).toBe('complete');
});
});
// v0.36.1.x (cherry-pick #1062): list, dry-run, and "all migrations up to
// date" paths must exit 0 so shell scripts gating on the exit code work.
// Pre-fix, these `return` statements left the CLI dispatcher's implicit
test/auto-think-phase.test.ts
+89 -4
View File
@@ -4,7 +4,7 @@ import { join } from 'node:path';
import { tmpdir } from 'node:os';
import { PGLiteEngine } from '../src/core/pglite-engine.ts';
import { runPhaseAutoThink } from '../src/core/cycle/auto-think.ts';
import { runPhaseDrift, __testing as driftTesting } from '../src/core/cycle/drift.ts';
import { runPhaseDrift, parseDriftOutput, __testing as driftTesting } from '../src/core/cycle/drift.ts';
import { _resetBudgetMeterWarningsForTest } from '../src/core/cycle/budget-meter.ts';
import type { ThinkLLMClient } from '../src/core/think/index.ts';
@@ -153,6 +153,18 @@ describe('runPhaseAutoThink', () => {
});
});
describe('parseDriftOutput', () => {
test('parses fenced JSON with clamped fields', () => {
const v = parseDriftOutput('```json\n{"drifted": true, "confidence": 1.7, "reasoning": "r", "suggested_weight": -0.2}\n```');
expect(v).toEqual({ drifted: true, confidence: 1, reasoning: 'r', suggested_weight: 0 });
});
test('returns null on garbage / missing drifted', () => {
expect(parseDriftOutput('not json at all')).toBeNull();
expect(parseDriftOutput('{"confidence": 0.5}')).toBeNull();
});
});
describe('runPhaseDrift', () => {
test('skipped when not enabled', async () => {
const r = await runPhaseDrift(engine, { dryRun: false, auditPath: join(tmpDir, 'd0.jsonl') });
@@ -166,16 +178,89 @@ describe('runPhaseDrift', () => {
expect(cands.every(c => c.weight >= 0.3 && c.weight <= 0.85)).toBe(true);
});
test('runs and surfaces candidates when enabled', async () => {
test('judges candidates and writes a report-only drift report page (#2653)', async () => {
await engine.setConfig('dream.drift.enabled', 'true');
await engine.setConfig('dream.drift.lookback_days', '30');
await engine.setConfig('dream.drift.budget', '1.0');
const r = await runPhaseDrift(engine, { dryRun: false, auditPath: join(tmpDir, 'd1.jsonl') });
const judgedRows: number[] = [];
const r = await runPhaseDrift(engine, {
dryRun: false,
auditPath: join(tmpDir, 'd1.jsonl'),
judge: async ({ candidate, evidence }) => {
judgedRows.push(candidate.rowNum);
expect(evidence).toContain('Funding round closed'); // real timeline evidence reached the judge
return candidate.rowNum === 2
? { drifted: true, confidence: 0.9, reasoning: 'evidence shifted', suggested_weight: 0.3 }
: { drifted: false, confidence: 0.7, reasoning: 'consistent' };
},
});
expect(r.status).toBe('complete');
expect((r.totals as { candidates?: number }).candidates).toBeGreaterThanOrEqual(0);
const totals = r.totals as { candidates: number; judged: number; drifted: number };
expect(totals.judged).toBeGreaterThanOrEqual(2);
expect(totals.drifted).toBe(1);
expect(judgedRows).toContain(2);
// Report page landed.
const date = new Date().toISOString().slice(0, 10);
const page = await engine.getPage(`reports/drift-${date}`);
expect(page).not.toBeNull();
expect(page!.compiled_truth).toContain('DRIFTED');
expect(page!.compiled_truth).toContain('Strong technical founder');
// Report-only v1: no take was mutated even though the judge suggested a weight.
const rows = await engine.executeRaw<{ weight: number; resolved_at: string | null }>(
'SELECT weight, resolved_at FROM takes WHERE page_id = $1 AND row_num = 2',
[alicePageId],
);
expect(Number(rows[0]!.weight)).toBe(0.6);
expect(rows[0]!.resolved_at).toBeNull();
await engine.setConfig('dream.drift.enabled', 'false');
});
test('auto_update mutates nothing in v1', async () => {
await engine.setConfig('dream.drift.enabled', 'true');
await engine.setConfig('dream.drift.auto_update', 'true');
const r = await runPhaseDrift(engine, {
dryRun: false,
auditPath: join(tmpDir, 'd-auto.jsonl'),
judge: async () => ({ drifted: true, confidence: 0.99, reasoning: 'x', suggested_weight: 0.1 }),
});
expect(r.status).toBe('complete');
const rows = await engine.executeRaw<{ weight: number }>(
'SELECT weight FROM takes WHERE page_id = $1 AND row_num = 3',
[alicePageId],
);
expect(Number(rows[0]!.weight)).toBe(0.5); // untouched
await engine.setConfig('dream.drift.auto_update', 'false');
await engine.setConfig('dream.drift.enabled', 'false');
});
test('budget exhaustion stops judging (partial, no judge calls)', async () => {
await engine.setConfig('dream.drift.enabled', 'true');
await engine.setConfig('dream.drift.budget', '0.0000001');
let judgeCalls = 0;
const r = await runPhaseDrift(engine, {
dryRun: false,
auditPath: join(tmpDir, 'd-budget.jsonl'),
judge: async () => { judgeCalls += 1; return { drifted: false, confidence: 0.5, reasoning: '' }; },
});
expect(r.status).toBe('partial');
expect(judgeCalls).toBe(0);
await engine.setConfig('dream.drift.budget', '1.0');
await engine.setConfig('dream.drift.enabled', 'false');
});
test('forceEnabled (--once) bypasses the dream.drift.enabled gate', async () => {
await engine.setConfig('dream.drift.enabled', 'false');
const r = await runPhaseDrift(engine, {
dryRun: false,
auditPath: join(tmpDir, 'd-once.jsonl'),
forceEnabled: true,
judge: async () => ({ drifted: false, confidence: 0.5, reasoning: 'ok' }),
});
expect(r.status).toBe('complete');
});
test('dry-run returns skipped with candidate count', async () => {
await engine.setConfig('dream.drift.enabled', 'true');
const r = await runPhaseDrift(engine, { dryRun: true, auditPath: join(tmpDir, 'd2.jsonl') });
test/autopilot-global-maintenance.test.ts
+15 -2
View File
@@ -11,6 +11,9 @@
*/
import { describe, test, expect, beforeAll, afterAll, beforeEach } from 'bun:test';
import { mkdtempSync } from 'node:fs';
import { tmpdir } from 'node:os';
import { join } from 'node:path';
import { PGLiteEngine } from '../src/core/pglite-engine.ts';
import { resetPgliteState } from './helpers/reset-pglite.ts';
import { registerBuiltinHandlers } from '../src/commands/jobs.ts';
@@ -130,13 +133,23 @@ describe('autopilot-global-maintenance handler stamps last_global_at (PGLite)',
test('runs global phases (no source_id) and stamps autopilot.last_global_at on success', async () => {
expect(await engine.getConfig(LAST_GLOBAL_AT_KEY)).toBeNull();
const repoPath = mkdtempSync(join(tmpdir(), 'gbrain-global-maintenance-'));
await engine.executeRaw(
`INSERT INTO sources (id, name, local_path) VALUES ($1, $2, $3)`,
['repo-a', 'repo-a', repoPath],
);
const handlers = await captureHandlers();
const handler = handlers.get('autopilot-global-maintenance');
expect(handler).toBeTruthy();
const result = await handler!({ data: { phases: ['orphans', 'embed'] }, signal: undefined });
const result = await handler!({
data: { phases: ['orphans', 'embed'], repoPath },
signal: undefined,
});
// The cycle ran the requested global phases (DB-only on an empty brain).
expect(result.report.phases.some((p: any) => p.phase === 'orphans')).toBe(true);
const orphans = result.report.phases.find((p: any) => p.phase === 'orphans');
expect(orphans).toBeTruthy();
expect(orphans.details.source_id).toBeUndefined();
expect(['ok', 'clean', 'partial']).toContain(result.report.status);
// Freshness stamped so the dispatch gate backs off.
const stamped = await engine.getConfig(LAST_GLOBAL_AT_KEY);
test/autopilot-install.test.ts
+1 -36
View File
@@ -20,7 +20,7 @@ import { mkdtempSync, rmSync, mkdirSync, writeFileSync, readFileSync, existsSync
import { join } from 'path';
import { tmpdir } from 'os';
import { detectInstallTarget, writeWrapperScript } from '../src/commands/autopilot.ts';
import { detectInstallTarget } from '../src/commands/autopilot.ts';
let tmp: string;
const envSnapshot: Record<string, string | undefined> = {};
@@ -85,41 +85,6 @@ describe('detectInstallTarget', () => {
// exported in zshrc never reach the LaunchAgent subprocess. Operators who
// exported GBRAIN_DATABASE_URL or {OPENAI,ANTHROPIC}_API_KEY in zshrc and
// expected autopilot to inherit them hit silent missing-secret failures.
// issue #2794: `--install --interval N` used to be silently dropped — the
// wrapper always exec'd bare `autopilot --repo ...` (default 300s). The
// wrapper's exec line must carry the interval when one is known.
describe('autopilot wrapper script — --interval threading (#2794)', () => {
// writeWrapperScript resolves the gbrain CLI via `which gbrain`; CI runners
// don't have the binary installed, so put a fake one on PATH to keep the
// test hermetic (deterministic on dev machines too — fake bin wins).
let pathSnapshot: string | undefined;
beforeEach(() => {
pathSnapshot = process.env.PATH;
const bin = join(tmp, 'bin');
mkdirSync(bin, { recursive: true });
writeFileSync(join(bin, 'gbrain'), '#!/bin/sh\n', { mode: 0o755 });
process.env.PATH = `${bin}:${process.env.PATH ?? ''}`;
});
afterEach(() => {
if (pathSnapshot === undefined) delete process.env.PATH;
else process.env.PATH = pathSnapshot;
});
test('exec line carries --interval when provided', () => {
const wrapperPath = writeWrapperScript('/tmp/some repo', 600);
const script = readFileSync(wrapperPath, 'utf8');
expect(script).toContain(`--repo '/tmp/some repo' --interval '600'`);
});
test('exec line omits --interval when not provided (default applies)', () => {
const wrapperPath = writeWrapperScript('/tmp/some-repo');
const script = readFileSync(wrapperPath, 'utf8');
expect(script).not.toContain('--interval');
});
});
describe('autopilot wrapper script — env source order (v0.36.1.x #966)', () => {
test('wrapper sources ~/.zshenv before ~/.zshrc', async () => {
const { readFileSync } = await import('fs');
test/brainstorm/model-config.test.ts
+65
View File
@@ -0,0 +1,65 @@
/**
* Brainstorm model configurability (takeover of PR #1855 by @starm2010).
*
* - The cost preview + hard cost ceiling price the model that will actually
* run: --model override → configured chat_model → gateway fallback. Before
* this, the preview always priced anthropic:claude-sonnet-4-6 even when
* the configured chat_model was something else.
* - The judge phase honors the `models.brainstorm.judge` config key when no
* --judge-model flag is passed.
*/
import { describe, test, expect } from 'bun:test';
import {
resolveBrainstormChatModel,
resolveBrainstormJudgeModel,
} from '../../src/core/brainstorm/orchestrator.ts';
import type { BrainEngine } from '../../src/core/engine.ts';
function mockEngine(configValues: Record<string, string>): { engine: BrainEngine; reads: string[] } {
const reads: string[] = [];
const engine = {
async getConfig(key: string): Promise<string | null> {
reads.push(key);
return configValues[key] ?? null;
},
} as unknown as BrainEngine;
return { engine, reads };
}
describe('resolveBrainstormChatModel', () => {
test('--model override wins over config', () => {
expect(resolveBrainstormChatModel({ chat_model: 'openai:gpt-5' }, 'anthropic:claude-opus-4-6'))
.toBe('anthropic:claude-opus-4-6');
});
test('configured chat_model wins over the hardcoded fallback', () => {
expect(resolveBrainstormChatModel({ chat_model: 'openai:gpt-5' }))
.toBe('openai:gpt-5');
});
test('falls back to the gateway default model when nothing is configured', () => {
expect(resolveBrainstormChatModel({})).toBe('anthropic:claude-sonnet-4-6');
});
});
describe('resolveBrainstormJudgeModel', () => {
test('--judge-model flag wins without touching config', async () => {
const { engine, reads } = mockEngine({ 'models.brainstorm.judge': 'openai:gpt-5' });
const out = await resolveBrainstormJudgeModel(engine, 'anthropic:claude-opus-4-6');
expect(out).toBe('anthropic:claude-opus-4-6');
expect(reads).toHaveLength(0);
});
test('models.brainstorm.judge config key is honored when no flag is passed', async () => {
const { engine, reads } = mockEngine({ 'models.brainstorm.judge': 'openai:gpt-5' });
const out = await resolveBrainstormJudgeModel(engine);
expect(out).toBe('openai:gpt-5');
expect(reads).toEqual(['models.brainstorm.judge']);
});
test('returns undefined (defer to modelOverride / gateway default) when unset', async () => {
const { engine } = mockEngine({});
expect(await resolveBrainstormJudgeModel(engine)).toBeUndefined();
});
});
test/claude-cli-recipe.test.ts
+535
View File
@@ -0,0 +1,535 @@
/**
* Tests for the claude-cli LanguageModelV2 implementation that the
* `claude-cli` recipe instantiates.
*
* Strategy: a POSIX shell stub at GBRAIN_CLAUDE_CLI_BIN emits scripted
* --output-format json envelopes. Tests exercise the LanguageModelV2
* doGenerate surface: text round trip, tool-call extraction (single +
* multiple parallel), abort semantics, context-isolation flags. No
* claude-cli installation or API credits required.
*
* Recipe registration is also smoke-tested: getRecipe('claude-cli')
* returns a chat-only Recipe with the right model list.
*
* Env isolation: GBRAIN_CLAUDE_CLI_BIN is set per-test via withEnv(),
* NOT in beforeAll. The provider reads the env var at spawn time so
* withEnv's save/restore in try/finally is sufficient; no leakage to
* sibling test files in the same bun-test process.
*/
import { describe, test, expect, beforeAll, afterAll } from 'bun:test';
import { writeFileSync, chmodSync, mkdirSync, rmSync } from 'node:fs';
import { join } from 'node:path';
import { tmpdir } from 'node:os';
import type { LanguageModelV2CallOptions } from '@ai-sdk/provider';
import { withEnv } from './helpers/with-env.ts';
const stubDir = join(tmpdir(), `claude-cli-recipe-stub-${process.pid}`);
const stubBin = join(stubDir, 'claude');
const stubResponsePath = join(stubDir, 'claude_response.json');
beforeAll(() => {
mkdirSync(stubDir, { recursive: true });
const stub = [
'#!/bin/sh',
'cat > /dev/null',
'case " $* " in',
' *" --print "*) ;;',
' *) echo "missing --print in argv: $*" >&2; exit 64 ;;',
'esac',
`cat "${stubResponsePath}"`,
].join('\n');
writeFileSync(stubBin, stub);
chmodSync(stubBin, 0o755);
});
afterAll(() => {
rmSync(stubDir, { recursive: true, force: true });
});
function withStubEnv<T>(fn: () => T | Promise<T>): Promise<T> {
return withEnv({ GBRAIN_CLAUDE_CLI_BIN: stubBin }, fn);
}
function stageResponse(envelope: Record<string, unknown>): void {
writeFileSync(stubResponsePath, JSON.stringify(envelope));
}
function baseEnvelope(result: string, overrides: Record<string, unknown> = {}): Record<string, unknown> {
return {
type: 'result',
subtype: 'success',
is_error: false,
result,
stop_reason: 'end_turn',
session_id: 'test-session',
num_turns: 1,
usage: {
input_tokens: 12,
output_tokens: 34,
cache_read_input_tokens: 0,
cache_creation_input_tokens: 0,
},
...overrides,
};
}
function userMessage(text: string): LanguageModelV2CallOptions['prompt'][number] {
return { role: 'user', content: [{ type: 'text', text }] };
}
describe('claude-cli recipe registration', () => {
test('getRecipe returns chat-only Recipe with the documented models', async () => {
const { getRecipe } = await import('../src/core/ai/recipes/index.ts');
const recipe = getRecipe('claude-cli');
expect(recipe).toBeDefined();
expect(recipe!.id).toBe('claude-cli');
expect(recipe!.implementation).toBe('claude-cli');
expect(recipe!.touchpoints.chat).toBeDefined();
expect(recipe!.touchpoints.chat!.supports_tools).toBe(true);
expect(recipe!.touchpoints.chat!.supports_subagent_loop).toBe(true);
expect(recipe!.touchpoints.chat!.models).toContain('claude-sonnet-4-6');
expect(recipe!.touchpoints.embedding).toBeUndefined();
expect(recipe!.touchpoints.expansion).toBeUndefined();
});
test('recipe aliases map short names to canonical model ids', async () => {
const { getRecipe } = await import('../src/core/ai/recipes/index.ts');
const recipe = getRecipe('claude-cli');
expect(recipe!.aliases!['sonnet']).toBe('claude-sonnet-4-6');
expect(recipe!.aliases!['haiku']).toBe('claude-haiku-4-5-20251001');
});
});
describe('claude-cli LanguageModel — text-only round trip', () => {
test('returns a single text content block with usage + stop finish reason', async () => {
await withStubEnv(async () => {
stageResponse(baseEnvelope('hello world'));
const { ClaudeCliLanguageModel } = await import('../src/core/ai/providers/claude-cli-language-model.ts');
const model = new ClaudeCliLanguageModel('claude-sonnet-4-6');
const result = await model.doGenerate({
prompt: [userMessage('hi')],
} as LanguageModelV2CallOptions);
expect(result.finishReason).toBe('stop');
expect(result.content).toHaveLength(1);
expect(result.content[0]).toEqual({ type: 'text', text: 'hello world' });
expect(result.usage.inputTokens).toBe(12);
expect(result.usage.outputTokens).toBe(34);
});
});
test('strips provider prefixes from the model id', async () => {
const { ClaudeCliLanguageModel } = await import('../src/core/ai/providers/claude-cli-language-model.ts');
const model = new ClaudeCliLanguageModel('anthropic:claude-sonnet-4-6');
expect(model.modelId).toBe('claude-sonnet-4-6');
});
});
describe('claude-cli LanguageModel — tool use', () => {
test('parses <use_tools> block into LanguageModelV2 tool-call content', async () => {
await withStubEnv(async () => {
stageResponse(
baseEnvelope(
[
'I will look up the pattern first.',
'<use_tools>',
'[{"id": "toolu_01ABC", "name": "search", "input": {"query": "n+1 query"}}]',
'</use_tools>',
].join('\n'),
),
);
const { ClaudeCliLanguageModel } = await import('../src/core/ai/providers/claude-cli-language-model.ts');
const model = new ClaudeCliLanguageModel('claude-sonnet-4-6');
const result = await model.doGenerate({
prompt: [userMessage('find n+1 queries')],
tools: [
{
type: 'function',
name: 'search',
description: 'Search the brain',
inputSchema: { type: 'object', properties: { query: { type: 'string' } } },
},
],
} as LanguageModelV2CallOptions);
expect(result.finishReason).toBe('tool-calls');
expect(result.content).toHaveLength(2);
expect(result.content[0]).toMatchObject({ type: 'text', text: 'I will look up the pattern first.' });
expect(result.content[1]).toMatchObject({
type: 'tool-call',
toolCallId: 'toolu_01ABC',
toolName: 'search',
input: '{"query":"n+1 query"}',
});
});
});
test('parses multiple parallel tool calls in a single block', async () => {
await withStubEnv(async () => {
stageResponse(
baseEnvelope(
[
'<use_tools>',
'[',
' {"id": "toolu_A", "name": "search", "input": {"query": "foo"}},',
' {"id": "toolu_B", "name": "get_page", "input": {"slug": "areas/x"}}',
']',
'</use_tools>',
].join('\n'),
),
);
const { ClaudeCliLanguageModel } = await import('../src/core/ai/providers/claude-cli-language-model.ts');
const model = new ClaudeCliLanguageModel('claude-sonnet-4-6');
const result = await model.doGenerate({
prompt: [userMessage('multi')],
tools: [
{ type: 'function', name: 'search', description: 's', inputSchema: { type: 'object', properties: {} } },
{ type: 'function', name: 'get_page', description: 'g', inputSchema: { type: 'object', properties: {} } },
],
} as LanguageModelV2CallOptions);
const calls = result.content.filter(c => c.type === 'tool-call');
expect(calls).toHaveLength(2);
expect(calls.map(c => (c as { toolName: string }).toolName)).toEqual(['search', 'get_page']);
expect(result.finishReason).toBe('tool-calls');
});
});
test('tolerates fenced JSON inside <use_tools>', async () => {
await withStubEnv(async () => {
stageResponse(
baseEnvelope(
[
'<use_tools>',
'```json',
'[{"id": "toolu_F", "name": "search", "input": {"q": "x"}}]',
'```',
'</use_tools>',
].join('\n'),
),
);
const { ClaudeCliLanguageModel } = await import('../src/core/ai/providers/claude-cli-language-model.ts');
const model = new ClaudeCliLanguageModel('claude-sonnet-4-6');
const result = await model.doGenerate({
prompt: [userMessage('fenced')],
tools: [{ type: 'function', name: 'search', description: '', inputSchema: { type: 'object', properties: {} } }],
} as LanguageModelV2CallOptions);
const calls = result.content.filter(c => c.type === 'tool-call');
expect(calls).toHaveLength(1);
});
});
test('synthesizes an id when the model omits it', async () => {
await withStubEnv(async () => {
stageResponse(
baseEnvelope(
[
'<use_tools>',
'[{"name": "search", "input": {"q": "x"}}]',
'</use_tools>',
].join('\n'),
),
);
const { ClaudeCliLanguageModel } = await import('../src/core/ai/providers/claude-cli-language-model.ts');
const model = new ClaudeCliLanguageModel('claude-sonnet-4-6');
const result = await model.doGenerate({
prompt: [userMessage('no id')],
tools: [{ type: 'function', name: 'search', description: '', inputSchema: { type: 'object', properties: {} } }],
} as LanguageModelV2CallOptions);
const call = result.content.find(c => c.type === 'tool-call') as { toolCallId: string } | undefined;
expect(call).toBeDefined();
expect(call!.toolCallId).toMatch(/^toolu_claude_cli_/);
});
});
test('falls back to text on malformed JSON', async () => {
await withStubEnv(async () => {
stageResponse(
baseEnvelope(
[
'<use_tools>',
'not valid json',
'</use_tools>',
].join('\n'),
),
);
const { ClaudeCliLanguageModel } = await import('../src/core/ai/providers/claude-cli-language-model.ts');
const model = new ClaudeCliLanguageModel('claude-sonnet-4-6');
const result = await model.doGenerate({
prompt: [userMessage('malformed')],
tools: [{ type: 'function', name: 'search', description: '', inputSchema: { type: 'object', properties: {} } }],
} as LanguageModelV2CallOptions);
expect(result.content.filter(c => c.type === 'tool-call')).toHaveLength(0);
expect(result.finishReason).toBe('stop');
});
});
test('returns text-only stop when tools are offered but model declines to call any', async () => {
// Real-world case: the model decides the user's request does not require
// a tool call, ignores the use_tools protocol, and answers directly.
// The recipe still must return clean LanguageModelV2 output so the
// caller (gateway.toolLoop) can treat the text as the final answer
// rather than wedge waiting for tool calls that never come.
await withStubEnv(async () => {
stageResponse(
baseEnvelope(
'I do not actually need to call any tools for this. The answer is 42.',
{ stop_reason: 'end_turn' },
),
);
const { ClaudeCliLanguageModel } = await import('../src/core/ai/providers/claude-cli-language-model.ts');
const model = new ClaudeCliLanguageModel('claude-sonnet-4-6');
const result = await model.doGenerate({
prompt: [userMessage('what is the meaning of life? you may use tools but do not need to')],
tools: [{ type: 'function', name: 'compute', description: 'Compute things', inputSchema: { type: 'object', properties: {} } }],
} as LanguageModelV2CallOptions);
// No tool-call content blocks; caller treats this as a final answer.
expect(result.content.filter(c => c.type === 'tool-call')).toHaveLength(0);
// Text block present with the full model reply.
const textBlocks = result.content.filter(c => c.type === 'text');
expect(textBlocks).toHaveLength(1);
expect((textBlocks[0] as { text: string }).text).toContain('42');
// finishReason 'stop' tells the gateway-loop this is terminal output,
// not a partial mid-tool-loop state.
expect(result.finishReason).toBe('stop');
});
});
test('drops the block when the close tag is missing', async () => {
await withStubEnv(async () => {
stageResponse(
baseEnvelope(
[
'<use_tools>',
'[{"id": "toolu_X", "name": "search", "input": {}}',
].join('\n'),
),
);
const { ClaudeCliLanguageModel } = await import('../src/core/ai/providers/claude-cli-language-model.ts');
const model = new ClaudeCliLanguageModel('claude-sonnet-4-6');
const result = await model.doGenerate({
prompt: [userMessage('unterminated')],
tools: [{ type: 'function', name: 'search', description: '', inputSchema: { type: 'object', properties: {} } }],
} as LanguageModelV2CallOptions);
expect(result.content.filter(c => c.type === 'tool-call')).toHaveLength(0);
expect(result.finishReason).toBe('stop');
});
});
});
describe('claude-cli LanguageModel — context isolation', () => {
test('argv includes --disable-slash-commands + --system-prompt and cwd is the dedicated tmpdir', async () => {
await withStubEnv(async () => {
const argvLog = join(stubDir, 'argv.log');
const cwdLog = join(stubDir, 'cwd.log');
const recordStub = [
'#!/bin/sh',
`printf "%s\\n" "$@" > "${argvLog}"`,
`pwd > "${cwdLog}"`,
'cat > /dev/null',
`cat "${stubResponsePath}"`,
].join('\n');
writeFileSync(stubBin, recordStub);
chmodSync(stubBin, 0o755);
stageResponse(baseEnvelope('ok'));
const { ClaudeCliLanguageModel } = await import('../src/core/ai/providers/claude-cli-language-model.ts');
const model = new ClaudeCliLanguageModel('claude-sonnet-4-6');
await model.doGenerate({
prompt: [
{ role: 'system', content: 'You are gbrain subagent.' },
userMessage('hi'),
],
} as LanguageModelV2CallOptions);
const fs = require('node:fs');
const argv = fs.readFileSync(argvLog, 'utf8').split('\n').filter(Boolean);
const cwd = fs.readFileSync(cwdLog, 'utf8').trim();
expect(argv).toContain('--print');
expect(argv).toContain('--output-format');
expect(argv).toContain('json');
expect(argv).toContain('--disable-slash-commands');
// Agent-isolation hardening: no built-in tools, no inherited MCP servers.
expect(argv).toContain('--tools');
expect(argv).toContain('--strict-mcp-config');
expect(argv).toContain('--system-prompt');
expect(argv).toContain('You are gbrain subagent.');
expect(cwd).toMatch(/gbrain-claude-cli-cwd-\d+$/);
const fastStub = [
'#!/bin/sh',
'cat > /dev/null',
`cat "${stubResponsePath}"`,
].join('\n');
writeFileSync(stubBin, fastStub);
chmodSync(stubBin, 0o755);
});
});
test('scrubs ANTHROPIC_* credentials from the child env (subscription-only auth)', async () => {
await withStubEnv(async () => {
await withEnv(
{
ANTHROPIC_API_KEY: 'sk-should-never-leak',
ANTHROPIC_AUTH_TOKEN: 'tok-should-never-leak',
ANTHROPIC_BASE_URL: 'https://proxy.should.never.leak',
},
async () => {
const envLog = join(stubDir, 'env.log');
const envStub = [
'#!/bin/sh',
`printf "key=%s\\ntoken=%s\\nbase=%s\\n" "\${ANTHROPIC_API_KEY:-UNSET}" "\${ANTHROPIC_AUTH_TOKEN:-UNSET}" "\${ANTHROPIC_BASE_URL:-UNSET}" > "${envLog}"`,
'cat > /dev/null',
`cat "${stubResponsePath}"`,
].join('\n');
writeFileSync(stubBin, envStub);
chmodSync(stubBin, 0o755);
stageResponse(baseEnvelope('ok'));
try {
const { ClaudeCliLanguageModel } = await import('../src/core/ai/providers/claude-cli-language-model.ts');
const model = new ClaudeCliLanguageModel('claude-sonnet-4-6');
await model.doGenerate({
prompt: [userMessage('hi')],
} as LanguageModelV2CallOptions);
const fs = require('node:fs');
const seen = fs.readFileSync(envLog, 'utf8');
expect(seen).toContain('key=UNSET');
expect(seen).toContain('token=UNSET');
expect(seen).toContain('base=UNSET');
} finally {
const fastStub = [
'#!/bin/sh',
'cat > /dev/null',
`cat "${stubResponsePath}"`,
].join('\n');
writeFileSync(stubBin, fastStub);
chmodSync(stubBin, 0o755);
}
},
);
});
});
});
describe('claude-cli LanguageModel — abort + error envelopes', () => {
test('SIGTERMs the child on AbortSignal', async () => {
await withStubEnv(async () => {
const slowStub = [
'#!/bin/sh',
'cat > /dev/null',
'sleep 30',
'echo "{}"',
].join('\n');
writeFileSync(stubBin, slowStub);
chmodSync(stubBin, 0o755);
try {
const { ClaudeCliLanguageModel } = await import('../src/core/ai/providers/claude-cli-language-model.ts');
const model = new ClaudeCliLanguageModel('claude-sonnet-4-6');
const ac = new AbortController();
const promise = model.doGenerate({
prompt: [userMessage('slow')],
abortSignal: ac.signal,
} as LanguageModelV2CallOptions);
setTimeout(() => ac.abort(), 30);
await expect(promise).rejects.toThrow(/aborted/);
} finally {
const fastStub = [
'#!/bin/sh',
'cat > /dev/null',
`cat "${stubResponsePath}"`,
].join('\n');
writeFileSync(stubBin, fastStub);
chmodSync(stubBin, 0o755);
}
});
});
test('rejects when stub reports is_error: true', async () => {
await withStubEnv(async () => {
stageResponse({ ...baseEnvelope('boom'), is_error: true });
const { ClaudeCliLanguageModel } = await import('../src/core/ai/providers/claude-cli-language-model.ts');
const model = new ClaudeCliLanguageModel('claude-sonnet-4-6');
await expect(
model.doGenerate({ prompt: [userMessage('x')] } as LanguageModelV2CallOptions),
).rejects.toThrow(/claude-cli reported error/);
});
});
test('rejects on non-JSON output', async () => {
await withStubEnv(async () => {
writeFileSync(stubResponsePath, 'this is not json');
const { ClaudeCliLanguageModel } = await import('../src/core/ai/providers/claude-cli-language-model.ts');
const model = new ClaudeCliLanguageModel('claude-sonnet-4-6');
await expect(
model.doGenerate({ prompt: [userMessage('x')] } as LanguageModelV2CallOptions),
).rejects.toThrow(/claude-cli output not JSON/);
});
});
test('accepts a verbose-mode JSON event array and picks the result event', async () => {
// With `"verbose": true` in ~/.claude/settings.json the CLI emits an array
// of events instead of the bare result object (no CLI flag disables it).
await withStubEnv(async () => {
writeFileSync(
stubResponsePath,
JSON.stringify([
{ type: 'system', subtype: 'init', session_id: 'test-session', tools: [], mcp_servers: [] },
baseEnvelope('hello from array'),
]),
);
const { ClaudeCliLanguageModel } = await import('../src/core/ai/providers/claude-cli-language-model.ts');
const model = new ClaudeCliLanguageModel('claude-sonnet-4-6');
const result = await model.doGenerate({
prompt: [userMessage('hi')],
} as LanguageModelV2CallOptions);
expect(result.finishReason).toBe('stop');
expect(result.content[0]).toEqual({ type: 'text', text: 'hello from array' });
});
});
test('rejects a verbose-mode event array that lacks a result event', async () => {
// Verbose mode emits an event array; a truncated stream (or one carrying
// only init/system events) has no result event to unwrap.
await withStubEnv(async () => {
writeFileSync(
stubResponsePath,
JSON.stringify([
{ type: 'system', subtype: 'init', session_id: 'test-session', tools: [], mcp_servers: [] },
{ type: 'assistant', message: { role: 'assistant', content: [] } },
]),
);
const { ClaudeCliLanguageModel } = await import('../src/core/ai/providers/claude-cli-language-model.ts');
const model = new ClaudeCliLanguageModel('claude-sonnet-4-6');
await expect(
model.doGenerate({ prompt: [userMessage('x')] } as LanguageModelV2CallOptions),
).rejects.toThrow(/had no "result" event/);
});
});
test('rejects cleanly when the claude binary is missing (no worker crash)', async () => {
// A missing binary must surface as a rejected promise via the spawn 'error'
// handler; the child stdin 'error' (EPIPE) handler swallows the pipe failure
// so it never escalates to an unhandled rejection that would down the worker.
await withEnv({ GBRAIN_CLAUDE_CLI_BIN: join(stubDir, 'nonexistent-claude') }, async () => {
const { ClaudeCliLanguageModel } = await import('../src/core/ai/providers/claude-cli-language-model.ts');
const model = new ClaudeCliLanguageModel('claude-sonnet-4-6');
await expect(
model.doGenerate({ prompt: [userMessage('x')] } as LanguageModelV2CallOptions),
).rejects.toThrow(/claude-cli spawn failed/);
});
});
test('doStream throws not-supported', async () => {
const { ClaudeCliLanguageModel } = await import('../src/core/ai/providers/claude-cli-language-model.ts');
const model = new ClaudeCliLanguageModel('claude-sonnet-4-6');
await expect(model.doStream()).rejects.toThrow(/does not support streaming/);
});
});
test/contextual-retrieval-service-pure.test.ts
+338 -6
View File
@@ -1,20 +1,38 @@
/**
* Pure-function tests for src/core/contextual-retrieval-service.ts.
*
* The full service test (PHASE 1 + PHASE 2 happy path, refusal restart,
* transient error propagation) needs a real PGLite + gateway stub seam.
* That lands in test/e2e/contextual-retrieval.test.ts. This file pins
* the service's pure helpers: corpus_generation hash composition + the
* expectedMode helper used by the T9 reindex sweep predicate.
* This file pins the service's pure helpers plus hermetic service behavior
* driven through fake engine + gateway seams. Full PGLite coverage lives in
* test/e2e/contextual-retrieval-pglite.test.ts.
*/
import { describe, test, expect } from 'bun:test';
import { afterEach, describe, test, expect } from 'bun:test';
import {
computeCorpusGeneration,
computeSourceTextHash,
expectedModeForPageSourceOnly,
reembedPageWithContextualRetrieval,
resolveContextualChunkConcurrency,
TITLE_WRAPPER_VERSION,
} from '../src/core/contextual-retrieval-service.ts';
import {
__setChatTransportForTests,
__setEmbedTransportForTests,
configureGateway,
resetGateway,
type ChatOpts,
type ChatResult,
} from '../src/core/ai/gateway.ts';
import type { ChunkInput } from '../src/core/types.ts';
import { withEnv } from './helpers/with-env.ts';
const TEST_DIMS = 1536;
afterEach(() => {
__setChatTransportForTests(null);
__setEmbedTransportForTests(null);
resetGateway();
});
describe('computeCorpusGeneration', () => {
test('returns 16-char hex hash', () => {
@@ -138,3 +156,317 @@ describe('expectedModeForPageSourceOnly (T9 reindex sweep helper)', () => {
}
});
});
describe('resolveContextualChunkConcurrency', () => {
test('defaults to 4 and reads the process env', async () => {
await withEnv({ GBRAIN_CONTEXTUAL_CHUNK_CONCURRENCY: undefined }, async () => {
expect(resolveContextualChunkConcurrency()).toBe(4);
});
await withEnv({ GBRAIN_CONTEXTUAL_CHUNK_CONCURRENCY: '7' }, async () => {
expect(resolveContextualChunkConcurrency()).toBe(7);
});
});
test('clamps to [1, 16] and ignores invalid values', () => {
expect(resolveContextualChunkConcurrency({
GBRAIN_CONTEXTUAL_CHUNK_CONCURRENCY: '0',
})).toBe(1);
expect(resolveContextualChunkConcurrency({
GBRAIN_CONTEXTUAL_CHUNK_CONCURRENCY: '-3',
})).toBe(1);
expect(resolveContextualChunkConcurrency({
GBRAIN_CONTEXTUAL_CHUNK_CONCURRENCY: '99',
})).toBe(16);
expect(resolveContextualChunkConcurrency({
GBRAIN_CONTEXTUAL_CHUNK_CONCURRENCY: '1.9',
})).toBe(1);
expect(resolveContextualChunkConcurrency({
GBRAIN_CONTEXTUAL_CHUNK_CONCURRENCY: 'not-a-number',
})).toBe(4);
});
});
describe('per-chunk synopsis concurrency', () => {
test('concurrency > 1 preserves chunk-order embed input', async () => {
const chunks = makeChunks(['alpha', 'beta', 'gamma', 'delta']);
const delays: Record<string, number> = { alpha: 30, beta: 5, gamma: 20, delta: 1 };
const sequential = await runWithChatStub({
chunks,
concurrency: 1,
delayForChunk: (chunk) => delays[chunk] ?? 1,
});
const parallel = await runWithChatStub({
chunks,
concurrency: 4,
delayForChunk: (chunk) => delays[chunk] ?? 1,
});
expect(parallel.result.kind).toBe('success');
expect(parallel.embedInputs).toEqual(sequential.embedInputs);
expect(parallel.embeddedChunks.map((c) => c.chunk_text)).toEqual(
chunks.map((c) => c.chunk_text),
);
});
test('concurrency is bounded', async () => {
let active = 0;
let maxActive = 0;
let leaseActive = 0;
let maxLeaseActive = 0;
let acquired = 0;
let released = 0;
const chunks = makeChunks(Array.from({ length: 8 }, (_, i) => `chunk-${i}`));
const out = await runWithChatStub({
chunks,
concurrency: 3,
acquireSynopsisLease: async () => {
acquired++;
leaseActive++;
maxLeaseActive = Math.max(maxLeaseActive, leaseActive);
return acquired;
},
releaseSynopsisLease: async () => {
released++;
leaseActive--;
},
chat: async (opts) => {
active++;
maxActive = Math.max(maxActive, active);
try {
await delay(20, opts.abortSignal);
return chatSuccess(`Synopsis for ${extractChunk(opts)}`);
} finally {
active--;
}
},
});
expect(out.result.kind).toBe('success');
expect(maxActive).toBeGreaterThan(1);
expect(maxActive).toBeLessThanOrEqual(3);
expect(maxLeaseActive).toBeLessThanOrEqual(3);
expect(acquired).toBe(8);
expect(released).toBe(8);
expect(leaseActive).toBe(0);
});
test('one chunk failure aborts queued work and falls back at page level', async () => {
let started = 0;
const chunks = makeChunks(Array.from({ length: 9 }, (_, i) => `chunk-${i}`));
const out = await runWithChatStub({
chunks,
concurrency: 3,
chat: async (opts) => {
started++;
const chunk = extractChunk(opts);
if (chunk === 'chunk-0') return chatSuccess('');
await delay(30, opts.abortSignal);
return chatSuccess(`Synopsis for ${chunk}`);
},
});
expect(out.result.kind).toBe('page_fallback');
expect(started).toBeLessThanOrEqual(3);
});
test('fenced code chunks bypass synopsis calls and leases', async () => {
let chatCalls = 0;
let leaseCalls = 0;
const chunks: ChunkInput[] = [
{ chunk_index: 0, chunk_text: 'intro', chunk_source: 'compiled_truth' },
{ chunk_index: 1, chunk_text: 'const x = 1;', chunk_source: 'fenced_code' },
{ chunk_index: 2, chunk_text: 'outro', chunk_source: 'compiled_truth' },
];
const out = await runWithChatStub({
chunks,
concurrency: 3,
acquireSynopsisLease: async () => {
leaseCalls++;
},
releaseSynopsisLease: async () => {},
chat: async (opts) => {
chatCalls++;
return chatSuccess(`Synopsis for ${extractChunk(opts)}`);
},
});
expect(out.result.kind).toBe('success');
expect(chatCalls).toBe(2);
expect(leaseCalls).toBe(2);
expect(out.embedInputs[1]).toBe('const x = 1;');
});
test('abortSignal cancels in-flight and queued synopsis work promptly', async () => {
const controller = new AbortController();
let started = 0;
const chunks = makeChunks(Array.from({ length: 20 }, (_, i) => `chunk-${i}`));
const startedAt = Date.now();
const promise = runWithChatStub({
chunks,
concurrency: 4,
abortSignal: controller.signal,
chat: async (opts) => {
started++;
await delay(1000, opts.abortSignal);
return chatSuccess(`Synopsis for ${extractChunk(opts)}`);
},
});
setTimeout(() => controller.abort(), 20);
const out = await promise;
expect(out.result.kind).toBe('transient_error');
if (out.result.kind === 'transient_error') {
expect(out.result.cause).toBe('timeout');
}
expect(started).toBeLessThanOrEqual(4);
expect(Date.now() - startedAt).toBeLessThan(300);
});
});
function makeChunks(texts: string[]): ChunkInput[] {
return texts.map((text, i) => ({
chunk_index: i,
chunk_text: text,
chunk_source: 'compiled_truth',
}));
}
async function runWithChatStub(opts: {
chunks: ChunkInput[];
concurrency: number;
abortSignal?: AbortSignal;
delayForChunk?: (chunk: string) => number;
chat?: (opts: ChatOpts) => Promise<ChatResult>;
acquireSynopsisLease?: () => Promise<unknown>;
releaseSynopsisLease?: (lease?: unknown) => Promise<void>;
}) {
configureGateway({
embedding_model: 'openai:text-embedding-3-large',
embedding_dimensions: TEST_DIMS,
env: { OPENAI_API_KEY: 'sk-test' },
});
const embedInputs: string[][] = [];
__setEmbedTransportForTests(async ({ values }: any) => {
embedInputs.push([...values]);
return {
embeddings: values.map((_: string, i: number) =>
Array.from({ length: TEST_DIMS }, () => 0.001 + i * 0.001),
),
usage: { tokens: 0 },
} as any;
});
__setChatTransportForTests(opts.chat ?? (async (chatOpts) => {
const chunk = extractChunk(chatOpts);
await delay(opts.delayForChunk?.(chunk) ?? 1, chatOpts.abortSignal);
return chatSuccess(`Synopsis for ${chunk}`);
}));
const engine = makeServiceEngine(opts.chunks);
const result = await reembedPageWithContextualRetrieval({
engine,
pageSlug: 'wiki/concepts/concurrency-test',
sourceId: 'default',
globalMode: 'per_chunk_synopsis',
chunkConcurrency: opts.concurrency,
abortSignal: opts.abortSignal,
...(opts.acquireSynopsisLease && { acquireSynopsisLease: opts.acquireSynopsisLease }),
...(opts.releaseSynopsisLease && { releaseSynopsisLease: opts.releaseSynopsisLease }),
});
return {
result,
embedInputs: embedInputs.flat(),
embeddedChunks: engine.embeddedChunks as ChunkInput[],
};
}
function makeServiceEngine(chunks: ChunkInput[]) {
const engine: any = {
embeddedChunks: [] as ChunkInput[],
async getPage() {
return {
id: 1,
slug: 'wiki/concepts/concurrency-test',
source_id: 'default',
type: 'concept',
title: 'Concurrency Test',
compiled_truth: chunks.map((c) => c.chunk_text).join('\n\n'),
timeline: '',
frontmatter: {},
created_at: new Date('2026-01-01T00:00:00Z'),
updated_at: new Date('2026-01-01T00:00:00Z'),
deleted_at: null,
};
},
async executeRaw() {
return [{
id: 'default',
name: 'Default',
local_path: null,
last_commit: null,
last_sync_at: null,
config: {},
created_at: new Date('2026-01-01T00:00:00Z'),
contextual_retrieval_mode: null,
trust_frontmatter_overrides: false,
}];
},
async getChunks() {
return chunks;
},
async transaction(fn: (tx: any) => Promise<void>) {
await fn({
upsertChunks: async (_slug: string, embedded: ChunkInput[]) => {
engine.embeddedChunks = embedded;
},
updatePageContextualRetrievalState: async () => {},
});
},
async updatePageContextualRetrievalState() {},
};
return engine;
}
function extractChunk(opts: ChatOpts): string {
const content = String(opts.messages[0]?.content ?? '');
return content.match(/<chunk>\n([\s\S]*?)\n<\/chunk>/)?.[1] ?? '';
}
function chatSuccess(text: string): ChatResult {
return {
text,
blocks: [],
stopReason: 'end',
usage: {
input_tokens: 0,
output_tokens: 0,
cache_read_tokens: 0,
cache_creation_tokens: 0,
},
model: 'stub:chat',
providerId: 'stub',
};
}
function delay(ms: number, signal?: AbortSignal): Promise<void> {
return new Promise((resolve, reject) => {
if (signal?.aborted) {
reject(abortError());
return;
}
const timer = setTimeout(resolve, ms);
signal?.addEventListener('abort', () => {
clearTimeout(timer);
reject(abortError());
}, { once: true });
});
}
function abortError(): Error {
const err = new Error('aborted');
err.name = 'AbortError';
return err;
}
test/core/cycle.serial.test.ts
+36 -3
View File
@@ -21,6 +21,7 @@ let syncCalls: Array<{ dryRun: boolean | undefined; noPull: boolean | undefined;
let extractCalls: Array<{ mode: string; dir: string; slugs: string[] | undefined }> = [];
let embedCalls: Array<{ stale: boolean | undefined; dryRun: boolean | undefined }> = [];
let orphansCalls: number = 0;
let orphansOpts: Array<{ sourceId?: string } | undefined> = [];
// Mock lint
mock.module('../../src/commands/lint.ts', () => ({
@@ -98,8 +99,9 @@ mock.module('../../src/commands/embed.ts', () => ({
// Mock orphans
mock.module('../../src/commands/orphans.ts', () => ({
findOrphans: async () => {
findOrphans: async (_engine: any, opts?: { sourceId?: string }) => {
orphansCalls++;
orphansOpts.push(opts);
return {
orphans: [],
total_orphans: 1,
@@ -148,6 +150,7 @@ beforeEach(() => {
extractCalls = [];
embedCalls = [];
orphansCalls = 0;
orphansOpts = [];
});
// ─── dryRun propagation (regression guards) ────────────────────────
@@ -215,6 +218,11 @@ describe('runCycle — phase selection', () => {
expect(orphansCalls).toBe(1);
expect(syncCalls.length).toBe(0);
});
test('--phase orphans preserves explicit source scope', async () => {
await runCycle(sharedEngine, { brainDir: '/tmp/brain', phases: ['orphans'], sourceId: 'source-a' });
expect(orphansOpts.at(-1)).toEqual({ sourceId: 'source-a' });
});
});
// ─── Lock-skip for non-DB-write phase selections ──────────────────
@@ -394,7 +402,9 @@ describe('runCycle — yieldBetweenPhases hook', () => {
// v0.41.11.0: 20 phases (added `conversation_facts_backfill` between consolidate and propose_takes).
// v0.41.39 (#1700) + v0.42.0.0: 22 phases (added `enrich_thin` AND `skillopt`
// between conversation_facts_backfill and embed — both landed in this merge).
expect(hookCalls).toBe(22);
// #2653: 23 phases (added `drift` between calibration_profile and
// conversation_facts_backfill).
expect(hookCalls).toBe(23);
});
test('hook exceptions do not abort the cycle', async () => {
@@ -409,7 +419,8 @@ describe('runCycle — yieldBetweenPhases hook', () => {
// v0.39.0.0: 17 phases (T12 schema-suggest phase between orphans and purge).
// v0.41.11.0: 20 phases (+extract_atoms, +synthesize_concepts, +conversation_facts_backfill).
// v0.41.39 (#1700) + v0.42.0.0: 22 phases (+enrich_thin, +skillopt).
expect(report.phases.length).toBe(22);
// #2653: 23 phases (+drift).
expect(report.phases.length).toBe(23);
});
});
@@ -499,6 +510,28 @@ describe('runCycle — sourceId resolution (regression #475)', () => {
expect(syncCalls.at(-1)?.sourceId).toBe('default');
});
test('seeded sources row → orphans phase receives matching sourceId', async () => {
await (sharedEngine as any).db.query(
`INSERT INTO sources (id, name, local_path) VALUES ($1, $2, $3)`,
['alpha', 'alpha', '/tmp/brain-2349-alpha'],
);
await runCycle(sharedEngine, { brainDir: '/tmp/brain-2349-alpha', phases: ['orphans'] });
expect(orphansOpts.at(-1)).toEqual({ sourceId: 'alpha' });
});
test('forceGlobalOrphans keeps orphans brain-wide even when brainDir maps to a source', async () => {
await (sharedEngine as any).db.query(
`INSERT INTO sources (id, name, local_path) VALUES ($1, $2, $3)`,
['global-source', 'global-source', '/tmp/brain-2349-global'],
);
await runCycle(sharedEngine, {
brainDir: '/tmp/brain-2349-global',
phases: ['embed', 'orphans', 'purge'],
forceGlobalOrphans: true,
});
expect(orphansOpts.at(-1)).toEqual({});
});
test('no matching sources row → performSync receives sourceId=undefined', async () => {
await runCycle(sharedEngine, { brainDir: '/tmp/brain-475-b' });
expect(syncCalls.at(-1)?.sourceId).toBeUndefined();
test/cycle-synthesize-slug-collection.test.ts
+43
View File
@@ -117,6 +117,22 @@ describe('C6: collectChildPutPageSlugs survives double-encoded jsonb (#745)', ()
expect(refs.length).toBeGreaterThan(0);
for (const r of refs) expect(r.source_id).toBe('default');
});
// #1978: refs carry the source transcript path when the orchestrator
// supplies a job_id → path map, so stampDreamProvenance can persist it.
test('stamps refs with raw_source from the jobRawSource map (#1978)', async () => {
const jobRawSource = new Map([[1001, '/transcripts/2026-07-01-standup.md']]);
const refs = await collectChildPutPageSlugs(engine as any, [1001], new Map(), 'default', jobRawSource);
const ref = refs.find((r: { slug: string }) => r.slug === 'wiki/agents/test/normal-shape');
expect(ref?.raw_source).toBe('/transcripts/2026-07-01-standup.md');
});
test('omits raw_source when no map entry exists for the job (#1978)', async () => {
const refs = await collectChildPutPageSlugs(engine as any, [1001], new Map(), 'default', new Map());
const ref = refs.find((r: { slug: string }) => r.slug === 'wiki/agents/test/normal-shape');
expect(ref).toBeDefined();
expect('raw_source' in (ref as object)).toBe(false);
});
});
describe('#2569: stampDreamProvenance persists the marker into DB frontmatter', () => {
@@ -151,4 +167,31 @@ describe('#2569: stampDreamProvenance persists the marker into DB frontmatter',
await stampDreamProvenance(engine as any, refs, '2026-07-17'); // no throw
await stampDreamProvenance(engine as any, refs, '2026-07-17'); // idempotent
});
// #1978: raw-source persistence — the stamp carries the transcript path
// the synthesis was derived from, when the ref supplies one.
test('persists raw_source into pages.frontmatter when the ref carries it (#1978)', async () => {
await engine.putPage('wiki/originals/ideas/2026-07-17-raw-src-def456', {
type: 'note',
title: 'Raw source stamp',
compiled_truth: 'body',
timeline: '',
frontmatter: {},
});
await stampDreamProvenance(
engine as any,
[{
slug: 'wiki/originals/ideas/2026-07-17-raw-src-def456',
source_id: 'default',
raw_source: '/transcripts/2026-07-17-standup.md',
}],
'2026-07-17',
);
const rows = await engine.executeRaw<{ fm: Record<string, unknown> }>(
`SELECT frontmatter AS fm FROM pages WHERE slug = 'wiki/originals/ideas/2026-07-17-raw-src-def456'`,
);
const fm = rows[0].fm as Record<string, unknown>;
expect(fm.dream_generated).toBe(true);
expect(fm.raw_source).toBe('/transcripts/2026-07-17-standup.md');
});
});
test/cycle/extract-atoms-synthesize-concepts.test.ts
+66
View File
@@ -370,3 +370,69 @@ describe('v0.41 T6: runPhaseSynthesizeConcepts via stubbed chat', () => {
expect((page[0].fm as Record<string, unknown>).tier).toBe('T1');
});
});
// #2123 — extract_atoms must stamp `concepts` so synthesize_concepts has
// material. The pre-fix pipeline was broken end-to-end: the extractor
// never wrote the field, and every synthesize_concepts cycle skipped with
// "no atoms with concept refs". The earlier describe blocks feed
// synthesize via the `_atoms` seam, which is exactly how the gap survived
// — so the last test here goes extractor → REAL frontmatter → real DB
// query path → concept page.
describe('#2123: concepts label parsing', () => {
test('keeps valid kebab-case labels', () => {
const raw = `[{"title":"T","atom_type":"insight","body":"b","concepts":["captive-portal","tls-certificates"]}]`;
expect(parseAtomsResponse(raw)[0].concepts).toEqual(['captive-portal', 'tls-certificates']);
});
test('filters non-kebab labels, keeps the rest', () => {
const raw = `[{"title":"T","atom_type":"insight","body":"b","concepts":["Captive Portal","tp_link","UPPER","valid-label"]}]`;
expect(parseAtomsResponse(raw)[0].concepts).toEqual(['valid-label']);
});
test('truncates to 3 labels', () => {
const raw = `[{"title":"T","atom_type":"insight","body":"b","concepts":["a","b","c","d","e"]}]`;
expect(parseAtomsResponse(raw)[0].concepts).toEqual(['a', 'b', 'c']);
});
test('absent / non-array / all-invalid → undefined', () => {
expect(parseAtomsResponse(`[{"title":"T","atom_type":"insight","body":"b"}]`)[0].concepts).toBeUndefined();
expect(parseAtomsResponse(`[{"title":"T","atom_type":"insight","body":"b","concepts":"not-an-array"}]`)[0].concepts).toBeUndefined();
expect(parseAtomsResponse(`[{"title":"T","atom_type":"insight","body":"b","concepts":["Bad Label!"]}]`)[0].concepts).toBeUndefined();
});
});
describe('#2123: extractor stamps concepts → synthesize_concepts consumes via real DB path', () => {
test('end-to-end: atoms with shared label materialize a concept page', async () => {
const chat = stubChat(`[
{"title":"Cert warning on guest wifi","atom_type":"insight","body":"Portal redirects to an IP-based HTTPS URL.","concepts":["captive-portal"]},
{"title":"iPhone portal popup is flaky","atom_type":"critique","body":"CNA probe behavior differs across iOS versions.","concepts":["captive-portal"]}
]`);
const extract = await runPhaseExtractAtoms(engine, {
_transcripts: [{ filePath: '/fake/notes.txt', content: 'content', contentHash: 'cc2123' }],
_pages: [],
_chat: chat,
});
expect(extract.status).toBe('ok');
expect(extract.details?.atoms_extracted).toBe(2);
// Frontmatter really carries the label (a jsonb array, not a string).
const stamped = await engine.executeRaw<{ concepts: unknown }>(
`SELECT frontmatter->'concepts' AS concepts FROM pages WHERE type = 'atom'`,
);
expect(stamped.length).toBe(2);
for (const row of stamped) {
const arr = typeof row.concepts === 'string' ? JSON.parse(row.concepts) : row.concepts;
expect(arr).toEqual(['captive-portal']);
}
// NO _atoms seam: synthesize discovers the atoms through its own
// DB query — this is the path that was dead before the fix.
const synth = await runPhaseSynthesizeConcepts(engine, { _chat: stubChat('unused — T3 is deterministic') });
expect(synth.status).toBe('ok');
expect(synth.details?.concepts_written).toBe(1);
const concept = await engine.executeRaw<{ slug: string }>(
`SELECT slug FROM pages WHERE slug = 'concepts/captive-portal' AND type = 'concept'`,
);
expect(concept.length).toBe(1);
});
});
test/doctor-raw-provenance.test.ts
+110
View File
@@ -0,0 +1,110 @@
/**
* #1978 — raw-source persistence guarantee (warn-only v1).
*
* `rawProvenanceCheck` flags synthesized/derived pages (dream_generated:true
* frontmatter or type:synthesis) that carry NO raw trace (raw_trace /
* raw_source / source_uri frontmatter, attached raw_data row, or
* synthesis_evidence rows) and NO explicit raw_trace_exempt marker.
*
* Runs against real PGLite so the SQL shape (`?|` key-existence operator +
* NOT EXISTS subqueries) is pinned on an actual engine, not a mock.
*/
import { describe, test, expect, beforeAll, afterAll } from 'bun:test';
import { PGLiteEngine } from '../src/core/pglite-engine.ts';
import { rawProvenanceCheck } from '../src/commands/doctor.ts';
import { categorizeCheck } from '../src/core/doctor-categories.ts';
import type { BrainEngine } from '../src/core/engine.ts';
let engine: PGLiteEngine;
beforeAll(async () => {
engine = new PGLiteEngine();
await engine.connect({});
await engine.initSchema();
});
afterAll(async () => {
await engine.disconnect();
});
describe('rawProvenanceCheck (#1978, warn-only v1)', () => {
test('empty brain → ok', async () => {
const result = await rawProvenanceCheck(engine as unknown as BrainEngine);
expect(result.name).toBe('raw_provenance');
expect(result.status).toBe('ok');
});
test('flags only the synthesized page without a trace; every trace/exemption shape passes', async () => {
// 1. VIOLATION: dream-generated, no trace, no exemption.
await engine.putPage('wiki/derived/no-trace', {
type: 'note', title: 'No trace', compiled_truth: 'body', timeline: '',
frontmatter: { dream_generated: true },
});
// 2. OK: dream-generated with raw_source frontmatter.
await engine.putPage('wiki/derived/with-raw-source', {
type: 'note', title: 'Has raw_source', compiled_truth: 'body', timeline: '',
frontmatter: { dream_generated: true, raw_source: '/transcripts/2026-07-01.md' },
});
// 3. OK: type:synthesis with explicit exemption.
await engine.putPage('synthesis/exempt-page', {
type: 'synthesis', title: 'Exempt', compiled_truth: 'body', timeline: '',
frontmatter: { raw_trace_exempt: true, raw_trace_exempt_reason: 'test' },
});
// 4. OK: hand-authored note — not synthesized, never flagged.
await engine.putPage('wiki/hand-authored', {
type: 'note', title: 'Hand authored', compiled_truth: 'body', timeline: '',
frontmatter: {},
});
// 5. OK: dream-generated with an attached raw_data row.
const withRaw = await engine.putPage('wiki/derived/with-raw-data', {
type: 'note', title: 'Has raw_data', compiled_truth: 'body', timeline: '',
frontmatter: { dream_generated: true },
});
await engine.executeRaw(
`INSERT INTO raw_data (page_id, source, data) VALUES ($1, 'test', '{}'::jsonb)`,
[withRaw.id],
);
const result = await rawProvenanceCheck(engine as unknown as BrainEngine);
expect(result.status).toBe('warn');
expect(result.message).toContain('1 synthesized page(s)');
expect(result.message).toContain('wiki/derived/no-trace');
expect(result.message).not.toContain('with-raw-source');
expect(result.message).not.toContain('exempt-page');
expect(result.message).not.toContain('hand-authored');
expect(result.message).not.toContain('with-raw-data');
});
test('stamping an exemption on the violator clears the warning', async () => {
await engine.executeRaw(
`UPDATE pages SET frontmatter = frontmatter || '{"raw_trace_exempt": true, "raw_trace_exempt_reason": "reviewed"}'::jsonb
WHERE slug = 'wiki/derived/no-trace'`,
);
const result = await rawProvenanceCheck(engine as unknown as BrainEngine);
expect(result.status).toBe('ok');
});
test('soft-deleted violators are not flagged', async () => {
await engine.putPage('wiki/derived/deleted-no-trace', {
type: 'note', title: 'Deleted violator', compiled_truth: 'body', timeline: '',
frontmatter: { dream_generated: true },
});
expect((await rawProvenanceCheck(engine as unknown as BrainEngine)).status).toBe('warn');
await engine.executeRaw(
`UPDATE pages SET deleted_at = now() WHERE slug = 'wiki/derived/deleted-no-trace'`,
);
expect((await rawProvenanceCheck(engine as unknown as BrainEngine)).status).toBe('ok');
});
test('query failure degrades to warn, never throws', async () => {
const broken = { executeRaw: async () => { throw new Error('boom'); } } as unknown as BrainEngine;
const result = await rawProvenanceCheck(broken);
expect(result.status).toBe('warn');
expect(result.message).toContain('Could not check');
});
test('raw_provenance is categorized as a brain check', () => {
expect(categorizeCheck('raw_provenance')).toBe('brain');
});
});
test/doctor-remediation-plan-render.test.ts
+103
View File
@@ -0,0 +1,103 @@
// Regression coverage for the `gbrain doctor --remediation-plan` verdict
// contradiction: when the brain was below target AND the target was
// unreachable, the human renderer printed "Target unreachable: max with
// autonomous remediation is N/100" followed immediately by "No
// remediations needed. Brain is at target." — two consecutive lines that
// contradicted each other and hid the real next step.
import { describe, test, expect } from 'bun:test';
import { renderRemediationPlanLines } from '../src/commands/doctor.ts';
type Plan = Parameters<typeof renderRemediationPlanLines>[0];
function planFixture(overrides: Partial<Plan>): Plan {
return {
brain_score_current: 0,
target_unreachable: false,
max_reachable_score: 100,
plan: [],
est_total_seconds: 0,
est_total_usd_cost: 0,
blocked: [],
...overrides,
};
}
describe('renderRemediationPlanLines', () => {
test('unreachable + brain below target — never claims "Brain is at target"', () => {
const plan = planFixture({
brain_score_current: 45,
target_unreachable: true,
max_reachable_score: 70,
plan: [],
blocked: [{ check: 'link_density', reason: 'no enrichment keys configured' }],
});
const text = renderRemediationPlanLines(plan, 90).join('\n');
expect(text).toContain('Brain score: 45/100');
expect(text).toContain('Target unreachable: max with autonomous remediation is 70/100');
expect(text).not.toContain('Brain is at target');
expect(text).toContain('Blocked checks');
});
test('reachable, brain at or above target, no plan — emits the "at target" line', () => {
const plan = planFixture({
brain_score_current: 95,
target_unreachable: false,
max_reachable_score: 100,
plan: [],
});
const text = renderRemediationPlanLines(plan, 90).join('\n');
expect(text).toContain('Brain is at target');
expect(text).not.toContain('Target unreachable');
});
test('brain at exact target with empty plan — still "at target"', () => {
const plan = planFixture({
brain_score_current: 90,
target_unreachable: false,
plan: [],
});
const text = renderRemediationPlanLines(plan, 90).join('\n');
expect(text).toContain('Brain is at target');
});
test('brain below target with plan steps — lists the plan, no "at target" line', () => {
const plan = planFixture({
brain_score_current: 60,
target_unreachable: false,
max_reachable_score: 100,
est_total_seconds: 120,
est_total_usd_cost: 0.4,
plan: [
{ step: 1, severity: 'high', job: 'embed-coverage', rationale: 'missing embeddings' },
{ step: 2, severity: 'med', job: 'consolidate', rationale: 'pending entity merges', est_usd_cost: 0.4 },
],
});
const lines = renderRemediationPlanLines(plan, 90);
const text = lines.join('\n');
expect(text).toContain('Plan: 2 step(s)');
expect(text).toContain('1. [high] embed-coverage');
expect(text).toContain('2. [med] consolidate');
expect(text).toContain('($0.40)');
expect(text).not.toContain('Brain is at target');
});
test('unreachable but a partial plan exists — plan prints, "at target" suppressed', () => {
const plan = planFixture({
brain_score_current: 30,
target_unreachable: true,
max_reachable_score: 55,
est_total_seconds: 90,
est_total_usd_cost: 0.2,
plan: [
{ step: 1, severity: 'high', job: 'embed-coverage', rationale: 'reach max_reachable' },
],
blocked: [{ check: 'enrichment', reason: 'no provider key configured' }],
});
const text = renderRemediationPlanLines(plan, 90).join('\n');
expect(text).toContain('Target unreachable: max with autonomous remediation is 55/100');
expect(text).toContain('Plan: 1 step(s)');
expect(text).toContain('Blocked checks');
expect(text).not.toContain('Brain is at target');
});
});
test/e2e/dream-cycle-phase-order-pglite.test.ts
+1
View File
@@ -127,6 +127,7 @@ const EXPECTED_PHASES: CyclePhase[] = [
'propose_takes', // v0.36.1.0 — hindsight calibration wave
'grade_takes', // v0.36.1.0
'calibration_profile', // v0.36.1.0
'drift', // #2653 — drift detection (default OFF, report-only)
'conversation_facts_backfill', // v0.41.11.0 — opt-in conversation backfill
'enrich_thin', // v0.41.39 (#1700) — brain-internal stub enrichment (default OFF)
'skillopt', // v0.42.0.0 — self-evolving skills (default OFF)
test/e2e/embedding-column-pglite.test.ts
+38
View File
@@ -216,6 +216,44 @@ describe('hybridSearch + resolver — unknown column at entry (D11)', () => {
});
});
describe('upsertChunks — model provenance uses gateway-resolved model, not compiled default', () => {
// Regression (zbrain-rfi): when a caller builds ChunkInputs without an
// explicit `model` (as src/commands/embed.ts does), the engine used to
// stamp the compile-time DEFAULT_EMBEDDING_MODEL ('zeroentropyai:zembed-1')
// onto content_chunks.model — even though the vector was produced by the
// config-resolved model. That corrupted provenance the signature-drift +
// dim-migration logic trusts. The engine must fall back to the model the
// gateway ACTUALLY resolves at write time.
test('unspecified chunk.model records the resolved model, not zeroentropyai:zembed-1', async () => {
configureGateway({
embedding_model: 'openai:text-embedding-3-large',
embedding_dimensions: 1536,
env: { OPENAI_API_KEY: 'sk-test' },
});
await engine.putPage('docs/provenance-page', {
type: 'concept',
title: 'Provenance test page',
compiled_truth: 'Chunk whose model column must reflect the resolved model.',
});
// No `model` field on the input — the write-side fallback must fill it.
await engine.upsertChunks('docs/provenance-page', [
{ chunk_index: 0, chunk_text: 'provenance chunk', chunk_source: 'compiled_truth' },
]);
const rows = await engine.executeRaw<{ model: string }>(
`SELECT cc.model FROM content_chunks cc
JOIN pages p ON p.id = cc.page_id
WHERE p.slug = 'docs/provenance-page'`,
);
expect(rows.length).toBe(1);
expect(rows[0].model).toBe('openai:text-embedding-3-large');
expect(rows[0].model).not.toBe('zeroentropyai:zembed-1');
resetGateway();
});
});
describe('buildVectorCastFragment — engine SQL composer (D3)', () => {
test('vector descriptor emits $1::vector', () => {
const r: ResolvedColumn = {
test/e2e/init-reinit-after-deferred.test.ts
-111
View File
@@ -1,111 +0,0 @@
/**
* #2301 — re-init with an explicit --embedding-model must recover a brain
* that was initialized with --no-embedding (deferred setup).
*
* Pre-fix: resolveAIOptions honored the persisted `embedding_disabled: true`
* sentinel BEFORE the explicit flag and never cleared noEmbedding, and the
* persistence merge carried the sentinel forward via ...existingFile. Result:
* every re-init (including the recovery command the deferred-setup error
* itself recommends) silently re-deferred embedding, forever.
*
* Hermetic: in-process runInit, GBRAIN_HOME pinned to a tmpdir (same pattern
* as test/e2e/fresh-install-pglite.test.ts).
*/
import { afterEach, beforeEach, describe, expect, test } from 'bun:test';
import { mkdtempSync, rmSync, readFileSync } from 'fs';
import { tmpdir } from 'os';
import { join } from 'path';
import { configureGateway, resetGateway } from '../../src/core/ai/gateway.ts';
describe('E2E: re-init with --embedding-model after --no-embedding init (#2301)', () => {
let tmpHome: string;
let origHome: string | undefined;
let origZeKey: string | undefined;
let origOpenaiKey: string | undefined;
let origVoyageKey: string | undefined;
beforeEach(() => {
tmpHome = mkdtempSync(join(tmpdir(), 'gbrain-e2e-reinit-'));
origHome = process.env.GBRAIN_HOME;
origZeKey = process.env.ZEROENTROPY_API_KEY;
origOpenaiKey = process.env.OPENAI_API_KEY;
origVoyageKey = process.env.VOYAGE_API_KEY;
delete process.env.OPENAI_API_KEY;
delete process.env.VOYAGE_API_KEY;
process.env.GBRAIN_HOME = tmpHome;
process.env.ZEROENTROPY_API_KEY = 'sk-test-ze';
resetGateway();
});
afterEach(() => {
rmSync(tmpHome, { recursive: true, force: true });
if (origHome === undefined) delete process.env.GBRAIN_HOME;
else process.env.GBRAIN_HOME = origHome;
if (origZeKey === undefined) delete process.env.ZEROENTROPY_API_KEY;
else process.env.ZEROENTROPY_API_KEY = origZeKey;
if (origOpenaiKey !== undefined) process.env.OPENAI_API_KEY = origOpenaiKey;
if (origVoyageKey !== undefined) process.env.VOYAGE_API_KEY = origVoyageKey;
// Restore legacy-preload gateway state (mirrors fresh-install-pglite.test.ts).
configureGateway({
embedding_model: 'openai:text-embedding-3-large',
embedding_dimensions: 1536,
env: { ...process.env },
});
});
async function runInitCapturing(args: string[]): Promise<string> {
const { runInit } = await import('../../src/commands/init.ts');
const origLog = console.log;
const origWarn = console.warn;
const stdoutBuf: string[] = [];
console.log = (...a: unknown[]) => {
stdoutBuf.push(a.map(x => (typeof x === 'string' ? x : JSON.stringify(x))).join(' '));
};
console.warn = () => {};
try {
await runInit(args);
} finally {
console.log = origLog;
console.warn = origWarn;
}
return stdoutBuf.join('\n');
}
const cfgPath = () => join(tmpHome, '.gbrain', 'config.json');
const readCfg = () => JSON.parse(readFileSync(cfgPath(), 'utf-8'));
test('explicit --embedding-model clears the persisted embedding_disabled sentinel', async () => {
// Step 1: deferred-setup init writes the sentinel.
const out1 = await runInitCapturing(['--pglite', '--non-interactive', '--no-embedding']);
expect(out1).toContain('deferred setup');
const cfg1 = readCfg();
expect(cfg1.embedding_disabled).toBe(true);
expect(cfg1.embedding_model).toBeUndefined();
// Step 2: re-init with an explicit embedding model — the recovery path.
// Pre-fix this printed the deferred-setup line again and re-persisted
// embedding_disabled: true.
const out2 = await runInitCapturing([
'--pglite', '--non-interactive', '--skip-embed-check',
'--embedding-model', 'zeroentropyai:zembed-1',
'--embedding-dimensions', '1280',
]);
expect(out2).not.toContain('deferred setup');
expect(out2).toContain('zeroentropyai:zembed-1');
const cfg2 = readCfg();
expect(cfg2.embedding_model).toBe('zeroentropyai:zembed-1');
expect(cfg2.embedding_dimensions).toBe(1280);
expect(cfg2.embedding_disabled).toBeUndefined();
}, 60000);
test('re-init WITHOUT flags still honors the deferred-setup sentinel (no regression)', async () => {
await runInitCapturing(['--pglite', '--non-interactive', '--no-embedding']);
const out = await runInitCapturing(['--pglite', '--non-interactive']);
expect(out).toContain('deferred setup');
const cfg = readCfg();
expect(cfg.embedding_disabled).toBe(true);
expect(cfg.embedding_model).toBeUndefined();
}, 60000);
});

Some files were not shown because too many files have changed in this diff Show More