diff --git a/docs/INSTALL.md b/docs/INSTALL.md index f3517161b..7c7df9851 100644 --- a/docs/INSTALL.md +++ b/docs/INSTALL.md @@ -107,7 +107,7 @@ Useful for: team mounts, brain-as-a-service deployments, dev machines without di ```bash gbrain doctor --json # full health check gbrain models # which AI models are configured for what -gbrain models doctor # 1-token probe per configured model +gbrain models doctor # minimal reachability probe per configured model ``` If anything's yellow, `gbrain doctor` names the fix command in the message. Most issues are missing API keys or stale schema (`gbrain upgrade --force-schema`). diff --git a/docs/architecture/KEY_FILES.md b/docs/architecture/KEY_FILES.md index c102a7ef5..6fbeb9023 100644 --- a/docs/architecture/KEY_FILES.md +++ b/docs/architecture/KEY_FILES.md @@ -132,11 +132,11 @@ per-release `**vX.Y.Z:**` narration — CI enforces this - `src/core/diarize/payload-fitter.ts` — generic fit-arbitrarily-large-items-into-per-call-token-budget utility. `'batch'` strategy is deterministic token-budgeted chunking with no LLM calls. `'summarize'` strategy embed-clusters into ceil(items/4) groups via cheap deterministic nearest-neighbor on cosine, Haiku-summarizes each cluster via `Promise.allSettled` at parallelism=4. Each Haiku call composes the active BudgetTracker via the AsyncLocalStorage. Quality gate: when `success_ratio < min_success_ratio` (default 0.75), result is flagged `degraded: true` — the fitter preserves the successful subset; the caller decides whether to surface a partial result or abort. - `src/core/brainstorm/checkpoint.ts` — crash-resilient checkpoint for `gbrain brainstorm` and `gbrain lsd`. Persists FULL idea bodies (~50KB/run) so resume MERGES pre-crash ideas with post-resume ideas before the judge runs (a resume that produces only second-run output is silent partial output). `run_id = sha256(question + profile + sort(close_slugs) + sort(far_slugs)).slice(0,16)` — NO embedding bits, stable across embedding-model swaps. Atomic write via `.tmp + rename`. ONE resume flag (`--resume ` covers both failed AND never-attempted crosses); `--list-runs` prints run_ids mtime-newest-first; `--force-resume` bypasses the 7-day staleness gate. Cycle purge phase (`gbrain dream --phase purge`) GCs checkpoints older than 7 days via `gcStaleCheckpoints(7)`. Pinned by `test/e2e/brainstorm-resume.test.ts` (20 unit + 3 E2E cases incl. the merge contract). - `src/core/remediation-checkpoint.ts` — `doctor --remediate` checkpoint at `~/.gbrain/remediation/.json`. `plan_hash = sha256(JSON.stringify(sorted recommendation ids)).slice(0,16)`. Schema-versioned, atomic `.tmp + rename`. `gbrain doctor --remediate --resume ` (no arg picks newest matching) loads it and skips completed steps. Mismatched plan_hash refuses with a paste-ready message. Cleared on clean completion. Pinned by 13 unit cases. -- `src/core/model-config.ts` — Model-string resolution (the seam every internal LLM call walks through). Four-tier system (`ModelTier = 'utility' | 'reasoning' | 'deep' | 'subagent'`) with `TIER_DEFAULTS` (utility→haiku-4-5, reasoning→sonnet-4-6, deep→opus-4-7, subagent→sonnet-4-6) and `tier?: ModelTier` on `ResolveModelOpts`. 8-step resolution chain: cliFlag → deprecated key → config key → `models.default` → `models.tier.` → env var → `TIER_DEFAULTS[tier]` → caller fallback. `isAnthropicProvider(modelString)` checks `provider:model` prefix OR `claude-` bare-id pattern (routes through `splitProviderModelId` from `src/core/model-id.ts` so slash-form ids like `anthropic/claude-sonnet-4-6` classify correctly). `enforceSubagentAnthropic()` is the layer-2 runtime guard: when `tier === 'subagent'` resolves non-Anthropic, it emits a once-per-`(source, model)` stderr warn AND falls back to `TIER_DEFAULTS.subagent` (the Anthropic Messages API tool-loop can't run on OpenAI/Gemini). `_resetDeprecationWarningsForTest()` also clears `_subagentTierWarningsEmitted`. Pinned by `test/model-config.serial.test.ts`. +- `src/core/model-config.ts` — Model-string resolution (the seam every internal LLM call walks through). Four-tier system (`ModelTier = 'utility' | 'reasoning' | 'deep' | 'subagent'`) with `TIER_DEFAULTS` (utility→haiku-4-5, reasoning→sonnet-4-6, deep→opus-4-7, subagent→sonnet-4-6) and `tier?: ModelTier` on `ResolveModelOpts`. 9-step resolution chain: cliFlag → config key → deprecated key → `models.default` → `models.tier.` → env var → `userFallback` (caller-attested explicit user config, e.g. the file-plane `chat_model` from config.json — ranks above the tier default so user settings can't be silently replaced by `TIER_DEFAULTS`; `reconfigureGatewayWithEngine` passes it only when the gateway value differs from the hardcoded default) → `TIER_DEFAULTS[tier]` → caller fallback. `isAnthropicProvider(modelString)` checks `provider:model` prefix OR `claude-` bare-id pattern (routes through `splitProviderModelId` from `src/core/model-id.ts` so slash-form ids like `anthropic/claude-sonnet-4-6` classify correctly). `enforceSubagentAnthropic()` is the layer-2 runtime guard: when `tier === 'subagent'` resolves non-Anthropic, it emits a once-per-`(source, model)` stderr warn AND falls back to `TIER_DEFAULTS.subagent` (the Anthropic Messages API tool-loop can't run on OpenAI/Gemini). `_resetDeprecationWarningsForTest()` also clears `_subagentTierWarningsEmitted`. Pinned by `test/model-config.serial.test.ts`. - `src/core/ai/model-resolver.ts` — Recipe-touchpoint validator. `assertTouchpoint(recipe, touchpoint, modelId, extendedModels?)` takes an optional 4th `extendedModels: ReadonlySet`: when the modelId is in that set the native-recipe allowlist throw is bypassed (user explicitly opted in via config, so provider rejection surfaces as `model_not_found` at HTTP call time and `gbrain models doctor` catches it earlier). Default code paths with hardcoded model strings MUST NOT pass `extendedModels` — source typos still fail fast (the fail-fast contract for chat + expand + embed stays intact). - `src/core/ai/gateway.ts` extension — module-scoped `_extendedModels: Map>` registry feeds `assertTouchpoint`'s 4th-arg path. `reconfigureGatewayWithEngine(engine)` (async, called from `cli.ts` after `engine.connect()`, before every command except `CLI_ONLY` no-DB commands) re-resolves expansion + chat defaults through `resolveModel()` so `models.tier.*` and `models.default` overrides apply to both. `DEFAULT_CHAT_MODEL` is `anthropic:claude-sonnet-4-6`. `__setChatTransportForTests` seam mirrors `__setEmbedTransportForTests` so tests drive `chat()` with a stubbed transport. - `src/core/minions/queue.ts` extension — `MinionQueue.add()` rejects `subagent` jobs whose `data.model` resolves via `isAnthropicProvider()` to a non-Anthropic provider. Lazy-imports `model-config.ts` to avoid pulling engine types into queue's eager-load surface. Layer 1 of the three-layer subagent provider enforcement (layers 2+3: `model-config.ts:enforceSubagentAnthropic` runtime fallback + `src/commands/doctor.ts` `subagent_provider` check). Pinned by `test/agent-cli.test.ts`. -- `src/commands/models.ts` — `gbrain models [--json]` read-only routing dashboard: prints tier defaults (`utility`/`reasoning`/`deep`/`subagent`), the resolved value for each (re-walking the resolution chain), every per-task override (11 `PER_TASK_KEYS`: `models.dream.synthesize`, `models.dream.patterns`, `models.drift`, `models.auto_think`, `models.think`, `models.subagent`, `facts.extraction_model`, `models.eval.longmemeval`, `models.expansion`, `models.chat`, `models.dream.synthesize_verdict`), the alias map, and a source-of-truth column (`default` / `config: ` / `env: `). `gbrain models doctor [--skip=] [--json]` fires a 1-token `gateway.chat()` probe against each configured chat + expansion model and classifies failures into `{model_not_found, auth, rate_limit, network, unknown}`. Wired into `cli.ts` dispatch + `CLI_ONLY` set. A zero-token `embedding_config` probe runs FIRST, before any chat/expansion probes spend money: `probeEmbeddingConfig()` reads `getEmbeddingModel()` + `getEmbeddingDimensions()` and (for Voyage flexible-dim models) checks `isValidVoyageOutputDim(dims)` against `VOYAGE_VALID_OUTPUT_DIMS`. `ProbeStatus` variant `'config'` + optional `fix?: string` on `ProbeResult` surface a paste-ready `gbrain config set ...` line in human + JSON output; touchpoint label `'embedding_config'` joins `'chat'` and `'expansion'`. +- `src/commands/models.ts` — `gbrain models [--json]` read-only routing dashboard: prints tier defaults (`utility`/`reasoning`/`deep`/`subagent`), the resolved value for each (re-walking the resolution chain), every per-task override (11 `PER_TASK_KEYS`: `models.dream.synthesize`, `models.dream.patterns`, `models.drift`, `models.auto_think`, `models.think`, `models.subagent`, `facts.extraction_model`, `models.eval.longmemeval`, `models.expansion`, `models.chat`, `models.dream.synthesize_verdict`), the alias map, and a source-of-truth column (`default` / `config: ` / `env: `). `gbrain models doctor [--skip=] [--json]` fires a small bounded `gateway.chat()` probe (`PROBE_MAX_OUTPUT_TOKENS`, enough headroom that reasoning models which burn output budget on internal reasoning don't falsely fail; a length-exhausted empty-text response counts as reachable with the limitation surfaced in the probe message) against each configured chat + expansion model AND each distinct per-task background route (`resolvePerTaskProbePlan` resolves every `PER_TASK_KEYS` route the same way `buildReport` does, dedups against the chat/expansion models and across routes, and probes each distinct resolved model once with the comma-joined route keys as the probe's `touchpoint` for attribution), classifying failures into `{model_not_found, auth, rate_limit, network, unknown}`. Wired into `cli.ts` dispatch + `CLI_ONLY` set. A zero-token `embedding_config` probe runs FIRST, before any chat/expansion probes spend money: `probeEmbeddingConfig()` reads `getEmbeddingModel()` + `getEmbeddingDimensions()` and (for Voyage flexible-dim models) checks `isValidVoyageOutputDim(dims)` against `VOYAGE_VALID_OUTPUT_DIMS`. `ProbeStatus` variant `'config'` + optional `fix?: string` on `ProbeResult` surface a paste-ready `gbrain config set ...` line in human + JSON output; touchpoint label `'embedding_config'` joins `'chat'` and `'expansion'`. - `src/core/init-embed-check.ts` — embedding-key validation at `gbrain init`. `runInitEmbedCheck(opts)` runs a config-only `diagnoseEmbedding` (catches a missing key for ANY provider) plus a best-effort `liveTestEmbed` (1-token `gateway.embed(['probe'], {inputType:'query', abortSignal})`, 5s `AbortController` timeout, never throws — catches an invalid/expired key). Loud warning to stderr; init still exits 0 (`--no-embedding` is the deferred-setup escape; `--skip-embed-check` / `GBRAIN_INIT_SKIP_EMBED_CHECK=1` skip the check). Builds the effective env (`process.env` + file-plane `openai/anthropic/zeroentropy_api_key` from `loadConfigFileOnly()` + `opts.apiKey`) and configures the gateway via `buildGatewayConfig` before diagnose/probe, so the check sees the same keys AND provider base URLs runtime will (no false "missing key" for config.json-keyed users; the probe hits the right endpoint). Init-specific warning text names `--no-embedding` / `--skip-embed-check`, not the sync-flavored `--no-embed`. Wired into `initPGLite` + `initPostgres` in `src/commands/init.ts`, with the result added to the `--json` envelope as `embedding_check {ok, reason?, live_ok?}`. Pinned by `test/init-embed-check.test.ts` (hermetic via the gateway embed-transport seam + `withEnv`). - `src/core/ai/build-gateway-config.ts` — `buildGatewayConfig(c: GBrainConfig): AIGatewayConfig`, extracted from `src/cli.ts` (which re-exports it for back-compat). Lets core modules (`init-embed-check.ts`) reuse it without importing the CLI entrypoint. Single owner of folding file-plane API keys (openai/anthropic/zeroentropy) into the gateway env and threading local-server `*_BASE_URL` env vars into base_urls. `process.env` wins EXCEPT empty-string / undefined values are dropped before the merge, so an injected empty `ANTHROPIC_API_KEY=''` (Claude Code neuters subprocess LLM calls this way) can't clobber a valid config-plane key; `'0'` / `'false'` are preserved. Pinned by `test/ai/build-gateway-config.test.ts`. - `src/commands/doctor.ts` extension — `subagent_provider` check (layer 3 of 3). Warns when `models.tier.subagent` is explicitly set non-Anthropic (message names the bad value + paste-ready fix `gbrain config set models.tier.subagent anthropic:claude-sonnet-4-6`); also warns when `models.default` would sneak `subagent` into a non-Anthropic provider via tier inheritance. OK when subagent tier resolves to Anthropic. Tests in `test/doctor.test.ts`. diff --git a/skills/conventions/model-routing.md b/skills/conventions/model-routing.md index 14a3d5cad..5f7760d05 100644 --- a/skills/conventions/model-routing.md +++ b/skills/conventions/model-routing.md @@ -47,7 +47,7 @@ Visibility: ```bash gbrain models # print current routing table -gbrain models doctor # 1-token probe to each configured model +gbrain models doctor # minimal reachability probe to each configured model ``` **Subagent tier exists because the loop is Anthropic-only.** The handler diff --git a/src/commands/models.ts b/src/commands/models.ts index b44e8f506..da3358e3b 100644 --- a/src/commands/models.ts +++ b/src/commands/models.ts @@ -9,8 +9,9 @@ * per-task overrides, alias map, and source-of-truth * column (default / config / env). * - * `gbrain models doctor` — opt-in probe. Fires a 1-token `gateway.chat()` - * call against each configured chat / expansion + * `gbrain models doctor` — opt-in probe. Fires a small bounded + * `gateway.chat()` call (PROBE_MAX_OUTPUT_TOKENS) + * against each configured chat / expansion * model and reports reachability with the * provider's error string. Catches the bug class * that motivated v0.31.12 (the v0.31.6 chat @@ -22,9 +23,10 @@ * --skip= — narrow `doctor` probe to skip a provider * (e.g. cost-sensitive operators with rate limits) * - * Per Codex F11 in plan review: no specific dollar cost claim. Probe uses - * `max_tokens: 1` against each configured model; actual cost depends on - * provider billing minimums. + * Per Codex F11 in plan review: no specific dollar cost claim. Probe caps + * output at PROBE_MAX_OUTPUT_TOKENS against each configured model (widened + * above any configured extended-thinking budget); actual cost depends on + * tokens actually generated and provider billing minimums. */ import type { BrainEngine } from '../core/engine.ts'; @@ -161,7 +163,14 @@ type ProbeStatus = 'ok' | 'model_not_found' | 'auth' | 'rate_limit' | 'network' interface ProbeResult { model: string; - touchpoint: 'chat' | 'expansion' | 'embedding_config' | 'embedding_reachability' | 'reranker_config'; + /** + * Which surface this probe covers. The fixed labels ('chat', 'expansion', + * 'embedding_config', 'embedding_reachability', 'reranker_config') are + * joined by per-task route keys (#3219): a probe covering one or more + * PER_TASK_KEYS routes carries the comma-joined route keys (e.g. + * 'models.dream.synthesize,models.think') for attribution. + */ + touchpoint: string; status: ProbeStatus; message: string; elapsed_ms: number; @@ -503,30 +512,110 @@ async function probeEmbeddingReachability(): Promise { } } -async function probeModel(modelStr: string, touchpoint: 'chat' | 'expansion'): Promise { +/** + * Output budget for the chat/expansion reachability probe. + * + * This was `maxTokens: 1`, which falsely fails reasoning models: they spend + * output budget on internal reasoning before emitting any text, so a 1-token + * cap is either exhausted with zero usable text (finishReason 'length') or + * rejected outright by providers that require the cap to exceed the model's + * minimum reasoning spend — and the probe then reported a config failure for + * a perfectly reachable model. 64 tokens gives every model class enough + * headroom to prove transport + auth + model routing, while staying a + * minimal-cost probe: providers bill actual tokens generated, not the cap, + * and non-reasoning models answer '.' in a handful of tokens and stop. + * + * @internal exported for tests (test/models-doctor-probe-token-budget.test.ts). + */ +export const PROBE_MAX_OUTPUT_TOKENS = 64; + +/** @internal exported for tests (test/models-doctor-probe-token-budget.test.ts). */ +export async function probeModel(modelStr: string, touchpoint: string): Promise { const start = Date.now(); try { - const { chat } = await import('../core/ai/gateway.ts'); + const { chat, getConfiguredThinkingBudget } = await import('../core/ai/gateway.ts'); + // Anthropic requires max_tokens > thinking.budgetTokens. When the user + // configured an extended-thinking budget for this model via + // provider_chat_options, a fixed small cap would be rejected outright — + // the same false-FAIL class this probe budget exists to prevent. Widen + // the cap above the configured budget; billing is still actual tokens. + const thinkingBudget = getConfiguredThinkingBudget(modelStr); + const maxTokens = (thinkingBudget ?? 0) + PROBE_MAX_OUTPUT_TOKENS; // Use AbortController so the 5s timeout doesn't hang on a stuck network. const controller = new AbortController(); const timeoutId = setTimeout(() => controller.abort(new Error('probe timed out after 5s')), 5000); try { - await chat({ + const res = await chat({ model: modelStr, messages: [{ role: 'user', content: '.' }], - maxTokens: 1, + maxTokens, abortSignal: controller.signal, }); - return { model: modelStr, touchpoint, status: 'ok', message: 'reachable', elapsed_ms: Date.now() - start }; + // A length-exhausted response with no text is still proof of + // reachability: the request survived transport + auth + model routing, + // and the model generated tokens — a reasoning model may spend the + // whole probe budget on internal reasoning. Report ok, but surface the + // generation limitation instead of a bare 'reachable'. + const message = + res.stopReason === 'length' && res.text.length === 0 + ? `reachable (spent the ${maxTokens}-token probe budget on internal reasoning without emitting text; transport verified, text generation not exercised)` + : 'reachable'; + return { model: modelStr, touchpoint, status: 'ok', message, elapsed_ms: Date.now() - start }; } finally { clearTimeout(timeoutId); } } catch (err) { + // #3249 coherence: when the gateway rejects contentless length-exhausted + // completions (AIConfigError "output budget exhausted before any content + // was emitted"), the request still survived transport + auth + model + // routing — the same reachable-but-reasoning-burned class the in-band + // stopReason === 'length' branch above handles. Classify it as reachable + // instead of a probe failure. + const raw = err instanceof Error ? err.message : String(err); + if (/output budget exhausted before any content/i.test(raw)) { + return { + model: modelStr, + touchpoint, + status: 'ok', + message: 'reachable (spent the probe budget on internal reasoning without emitting text; transport verified, text generation not exercised)', + elapsed_ms: Date.now() - start, + }; + } const { status, message } = classifyError(err); return { model: modelStr, touchpoint, status, message, elapsed_ms: Date.now() - start }; } } +/** + * #3219: resolve every PER_TASK_KEYS route (dream synthesis, think, facts + * extraction, …) the same way `buildReport` does, drop the routes whose + * resolved model is already covered by another probe, and group the rest by + * resolved model so each distinct provider:model is probed exactly once with + * the route keys retained for attribution. Pre-fix, doctor probed only the + * global chat + expansion models, so a per-task route configured to a + * distinct unreachable provider still reported green. + * + * `models.chat` / `models.expansion` are excluded up front — those routes ARE + * the chat/expansion probes. + * + * @internal exported for tests (test/models-doctor-per-task-routes.test.ts). + */ +export async function resolvePerTaskProbePlan( + engine: BrainEngine, + alreadyProbed: ReadonlySet, +): Promise> { + const byModel = new Map(); + for (const { key, tier } of PER_TASK_KEYS) { + if (key === 'models.chat' || key === 'models.expansion') continue; + const resolved = await resolveModel(engine, { configKey: key, tier, fallback: TIER_DEFAULTS[tier] }); + if (alreadyProbed.has(resolved)) continue; + const routes = byModel.get(resolved) ?? []; + routes.push(key); + byModel.set(resolved, routes); + } + return [...byModel.entries()].map(([model, routes]) => ({ model, routes })); +} + function shouldSkipProvider(modelStr: string, skip: string[]): boolean { if (skip.length === 0) return false; const colon = modelStr.indexOf(':'); @@ -555,7 +644,7 @@ export async function runModels(engine: BrainEngine, args: string[]): Promise.thinking.budgetTokens`, provider- + * or model-scoped). The Anthropic API requires `max_tokens` to exceed + * `thinking.budgetTokens`, so callers that cap output tightly (the + * `gbrain models doctor` reachability probe) must widen their cap above this + * budget or the provider rejects the request outright — falsely failing a + * reachable model. Returns undefined when no budget is configured, the model + * string is malformed, or the gateway is unconfigured. Read-only, never throws. + * + * @internal exported for the doctor probe + tests; not part of the public gateway API. + */ +export function getConfiguredThinkingBudget(modelStr: string): number | undefined { + try { + if (!_config) return undefined; + // Mirror chat()'s resolution (resolveChatProvider → resolveRecipe): map + // recipe aliases to the canonical model id BEFORE the model-scoped + // provider_chat_options lookup, so a request made under a stale alias + // still sees the budget configured under the canonical id. + const { parsed, recipe } = resolveRecipe(modelStr); + const merged: Record = {}; + applyConfiguredChatProviderOptions(merged, _config, recipe.id, parsed.modelId); + const budget = merged[recipe.id]?.thinking?.budgetTokens; + return typeof budget === 'number' && Number.isFinite(budget) && budget > 0 + ? budget + : undefined; + } catch { + return undefined; + } +} + function applyConfiguredChatProviderOptions( providerOptions: Record, cfg: AIGatewayConfig, diff --git a/src/core/facts/classify.ts b/src/core/facts/classify.ts index 259882eb2..790cc7c05 100644 --- a/src/core/facts/classify.ts +++ b/src/core/facts/classify.ts @@ -95,7 +95,11 @@ export async function classifyAgainstCandidates( } // Try the classifier. On failure, fall back to cosine ≥ 0.92 → DUPLICATE. - if (!isAvailable('chat')) { + // #3206: gate on the model this call will ACTUALLY use — an opts.model + // pointing at a non-default provider must not be vetoed by an unavailable + // global chat default (and vice versa). + const classifierModel = opts.model ?? 'anthropic:claude-haiku-4-5-20251001'; + if (!isAvailable('chat', classifierModel)) { if (topId !== null && topScore >= fallback) { return { decision: 'duplicate', matched_id: topId, reason: 'cosine_fallback' }; } @@ -105,7 +109,7 @@ export async function classifyAgainstCandidates( let classifierResult: ChatResult | null = null; try { classifierResult = await chat({ - model: opts.model ?? 'anthropic:claude-haiku-4-5-20251001', + model: classifierModel, system: CLASSIFIER_SYSTEM, messages: [ { diff --git a/src/core/facts/extract.ts b/src/core/facts/extract.ts index 0ee9930ec..c14fd9870 100644 --- a/src/core/facts/extract.ts +++ b/src/core/facts/extract.ts @@ -173,16 +173,22 @@ export async function extractFactsFromTurn(input: ExtractInput): Promise\n${cleaned}\n\n\nExtract up to ${cap} facts.${ input.entityHints && input.entityHints.length ? ` Known entity slugs the user already mentioned: ${input.entityHints.slice(0, 5).join(', ')}.` diff --git a/src/core/model-config.ts b/src/core/model-config.ts index 9b993c8d0..ee0996acf 100644 --- a/src/core/model-config.ts +++ b/src/core/model-config.ts @@ -10,7 +10,10 @@ * — read with stderr deprecation warning, one-per-process * 4. Global default (models.default) * 5. Env var (process.env[envVar] or GBRAIN_MODEL) - * 6. Hardcoded fallback (caller-supplied) + * 6. User-configured fallback (`userFallback` — caller-attested explicit + * user config, e.g. file-plane chat_model; beats tier defaults, #3206) + * 7. Tier default (TIER_DEFAULTS[tier]) + * 8. Hardcoded fallback (caller-supplied) * * Aliases (`opus`, `sonnet`, `haiku`, `gemini`, `gpt`) resolve at the end so any * tier can use a short name. Unknown alias passes through unchanged so users can @@ -44,6 +47,16 @@ export interface ResolveModelOpts { * with a one-shot stderr warn instead). */ tier?: ModelTier; + /** + * A value the CALLER knows was explicitly user-configured (e.g. the + * file-plane `chat_model` from `~/.gbrain/config.json`), as opposed to a + * hardcoded caller default (#3206). Ranks above the tier default (step 7) + * but below every config-key / env override — so a user's file-plane + * setting can't be silently replaced by `TIER_DEFAULTS`, while incident + * escape hatches (`GBRAIN_MODEL`) still win. Callers MUST NOT pass their + * own hardcoded defaults here; those belong in `fallback`. + */ + userFallback?: string; /** Hardcoded last-resort fallback. */ fallback: string; } @@ -190,6 +203,15 @@ export async function resolveModel( return enforceSubagentCapable(resolved, opts.tier, `env:${envVar}`); } + // 6.5. Explicitly user-configured fallback (#3206). Beats the tier default: + // a user who set `chat_model` in config.json meant it, and the tier + // default silently overriding it disabled every chat-gated feature on + // non-Anthropic brains with no error. + if (opts.userFallback && opts.userFallback.trim()) { + const resolved = await resolveAlias(engine, opts.userFallback.trim()); + return enforceSubagentCapable(resolved, opts.tier, 'userFallback'); + } + // 7. Tier default (v0.31.12 — when no override beats us, the tier's // canonical model wins over caller-supplied fallback) if (opts.tier && TIER_DEFAULTS[opts.tier]) { diff --git a/test/facts-model-gate.test.ts b/test/facts-model-gate.test.ts new file mode 100644 index 000000000..78fb778ef --- /dev/null +++ b/test/facts-model-gate.test.ts @@ -0,0 +1,104 @@ +/** + * #3206 (silent-zero half) — the facts chat-availability gates must check the + * model the call will ACTUALLY use, not the global chat default. + * + * Repro class: a brain configured against a self-hosted OpenAI-compatible + * endpoint (litellm recipe, no auth required) with no ANTHROPIC_API_KEY. The + * extraction/classifier model resolves correctly to the self-hosted provider, + * but the pre-fix gate `isAvailable('chat')` consulted `getChatModel()` — the + * (unavailable) Anthropic default — and silently returned zero facts / + * cosine-fallback with no error. + * + * Hermetic: `__setGenerateTextTransportForTests` stubs the SDK call AFTER + * provider resolution, so the litellm recipe + auth check stay live while no + * network is touched. (`__setChatTransportForTests` can't be used here — it + * makes `isAvailable('chat')` unconditionally true, which would hide the bug.) + */ + +import { describe, test, expect, beforeEach, afterEach } from 'bun:test'; +import { + configureGateway, + resetGateway, + __setGenerateTextTransportForTests, +} from '../src/core/ai/gateway.ts'; +import { extractFactsFromTurn } from '../src/core/facts/extract.ts'; +import { classifyAgainstCandidates } from '../src/core/facts/classify.ts'; +import type { FactRow } from '../src/core/engine.ts'; + +function installTextTransport(text: string): void { + __setGenerateTextTransportForTests(async () => ({ + content: [{ type: 'text', text }], + finishReason: 'stop', + usage: { inputTokens: 10, outputTokens: 10 }, + }) as any); +} + +beforeEach(() => { + resetGateway(); + // No API keys at all: the Anthropic default chat model is UNAVAILABLE, + // while the auth-free litellm recipe is available. + configureGateway({ + embedding_model: 'openai:text-embedding-3-large', + embedding_dimensions: 1536, + env: {}, + }); +}); + +afterEach(() => { + __setGenerateTextTransportForTests(null); + resetGateway(); +}); + +describe('#3206 — facts gates check the model actually used', () => { + test('extractFactsFromTurn extracts via a non-default provider when the global chat default is unavailable', async () => { + installTextTransport(JSON.stringify({ + facts: [{ fact: 'User moved to Tokyo', kind: 'event', confidence: 1.0, notability: 'high' }], + })); + + const facts = await extractFactsFromTurn({ + turnText: 'I moved to Tokyo last month and it changed everything.', + source: 'test:model-gate', + model: 'litellm:qwen-test', + }); + + // Pre-fix: isAvailable('chat') consulted the unavailable Anthropic + // default and this silently returned []. + expect(facts.length).toBe(1); + expect(facts[0]!.fact).toContain('Tokyo'); + }); + + test('extractFactsFromTurn still bails when the model it would use is unavailable', async () => { + installTextTransport(JSON.stringify({ facts: [{ fact: 'x', kind: 'event' }] })); + + const facts = await extractFactsFromTurn({ + turnText: 'Some meaningful turn text about a life event.', + source: 'test:model-gate', + model: 'anthropic:claude-sonnet-4-6', // requires ANTHROPIC_API_KEY — absent + }); + + expect(facts).toEqual([]); + }); + + test('classifyAgainstCandidates runs the classifier via a non-default provider when the chat default is unavailable', async () => { + installTextTransport('{"decision":"duplicate","matched_id":7}'); + + const candidate = { + id: 7, + fact: 'User lives in Tokyo', + kind: 'fact', + embedding: null, + } as unknown as FactRow; + + const result = await classifyAgainstCandidates( + { fact: 'User moved to Tokyo', kind: 'fact', embedding: null }, + [candidate], + { model: 'litellm:qwen-test' }, + ); + + // Pre-fix: the gate consulted the unavailable Anthropic default and this + // degraded to reason 'cosine_fallback' (decision 'independent', since no + // embeddings) instead of running the classifier. + expect(result.reason).toBe('classifier'); + expect(result.decision).toBe('duplicate'); + }); +}); diff --git a/test/gateway-tier-extended-models.test.ts b/test/gateway-tier-extended-models.test.ts index 717ab2baf..7320317ce 100644 --- a/test/gateway-tier-extended-models.test.ts +++ b/test/gateway-tier-extended-models.test.ts @@ -15,6 +15,8 @@ import { describe, test, expect, afterEach } from 'bun:test'; import { configureGateway, + getChatModel, + getExpansionModel, reconfigureGatewayWithEngine, resetGateway, validateModelId, @@ -61,3 +63,56 @@ describe('reconfigureGatewayWithEngine — tier models extend the allowlist', () expect(validateModelId('anthropic:claude-hypothetical-10').ok).toBe(true); }); }); + +describe('#3206 — reconfigure keeps explicitly configured file-plane models', () => { + const savedGbrainModel = process.env.GBRAIN_MODEL; + afterEach(() => { + if (savedGbrainModel === undefined) delete process.env.GBRAIN_MODEL; + else process.env.GBRAIN_MODEL = savedGbrainModel; + }); + + test('config.json chat_model / expansion_model survive reconfigure with no models.* keys set', async () => { + delete process.env.GBRAIN_MODEL; + configureGateway({ + embedding_model: 'openai:text-embedding-3-large', + embedding_dimensions: 1536, + chat_model: 'litellm:qwen-test', + expansion_model: 'litellm:expander-test', + env: {}, + }); + + await reconfigureGatewayWithEngine(stubEngine({})); + + // Pre-fix: cfg.chat_model rode resolveModel's bottom-rung `fallback` and + // TIER_DEFAULTS silently replaced it with anthropic:claude-sonnet-4-6. + expect(getChatModel()).toBe('litellm:qwen-test'); + expect(getExpansionModel()).toBe('litellm:expander-test'); + }); + + test('DB-plane models.chat still beats the file-plane chat_model', async () => { + delete process.env.GBRAIN_MODEL; + configureGateway({ + embedding_model: 'openai:text-embedding-3-large', + embedding_dimensions: 1536, + chat_model: 'litellm:qwen-test', + env: { ANTHROPIC_API_KEY: 'sk-fake' }, + }); + + await reconfigureGatewayWithEngine(stubEngine({ 'models.chat': 'anthropic:claude-sonnet-4-6' })); + + expect(getChatModel()).toBe('anthropic:claude-sonnet-4-6'); + }); + + test('unset chat_model still resolves through the tier default', async () => { + delete process.env.GBRAIN_MODEL; + configureGateway({ + embedding_model: 'openai:text-embedding-3-large', + embedding_dimensions: 1536, + env: { ANTHROPIC_API_KEY: 'sk-fake' }, + }); + + await reconfigureGatewayWithEngine(stubEngine({})); + + expect(getChatModel()).toBe('anthropic:claude-sonnet-4-6'); + }); +}); diff --git a/test/model-config.serial.test.ts b/test/model-config.serial.test.ts index 94f28bc19..e9d2d70e1 100644 --- a/test/model-config.serial.test.ts +++ b/test/model-config.serial.test.ts @@ -175,6 +175,34 @@ describe('resolveModel — v0.31.12 tier system', () => { expect(m).toBe(TIER_DEFAULTS.reasoning); }); + test('#3206: userFallback beats the tier default (explicit user config survives)', async () => { + const m = await resolveModel(stub as never, { + tier: 'reasoning', + userFallback: 'litellm:qwen-test', + fallback: 'anthropic:claude-sonnet-4-6', + }); + expect(m).toBe('litellm:qwen-test'); + }); + + test('#3206: config keys and env still beat userFallback', async () => { + stub.set('models.chat', 'opus'); + const viaConfig = await resolveModel(stub as never, { + configKey: 'models.chat', + tier: 'reasoning', + userFallback: 'litellm:qwen-test', + fallback: 'sonnet', + }); + expect(viaConfig).toBe(DEFAULT_ALIASES.opus); + + process.env.GBRAIN_MODEL = 'haiku'; + const viaEnv = await resolveModel(stub as never, { + tier: 'reasoning', + userFallback: 'litellm:qwen-test', + fallback: 'sonnet', + }); + expect(viaEnv).toBe(DEFAULT_ALIASES.haiku); + }); + test('v0.38 D7: tier.subagent accepts non-Anthropic models that support tools (with cost warn)', async () => { // Pre-v0.38 the resolver hard-fell-back to TIER_DEFAULTS.subagent for any // non-Anthropic model. v0.38 (D6/D7) replaces that with a capability check: diff --git a/test/models-doctor-per-task-routes.test.ts b/test/models-doctor-per-task-routes.test.ts new file mode 100644 index 000000000..614d0356a --- /dev/null +++ b/test/models-doctor-per-task-routes.test.ts @@ -0,0 +1,94 @@ +/** + * #3219 — `gbrain models doctor` must probe configured per-task routes, not + * only the global chat + expansion models. A background route (e.g. + * `models.dream.synthesize`) can point at a distinct unreachable provider + * while chat/expansion are green. + * + * Pins `resolvePerTaskProbePlan`: same resolution path `buildReport` uses, + * dedup against already-probed models, dedup identical resolved models across + * routes, route-key attribution retained. + */ + +import { describe, test, expect, beforeEach } from 'bun:test'; +import { resolvePerTaskProbePlan, probeModel } from '../src/commands/models.ts'; +import { TIER_DEFAULTS } from '../src/core/model-config.ts'; +import type { BrainEngine } from '../src/core/engine.ts'; +import { __setChatTransportForTests } from '../src/core/ai/gateway.ts'; +import { afterEach } from 'bun:test'; + +function stubEngine(config: Record): BrainEngine { + return { getConfig: async (k: string) => config[k] ?? null } as unknown as BrainEngine; +} + +beforeEach(() => { + delete process.env.GBRAIN_MODEL; +}); + +afterEach(() => { + __setChatTransportForTests(null); +}); + +describe('#3219 — doctor probes per-task background routes', () => { + test('a distinctly configured background route lands in the probe plan with route attribution', async () => { + const plan = await resolvePerTaskProbePlan( + stubEngine({ 'models.dream.synthesize': 'litellm:qwen-test' }), + new Set([TIER_DEFAULTS.reasoning, TIER_DEFAULTS.utility, TIER_DEFAULTS.deep]), + ); + + const entry = plan.find(p => p.model === 'litellm:qwen-test'); + expect(entry).toBeDefined(); + expect(entry!.routes).toEqual(['models.dream.synthesize']); + }); + + test('routes resolving to already-probed chat/expansion models are excluded', async () => { + // Nothing configured: every route resolves to its tier default. With all + // tier defaults already probed, the plan must be empty (no double-probes). + const plan = await resolvePerTaskProbePlan( + stubEngine({}), + new Set(Object.values(TIER_DEFAULTS)), + ); + expect(plan).toEqual([]); + }); + + test('multiple routes on the same distinct model dedup into ONE probe carrying all route keys', async () => { + const plan = await resolvePerTaskProbePlan( + stubEngine({}), + // Only reasoning + utility probed (the doctor default when chat + + // expansion are unconfigured) — the deep tier default is a distinct + // model shared by models.auto_think and models.think. + new Set([TIER_DEFAULTS.reasoning, TIER_DEFAULTS.utility]), + ); + + const deep = plan.find(p => p.model === TIER_DEFAULTS.deep); + expect(deep).toBeDefined(); + expect(deep!.routes).toContain('models.auto_think'); + expect(deep!.routes).toContain('models.think'); + // Exactly one plan entry per distinct model. + expect(plan.filter(p => p.model === TIER_DEFAULTS.deep).length).toBe(1); + }); + + test('models.chat / models.expansion route keys never enter the plan (they ARE the chat/expansion probes)', async () => { + const plan = await resolvePerTaskProbePlan( + stubEngine({ 'models.chat': 'litellm:bare-chat', 'models.expansion': 'litellm:bare-expand' }), + new Set(Object.values(TIER_DEFAULTS)), + ); + const routes = plan.flatMap(p => p.routes); + expect(routes).not.toContain('models.chat'); + expect(routes).not.toContain('models.expansion'); + }); + + test('probeModel carries the route-key touchpoint through for attribution', async () => { + __setChatTransportForTests(async () => ({ + text: 'ok', + blocks: [{ type: 'text', text: 'ok' }], + stopReason: 'end', + usage: { input_tokens: 1, output_tokens: 1, cache_read_tokens: 0, cache_creation_tokens: 0 }, + model: 'test:stub', + providerId: 'test', + })); + + const r = await probeModel('litellm:qwen-test', 'models.dream.synthesize,models.think'); + expect(r.status).toBe('ok'); + expect(r.touchpoint).toBe('models.dream.synthesize,models.think'); + }); +}); diff --git a/test/models-doctor-probe-token-budget.test.ts b/test/models-doctor-probe-token-budget.test.ts new file mode 100644 index 000000000..e79636085 --- /dev/null +++ b/test/models-doctor-probe-token-budget.test.ts @@ -0,0 +1,215 @@ +/** + * #3221 — one-token doctor probes falsely fail reasoning models. + * + * The `gbrain models doctor` chat/expansion reachability probe used + * `maxTokens: 1`. Reasoning models spend output budget on internal reasoning + * before emitting any text, so a 1-token cap is either exhausted with zero + * usable text (finishReason 'length') or rejected outright by providers that + * require the cap to exceed the model's minimum reasoning spend — and the + * probe then reported a failure for a perfectly reachable model. + * + * Pins (behavioral, via the gateway's `__setChatTransportForTests` seam — + * no network, no gateway config needed since the transport branch returns + * before provider resolution): + * + * 1. The probe grants a sufficient diagnostic output budget (the shared + * PROBE_MAX_OUTPUT_TOKENS constant, strictly greater than 1). + * 2. A length-exhausted response with NO text still counts as reachable + * (status 'ok'), with the generation limitation surfaced in the message + * rather than a bare 'reachable'. + * 3. A normal completion keeps the plain 'reachable' message (no noise for + * the common case). + * 4. Provider errors are still classified as failures (the wider budget + * must not swallow real auth/config problems). + */ + +import { describe, test, expect, afterEach } from 'bun:test'; +import { readFileSync } from 'node:fs'; +import { join } from 'node:path'; +import { + __setChatTransportForTests, + __setGenerateTextTransportForTests, + configureGateway, + resetGateway, + type ChatOpts, + type ChatResult, +} from '../src/core/ai/gateway.ts'; +import { probeModel, PROBE_MAX_OUTPUT_TOKENS } from '../src/commands/models.ts'; + +afterEach(() => { + __setChatTransportForTests(null); + __setGenerateTextTransportForTests(null); + resetGateway(); +}); + +function stubResult(over: Partial = {}): ChatResult { + return { + text: 'ok', + blocks: [{ type: 'text', text: 'ok' }], + stopReason: 'end', + usage: { input_tokens: 1, output_tokens: 1, cache_read_tokens: 0, cache_creation_tokens: 0 }, + model: 'test:stub', + providerId: 'test', + ...over, + }; +} + +describe('models doctor — probe token budget (#3221)', () => { + test('probe requests a sufficient output budget, not 1 token', async () => { + let seenMaxTokens: number | undefined; + __setChatTransportForTests(async (opts: ChatOpts) => { + seenMaxTokens = opts.maxTokens; + return stubResult(); + }); + + const r = await probeModel('test:reasoner', 'chat'); + + expect(r.status).toBe('ok'); + expect(seenMaxTokens).toBe(PROBE_MAX_OUTPUT_TOKENS); + // The bug class: a tiny budget cannot accommodate reasoning burn. Pin a + // real floor (not just > 1) so the constant can't quietly regress to a + // value that re-triggers the false-FAIL. + expect(PROBE_MAX_OUTPUT_TOKENS).toBeGreaterThanOrEqual(64); + }); + + test('length-exhausted empty completion is reachable, with the limitation surfaced', async () => { + // A direct reasoning model spends the whole probe budget on internal + // reasoning: valid HTTP response, finishReason 'length', empty text. + __setChatTransportForTests(async () => + stubResult({ text: '', blocks: [], stopReason: 'length' }), + ); + + const r = await probeModel('test:reasoner', 'chat'); + + expect(r.status).toBe('ok'); + expect(r.message).toContain('reachable'); + // The generation limitation is reported, not silently folded into a + // bare 'reachable' (the issue's "clearly reporting the generation + // limitation" requirement). + expect(r.message).not.toBe('reachable'); + expect(r.message).toContain('reasoning'); + }); + + test('length-exhausted completion WITH text keeps the plain reachable message', async () => { + // Non-reasoning model that simply hit the cap mid-sentence: it emitted + // text, so generation itself was exercised — no caveat needed. + __setChatTransportForTests(async () => + stubResult({ text: 'partial answ', stopReason: 'length' }), + ); + + const r = await probeModel('test:small-model', 'chat'); + + expect(r.status).toBe('ok'); + expect(r.message).toBe('reachable'); + }); + + test('normal completion reports plain reachable', async () => { + __setChatTransportForTests(async () => stubResult()); + + const r = await probeModel('test:small-model', 'expansion'); + + expect(r.status).toBe('ok'); + expect(r.message).toBe('reachable'); + expect(r.touchpoint).toBe('expansion'); + }); + + test('#3249 coherence: a gateway-side contentless-length rejection still counts as reachable', async () => { + // If the gateway grows a contentless-completion guard (#3249/#3217), + // chat() THROWS an AIConfigError for a length-exhausted empty completion + // instead of returning it. The probe must classify that throw as + // reachable — same class as the in-band stopReason 'length' branch. + __setChatTransportForTests(async () => { + throw new Error( + 'chat(anthropic:claude-sonnet-4-6): output budget exhausted before any content was emitted (maxOutputTokens=64)', + ); + }); + + const r = await probeModel('anthropic:claude-sonnet-4-6', 'chat'); + + expect(r.status).toBe('ok'); + expect(r.message).toContain('reachable'); + }); + + test('provider errors are still classified as failures', async () => { + __setChatTransportForTests(async () => { + throw Object.assign(new Error('401 unauthorized: bad api key'), { status: 401 }); + }); + + const r = await probeModel('test:reasoner', 'chat'); + + expect(r.status).toBe('auth'); + }); + + test('probe cap widens above a configured extended-thinking budget', async () => { + // Anthropic rejects requests where max_tokens <= thinking.budgetTokens. + // A user who configured `provider_chat_options.anthropic.thinking` + // (e.g. budgetTokens: 1024) would have the fixed 64-token probe cap + // rejected outright — the same false-FAIL class. Exercised through the + // generate-text seam so provider-option resolution stays live. + let capturedMaxOutputTokens: number | undefined; + __setGenerateTextTransportForTests(async (args: any) => { + capturedMaxOutputTokens = args.maxOutputTokens; + return { + content: [{ type: 'text', text: 'ok' }], + finishReason: 'stop', + usage: { inputTokens: 1, outputTokens: 1 }, + } as any; + }); + configureGateway({ + embedding_model: 'openai:text-embedding-3-large', + embedding_dimensions: 1536, + chat_model: 'anthropic:claude-sonnet-4-6', + provider_chat_options: { + anthropic: { thinking: { type: 'enabled', budgetTokens: 1024 } }, + }, + env: { ANTHROPIC_API_KEY: 'sk-fake', OPENAI_API_KEY: 'sk-fake' }, + }); + + const r = await probeModel('anthropic:claude-sonnet-4-6', 'chat'); + + expect(r.status).toBe('ok'); + expect(capturedMaxOutputTokens).toBe(1024 + PROBE_MAX_OUTPUT_TOKENS); + }); + + test('probe cap widens for a model-scoped budget reached through a recipe alias', async () => { + // chat() resolves recipe aliases to the canonical model id before the + // model-scoped provider_chat_options lookup. The budget helper must + // mirror that: probing via the stale alias claude-sonnet-4-6-20250929 + // still sees the budget configured under the canonical id. + let capturedMaxOutputTokens: number | undefined; + __setGenerateTextTransportForTests(async (args: any) => { + capturedMaxOutputTokens = args.maxOutputTokens; + return { + content: [{ type: 'text', text: 'ok' }], + finishReason: 'stop', + usage: { inputTokens: 1, outputTokens: 1 }, + } as any; + }); + configureGateway({ + embedding_model: 'openai:text-embedding-3-large', + embedding_dimensions: 1536, + chat_model: 'anthropic:claude-sonnet-4-6', + provider_chat_options: { + 'anthropic:claude-sonnet-4-6': { thinking: { type: 'enabled', budgetTokens: 2048 } }, + }, + env: { ANTHROPIC_API_KEY: 'sk-fake', OPENAI_API_KEY: 'sk-fake' }, + }); + + const r = await probeModel('anthropic:claude-sonnet-4-6-20250929', 'chat'); + + expect(r.status).toBe('ok'); + expect(capturedMaxOutputTokens).toBe(2048 + PROBE_MAX_OUTPUT_TOKENS); + }); + + test('human output renders ok-probe caveat messages, not only failure messages', () => { + // runModels needs a live engine, so pin the render branch structurally + // (same source-text convention as test/models-doctor-embed.test.ts): the + // non-json output path must print the message for an ok probe whose + // message deviates from the bare 'reachable' (the reasoning-burn caveat + // would otherwise be visible only under --json). + const src = readFileSync(join(__dirname, '..', 'src', 'commands', 'models.ts'), 'utf-8'); + const runIdx = src.indexOf('export async function runModels'); + expect(runIdx).toBeGreaterThan(0); + expect(src.slice(runIdx)).toMatch(/else if \(r\.message !== 'reachable'\)\s*\{[^}]*process\.stdout\.write\(`\s+\$\{r\.message\}\\n`\)/); + }); +});