diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 2d00c5808..d2c215191 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -88,8 +88,13 @@ jobs: } EOF - name: Run Tier 2 skill tests - run: bun test test/e2e/skills.test.ts + run: bun test test/e2e/skills.test.ts test/e2e/zeroentropy-live.test.ts env: DATABASE_URL: postgresql://postgres:postgres@localhost:5432/gbrain_test OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} + # v0.33.3.0: ZE live API tests skip gracefully when this is unset, + # so forks without the secret stay green. The test exercises the + # zeroEntropyCompatFetch response-rewriter + URL rewrite + flexible + # dim handling + gateway.rerank against the real provider. + ZEROENTROPY_API_KEY: ${{ secrets.ZEROENTROPY_API_KEY }} diff --git a/CHANGELOG.md b/CHANGELOG.md index cf950b4d8..f8b93623b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,66 @@ All notable changes to GBrain will be documented in this file. +## [0.35.0.0] - 2026-05-15 + +**ZeroEntropy in the box: zembed-1 embeddings + zerank-2 cross-encoder reranking, on by default for tokenmax mode.** + +ZeroEntropy ships two specialized small models that target the two weakest retrieval moments in a gbrain pipeline: `zembed-1` (a 32K-context embedding distilled from zerank-2 with flexible Matryoshka dims at 2560/1280/640/320/160/80/40) and `zerank-2` (a multilingual cross-encoder reranker, $0.025/1M tokens, ~50% cheaper than Cohere/Voyage rerankers). Both land as a new openai-compatible recipe alongside OpenAI/Voyage. The reranker is the bigger story: search had no reranker stage before this release. Hybrid search now ends with `RRF → dedup → reranker → token-budget` when reranker is enabled, with one configuration flip to opt in. + +### What you can now do + +**Switch to ZeroEntropy embeddings on either supported plane.** Set `embedding_model: zeroentropyai:zembed-1` and `embedding_dimensions: 2560` (or any of the 7 Matryoshka dims: 1280/640/320/160/80/40) in `~/.gbrain/config.json`, or export `GBRAIN_EMBEDDING_MODEL=zeroentropyai:zembed-1` + `GBRAIN_EMBEDDING_DIMENSIONS=2560`. The `gbrain config set` plane intentionally does NOT live-switch embedding models — they size the schema and must stay stable across engine connects. `gbrain models doctor` now probes the ZE config before spending any tokens, so an invalid `embedding_dimensions` value (the most common trip: leaving it unset and falling back to 1536, which ZE rejects) gets caught at install time with a paste-ready fix hint. + +**Get cross-encoder reranking on every tokenmax query, automatically.** `tokenmax` mode now defaults `search.reranker.enabled = true` with `zerank-2` as the model. The cost is ~$0.0003/query at 30-document topNIn (~12K tokens × $0.025/1M) — rounding error against the tier's existing $700/mo @ Opus pairing per the CLAUDE.md cost matrix. `balanced` and `conservative` modes default reranker off; opt in with `gbrain config set search.reranker.enabled true`. The reranker re-orders the top 30 deduped candidates by cross-encoder relevance and preserves the un-reranked long tail in its original RRF order, so recall is protected even when the reranker drops items. + +**Search keeps working when the reranker is flaky.** Every error class — auth, rate-limit, network, timeout, payload-too-large — fails open. The original RRF order passes through unchanged and the failure logs to `~/.gbrain/audit/rerank-failures-YYYY-Www.jsonl` (ISO-week rotation, mirrors the slug-fallback audit). `gbrain doctor` reads the audit and warns on any auth failure (config-time problem doctor's own probe should have caught) and on >=5 transient failures in the last 7 days. The check reads `search.reranker.enabled` first so "no events" means different things when reranker is on vs off (disabled = healthy by definition; enabled = healthy because nothing has failed yet). + +**Use asymmetric query/document encoding where the provider supports it.** ZE zembed-1 and Voyage v3+ models accept an `input_type: 'query' | 'document'` knob for asymmetric retrieval. Hybrid search's two query-side embed sites (`hybrid.ts:400` vector seed and `hybrid.ts:629` cache lookup) now call a new `embedQuery()` companion that threads `input_type: 'query'`. All ingest paths (sync, import, embed CLI) continue using `embed()` which defaults to document encoding. Symmetric providers (OpenAI text-3, DashScope, Zhipu) ignore the field — no behavior change. MiniMax embo-01's asymmetric quirk stays as it was; opting it into the new seam is a deferred follow-up. + +**Cache key versioning is bumped to v=2.** `KNOBS_HASH_VERSION` rolls 1 → 2 to fold the five new reranker fields (`reranker_enabled`, `reranker_model`, `reranker_top_n_in`, `reranker_top_n_out`, `reranker_timeout_ms`) into the `query_cache.knobs_hash` column. A tokenmax-with-reranker cache write can't be served to a reranker-off lookup. Mid-rolling-deploy operators should expect a temporary cache hit-rate dip and a brief doubling of cache rows for hot queries — v=1 rows TTL out within `cache.ttl_seconds` (default 3600s), then the hit rate recovers. + +### Itemized changes + +- `src/core/ai/recipes/zeroentropyai.ts` declares the new recipe with `implementation: 'openai-compatible'` (NOT the misspelled `'openai-compat'` the original plan draft had — pinned by F1 regression test). Both `touchpoints.embedding` (`zembed-1`, 7 Matryoshka dims) and `touchpoints.reranker` (`zerank-2` / `zerank-1` / `zerank-1-small`, 5MB payload cap) are declared. +- `src/core/ai/gateway.ts` adds `zeroEntropyCompatFetch` — a fetch wrapper that rewrites the request URL from `/embeddings` to `/models/embed` (since ZE is NOT OpenAI-compatible at the wire level), injects `input_type` + explicit `encoding_format: 'float'`, and rewrites the response from `{results: [{embedding}]}` to `{data: [{embedding, index}]}` with `usage.prompt_tokens` added (Voyage's shim hit the same SDK schema requirement at `:655`). Layer 1 (Content-Length) + Layer 2 (per-embedding) OOM caps mirror Voyage's pattern via a new `ZeroEntropyResponseTooLargeError` class. +- `gateway.rerank()` is the new native HTTP path — no AI-SDK reranking abstraction exists. Returns `RerankResult[]` sorted by `relevanceScore`; errors classify into `RerankError.reason` (`auth | rate_limit | network | timeout | payload_too_large | unknown`). 5-second default timeout (search hot path; long stalls degrade UX worse than fallthrough). Pre-flight payload guard rejects bodies over the recipe's `max_payload_bytes` cap so the caller can fail-open without an HTTP call. `_rerankTransport` test seam mirrors `_embedTransport`. +- `gateway.embedQuery(text)` companion routes `inputType: 'query'` through `dimsProviderOptions()` (now a 4-arg signature; the 4th `inputType` param defaults to undefined for back-compat). `src/core/embedding.ts` re-exports it; `hybrid.ts` flips the two query-side embed call sites. +- `src/core/search/rerank.ts` is the call-site abstraction. `applyReranker(query, results, opts)` slices the top `opts.topNIn` candidates, re-orders by reranker score, appends the un-reranked tail, and optionally truncates to `opts.topNOut`. `topNOut: null` is the explicit "don't truncate" signal — distinct from undefined which means "fall through to mode bundle" (per the CDX2-F16 null-vs-undefined contract). +- `src/core/rerank-audit.ts` is the JSONL audit (failure-only — `logRerankSuccess` was deliberately dropped per CDX2-F22 to avoid hot-path I/O churn + query-volume leaks). Mirrors `src/core/audit-slug-fallback.ts` shape with ISO-week filename rotation. +- `src/core/search/mode.ts` extends `ModeBundle`, `MODE_BUNDLES`, `SearchKeyOverrides`, `SearchPerCallOpts`, `loadOverridesFromConfig`, and `SEARCH_MODE_CONFIG_KEYS` with the five new reranker fields. `KNOBS_HASH_VERSION` bumps 1→2 (append-only convention per CDX2-F13). `loadOverridesFromConfig` parses `top_n_out` with the three-shape distinction (absent → undefined, `'null'`/`'none'`/`''` → null, positive integer → number). +- `src/core/ai/types.ts` widens `TouchpointKind` with `'reranker'`, adds `RerankerTouchpoint` interface, and extends `Recipe.touchpoints` and `AIGatewayConfig` with the reranker fields. `src/core/ai/model-resolver.ts` widens `KnownTouchpointKey` and `getTouchpoint()` to thread reranker. `assertTouchpoint()` does NOT enforce allowlists for openai-compatible recipes (existing v0.31.12 behavior); `rerank()` and `probeRerankerConfig` do the allowlist check directly so a typo'd `search.reranker.model` surfaces at probe time, not first call. +- `src/commands/models.ts` adds `probeRerankerConfig` (zero-network: model + touchpoint + allowlist) and `probeRerankerReachability` (1-token-equivalent: minimal rerank against real API). Both surface paste-ready `gbrain config set` fix hints. `ProbeResult.touchpoint` widens to include `'reranker_config'`. +- `src/commands/doctor.ts` adds `checkRerankerHealth`. Reads `search.reranker.enabled` first; warns on any auth failure (singular signal) or >=5 transient failures (volume signal) in the last 7 days. Engine-agnostic (file-based + one config-key read). +- New tests: `test/ai/zeroentropy-recipe.test.ts`, `test/ai/dims-zeroentropy.test.ts`, `test/search/rerank.test.ts`, `test/rerank-audit.test.ts` — 42 cases pinning recipe shape, dim allowlist, 4th-arg inputType plumbing, reranker fail-open across every error class, null-vs-undefined topNOut semantics, and the no-success-logging contract. +- The plan that drove this release went through two Codex rounds (47 source-grounded findings across consult + adversarial challenge) before any code was written. Every must-fix landed; nine deferred bugs (the cache-wrapper double-embed, MiniMax embo-01 asymmetry, openai-compat allowlist gap, cache-hit limit/budget divergence) are documented in the plan file at `~/.claude/plans/system-instruction-you-are-working-linked-moonbeam.md`. + +## To take advantage of v0.35.0.0 + +`gbrain upgrade` does NOT auto-switch your embedding or reranker model. ZeroEntropy is opt-in; you choose when to flip. To try it: + +1. **Get an API key:** `https://dashboard.zeroentropy.dev` → create key → `export ZEROENTROPY_API_KEY=...` +2. **(Optional) Switch embedding to zembed-1.** Note: switching embedding models invalidates the vector index; you'll need to re-embed. Add to `~/.gbrain/config.json`: + ```json + { + "embedding_model": "zeroentropyai:zembed-1", + "embedding_dimensions": 2560 + } + ``` +3. **(Optional) Enable reranker on a non-tokenmax mode:** + ```bash + gbrain config set search.reranker.enabled true + ``` + Skip this step if you're already on `tokenmax` — reranker is on by default there. +4. **Verify:** + ```bash + gbrain models doctor --json | jq '.probes[] | select(.touchpoint=="reranker_config" or .touchpoint=="embedding_config")' + ``` + Both should report `status: "ok"`. +5. **If anything fails,** please file an issue: https://github.com/garrytan/gbrain/issues with: + - output of `gbrain models doctor --json` + - contents of `~/.gbrain/audit/rerank-failures-*.jsonl` if present + - which step broke + ## [0.34.4.0] - 2026-05-14 **`gbrain embed --stale` now actually works on large brains. `--source` flag stops silently dropping. The retry loop stops storming itself.** diff --git a/CLAUDE.md b/CLAUDE.md index 121bd5ccf..2be8b7ed6 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -94,7 +94,10 @@ strict behavior when unset. - `src/core/embedding.ts` — OpenAI text-embedding-3-large, batch, retry, backoff. **v0.28.7:** `BATCH_SIZE` reverted 50→100 — the original Voyage safety guard halved OpenAI throughput on every page. Per-recipe pre-split + recursive halving + adaptive shrink-on-miss now live in the gateway, so the outer paginator goes back to its original purpose: progress-callback granularity, not batch protection. - `src/core/ai/dims.ts` (v0.33.1.1, PR #962 + #866) — per-provider `providerOptions` resolver for embed-time dimension passthrough. The single source of truth for "which provider needs which knob to produce `vector(N)`". Exports `dimsProviderOptions(implementation, modelId, dims)` (called by `embed()` in `gateway.ts`), `VOYAGE_OUTPUT_DIMENSION_MODELS` (private const — the 7 hosted Voyage models that accept `output_dimension`: `voyage-4-large`, `voyage-4`, `voyage-4-lite`, `voyage-3-large`, `voyage-3.5`, `voyage-3.5-lite`, `voyage-code-3` — nano deliberately excluded), `VOYAGE_VALID_OUTPUT_DIMS = [256, 512, 1024, 2048] as const`, `supportsVoyageOutputDimension(modelId)`, and `isValidVoyageOutputDim(dims)`. **Voyage path uses the SDK-supported `dimensions` field** (`{ openaiCompatible: { dimensions: N } }`), NOT Voyage's `output_dimension` wire-key — the existing `voyageCompatFetch` shim in `gateway.ts:541` translates `dimensions → output_dimension` before the HTTP body is built. The reverse (sending `output_dimension` from here) was the v0.33.1.0 bug class: the AI SDK's openai-compatible adapter doesn't recognize the wire-key so it was silently dropped, Voyage returned its default 1024-dim, and the gateway dimension check threw on every embed call. Runtime guard: when a Voyage flexible-dim model is configured with `dims` outside `VOYAGE_VALID_OUTPUT_DIMS`, throws `AIConfigError` with a paste-ready `gbrain config set embedding_dimensions <256|512|1024|2048>` hint at the embed boundary — fail-loud instead of opaque Voyage HTTP 400. Most common trigger: `embedding_model: voyage:voyage-4-large` configured without `embedding_dimensions` (falls back to `DEFAULT_EMBEDDING_DIMENSIONS=1536`, an OpenAI default not a Voyage one). Eva (@100yenadmin) shipped the wire-key fix in #866; Codex P3 follow-up landed the validator + valid-dims allowlist in #962. - `src/core/ai/types.ts` — provider/recipe types. **v0.28.7 (#680):** `EmbeddingTouchpoint` extended with optional `chars_per_token` (default 4 chars/token, matching OpenAI tiktoken on English) and `safety_factor` (default 0.8, budget-utilization ceiling). Both consulted only when `max_batch_tokens` is also set. Voyage declares `chars_per_token=1` + `safety_factor=0.5` to handle dense payloads (CJK/JSON/base64) that overshoot tiktoken. The pre-split budget is `max_batch_tokens × safety_factor / chars_per_token`. **v0.28.11 (#719):** `EmbeddingTouchpoint.multimodal_models?: string[]` model-level allow-list for recipes that mix text-only + multimodal models under one touchpoint (Voyage's 12 models share `supports_multimodal: true` but only `voyage-multimodal-3` accepts `/multimodalembeddings`). When omitted, recipe-level `supports_multimodal` is sufficient. `AIGatewayConfig.embedding_multimodal_model?: string` lets `embedMultimodal()` route to a different model than `embedding_model` — brains using OpenAI for text can use Voyage for images without flipping the primary embedding pipeline. -- `src/core/ai/gateway.ts` — unified seam for every AI call. **v0.28.7 (#680):** module-scoped `_embedTransport` defaulting to AI SDK `embedMany`, with `__setEmbedTransportForTests(fn)` test seam so tests drive the public `embed()` function with a stubbed transport instead of probing private helpers. `splitByTokenBudget` and `isTokenLimitError` are now exported `@internal` — pure functions reused directly by the test file. Module-level `_shrinkState: Map` halves the recipe's effective `safety_factor` on token-limit miss (floor 0.05) and heals back ×1.5 toward the ceiling after `SHRINK_HEAL_AFTER=10` consecutive successes. `configureGateway()` walks every registered recipe at construction time and emits a once-per-process stderr warning for any embedding touchpoint missing `max_batch_tokens` (excluding the canonical OpenAI fast-path recipe). `resetGateway()` clears `_shrinkState`, the warned-set, and restores the real transport. ASCII flow diagram embedded in the `embed()` JSDoc covers the routing decision, recursion + halving, and shrinkState lifecycle. **v0.28.11 (#719):** `embedMultimodal()` reads `cfg.embedding_multimodal_model` first (falls back to `cfg.embedding_model` for single-model setups). After the existing recipe-level `supports_multimodal` fast-fail, validates the resolved model against `touchpoint.multimodal_models` when declared — closes the Voyage-text-only-model-into-multimodal-endpoint footgun before any HTTP call (Codex F1 from PR review). New `getMultimodalModel()` accessor mirrors `getEmbeddingModel` / `getChatModel` so doctor and integration tests can read the gateway state. **v0.33.1.1 (#962, Codex P3 follow-up):** new exported `VoyageResponseTooLargeError` tagged class at the top of the file. `voyageCompatFetch`'s two OOM-defense caps (Layer 1 Content-Length check at `:595`, Layer 2 per-embedding base64 cap at `:619`) now throw `VoyageResponseTooLargeError` instead of a generic `Error`. The inbound response-rewriter's surrounding try/catch (which intentionally swallows parse failures so misshaped Voyage responses fall through to the SDK's JSON parser) checks `instanceof VoyageResponseTooLargeError` and rethrows. Pre-fix, the Layer 2 throw was silently swallowed and the oversized response returned to the AI SDK anyway — Layer 2 was theatrical. Source-shape regression assertion in `test/voyage-response-cap.test.ts` pins the `instanceof ⇒ throw err` line. **v0.34.1.0 (#875):** new `embedMultimodalOpenAICompat()` routes recipes with `implementation: 'openai-compatible'` (LiteLLM, Anyscale, vLLM, Gemini multimodal via proxy) through the standard `/embeddings` endpoint with content arrays carrying `image_url` entries. The pre-existing Voyage `/multimodalembeddings` path is unchanged; the gateway selects by recipe `implementation` tag. Runtime dimension validation throws `AIConfigError` (with model id + observed + expected) before the vector reaches storage when the provider returns a width that doesn't match the recipe's `default_dims` or the brain's `embedding_dimensions` config — no more cryptic `vector dimension mismatch` at INSERT time. Pinned by 11 cases in `test/openai-compat-multimodal.test.ts`. +- `src/core/ai/gateway.ts` — unified seam for every AI call. **v0.35.0.0:** ZeroEntropy support lands. New `zeroEntropyCompatFetch` shim (sibling to `voyageCompatFetch`) handles ZE's non-OpenAI-compatible wire shape — rewrites the request URL from `/embeddings` to `/models/embed`, injects `input_type` (default `'document'`; `'query'` when threaded via `providerOptions.openaiCompatible.input_type`) and explicit `encoding_format: 'float'`, and rewrites the response from `{results: [{embedding}], usage: {total_bytes, total_tokens}}` to `{data: [{embedding, index}], usage: {prompt_tokens, total_tokens}}` so the SDK's openai-compatible Zod schema validates (Voyage's shim hit the same `prompt_tokens` requirement at `:655`). Layer 1 (Content-Length) + Layer 2 (per-embedding) OOM caps via a new tagged `ZeroEntropyResponseTooLargeError` class (kept separate from `VoyageResponseTooLargeError` because `test/voyage-response-cap.test.ts` does structural source-text greps pinning the Voyage name — class unification is a deferred cleanup). Wired in `instantiateEmbedding()` via the same `recipe.id === 'zeroentropyai'` branch pattern Voyage uses. New `gateway.rerank()` native HTTP path (no AI-SDK reranking abstraction): resolves the configured reranker model via `getRerankerModel()`, posts to `${recipe.base_url}/models/rerank` with bearer auth, returns `RerankResult[]` sorted by relevance score. `RerankError.reason` classifier: `auth | rate_limit | network | timeout | payload_too_large | unknown`. 5s default timeout (search hot path). Pre-flight payload guard rejects bodies over `recipe.touchpoints.reranker.max_payload_bytes` with `reason: 'payload_too_large'` so callers can fail-open without an HTTP call. `_rerankTransport` test seam mirrors `_embedTransport`. New `gateway.embedQuery(text)` companion threads `inputType: 'query'` through `dimsProviderOptions()` (now 4-arg). `getRerankerModel()` accessor + `isAvailable('reranker')` branch added. `configureGateway` + `reconfigureGatewayWithEngine` thread `reranker_model` through the same path as embedding/expansion/chat. `applyResolveAuth` + `defaultResolveAuth` widen touchpoint param to include `'reranker'`. **v0.34.1.0 (#875):** new `embedMultimodalOpenAICompat()` routes recipes with `implementation: 'openai-compatible'` (LiteLLM, Anyscale, vLLM, Gemini multimodal via proxy) through the standard `/embeddings` endpoint with content arrays carrying `image_url` entries. The pre-existing Voyage `/multimodalembeddings` path is unchanged; the gateway selects by recipe `implementation` tag. Runtime dimension validation throws `AIConfigError` (with model id + observed + expected) before the vector reaches storage when the provider returns a width that doesn't match the recipe's `default_dims` or the brain's `embedding_dimensions` config — no more cryptic `vector dimension mismatch` at INSERT time. Pinned by 11 cases in `test/openai-compat-multimodal.test.ts`. **v0.28.7 (#680):** module-scoped `_embedTransport` defaulting to AI SDK `embedMany`, with `__setEmbedTransportForTests(fn)` test seam so tests drive the public `embed()` function with a stubbed transport instead of probing private helpers. `splitByTokenBudget` and `isTokenLimitError` are now exported `@internal` — pure functions reused directly by the test file. Module-level `_shrinkState: Map` halves the recipe's effective `safety_factor` on token-limit miss (floor 0.05) and heals back ×1.5 toward the ceiling after `SHRINK_HEAL_AFTER=10` consecutive successes. `configureGateway()` walks every registered recipe at construction time and emits a once-per-process stderr warning for any embedding touchpoint missing `max_batch_tokens` (excluding the canonical OpenAI fast-path recipe). `resetGateway()` clears `_shrinkState`, the warned-set, and restores the real transport. ASCII flow diagram embedded in the `embed()` JSDoc covers the routing decision, recursion + halving, and shrinkState lifecycle. **v0.28.11 (#719):** `embedMultimodal()` reads `cfg.embedding_multimodal_model` first (falls back to `cfg.embedding_model` for single-model setups). After the existing recipe-level `supports_multimodal` fast-fail, validates the resolved model against `touchpoint.multimodal_models` when declared — closes the Voyage-text-only-model-into-multimodal-endpoint footgun before any HTTP call (Codex F1 from PR review). New `getMultimodalModel()` accessor mirrors `getEmbeddingModel` / `getChatModel` so doctor and integration tests can read the gateway state. **v0.33.1.1 (#962, Codex P3 follow-up):** new exported `VoyageResponseTooLargeError` tagged class at the top of the file. `voyageCompatFetch`'s two OOM-defense caps (Layer 1 Content-Length check at `:595`, Layer 2 per-embedding base64 cap at `:619`) now throw `VoyageResponseTooLargeError` instead of a generic `Error`. The inbound response-rewriter's surrounding try/catch (which intentionally swallows parse failures so misshaped Voyage responses fall through to the SDK's JSON parser) checks `instanceof VoyageResponseTooLargeError` and rethrows. Pre-fix, the Layer 2 throw was silently swallowed and the oversized response returned to the AI SDK anyway — Layer 2 was theatrical. Source-shape regression assertion in `test/voyage-response-cap.test.ts` pins the `instanceof ⇒ throw err` line. +- `src/core/ai/recipes/zeroentropyai.ts` (v0.35.0.0) — ZeroEntropy openai-compatible recipe declaring BOTH `embedding` (`zembed-1`, 7 Matryoshka dims: 2560/1280/640/320/160/80/40) AND `reranker` (`zerank-2` flagship + `zerank-1` + `zerank-1-small`, 5MB payload cap) touchpoints. `implementation: 'openai-compatible'` (NOT the misspelled `'openai-compat'` the original plan draft had — pinned by F1 regression in `test/ai/zeroentropy-recipe.test.ts`). `base_url_default: 'https://api.zeroentropy.dev/v1'` already ends with `/v1`, so the `zeroEntropyCompatFetch` URL rewrite `/embeddings → /models/embed` produces `…/v1/models/embed` (NOT `…/v1/v1/…` — pinned by F2 regression). `chars_per_token: 1` + `safety_factor: 0.5` match Voyage's dense-content hedge. +- `src/core/rerank-audit.ts` (v0.35.0.0) — failure-only JSONL audit at `~/.gbrain/audit/rerank-failures-YYYY-Www.jsonl` (ISO-week rotation, mirrors `src/core/audit-slug-fallback.ts`). Exports `logRerankFailure({reason, model, query_hash, doc_count, error_summary})` + `readRecentRerankFailures(days)`. **Deliberately no `logRerankSuccess`** (CDX2-F22 in plan review): writing once per tokenmax search is hot-path I/O churn AND success events leak query volume + timing into a local audit file. `gbrain doctor`'s `reranker_health` check reads `search.reranker.enabled` first so "no events in window" is interpreted correctly (disabled → ok; enabled → ok). Query text is SHA-256-prefix-hashed (8 hex chars) for privacy. `GBRAIN_AUDIT_DIR` env override honored via the shared `resolveAuditDir()`. +- `src/core/search/rerank.ts` (v0.35.0.0) — the call-site abstraction. `applyReranker(query, results, opts)` slots between `dedupResults()` and `enforceTokenBudget()` in `src/core/search/hybrid.ts`. Slices `opts.topNIn` (default 30) by current RRF order, sends to `gateway.rerank()`, reorders by `relevanceScore` desc, and appends the un-reranked tail unchanged (recall protection). **Fail-open on every `RerankError.reason`**: any error logs via `logRerankFailure` and returns the input array unchanged. Stamps `rerank_score` onto reordered items so downstream telemetry sees the new ordering signal. `topNOut: null` is the explicit "don't truncate" signal — semantically distinct from `undefined` which means "fall through to mode bundle" (CDX2-F16). Test seam: `opts.rerankerFn` lets tests stub `gateway.rerank` without touching the network. - `src/core/ai/recipes/voyage.ts` — Voyage AI openai-compatible recipe. **v0.28.7 (#680):** declares `chars_per_token=1` + `safety_factor=0.5` so the gateway pre-splits Voyage batches at a 60K-character budget (50% of 120K-token cap with the dense-tokenizer ratio). Closes the v0.27 backfill loop where ~26% of the corpus stayed un-embedded because tiktoken-grounded budgeting silently undercounted Voyage's actual token usage. **v0.28.11 (#719):** declares `multimodal_models: ['voyage-multimodal-3']` so the gateway rejects text-only Voyage models pointed at the multimodal endpoint with a clear `AIConfigError` instead of waiting for Voyage's HTTP 400. **v0.33.1.1 (#962, fixup):** recipe docstring at `:7-16` tightened to name the seven hosted flexible-dim models that accept `output_dimension` explicitly (`voyage-4-large`, `voyage-4`, `voyage-4-lite`, `voyage-3-large`, `voyage-3.5`, `voyage-3.5-lite`, `voyage-code-3`) and call out that `voyage-4-nano` is the open-weight variant listed separately by Voyage as fixed 1024-dim — does NOT accept the parameter. The "all v4 variants are flexible" misread is what caused the original PR to include nano in `VOYAGE_OUTPUT_DIMENSION_MODELS`; the negative regression assertion in `test/ai/gateway.test.ts` (`dimsProviderOptions` returns `undefined` for `voyage-4-nano`) pins the contract. - `src/core/ai/recipes/anthropic.ts` — Anthropic recipe (chat + expansion touchpoints). **v0.31.12:** chat and expansion `models:` lists drop the v0.31.6 phantom `claude-sonnet-4-6-20250929` date suffix — canonical id is `claude-sonnet-4-6`. The wrong-direction alias `claude-sonnet-4-6 → claude-sonnet-4-6-20250929` is removed; a reverse alias `claude-sonnet-4-6-20250929 → claude-sonnet-4-6` keeps stale user configs working (rescues `facts.extraction_model` and `models.dream.synthesize` set by v0.31.6 installs). Recipe-shape regression pinned by `test/anthropic-model-ids.test.ts` (6 cases, verbatim cherry-pick of PR #830 plus the reverse-alias rescue case). - `src/core/anthropic-pricing.ts` — Single source of truth for Anthropic model pricing (per-MTok input/output). **v0.31.12:** Opus 4.7 corrected from `$15/$75` to `$5/$25` (the old number was from Opus 4 generation, never refreshed when 4.7 shipped); Opus 4.6 also corrected. Consumed by `src/core/budget-meter.ts` and `src/core/cross-modal-eval/runner.ts` — the cross-modal estimator now reads `ANTHROPIC_PRICING` for Anthropic models instead of duplicating the table, killing the v0.31.6 drift bug class. diff --git a/VERSION b/VERSION index 97745b5df..ba8c40ae0 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.34.4.0 +0.35.0.0 diff --git a/docs/ai-providers/zeroentropy.md b/docs/ai-providers/zeroentropy.md new file mode 100644 index 000000000..b465c0e3f --- /dev/null +++ b/docs/ai-providers/zeroentropy.md @@ -0,0 +1,173 @@ +# ZeroEntropy — zembed-1 + zerank-2 + +[ZeroEntropy](https://zeroentropy.dev) ships two specialized small models +for retrieval pipelines: + +- **`zembed-1`** — multilingual embedding distilled from zerank-2. + Flexible Matryoshka dims (2560/1280/640/320/160/80/40), 32K context, + asymmetric `input_type: query|document` encoding. $0.025/1M tokens + (sale) / $0.05 regular. +- **`zerank-2`** — SOTA multilingual cross-encoder reranker. + $0.025/1M tokens (~50% cheaper than Cohere/Voyage rerankers). + Plus `zerank-1` and `zerank-1-small` for legacy / open-source needs. + +Both land in gbrain v0.35.0.0 behind the openai-compatible recipe path, +alongside OpenAI and Voyage. + +## Setup + +1. Get an API key at + [dashboard.zeroentropy.dev](https://dashboard.zeroentropy.dev). +2. Export it: + ```bash + export ZEROENTROPY_API_KEY= + ``` + +## Embedding switch — zembed-1 + +**Important:** `gbrain config set embedding_model …` is NOT a live +gateway switch. `embedding_model` and `embedding_dimensions` size the +schema and must be stable across engine connects, so they only resolve +from the **file plane** (`~/.gbrain/config.json`) and the **env plane** +(`GBRAIN_EMBEDDING_MODEL` / `GBRAIN_EMBEDDING_DIMENSIONS`). The DB plane +is intentionally ignored for these two keys (same posture as today's +Voyage setup). + +### Option A — file plane (recommended for stable installs) + +Edit `~/.gbrain/config.json`: + +```json +{ + "embedding_model": "zeroentropyai:zembed-1", + "embedding_dimensions": 2560 +} +``` + +Valid dims: `2560` (default), `1280`, `640`, `320`, `160`, `80`, `40`. +Matryoshka-style — smaller trades quality for storage monotonically. +Pick the largest that fits your column width. + +### Option B — env plane (CI / Docker) + +```bash +export GBRAIN_EMBEDDING_MODEL=zeroentropyai:zembed-1 +export GBRAIN_EMBEDDING_DIMENSIONS=2560 +``` + +### Re-embed + +Switching embedding models invalidates the vector index. Re-embed: + +```bash +gbrain embed --stale --limit 50 # smoke a small batch +gbrain embed --stale # full re-embed +``` + +### Verify + +```bash +gbrain models doctor --json | jq '.probes[] | select(.touchpoint=="embedding_config")' +``` + +Expected: `status: "ok"`. Invalid dims (e.g. `1024`, `1536`, `3072`) +surface as `status: "config"` with a paste-ready +`gbrain config set embedding_dimensions ` fix hint. + +## Reranker switch — zerank-2 + +The reranker is the bigger story: gbrain had no cross-encoder reranker +stage before v0.35.0.0. It slots between RRF dedup and token-budget +enforcement in hybrid search. + +### Default-on with `tokenmax` mode + +`tokenmax` mode now defaults `search.reranker.enabled = true` with +`zerank-2`. If you already use `tokenmax` AND have `ZEROENTROPY_API_KEY` +set, reranker fires automatically. Without the key, every rerank call +fails-open (audit-logged) and search returns RRF order — same UX as +before, just with an observable failure surfaced via `gbrain doctor`. + +### Opt-in on `conservative` or `balanced` mode + +```bash +gbrain config set search.reranker.enabled true +``` + +The override sits above the mode-bundle default; opt-out is one flip. + +### Cost anchor + +At 30 candidates × ~400 tokens/chunk × $0.025/1M = **~$0.0003/query**. +Rounding error against the `tokenmax + Opus` pairing's ~$700/mo at +single-user volume per the CLAUDE.md cost matrix. + +### Verify + +```bash +gbrain models doctor --json | jq '.probes[] | select(.touchpoint=="reranker_config")' +``` + +Two probes run for reranker: +- `reranker_config` (zero-network) — validates the model resolves + through the recipe registry and is in the touchpoint's allowlist. +- A reachability probe sends a minimal `{query: "probe", documents: + ["probe"]}` rerank to verify auth + URL. + +## Knobs reference + +| Config key | Default | Notes | +|---|---|---| +| `search.reranker.enabled` | `true` for tokenmax, `false` for others | One-flip opt-in/out | +| `search.reranker.model` | `zeroentropyai:zerank-2` | Try `zerank-1` (older SOTA) or `zerank-1-small` (Apache-2.0 open) | +| `search.reranker.top_n_in` | `30` | Candidates sent to reranker (caps API spend) | +| `search.reranker.top_n_out` | `null` (no truncate) | Truncate reranked output to this many; `null` preserves full length | +| `search.reranker.timeout_ms` | `5000` | HTTP timeout; long stalls degrade UX worse than RRF fallback | + +## Failure observability + +Reranker is fail-open by construction: every error class (auth, rate-limit, +network, timeout, payload-too-large, unknown) returns the original RRF +order unchanged. Failures log to +`~/.gbrain/audit/rerank-failures-YYYY-Www.jsonl` (ISO-week rotation). + +`gbrain doctor` reads the audit and surfaces: +- **auth failures** — any single one warns (config-time problem doctor's + own probe should have caught) +- **payload-too-large** — any single one warns (workload-mismatch signal) +- **transient (network/timeout/rate_limit)** — warns at >=5 in 7 days + +Query text is SHA-256 hashed in the audit; never logged raw. + +## Asymmetric input_type + +ZE zembed-1 (and Voyage v3+) use asymmetric query/document encoding for +better retrieval. The gateway's `embedQuery(text)` companion threads +`input_type: 'query'`; standard `embed(texts)` defaults to +`'document'`. Hybrid search's two query-side embed sites use +`embedQuery()` automatically; all ingest paths use `embed()`. + +Symmetric providers (OpenAI text-embedding-3, fixed-dim Voyage models) +ignore the field — no behavior change. + +## Cache key versioning + +v0.35.0.0 bumped `KNOBS_HASH_VERSION` 1 → 2 to fold reranker config into +the `query_cache.knobs_hash` column. During a rolling deploy: + +- Expect a temporary cache hit-rate dip (~1 hour at default + `cache.ttl_seconds = 3600s`) +- Hot queries may briefly double their cache row count (one row per + version) + +Both clear naturally; no operator action required. + +## Troubleshooting + +| Symptom | Likely cause | Fix | +|---|---|---| +| `embedding_config` probe says invalid dim | Defaulting to 1536 (OpenAI default) | Set `embedding_dimensions` to one of 2560/1280/640/320/160/80/40 | +| `reranker_config` probe says model not in allowlist | Typo in `search.reranker.model` | Use one of `zerank-2` / `zerank-1` / `zerank-1-small` | +| `reranker_health` doctor warns about auth | `ZEROENTROPY_API_KEY` not set or invalid | Re-export the env var; `gbrain models doctor` to verify | +| `reranker_health` doctor warns about transient failures | Upstream flake or rate limit | Reranker fails open to RRF; check ZE status page if persistent | +| Cache hit rate dipped after upgrade | Expected during rolling deploy | Clears within `cache.ttl_seconds` (default 3600s) | diff --git a/llms-full.txt b/llms-full.txt index d1fc495ad..7c0641d5e 100644 --- a/llms-full.txt +++ b/llms-full.txt @@ -202,7 +202,10 @@ strict behavior when unset. - `src/core/embedding.ts` — OpenAI text-embedding-3-large, batch, retry, backoff. **v0.28.7:** `BATCH_SIZE` reverted 50→100 — the original Voyage safety guard halved OpenAI throughput on every page. Per-recipe pre-split + recursive halving + adaptive shrink-on-miss now live in the gateway, so the outer paginator goes back to its original purpose: progress-callback granularity, not batch protection. - `src/core/ai/dims.ts` (v0.33.1.1, PR #962 + #866) — per-provider `providerOptions` resolver for embed-time dimension passthrough. The single source of truth for "which provider needs which knob to produce `vector(N)`". Exports `dimsProviderOptions(implementation, modelId, dims)` (called by `embed()` in `gateway.ts`), `VOYAGE_OUTPUT_DIMENSION_MODELS` (private const — the 7 hosted Voyage models that accept `output_dimension`: `voyage-4-large`, `voyage-4`, `voyage-4-lite`, `voyage-3-large`, `voyage-3.5`, `voyage-3.5-lite`, `voyage-code-3` — nano deliberately excluded), `VOYAGE_VALID_OUTPUT_DIMS = [256, 512, 1024, 2048] as const`, `supportsVoyageOutputDimension(modelId)`, and `isValidVoyageOutputDim(dims)`. **Voyage path uses the SDK-supported `dimensions` field** (`{ openaiCompatible: { dimensions: N } }`), NOT Voyage's `output_dimension` wire-key — the existing `voyageCompatFetch` shim in `gateway.ts:541` translates `dimensions → output_dimension` before the HTTP body is built. The reverse (sending `output_dimension` from here) was the v0.33.1.0 bug class: the AI SDK's openai-compatible adapter doesn't recognize the wire-key so it was silently dropped, Voyage returned its default 1024-dim, and the gateway dimension check threw on every embed call. Runtime guard: when a Voyage flexible-dim model is configured with `dims` outside `VOYAGE_VALID_OUTPUT_DIMS`, throws `AIConfigError` with a paste-ready `gbrain config set embedding_dimensions <256|512|1024|2048>` hint at the embed boundary — fail-loud instead of opaque Voyage HTTP 400. Most common trigger: `embedding_model: voyage:voyage-4-large` configured without `embedding_dimensions` (falls back to `DEFAULT_EMBEDDING_DIMENSIONS=1536`, an OpenAI default not a Voyage one). Eva (@100yenadmin) shipped the wire-key fix in #866; Codex P3 follow-up landed the validator + valid-dims allowlist in #962. - `src/core/ai/types.ts` — provider/recipe types. **v0.28.7 (#680):** `EmbeddingTouchpoint` extended with optional `chars_per_token` (default 4 chars/token, matching OpenAI tiktoken on English) and `safety_factor` (default 0.8, budget-utilization ceiling). Both consulted only when `max_batch_tokens` is also set. Voyage declares `chars_per_token=1` + `safety_factor=0.5` to handle dense payloads (CJK/JSON/base64) that overshoot tiktoken. The pre-split budget is `max_batch_tokens × safety_factor / chars_per_token`. **v0.28.11 (#719):** `EmbeddingTouchpoint.multimodal_models?: string[]` model-level allow-list for recipes that mix text-only + multimodal models under one touchpoint (Voyage's 12 models share `supports_multimodal: true` but only `voyage-multimodal-3` accepts `/multimodalembeddings`). When omitted, recipe-level `supports_multimodal` is sufficient. `AIGatewayConfig.embedding_multimodal_model?: string` lets `embedMultimodal()` route to a different model than `embedding_model` — brains using OpenAI for text can use Voyage for images without flipping the primary embedding pipeline. -- `src/core/ai/gateway.ts` — unified seam for every AI call. **v0.28.7 (#680):** module-scoped `_embedTransport` defaulting to AI SDK `embedMany`, with `__setEmbedTransportForTests(fn)` test seam so tests drive the public `embed()` function with a stubbed transport instead of probing private helpers. `splitByTokenBudget` and `isTokenLimitError` are now exported `@internal` — pure functions reused directly by the test file. Module-level `_shrinkState: Map` halves the recipe's effective `safety_factor` on token-limit miss (floor 0.05) and heals back ×1.5 toward the ceiling after `SHRINK_HEAL_AFTER=10` consecutive successes. `configureGateway()` walks every registered recipe at construction time and emits a once-per-process stderr warning for any embedding touchpoint missing `max_batch_tokens` (excluding the canonical OpenAI fast-path recipe). `resetGateway()` clears `_shrinkState`, the warned-set, and restores the real transport. ASCII flow diagram embedded in the `embed()` JSDoc covers the routing decision, recursion + halving, and shrinkState lifecycle. **v0.28.11 (#719):** `embedMultimodal()` reads `cfg.embedding_multimodal_model` first (falls back to `cfg.embedding_model` for single-model setups). After the existing recipe-level `supports_multimodal` fast-fail, validates the resolved model against `touchpoint.multimodal_models` when declared — closes the Voyage-text-only-model-into-multimodal-endpoint footgun before any HTTP call (Codex F1 from PR review). New `getMultimodalModel()` accessor mirrors `getEmbeddingModel` / `getChatModel` so doctor and integration tests can read the gateway state. **v0.33.1.1 (#962, Codex P3 follow-up):** new exported `VoyageResponseTooLargeError` tagged class at the top of the file. `voyageCompatFetch`'s two OOM-defense caps (Layer 1 Content-Length check at `:595`, Layer 2 per-embedding base64 cap at `:619`) now throw `VoyageResponseTooLargeError` instead of a generic `Error`. The inbound response-rewriter's surrounding try/catch (which intentionally swallows parse failures so misshaped Voyage responses fall through to the SDK's JSON parser) checks `instanceof VoyageResponseTooLargeError` and rethrows. Pre-fix, the Layer 2 throw was silently swallowed and the oversized response returned to the AI SDK anyway — Layer 2 was theatrical. Source-shape regression assertion in `test/voyage-response-cap.test.ts` pins the `instanceof ⇒ throw err` line. **v0.34.1.0 (#875):** new `embedMultimodalOpenAICompat()` routes recipes with `implementation: 'openai-compatible'` (LiteLLM, Anyscale, vLLM, Gemini multimodal via proxy) through the standard `/embeddings` endpoint with content arrays carrying `image_url` entries. The pre-existing Voyage `/multimodalembeddings` path is unchanged; the gateway selects by recipe `implementation` tag. Runtime dimension validation throws `AIConfigError` (with model id + observed + expected) before the vector reaches storage when the provider returns a width that doesn't match the recipe's `default_dims` or the brain's `embedding_dimensions` config — no more cryptic `vector dimension mismatch` at INSERT time. Pinned by 11 cases in `test/openai-compat-multimodal.test.ts`. +- `src/core/ai/gateway.ts` — unified seam for every AI call. **v0.35.0.0:** ZeroEntropy support lands. New `zeroEntropyCompatFetch` shim (sibling to `voyageCompatFetch`) handles ZE's non-OpenAI-compatible wire shape — rewrites the request URL from `/embeddings` to `/models/embed`, injects `input_type` (default `'document'`; `'query'` when threaded via `providerOptions.openaiCompatible.input_type`) and explicit `encoding_format: 'float'`, and rewrites the response from `{results: [{embedding}], usage: {total_bytes, total_tokens}}` to `{data: [{embedding, index}], usage: {prompt_tokens, total_tokens}}` so the SDK's openai-compatible Zod schema validates (Voyage's shim hit the same `prompt_tokens` requirement at `:655`). Layer 1 (Content-Length) + Layer 2 (per-embedding) OOM caps via a new tagged `ZeroEntropyResponseTooLargeError` class (kept separate from `VoyageResponseTooLargeError` because `test/voyage-response-cap.test.ts` does structural source-text greps pinning the Voyage name — class unification is a deferred cleanup). Wired in `instantiateEmbedding()` via the same `recipe.id === 'zeroentropyai'` branch pattern Voyage uses. New `gateway.rerank()` native HTTP path (no AI-SDK reranking abstraction): resolves the configured reranker model via `getRerankerModel()`, posts to `${recipe.base_url}/models/rerank` with bearer auth, returns `RerankResult[]` sorted by relevance score. `RerankError.reason` classifier: `auth | rate_limit | network | timeout | payload_too_large | unknown`. 5s default timeout (search hot path). Pre-flight payload guard rejects bodies over `recipe.touchpoints.reranker.max_payload_bytes` with `reason: 'payload_too_large'` so callers can fail-open without an HTTP call. `_rerankTransport` test seam mirrors `_embedTransport`. New `gateway.embedQuery(text)` companion threads `inputType: 'query'` through `dimsProviderOptions()` (now 4-arg). `getRerankerModel()` accessor + `isAvailable('reranker')` branch added. `configureGateway` + `reconfigureGatewayWithEngine` thread `reranker_model` through the same path as embedding/expansion/chat. `applyResolveAuth` + `defaultResolveAuth` widen touchpoint param to include `'reranker'`. **v0.34.1.0 (#875):** new `embedMultimodalOpenAICompat()` routes recipes with `implementation: 'openai-compatible'` (LiteLLM, Anyscale, vLLM, Gemini multimodal via proxy) through the standard `/embeddings` endpoint with content arrays carrying `image_url` entries. The pre-existing Voyage `/multimodalembeddings` path is unchanged; the gateway selects by recipe `implementation` tag. Runtime dimension validation throws `AIConfigError` (with model id + observed + expected) before the vector reaches storage when the provider returns a width that doesn't match the recipe's `default_dims` or the brain's `embedding_dimensions` config — no more cryptic `vector dimension mismatch` at INSERT time. Pinned by 11 cases in `test/openai-compat-multimodal.test.ts`. **v0.28.7 (#680):** module-scoped `_embedTransport` defaulting to AI SDK `embedMany`, with `__setEmbedTransportForTests(fn)` test seam so tests drive the public `embed()` function with a stubbed transport instead of probing private helpers. `splitByTokenBudget` and `isTokenLimitError` are now exported `@internal` — pure functions reused directly by the test file. Module-level `_shrinkState: Map` halves the recipe's effective `safety_factor` on token-limit miss (floor 0.05) and heals back ×1.5 toward the ceiling after `SHRINK_HEAL_AFTER=10` consecutive successes. `configureGateway()` walks every registered recipe at construction time and emits a once-per-process stderr warning for any embedding touchpoint missing `max_batch_tokens` (excluding the canonical OpenAI fast-path recipe). `resetGateway()` clears `_shrinkState`, the warned-set, and restores the real transport. ASCII flow diagram embedded in the `embed()` JSDoc covers the routing decision, recursion + halving, and shrinkState lifecycle. **v0.28.11 (#719):** `embedMultimodal()` reads `cfg.embedding_multimodal_model` first (falls back to `cfg.embedding_model` for single-model setups). After the existing recipe-level `supports_multimodal` fast-fail, validates the resolved model against `touchpoint.multimodal_models` when declared — closes the Voyage-text-only-model-into-multimodal-endpoint footgun before any HTTP call (Codex F1 from PR review). New `getMultimodalModel()` accessor mirrors `getEmbeddingModel` / `getChatModel` so doctor and integration tests can read the gateway state. **v0.33.1.1 (#962, Codex P3 follow-up):** new exported `VoyageResponseTooLargeError` tagged class at the top of the file. `voyageCompatFetch`'s two OOM-defense caps (Layer 1 Content-Length check at `:595`, Layer 2 per-embedding base64 cap at `:619`) now throw `VoyageResponseTooLargeError` instead of a generic `Error`. The inbound response-rewriter's surrounding try/catch (which intentionally swallows parse failures so misshaped Voyage responses fall through to the SDK's JSON parser) checks `instanceof VoyageResponseTooLargeError` and rethrows. Pre-fix, the Layer 2 throw was silently swallowed and the oversized response returned to the AI SDK anyway — Layer 2 was theatrical. Source-shape regression assertion in `test/voyage-response-cap.test.ts` pins the `instanceof ⇒ throw err` line. +- `src/core/ai/recipes/zeroentropyai.ts` (v0.35.0.0) — ZeroEntropy openai-compatible recipe declaring BOTH `embedding` (`zembed-1`, 7 Matryoshka dims: 2560/1280/640/320/160/80/40) AND `reranker` (`zerank-2` flagship + `zerank-1` + `zerank-1-small`, 5MB payload cap) touchpoints. `implementation: 'openai-compatible'` (NOT the misspelled `'openai-compat'` the original plan draft had — pinned by F1 regression in `test/ai/zeroentropy-recipe.test.ts`). `base_url_default: 'https://api.zeroentropy.dev/v1'` already ends with `/v1`, so the `zeroEntropyCompatFetch` URL rewrite `/embeddings → /models/embed` produces `…/v1/models/embed` (NOT `…/v1/v1/…` — pinned by F2 regression). `chars_per_token: 1` + `safety_factor: 0.5` match Voyage's dense-content hedge. +- `src/core/rerank-audit.ts` (v0.35.0.0) — failure-only JSONL audit at `~/.gbrain/audit/rerank-failures-YYYY-Www.jsonl` (ISO-week rotation, mirrors `src/core/audit-slug-fallback.ts`). Exports `logRerankFailure({reason, model, query_hash, doc_count, error_summary})` + `readRecentRerankFailures(days)`. **Deliberately no `logRerankSuccess`** (CDX2-F22 in plan review): writing once per tokenmax search is hot-path I/O churn AND success events leak query volume + timing into a local audit file. `gbrain doctor`'s `reranker_health` check reads `search.reranker.enabled` first so "no events in window" is interpreted correctly (disabled → ok; enabled → ok). Query text is SHA-256-prefix-hashed (8 hex chars) for privacy. `GBRAIN_AUDIT_DIR` env override honored via the shared `resolveAuditDir()`. +- `src/core/search/rerank.ts` (v0.35.0.0) — the call-site abstraction. `applyReranker(query, results, opts)` slots between `dedupResults()` and `enforceTokenBudget()` in `src/core/search/hybrid.ts`. Slices `opts.topNIn` (default 30) by current RRF order, sends to `gateway.rerank()`, reorders by `relevanceScore` desc, and appends the un-reranked tail unchanged (recall protection). **Fail-open on every `RerankError.reason`**: any error logs via `logRerankFailure` and returns the input array unchanged. Stamps `rerank_score` onto reordered items so downstream telemetry sees the new ordering signal. `topNOut: null` is the explicit "don't truncate" signal — semantically distinct from `undefined` which means "fall through to mode bundle" (CDX2-F16). Test seam: `opts.rerankerFn` lets tests stub `gateway.rerank` without touching the network. - `src/core/ai/recipes/voyage.ts` — Voyage AI openai-compatible recipe. **v0.28.7 (#680):** declares `chars_per_token=1` + `safety_factor=0.5` so the gateway pre-splits Voyage batches at a 60K-character budget (50% of 120K-token cap with the dense-tokenizer ratio). Closes the v0.27 backfill loop where ~26% of the corpus stayed un-embedded because tiktoken-grounded budgeting silently undercounted Voyage's actual token usage. **v0.28.11 (#719):** declares `multimodal_models: ['voyage-multimodal-3']` so the gateway rejects text-only Voyage models pointed at the multimodal endpoint with a clear `AIConfigError` instead of waiting for Voyage's HTTP 400. **v0.33.1.1 (#962, fixup):** recipe docstring at `:7-16` tightened to name the seven hosted flexible-dim models that accept `output_dimension` explicitly (`voyage-4-large`, `voyage-4`, `voyage-4-lite`, `voyage-3-large`, `voyage-3.5`, `voyage-3.5-lite`, `voyage-code-3`) and call out that `voyage-4-nano` is the open-weight variant listed separately by Voyage as fixed 1024-dim — does NOT accept the parameter. The "all v4 variants are flexible" misread is what caused the original PR to include nano in `VOYAGE_OUTPUT_DIMENSION_MODELS`; the negative regression assertion in `test/ai/gateway.test.ts` (`dimsProviderOptions` returns `undefined` for `voyage-4-nano`) pins the contract. - `src/core/ai/recipes/anthropic.ts` — Anthropic recipe (chat + expansion touchpoints). **v0.31.12:** chat and expansion `models:` lists drop the v0.31.6 phantom `claude-sonnet-4-6-20250929` date suffix — canonical id is `claude-sonnet-4-6`. The wrong-direction alias `claude-sonnet-4-6 → claude-sonnet-4-6-20250929` is removed; a reverse alias `claude-sonnet-4-6-20250929 → claude-sonnet-4-6` keeps stale user configs working (rescues `facts.extraction_model` and `models.dream.synthesize` set by v0.31.6 installs). Recipe-shape regression pinned by `test/anthropic-model-ids.test.ts` (6 cases, verbatim cherry-pick of PR #830 plus the reverse-alias rescue case). - `src/core/anthropic-pricing.ts` — Single source of truth for Anthropic model pricing (per-MTok input/output). **v0.31.12:** Opus 4.7 corrected from `$15/$75` to `$5/$25` (the old number was from Opus 4 generation, never refreshed when 4.7 shipped); Opus 4.6 also corrected. Consumed by `src/core/budget-meter.ts` and `src/core/cross-modal-eval/runner.ts` — the cross-modal estimator now reads `ANTHROPIC_PRICING` for Anthropic models instead of duplicating the table, killing the v0.31.6 drift bug class. diff --git a/package.json b/package.json index e9254752b..525cdca13 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "gbrain", - "version": "0.34.4.0", + "version": "0.35.0.0", "description": "Postgres-native personal knowledge brain with hybrid RAG search", "type": "module", "main": "src/core/index.ts", diff --git a/skills/migrations/v0.35.0.0.md b/skills/migrations/v0.35.0.0.md new file mode 100644 index 000000000..3fae08669 --- /dev/null +++ b/skills/migrations/v0.35.0.0.md @@ -0,0 +1,84 @@ +--- +feature_pitch: ZeroEntropy zembed-1 embeddings + zerank-2 cross-encoder reranking +required_action: no # purely opt-in +--- + +# v0.35.0.0 migration notes + +ZeroEntropy support landed. **No required user action.** Reranker is on by +default for `tokenmax` mode only; embedding model is unchanged for everyone +unless the user explicitly opts in via config file or env var. + +## What changed automatically + +- `KNOBS_HASH_VERSION` bumped 1 → 2 to fold reranker config into the + `query_cache.knobs_hash` column. Expect a temporary cache hit-rate dip + for ~1 hour (the default `cache.ttl_seconds`) as v=1 rows TTL out and + v=2 rows backfill. Search still works during the dip; only cache hits + are affected. +- `tokenmax` mode now defaults `search.reranker.enabled = true`. If the + user has `ZEROENTROPY_API_KEY` set AND uses tokenmax, reranker fires. + Without the API key, the rerank attempt fails-open (audit-logged) and + search returns the RRF order — same UX as before, just with an + observable failure in `gbrain doctor`. +- `conservative` and `balanced` modes default reranker = false. Nothing + changes for those users without an explicit opt-in. + +## What the user can do (optional) + +### Try zembed-1 embeddings + +Switching embedding models invalidates the vector index — you'll need to +re-embed. Edit `~/.gbrain/config.json`: + +```json +{ + "embedding_model": "zeroentropyai:zembed-1", + "embedding_dimensions": 2560 +} +``` + +Valid dims: 2560, 1280, 640, 320, 160, 80, 40 (Matryoshka-style; smaller +trades quality for storage). Then: + +```bash +export ZEROENTROPY_API_KEY=... +gbrain models doctor # verify config +gbrain embed --stale --limit 50 # smoke a small re-embed +gbrain embed --stale # full re-embed +``` + +### Try zerank-2 on conservative/balanced + +```bash +export ZEROENTROPY_API_KEY=... +gbrain config set search.reranker.enabled true +gbrain models doctor # verify reranker_config + reachability +gbrain query "some query that previously misranked" +``` + +To opt out: +```bash +gbrain config set search.reranker.enabled false +``` + +### Opt out of reranker on tokenmax + +If you're on `tokenmax` mode and don't want reranker spend: + +```bash +gbrain config set search.reranker.enabled false +``` + +The override sticks above the mode bundle default. + +## Diagnostics + +- `gbrain models doctor --json` — probes `embedding_config`, `reranker_config`, + and reranker reachability. Surfaces config issues with paste-ready fix hints. +- `gbrain doctor` — runs `reranker_health` against the JSONL audit at + `~/.gbrain/audit/rerank-failures-YYYY-Www.jsonl`. Auth failures warn + immediately; transient failures warn at >=5 in 7 days. +- `tail -100 ~/.gbrain/audit/rerank-failures-*.jsonl` — raw failure log + for direct inspection (privacy: query text is SHA-256 hashed; never + raw). diff --git a/src/commands/doctor.ts b/src/commands/doctor.ts index 630012766..74e03e3cd 100644 --- a/src/commands/doctor.ts +++ b/src/commands/doctor.ts @@ -414,9 +414,94 @@ export async function doctorReportRemote(engine: BrainEngine): Promise=5 in window. + * Below that they're noise; reranker fails open anyway. + * 5) Payload-too-large failures: warn at >=1 (indicates a workload + * mismatch that the operator should know about). + * + * Engine-agnostic (file-based + one config-key read). + */ +export async function checkRerankerHealth(engine: BrainEngine): Promise { + try { + const { readRecentRerankFailures } = await import('../core/rerank-audit.ts'); + const cfg = await engine.getConfig('search.reranker.enabled'); + const rerankerEnabled = cfg === 'true' || cfg === '1'; + + const failures = readRecentRerankFailures(7); + if (failures.length === 0) { + return { + name: 'reranker_health', + status: 'ok', + message: rerankerEnabled + ? 'No rerank failures in last 7 days' + : 'Reranker disabled — no failures expected', + }; + } + + const authFails = failures.filter((f) => f.reason === 'auth'); + if (authFails.length > 0) { + return { + name: 'reranker_health', + status: 'warn', + message: `${authFails.length} reranker auth failure(s) in last 7 days. Fix: verify ZEROENTROPY_API_KEY and run \`gbrain models doctor\`.`, + }; + } + + const payloadFails = failures.filter((f) => f.reason === 'payload_too_large'); + if (payloadFails.length > 0) { + return { + name: 'reranker_health', + status: 'warn', + message: `${payloadFails.length} reranker payload-too-large failure(s) in last 7 days. Fix: lower \`search.reranker.top_n_in\` (default 30) or split very large documents.`, + }; + } + + const transientFails = failures.filter( + (f) => f.reason === 'network' || f.reason === 'timeout' || f.reason === 'rate_limit', + ); + if (transientFails.length >= 5) { + return { + name: 'reranker_health', + status: 'warn', + message: `${transientFails.length} transient reranker failure(s) in last 7 days. Search fails open to RRF order; check ZE status if persistent.`, + }; + } + + return { + name: 'reranker_health', + status: 'ok', + message: `${failures.length} reranker failure(s) in last 7 days (below threshold)`, + }; + } catch (e) { + const msg = e instanceof Error ? e.message : String(e); + return { + name: 'reranker_health', + status: 'warn', + message: `Could not check reranker audit: ${msg}`, + }; + } +} + /** * v0.32.3 [CDX-20]: surface mode + per-key override drift. * @@ -2420,6 +2505,9 @@ export async function runDoctor(engine: BrainEngine | null, args: string[], dbSo checks.push(await checkSearchMode(engine)); progress.heartbeat('eval_drift'); checks.push(await checkEvalDrift(engine)); + // v0.35.0.0+ reranker_health — read JSONL audit; warn on auth or volume. + progress.heartbeat('reranker_health'); + checks.push(await checkRerankerHealth(engine)); } progress.finish(); diff --git a/src/commands/models.ts b/src/commands/models.ts index 1fee07ac9..9a8ba084c 100644 --- a/src/commands/models.ts +++ b/src/commands/models.ts @@ -160,7 +160,7 @@ type ProbeStatus = 'ok' | 'model_not_found' | 'auth' | 'rate_limit' | 'network' interface ProbeResult { model: string; - touchpoint: 'chat' | 'expansion' | 'embedding_config'; + touchpoint: 'chat' | 'expansion' | 'embedding_config' | 'reranker_config'; status: ProbeStatus; message: string; elapsed_ms: number; @@ -196,8 +196,10 @@ async function probeEmbeddingConfig(): Promise { const start = Date.now(); const { getEmbeddingModel, getEmbeddingDimensions } = await import('../core/ai/gateway.ts'); const { parseModelId } = await import('../core/ai/model-resolver.ts'); - const { supportsVoyageOutputDimension, isValidVoyageOutputDim, VOYAGE_VALID_OUTPUT_DIMS } = - await import('../core/ai/dims.ts'); + const { + supportsVoyageOutputDimension, isValidVoyageOutputDim, VOYAGE_VALID_OUTPUT_DIMS, + supportsZeroEntropyDimension, isValidZeroEntropyDim, ZEROENTROPY_VALID_DIMS, + } = await import('../core/ai/dims.ts'); const modelStr = getEmbeddingModel(); const dims = getEmbeddingDimensions(); @@ -223,6 +225,26 @@ async function probeEmbeddingConfig(): Promise { } } + // ZeroEntropy zembed-1 flexible-dim check. Same bug class as Voyage: + // `embedding_model: zeroentropyai:zembed-1` configured without + // `embedding_dimensions` falls back to DEFAULT_EMBEDDING_DIMENSIONS=1536 + // (an OpenAI default) which ZE doesn't accept. + if (providerId === 'zeroentropyai' && supportsZeroEntropyDimension(modelId)) { + if (!isValidZeroEntropyDim(dims)) { + return { + model: modelStr, + touchpoint: 'embedding_config', + status: 'config', + message: + `embedding_dimensions=${dims} is not a valid ZeroEntropy dimensions ` + + `for "${modelId}" (allowed: ${ZEROENTROPY_VALID_DIMS.join('/')}).`, + fix: + `gbrain config set embedding_dimensions <${ZEROENTROPY_VALID_DIMS.join('|')}>.`, + elapsed_ms: Date.now() - start, + }; + } + } + return { model: modelStr, touchpoint: 'embedding_config', @@ -246,6 +268,129 @@ async function probeEmbeddingConfig(): Promise { } } +/** + * v0.35.0.0+: zero-network reranker config probe. Validates that the + * configured reranker model resolves through the recipe registry, that the + * recipe declares a `reranker` touchpoint, and that the model is in the + * touchpoint's `models[]` allowlist. + * + * CDX2-F11: `assertTouchpoint()` does NOT enforce allowlists for + * openai-compatible recipes — the probe does it directly here. Without + * this, `search.reranker.model=zeroentropyai:made-up-name` would silently + * pass config probes and fail at first rerank call. + * + * Returns 'ok' when reranker is unconfigured (default state — opt-in + * feature). Surfaces `status: 'config'` with paste-ready fix hint when + * model is invalid. + */ +async function probeRerankerConfig(): Promise { + const start = Date.now(); + const { getRerankerModel } = await import('../core/ai/gateway.ts'); + const { resolveRecipe } = await import('../core/ai/model-resolver.ts'); + + const modelStr = getRerankerModel(); + if (!modelStr) { + // Reranker not configured. Default state for fresh installs and any + // brain that hasn't opted in. Not an error; doctor reports 'ok' so the + // probe row is informational. + return { + model: '(none)', + touchpoint: 'reranker_config', + status: 'ok', + message: 'reranker not configured (set GBRAIN_RERANKER_MODEL or `gbrain config set search.reranker.enabled true`)', + elapsed_ms: Date.now() - start, + }; + } + + try { + const { parsed, recipe } = resolveRecipe(modelStr); + const tp = recipe.touchpoints.reranker; + if (!tp) { + return { + model: modelStr, + touchpoint: 'reranker_config', + status: 'config', + message: `Provider "${recipe.id}" does not declare a reranker touchpoint.`, + fix: 'Switch to a provider that does (e.g. zeroentropyai:zerank-2).', + elapsed_ms: Date.now() - start, + }; + } + if (tp.models.length > 0 && !tp.models.includes(parsed.modelId)) { + return { + model: modelStr, + touchpoint: 'reranker_config', + status: 'config', + message: `Model "${parsed.modelId}" is not in ${recipe.name}'s reranker allowlist.`, + fix: `gbrain config set search.reranker.model ${recipe.id}:`, + elapsed_ms: Date.now() - start, + }; + } + return { + model: modelStr, + touchpoint: 'reranker_config', + status: 'ok', + message: `reranker configured: ${modelStr}`, + elapsed_ms: Date.now() - start, + }; + } catch (err) { + const msg = err instanceof Error ? err.message : String(err); + return { + model: modelStr, + touchpoint: 'reranker_config', + status: 'config', + message: msg, + elapsed_ms: Date.now() - start, + }; + } +} + +/** + * v0.35.0.0+: 1-token-equivalent reranker reachability probe. Sends a minimal + * `{query, documents: [doc]}` request to verify auth + URL. Uses the same + * AbortController + 5s timeout pattern as probeModel. + * + * Returns 'ok' silently when reranker is unconfigured (no probe needed) — + * probeRerankerConfig already surfaced the missing-config state. + */ +async function probeRerankerReachability(): Promise { + const { getRerankerModel } = await import('../core/ai/gateway.ts'); + const modelStr = getRerankerModel(); + if (!modelStr) return null; + + const start = Date.now(); + try { + const { rerank } = await import('../core/ai/gateway.ts'); + const controller = new AbortController(); + const timeoutId = setTimeout(() => controller.abort(new Error('probe timed out after 5s')), 5000); + try { + await rerank({ + query: 'probe', + documents: ['probe document'], + signal: controller.signal, + timeoutMs: 5000, + }); + return { + model: modelStr, + touchpoint: 'reranker_config', + status: 'ok', + message: 'reachable', + elapsed_ms: Date.now() - start, + }; + } finally { + clearTimeout(timeoutId); + } + } catch (err) { + const { status, message } = classifyError(err); + return { + model: modelStr, + touchpoint: 'reranker_config', + status, + message, + elapsed_ms: Date.now() - start, + }; + } +} + async function probeModel(modelStr: string, touchpoint: 'chat' | 'expansion'): Promise { const start = Date.now(); try { @@ -327,6 +472,8 @@ Tiers: utility (haiku-class) | reasoning (sonnet) | deep (opus) | subagent (Anth // brain misconfigured for Voyage with the wrong embedding_dimensions would // 400 on first embed. Fast feedback before we spend a single token. results.push(await probeEmbeddingConfig()); + // v0.35.0.0+ reranker config probe — same zero-network model as embedding. + results.push(await probeRerankerConfig()); for (const [modelStr, touchpoint] of [[chatModel, 'chat'], [expansionModel, 'expansion']] as const) { if (shouldSkipProvider(modelStr, skip)) { @@ -336,6 +483,14 @@ Tiers: utility (haiku-class) | reasoning (sonnet) | deep (opus) | subagent (Anth results.push(await probeModel(modelStr, touchpoint)); } + // v0.35.0.0+: reranker reachability (only when configured + provider not in --skip). + const { getRerankerModel } = await import('../core/ai/gateway.ts'); + const rerankerModel = getRerankerModel(); + if (rerankerModel && !shouldSkipProvider(rerankerModel, skip)) { + const r = await probeRerankerReachability(); + if (r) results.push(r); + } + const report = { schema_version: 1 as const, probes: results, diff --git a/src/commands/search.ts b/src/commands/search.ts index c26a12b8d..b317b2aac 100644 --- a/src/commands/search.ts +++ b/src/commands/search.ts @@ -48,6 +48,11 @@ const KNOB_DESCRIPTIONS: Record = { tokenBudget: 'Per-call token-budget cap (undefined = no cap)', expansion: 'LLM multi-query expansion (Haiku call per search)', searchLimit: 'Default `limit` for the operation layer', + reranker_enabled: 'Cross-encoder reranker (ZE zerank-2) on/off', + reranker_model: 'Provider:model for the reranker', + reranker_top_n_in: 'Candidates sent to reranker per call', + reranker_top_n_out: 'Cap on reranked output (null = no truncate)', + reranker_timeout_ms: 'HTTP timeout for the reranker call', }; interface SearchModesReport { diff --git a/src/core/ai/dims.ts b/src/core/ai/dims.ts index 6b92cd1d5..558f347ff 100644 --- a/src/core/ai/dims.ts +++ b/src/core/ai/dims.ts @@ -47,6 +47,24 @@ export function isValidVoyageOutputDim(dims: number): boolean { return (VOYAGE_VALID_OUTPUT_DIMS as readonly number[]).includes(dims); } +// v0.35.0.0+ ZeroEntropy zembed-1 flexible-dim allowlist. zembed-1 distills +// from zerank-2 (Matryoshka-style); smaller dims trade quality for storage. +// ZE rejects any other value with HTTP 400; catching it locally produces a +// clearer error with the valid-values hint. Same failure mode as the Voyage +// case: `embedding_model: zeroentropyai:zembed-1` configured without +// `embedding_dimensions` falls back to DEFAULT_EMBEDDING_DIMENSIONS=1536 +// (an OpenAI default), which ZE doesn't accept. +const ZEROENTROPY_DIM_MODELS = new Set(['zembed-1']); +export const ZEROENTROPY_VALID_DIMS = [2560, 1280, 640, 320, 160, 80, 40] as const; + +export function supportsZeroEntropyDimension(modelId: string): boolean { + return ZEROENTROPY_DIM_MODELS.has(modelId); +} + +export function isValidZeroEntropyDim(dims: number): boolean { + return (ZEROENTROPY_VALID_DIMS as readonly number[]).includes(dims); +} + /** * Build the providerOptions blob for embedMany() that pins output dimensions. * @@ -56,15 +74,26 @@ export function isValidVoyageOutputDim(dims: number): boolean { * endpoint accepts `output_dimension`, but the AI SDK openai-compatible * adapter only forwards `dimensions`; gateway.ts translates that field to * Voyage's wire name in voyageCompatFetch. + * + * v0.35.0.0+ 4th param `inputType`: 'query' | 'document' for asymmetric + * providers (ZE zembed-1, Voyage v3+, MiniMax embo-01). When omitted, the + * existing document-encoding behavior is preserved (no `input_type` field + * emitted for symmetric providers; legacy hardcoded `type:'db'` for + * embo-01). gateway.embedQuery() threads `'query'`; gateway.embed() threads + * `'document'`. Per-model filtering happens INSIDE the switch — the field + * is NEVER emitted for providers that don't accept it (OpenAI text-3, + * DashScope, Zhipu) so the request body stays clean for those endpoints. */ export function dimsProviderOptions( implementation: Implementation, modelId: string, dims: number, + inputType?: 'query' | 'document', ): Record | undefined { switch (implementation) { case 'native-openai': { // text-embedding-3-* supports dimensions; text-embedding-ada-002 does not. + // OpenAI embeddings are symmetric — inputType ignored. if (modelId.startsWith('text-embedding-3')) { return { openai: { dimensions: dims } }; } @@ -80,10 +109,31 @@ export function dimsProviderOptions( // Anthropic has no embedding model. return undefined; case 'openai-compatible': - // Most openai-compatible providers (Ollama, LM Studio, vLLM, LiteLLM) - // do not expose a standard dimensions knob. Voyage is the exception, - // but it needs the SDK-supported field here so voyageCompatFetch can - // translate it to `output_dimension` before the HTTP request is sent. + // ZE zembed-1 — flexible Matryoshka dims + asymmetric input_type. + // Lives BEFORE the generic openai-compatible fall-through to avoid + // sending input_type to providers (Azure/DashScope/Zhipu) that + // would reject it. + if (supportsZeroEntropyDimension(modelId)) { + if (!isValidZeroEntropyDim(dims)) { + throw new AIConfigError( + `ZeroEntropy model "${modelId}" supports dimensions only in ` + + `{${ZEROENTROPY_VALID_DIMS.join(', ')}}, got ${dims}.`, + `Set \`embedding_dimensions\` to one of ` + + `${ZEROENTROPY_VALID_DIMS.join('/')} in your gbrain config.`, + ); + } + return { + openaiCompatible: { + dimensions: dims, + input_type: inputType ?? 'document', + }, + }; + } + // Voyage hosted flexible-dim models — accept `output_dimension` + // (translated by voyageCompatFetch) AND `input_type: query|document` + // for asymmetric retrieval. inputType is opt-in: when undefined, + // emit no field (preserves pre-v0.35.0.0 callers + existing tests). + // When threaded explicitly by embedQuery()/embed(), it reaches Voyage. if (supportsVoyageOutputDimension(modelId)) { // Fail-loud at the embed boundary if the user configured a dim // Voyage doesn't accept. The most common path here: a brain with @@ -101,13 +151,19 @@ export function dimsProviderOptions( `switch to a fixed-dim Voyage model (e.g. voyage-3, voyage-3-lite).`, ); } - return { openaiCompatible: { dimensions: dims } }; + return { + openaiCompatible: { + dimensions: dims, + ...(inputType ? { input_type: inputType } : {}), + }, + }; } // OpenAI text-embedding-3 family on the openai-compatible adapter // (Azure OpenAI hosts these via its OpenAI-compatible /embeddings // endpoint). The provider defaults to the model's native size (3072 // for `-large`, 1536 for `-small`); without `dimensions`, brains // configured for a smaller width (e.g. 1536) hard-fail at first embed. + // Azure/OpenAI-compat embeddings are symmetric — inputType ignored. if (modelId.startsWith('text-embedding-3')) { return { openaiCompatible: { dimensions: dims } }; } @@ -115,14 +171,15 @@ export function dimsProviderOptions( // embedding-3 (Matryoshka 256-2048) both accept `dimensions` on the // OpenAI-compat path. Without this, user-selected non-default dims are // silently ignored and the provider returns its default size. + // Symmetric retrieval — inputType ignored. if (modelId === 'text-embedding-v3' || modelId === 'embedding-3') { return { openaiCompatible: { dimensions: dims } }; } // MiniMax embo-01 takes a `type: 'db' | 'query'` field for asymmetric - // retrieval. Default to 'db' (the indexing path) so embed() works for - // import. Queries also embed with type:'db', making retrieval - // symmetric. Asymmetric query support is a follow-up TODO that needs - // a query/document signal threaded through the embed seam. + // retrieval. Today still hardcoded to 'db' for back-compat — opting + // into the new inputType seam is a follow-up (see plan's deferred + // section "Fix MiniMax embo-01 asymmetry"). When fixed: map + // inputType==='query' → type:'query', else 'db'. if (modelId === 'embo-01') { return { openaiCompatible: { type: 'db' } }; } diff --git a/src/core/ai/gateway.ts b/src/core/ai/gateway.ts index 406497d9f..e65b08948 100644 --- a/src/core/ai/gateway.ts +++ b/src/core/ai/gateway.ts @@ -46,6 +46,10 @@ const DEFAULT_EMBEDDING_MODEL = 'openai:text-embedding-3-large'; const DEFAULT_EMBEDDING_DIMENSIONS = 1536; const DEFAULT_EXPANSION_MODEL = 'anthropic:claude-haiku-4-5-20251001'; const DEFAULT_CHAT_MODEL = 'anthropic:claude-sonnet-4-6'; +// v0.35.0.0+: reranker default. Used only when search.reranker.enabled is set +// AND no explicit reranker_model is configured. Mode bundles' per-mode +// `reranker_model` default to this same value but can be overridden. +const DEFAULT_RERANKER_MODEL = 'zeroentropyai:zerank-2'; let _config: AIGatewayConfig | null = null; const _modelCache = new Map(); @@ -157,6 +161,24 @@ export class VoyageResponseTooLargeError extends Error { } } +/** + * v0.35.0.0+: same defense pattern as Voyage's cap but tagged separately so the + * `instanceof` rethrow inside zeroEntropyCompatFetch only matches its own + * throws (avoids cross-recipe entanglement if both shims fire in the same + * process). Plan called for unifying these into one + * `EmbeddingResponseTooLargeError` class — descoped because + * `test/voyage-response-cap.test.ts` does structural source-text greps + * pinning the Voyage name. Unification is a follow-up cleanup. + */ +const MAX_ZEROENTROPY_RESPONSE_BYTES = 256 * 1024 * 1024; + +export class ZeroEntropyResponseTooLargeError extends Error { + constructor(message: string) { + super(message); + this.name = 'ZeroEntropyResponseTooLargeError'; + } +} + // ---- Unified auth resolution (D12=A) ---- // // Pre-v0.32, openai-compatible auth was duplicated across instantiateEmbedding, @@ -181,7 +203,7 @@ export class VoyageResponseTooLargeError extends Error { export function defaultResolveAuth( recipe: Recipe, env: Record, - touchpoint: 'embedding' | 'expansion' | 'chat', + touchpoint: 'embedding' | 'expansion' | 'chat' | 'reranker', ): { headerName: string; token: string } { const required = recipe.auth_env?.required ?? []; const optional = recipe.auth_env?.optional ?? []; @@ -220,7 +242,7 @@ export function defaultResolveAuth( export function applyResolveAuth( recipe: Recipe, cfg: AIGatewayConfig, - touchpoint: 'embedding' | 'expansion' | 'chat', + touchpoint: 'embedding' | 'expansion' | 'chat' | 'reranker', ): { apiKey?: string; headers?: Record } { const resolved = recipe.resolveAuth ? recipe.resolveAuth(cfg.env) @@ -276,6 +298,11 @@ export function configureGateway(config: AIGatewayConfig): void { expansion_model: config.expansion_model ?? DEFAULT_EXPANSION_MODEL, chat_model: config.chat_model ?? DEFAULT_CHAT_MODEL, chat_fallback_chain: config.chat_fallback_chain, + // v0.35.0.0+: reranker_model stays undefined when unset — reranker is + // opt-in and pulling DEFAULT_RERANKER_MODEL into every gateway start + // would silently register a third-party model id on brains that never + // wanted it. isAvailable('reranker') returns false when unset. + reranker_model: config.reranker_model, base_urls: config.base_urls, env: config.env, }; @@ -289,6 +316,7 @@ export function configureGateway(config: AIGatewayConfig): void { _config.embedding_multimodal_model, _config.expansion_model, _config.chat_model, + _config.reranker_model, ...(_config.chat_fallback_chain ?? []), ]) { if (m) registerExtendedModel(m); @@ -347,6 +375,7 @@ export async function reconfigureGatewayWithEngine(engine: BrainEngine): Promise _config.embedding_multimodal_model, _config.expansion_model, _config.chat_model, + _config.reranker_model, ...(_config.chat_fallback_chain ?? []), ]) { if (m) registerExtendedModel(m); @@ -483,6 +512,17 @@ export function getChatFallbackChain(): string[] { return requireConfig().chat_fallback_chain ?? []; } +/** + * v0.35.0.0+: configured reranker model. Returns undefined when no reranker + * is configured (default for installs that haven't opted in). Callers must + * check before invoking gateway.rerank() — `applyReranker` in + * src/core/search/rerank.ts does the existence check via isAvailable + * ('reranker') first. + */ +export function getRerankerModel(): string | undefined { + return requireConfig().reranker_model; +} + /** * Check whether a touchpoint can be served given the current config. * Replaces scattered `!process.env.OPENAI_API_KEY` checks (Codex C3). @@ -503,6 +543,8 @@ export function isAvailable(touchpoint: TouchpointKind): boolean { ? getExpansionModel() : touchpoint === 'chat' ? getChatModel() + : touchpoint === 'reranker' + ? getRerankerModel() ?? null : null; if (!modelStr) return false; const { recipe } = resolveRecipe(modelStr); @@ -510,7 +552,7 @@ export function isAvailable(touchpoint: TouchpointKind): boolean { // Recipe must actually support the requested touchpoint. // Anthropic declares only expansion + chat (no embedding model); requesting // embedding from an anthropic-configured brain is unavailable regardless of auth. - const touchpointConfig = recipe.touchpoints[touchpoint as 'embedding' | 'expansion' | 'chat']; + const touchpointConfig = recipe.touchpoints[touchpoint as 'embedding' | 'expansion' | 'chat' | 'reranker']; if (!touchpointConfig) return false; // Openai-compat recipes with empty models list require a user-provided // model. Either the recipe explicitly opts in via @@ -677,6 +719,165 @@ const voyageCompatFetch = (async (input: RequestInfo | URL, init?: RequestInit) } }) as unknown as typeof fetch; +/** + * ZeroEntropy compatibility shim. ZE's `/v1/models/embed` endpoint is NOT + * OpenAI-compatible at the wire level: + * - Path: AI SDK adapter calls `${base_url}/embeddings`; ZE wants + * `${base_url}/models/embed`. Rewrite the URL path. + * - Body: inject `input_type: 'document'` (or `'query'` when threaded via + * providerOptions.openaiCompatible.input_type) and `encoding_format: + * 'float'` (don't trust SDK default; strip any base64 caller injected + * to keep the response rewriter simple). + * - Response: ZE returns `{results: [{embedding: float[]}], usage: + * {total_bytes, total_tokens}}`. AI SDK's openai-compatible Zod schema + * expects `{data: [{embedding, index}], usage: {prompt_tokens, ...}}`. + * Rewrite both shapes. + * + * Layer 1 / Layer 2 OOM caps mirror the Voyage pattern; ZE embeddings are + * float[] (not base64), so the Layer 2 cap compares against the JSON + * payload size of each embedding rather than a base64 string length. + */ +const zeroEntropyCompatFetch = (async (input: RequestInfo | URL, init?: RequestInit) => { + // OUTBOUND: normalize URL, rewrite path /embeddings → /models/embed, then + // rewrite body. fetch accepts RequestInfo (string | Request) | URL; we + // handle all three so a `new Request(...)`-shaped caller works. + let urlString: string; + let baseInit: RequestInit = init ?? {}; + if (typeof input === 'string') { + urlString = input; + } else if (input instanceof URL) { + urlString = input.toString(); + } else { + // input is a Request — pull URL + headers + method + body off it. + urlString = input.url; + baseInit = { + method: input.method, + headers: input.headers, + // Reading body off a Request consumes it; the test seam passes + // string/URL so this branch is rarely hit in practice. When it is, + // we copy what we can and trust the caller passes the body via init. + ...(init ?? {}), + }; + } + try { + const u = new URL(urlString); + // Replace the trailing path segment '/embeddings' with '/models/embed'. + // `base_url_default` ends with `/v1`, so the SDK calls `/v1/embeddings` + // and we rewrite to `/v1/models/embed`. Use endsWith to avoid mangling + // any future ZE endpoints that happen to contain 'embeddings' as a + // substring. + if (u.pathname.endsWith('/embeddings')) { + u.pathname = u.pathname.slice(0, -'/embeddings'.length) + '/models/embed'; + urlString = u.toString(); + } + } catch { + // Malformed URL — let fetch handle the error. + } + + // Rewrite request body: inject input_type + encoding_format, strip any + // base64 the caller smuggled in. + if (baseInit.body && typeof baseInit.body === 'string') { + try { + const parsed = JSON.parse(baseInit.body); + if (parsed && typeof parsed === 'object') { + let mutated = false; + // Force encoding_format: 'float' so the response is a plain + // float[] and the response-rewriter doesn't need to base64-decode. + if (parsed.encoding_format !== 'float') { + parsed.encoding_format = 'float'; + mutated = true; + } + // Default input_type when caller didn't thread one (document-side + // embedding is the correct default for ingest paths). + if (parsed.input_type === undefined) { + parsed.input_type = 'document'; + mutated = true; + } + if (mutated) { + const headers = new Headers(baseInit.headers ?? {}); + headers.delete('content-length'); + baseInit = { ...baseInit, body: JSON.stringify(parsed), headers }; + } + } + } catch { + // Body wasn't JSON — pass through untouched. + } + } + + const resp = await fetch(urlString, baseInit); + if (!resp.ok) return resp; + const ct = resp.headers.get('content-type') ?? ''; + if (!ct.toLowerCase().includes('application/json')) return resp; + + // Layer 1 OOM cap (Content-Length pre-check). Same sizing rationale as + // Voyage — 256 MB is "unambiguously not a real ZE response" given + // zembed-1's max 2560-dim × 4 bytes × 16K embeddings = ~160 MB raw. + const contentLengthHeader = resp.headers.get('content-length'); + if (contentLengthHeader) { + const len = parseInt(contentLengthHeader, 10); + if (Number.isFinite(len) && len > MAX_ZEROENTROPY_RESPONSE_BYTES) { + throw new ZeroEntropyResponseTooLargeError( + `ZeroEntropy response Content-Length=${len} exceeds ` + + `${MAX_ZEROENTROPY_RESPONSE_BYTES} bytes — likely compromised endpoint`, + ); + } + } + + // INBOUND: rewrite response shape from {results:[{embedding}]} to + // {data:[{embedding, index}]} so the AI SDK's openai-compatible schema + // validates. Also map usage.total_tokens → prompt_tokens (SDK requires + // prompt_tokens when `usage` is present — same divergence Voyage hit at + // gateway.ts:655). + try { + const json: any = await resp.clone().json(); + if (!json || typeof json !== 'object') return resp; + let modified = false; + if (Array.isArray(json.results) && !Array.isArray(json.data)) { + // Layer 2 OOM cap — per-embedding size. ZE returns float[] arrays, + // so we count the elements × 4 bytes (the float32 width). + for (const item of json.results) { + if (item && Array.isArray(item.embedding)) { + const estBytes = item.embedding.length * 4; + if (estBytes > MAX_ZEROENTROPY_RESPONSE_BYTES) { + throw new ZeroEntropyResponseTooLargeError( + `ZeroEntropy embedding exceeds ${MAX_ZEROENTROPY_RESPONSE_BYTES} ` + + `bytes (estimated ${estBytes} from ${item.embedding.length} floats)`, + ); + } + } + } + json.data = json.results.map((r: any, i: number) => ({ + object: 'embedding', + embedding: r?.embedding ?? [], + index: i, + })); + delete json.results; + modified = true; + } + if ( + json.usage && + typeof json.usage === 'object' && + json.usage.prompt_tokens === undefined + ) { + json.usage.prompt_tokens = + typeof json.usage.total_tokens === 'number' ? json.usage.total_tokens : 0; + // SDK also expects total_tokens; ZE provides it directly. + modified = true; + } + if (!modified) return resp; + return new Response(JSON.stringify(json), { + status: resp.status, + statusText: resp.statusText, + headers: resp.headers, + }); + } catch (err) { + // OOM-cap throws MUST propagate. Voyage's pattern: instanceof check on + // its own tagged class. Same here — only rethrow our own cap class. + if (err instanceof ZeroEntropyResponseTooLargeError) throw err; + return resp; + } +}) as unknown as typeof fetch; + async function resolveEmbeddingProvider(modelStr: string): Promise<{ model: any; recipe: Recipe; modelId: string }> { const { parsed, recipe } = resolveRecipe(modelStr); assertTouchpoint(recipe, 'embedding', parsed.modelId, getExtendedModelsForProvider(parsed.providerId)); @@ -729,7 +930,16 @@ function instantiateEmbedding(recipe: Recipe, modelId: string, cfg: AIGatewayCon // request/response shape) when the recipe doesn't ship its own fetch // wrapper via resolveOpenAICompatConfig. Azure recipes ship their own // fetch (api-version splice); voyage doesn't — use voyageCompatFetch. - const fetchWrapper = compat.fetch ?? (recipe.id === 'voyage' ? voyageCompatFetch : undefined); + // ZeroEntropy needs zeroEntropyCompatFetch (URL path + body input_type + // + response shape rewrite + OOM caps). Same per-recipe-id branch + // pattern as voyage so adding a third compat shim is one more case. + const fetchWrapper = + compat.fetch ?? + (recipe.id === 'voyage' + ? voyageCompatFetch + : recipe.id === 'zeroentropyai' + ? zeroEntropyCompatFetch + : undefined); const client = createOpenAICompatible({ name: recipe.id, baseURL: compat.baseURL, @@ -786,30 +996,39 @@ const MIN_SUB_BATCH = 1; * throughput. */ /** - * Per-call passthroughs for `embed()`. v0.33.4 added both fields so the - * `embed --stale` retry wrapper can (a) cancel mid-fetch when the wall-clock - * budget fires, and (b) suppress the AI SDK's default 2-retry stack so the - * wrapper's retry-after-aware loop is the single source of truth. + * Per-call passthroughs for `embed()`. Unifies v0.33.4 cancellation/retry + * controls and v0.35.0.0 asymmetric-input plumbing into one interface so + * a future passthrough doesn't churn the call signature again. * - * Both are optional; production callers that don't pass them get unchanged - * pre-v0.33.4 behavior. + * All fields are optional; production callers that don't pass them get + * unchanged pre-v0.33.4 behavior with document-side encoding (ZE / Voyage + * v3+ semantics) as the default. */ export interface EmbedOpts { /** - * Propagated to Vercel AI SDK's `embedMany({abortSignal})`. When the - * caller's wall-clock budget fires, an in-flight HTTP request is - * cancelled within seconds instead of waiting out the provider's HTTP - * timeout (~30s on OpenAI). + * v0.33.4: propagated to Vercel AI SDK's `embedMany({abortSignal})`. + * When the caller's wall-clock budget fires, an in-flight HTTP request + * is cancelled within seconds instead of waiting out the provider's + * HTTP timeout (~30s on OpenAI). */ abortSignal?: AbortSignal; /** - * Propagated to Vercel AI SDK's `embedMany({maxRetries})`. Default in - * the SDK is 2 (so up to 3 attempts per call). Pass `0` to disable - * SDK retries when a higher-level wrapper owns the retry policy — - * otherwise SDK and wrapper retries stack and amplify rate-limit - * pressure (3 × N wrapper attempts). + * v0.33.4: propagated to Vercel AI SDK's `embedMany({maxRetries})`. + * Default in the SDK is 2 (so up to 3 attempts per call). Pass `0` to + * disable SDK retries when a higher-level wrapper owns the retry + * policy — otherwise SDK and wrapper retries stack and amplify + * rate-limit pressure (3 × N wrapper attempts). */ maxRetries?: number; + /** + * v0.35.0.0: asymmetric retrieval signal. `'query'` routes through + * `dimsProviderOptions` so providers that accept query/document + * encoding (ZE zembed-1, Voyage v3+) produce query-side vectors. + * Symmetric providers (OpenAI text-3, DashScope, Zhipu) ignore the + * field. Defaults to undefined (treated as 'document' by the dim + * resolver — the correct default for indexing paths). + */ + inputType?: 'query' | 'document'; } export async function embed(texts: string[], opts?: EmbedOpts): Promise { @@ -818,7 +1037,12 @@ export async function embed(texts: string[], opts?: EmbedOpts): Promise (t ?? '').slice(0, MAX_CHARS)); - const providerOpts = dimsProviderOptions(recipe.implementation, modelId, cfg.embedding_dimensions ?? DEFAULT_EMBEDDING_DIMENSIONS); + const providerOpts = dimsProviderOptions( + recipe.implementation, + modelId, + cfg.embedding_dimensions ?? DEFAULT_EMBEDDING_DIMENSIONS, + opts?.inputType, + ); const expected = cfg.embedding_dimensions ?? DEFAULT_EMBEDDING_DIMENSIONS; const embedding = recipe.touchpoints?.embedding; @@ -997,6 +1221,26 @@ export async function embedOne(text: string): Promise { return v; } +/** + * v0.35.0.0+: embed a single text on the QUERY side of an asymmetric retrieval + * pipeline. Threads `inputType: 'query'` into `dimsProviderOptions`, which + * for ZE (`zembed-1`) and Voyage v3+ models emits `input_type: 'query'` into + * the request body so the provider returns query-side vectors. For + * symmetric providers (OpenAI text-3, DashScope, Zhipu) the field is dropped + * — no behavior change. + * + * Two call sites in v0.33.2: vector seed embed at hybrid.ts:400 (cache miss + * path) and cache lookup embed at hybrid.ts:629. All ingest paths (sync, + * import, embed CLI) continue to use `embed()` which defaults to document + * encoding. + * + * Returns a single Float32Array (not a batch). + */ +export async function embedQuery(text: string): Promise { + const [v] = await embed([text], { inputType: 'query' }); + return v; +} + // ---- Multimodal embedding (v0.27.1) ---- /** Voyage multimodal API caps at 32 inputs per request. */ @@ -1689,6 +1933,195 @@ export async function chat(opts: ChatOpts): Promise { } } +// ---- Reranker (v0.35.0.0+) ---- + +/** Tagged error class for gateway.rerank() failures. `reason` classifies into the + * shape applyReranker uses to decide between fail-open (network/timeout) and + * loud-fail (auth — should have been caught by doctor). Mirror of the + * RemoteMcpError pattern in src/core/mcp-client.ts. */ +export class RerankError extends Error { + reason: 'auth' | 'rate_limit' | 'network' | 'timeout' | 'payload_too_large' | 'unknown'; + status?: number; + constructor(message: string, reason: RerankError['reason'], status?: number) { + super(message); + this.name = 'RerankError'; + this.reason = reason; + this.status = status; + } +} + +export interface RerankInput { + query: string; + documents: string[]; + topN?: number; + /** Override the gateway-configured reranker model for this single call. */ + model?: string; + signal?: AbortSignal; + /** Timeout in ms (default 5000). Search hot path; long stalls degrade UX. */ + timeoutMs?: number; +} + +export interface RerankResult { + index: number; + relevanceScore: number; +} + +/** + * Test seam — same pattern as `_embedTransport` / `_chatTransport`. Tests + * install a stub via `__setRerankTransportForTests` to exercise the call-site + * pipeline without hitting the network. Production never reads the override. + */ +type RerankTransport = ( + url: string, + init: RequestInit, +) => Promise; +let _rerankTransport: RerankTransport | null = null; +export function __setRerankTransportForTests(fn: RerankTransport | null): void { + _rerankTransport = fn; +} + +const DEFAULT_RERANK_TIMEOUT_MS = 5000; + +/** + * Submit a query + N documents to the configured reranker. Returns a list of + * `{index, relevanceScore}` sorted by relevanceScore descending (per upstream + * convention). + * + * Resolution order: `input.model` → `getRerankerModel()` → `DEFAULT_RERANKER_MODEL`. + * + * Pre-flight: rejects payloads that would exceed + * `recipe.touchpoints.reranker.max_payload_bytes` (default 5MB for ZE) with + * `RerankError(reason: 'payload_too_large')`. applyReranker catches this in + * the fail-open path so search never throws. + * + * Errors classified into RerankError.reason for the caller's fail-open + * decision table. The model allowlist check is done HERE (not via + * assertTouchpoint), because assertTouchpoint doesn't enforce allowlists for + * openai-compatible recipes — CDX2-F11 in the plan. + */ +export async function rerank(input: RerankInput): Promise { + if (!input.query) { + throw new RerankError('rerank: query is required', 'unknown'); + } + if (!input.documents || input.documents.length === 0) { + return []; + } + + const modelStr = + input.model ?? + getRerankerModel() ?? + DEFAULT_RERANKER_MODEL; + const { parsed, recipe } = resolveRecipe(modelStr); + const tp = recipe.touchpoints.reranker; + if (!tp) { + throw new RerankError( + `Provider "${recipe.id}" does not declare a reranker touchpoint.`, + 'unknown', + ); + } + if (tp.models.length > 0 && !tp.models.includes(parsed.modelId)) { + throw new RerankError( + `Model "${parsed.modelId}" is not listed for ${recipe.name} reranker. ` + + `Known: ${tp.models.join(', ')}.`, + 'unknown', + ); + } + + // Resolve base URL + auth from the recipe (same path Voyage/ZE embeddings use). + const cfg = requireConfig(); + const compat = applyOpenAICompatConfig(recipe, cfg); + const url = `${compat.baseURL.replace(/\/$/, '')}/models/rerank`; + const auth = applyResolveAuth(recipe, cfg, 'reranker'); + // applyResolveAuth returns { apiKey } for Bearer-style auth (SDK's native + // path) or { headers } for custom-header providers (Azure). gateway.rerank + // builds the HTTP request directly (no SDK adapter), so we materialize + // both shapes into a Headers map. + const authHeaders: Record = auth.headers + ? { ...auth.headers } + : auth.apiKey + ? { Authorization: `Bearer ${auth.apiKey}` } + : {}; + const body = JSON.stringify({ + model: parsed.modelId, + query: input.query, + documents: input.documents, + ...(input.topN !== undefined ? { top_n: input.topN } : {}), + }); + + // Pre-flight payload size guard (CDX1-F17 / plan Phase 3 cost guard). The + // 5MB cap matches ZE's upstream limit; over-cap returns payload_too_large + // so applyReranker can fail-open without ever issuing the HTTP request. + const bodyBytes = Buffer.byteLength(body, 'utf8'); + if (bodyBytes > tp.max_payload_bytes) { + throw new RerankError( + `Rerank payload ${bodyBytes} bytes exceeds ${tp.max_payload_bytes} ` + + `byte cap for ${recipe.name}`, + 'payload_too_large', + ); + } + + // Build headers from resolveAuth (default applies Bearer-style header). + const headers = new Headers(authHeaders); + headers.set('Content-Type', 'application/json'); + + // Timeout via AbortController; merges with caller-supplied signal. + const ctrl = new AbortController(); + const timeoutMs = input.timeoutMs ?? DEFAULT_RERANK_TIMEOUT_MS; + const t = setTimeout(() => ctrl.abort(new Error('rerank timed out')), timeoutMs); + if (input.signal) { + if (input.signal.aborted) ctrl.abort(input.signal.reason); + else input.signal.addEventListener('abort', () => ctrl.abort(input.signal!.reason), { once: true }); + } + + try { + const transport: RerankTransport = _rerankTransport ?? ((u, init) => fetch(u, init)); + const resp = await transport(url, { + method: 'POST', + headers, + body, + signal: ctrl.signal, + }); + if (!resp.ok) { + let msg = `rerank HTTP ${resp.status}`; + try { + const txt = await resp.text(); + if (txt) msg = `${msg}: ${txt.slice(0, 500)}`; + } catch { + // Body read failed — preserve status-only message. + } + const reason: RerankError['reason'] = + resp.status === 401 || resp.status === 403 + ? 'auth' + : resp.status === 429 + ? 'rate_limit' + : resp.status >= 500 + ? 'network' + : 'unknown'; + throw new RerankError(msg, reason, resp.status); + } + const json: any = await resp.json(); + if (!json || !Array.isArray(json.results)) { + throw new RerankError('rerank: malformed response (no results array)', 'unknown'); + } + return json.results.map((r: any) => ({ + index: typeof r.index === 'number' ? r.index : 0, + relevanceScore: typeof r.relevance_score === 'number' ? r.relevance_score : 0, + })); + } catch (err) { + if (err instanceof RerankError) throw err; + // AbortError on timeout — classify cleanly. + if (err && typeof err === 'object' && (err as any).name === 'AbortError') { + const msg = (err as Error).message || 'rerank aborted'; + throw new RerankError(msg, msg.toLowerCase().includes('timed out') ? 'timeout' : 'unknown'); + } + // Network errors (DNS, connection refused, etc.) become network class. + const msg = err instanceof Error ? err.message : String(err); + throw new RerankError(`rerank: ${msg}`, 'network'); + } finally { + clearTimeout(t); + } +} + // ---- Future touchpoint stubs ---- class NotMigratedYet extends AIConfigError { diff --git a/src/core/ai/model-resolver.ts b/src/core/ai/model-resolver.ts index 3da79b62b..cc668c67a 100644 --- a/src/core/ai/model-resolver.ts +++ b/src/core/ai/model-resolver.ts @@ -2,7 +2,7 @@ * Parse and validate `provider:model` strings against the recipe registry. */ -import type { ParsedModelId, Recipe, TouchpointKind, ChatTouchpoint, EmbeddingTouchpoint, ExpansionTouchpoint } from './types.ts'; +import type { ParsedModelId, Recipe, TouchpointKind, ChatTouchpoint, EmbeddingTouchpoint, ExpansionTouchpoint, RerankerTouchpoint } from './types.ts'; import { getRecipe, RECIPES } from './recipes/index.ts'; import { AIConfigError } from './errors.ts'; @@ -54,10 +54,10 @@ export function resolveRecipe(modelId: string): { parsed: ParsedModelId; recipe: return { parsed, recipe }; } -type KnownTouchpointKey = 'embedding' | 'expansion' | 'chat'; +type KnownTouchpointKey = 'embedding' | 'expansion' | 'chat' | 'reranker'; -function getTouchpoint(recipe: Recipe, touchpoint: TouchpointKind): EmbeddingTouchpoint | ExpansionTouchpoint | ChatTouchpoint | undefined { - if (touchpoint === 'embedding' || touchpoint === 'expansion' || touchpoint === 'chat') { +function getTouchpoint(recipe: Recipe, touchpoint: TouchpointKind): EmbeddingTouchpoint | ExpansionTouchpoint | ChatTouchpoint | RerankerTouchpoint | undefined { + if (touchpoint === 'embedding' || touchpoint === 'expansion' || touchpoint === 'chat' || touchpoint === 'reranker') { return recipe.touchpoints[touchpoint as KnownTouchpointKey]; } return undefined; diff --git a/src/core/ai/recipes/index.ts b/src/core/ai/recipes/index.ts index 1682fa115..5915c9a92 100644 --- a/src/core/ai/recipes/index.ts +++ b/src/core/ai/recipes/index.ts @@ -20,6 +20,7 @@ import { minimax } from './minimax.ts'; import { dashscope } from './dashscope.ts'; import { zhipu } from './zhipu.ts'; import { azureOpenAI } from './azure-openai.ts'; +import { zeroentropyai } from './zeroentropyai.ts'; const ALL: Recipe[] = [ openai, @@ -36,6 +37,7 @@ const ALL: Recipe[] = [ dashscope, zhipu, azureOpenAI, + zeroentropyai, ]; /** Map from `provider:id` key to recipe. */ diff --git a/src/core/ai/recipes/zeroentropyai.ts b/src/core/ai/recipes/zeroentropyai.ts new file mode 100644 index 000000000..a5367df48 --- /dev/null +++ b/src/core/ai/recipes/zeroentropyai.ts @@ -0,0 +1,67 @@ +import type { Recipe } from '../types.ts'; + +/** + * ZeroEntropy ships two specialized small models that target the two weakest + * retrieval moments in a gbrain pipeline: + * + * - zembed-1 — flexible-dim embedding (2560 default; also 1280/640/320/160/80/40), + * distilled from zerank-2, 32K context. Asymmetric `input_type: query|document` + * encoding (like Voyage and MiniMax). $0.025/1M tokens (sale) / $0.05 regular. + * + * - zerank-{2,1,1-small} — cross-encoder rerankers. zerank-2 is flagship; + * multilingual + instruction-following; $0.025/1M tokens. + * + * Endpoints (from docs.zeroentropy.dev): + * POST https://api.zeroentropy.dev/v1/models/embed + * POST https://api.zeroentropy.dev/v1/models/rerank + * + * The AI-SDK openai-compatible adapter calls `${base_url}/embeddings` — wrong + * path for ZE. `zeroEntropyCompatFetch` in gateway.ts rewrites the URL path to + * `/models/embed`, injects `input_type` (default 'document') + explicit + * `encoding_format: 'float'`, and rewrites the response shape from + * `{results: [{embedding}]}` to `{data: [{embedding, index}]}` so the SDK's + * Zod schema validates. Response carries `usage.prompt_tokens = total_tokens` + * because ZE's response has no prompt_tokens field (Voyage's shim hit the + * same SDK schema requirement at gateway.ts:655). + * + * The reranker side uses gateway.rerank() (a native HTTP path) — the AI SDK + * has no reranking abstraction so there's no openai-compat seam to plug into. + * `touchpoints.reranker` declares the model allowlist and 5MB payload cap. + */ +export const zeroentropyai: Recipe = { + id: 'zeroentropyai', + name: 'ZeroEntropy', + tier: 'openai-compat', + implementation: 'openai-compatible', + base_url_default: 'https://api.zeroentropy.dev/v1', + auth_env: { + required: ['ZEROENTROPY_API_KEY'], + setup_url: 'https://dashboard.zeroentropy.dev', + }, + touchpoints: { + embedding: { + models: ['zembed-1'], + default_dims: 2560, + // ZE rate-limits free tier at 500KB/min input; max payload 5MB/request. + // Pre-split budget = 120K tokens × 0.5 safety × 1 char/token ≈ 60K chars + // per batch, same dense-content hedge Voyage uses. + max_batch_tokens: 120_000, + chars_per_token: 1, + safety_factor: 0.5, + supports_multimodal: false, + cost_per_1m_tokens_usd: 0.05, + price_last_verified: '2026-05-14', + }, + reranker: { + models: ['zerank-2', 'zerank-1', 'zerank-1-small'], + default_model: 'zerank-2', + cost_per_1m_tokens_usd: 0.025, + price_last_verified: '2026-05-14', + // ZE enforces 5MB per /v1/models/rerank request. gateway.rerank() + // pre-flights the body size and fails open (no throw to caller). + max_payload_bytes: 5_000_000, + }, + }, + setup_hint: + 'Get an API key at https://dashboard.zeroentropy.dev, then `export ZEROENTROPY_API_KEY=...`', +}; diff --git a/src/core/ai/types.ts b/src/core/ai/types.ts index e88867993..5ed895dad 100644 --- a/src/core/ai/types.ts +++ b/src/core/ai/types.ts @@ -15,7 +15,8 @@ export type TouchpointKind = | 'chunking' | 'transcription' | 'enrichment' - | 'improve'; + | 'improve' + | 'reranker'; export type Implementation = | 'native-openai' @@ -126,6 +127,30 @@ export interface ExpansionTouchpoint { * unstable tool_call_id behavior across replays. supports_subagent_loop is the * stricter signal that subagent.ts asserts. */ +/** + * Reranker touchpoint (v0.35.0.0+): cross-encoder rerankers that take a query + * + N documents and return a relevance-score-sorted index list. Slots into + * `applyReranker()` in src/core/search/rerank.ts between RRF dedup and + * token-budget enforcement. + * + * Reranking is NOT in the AI SDK's abstraction — `gateway.rerank()` makes a + * native HTTP call. The recipe carries auth + base URL + model allowlist; the + * gateway uses `recipe.auth_env.required[0]` for the Bearer token and posts to + * `${recipe.base_url_default}/models/rerank` (or the recipe-specific path). + * + * `max_payload_bytes` is the upstream's per-request size cap. gateway.rerank() + * pre-flights the body size and throws RerankError with reason + * 'payload_too_large' when over-cap; applyReranker catches this and falls + * back to RRF order (fail-open). + */ +export interface RerankerTouchpoint { + models: string[]; + default_model: string; + cost_per_1m_tokens_usd?: number; + price_last_verified?: string; + max_payload_bytes: number; +} + export interface ChatTouchpoint { models: string[]; /** Provider returns native function/tool calling. */ @@ -164,6 +189,7 @@ export interface Recipe { embedding?: EmbeddingTouchpoint; expansion?: ExpansionTouchpoint; chat?: ChatTouchpoint; + reranker?: RerankerTouchpoint; }; /** * Optional alias map for friendlier `provider:model` strings. @@ -251,6 +277,13 @@ export interface AIGatewayConfig { expansion_model?: string; /** Default chat model for `gateway.chat()` callers (subagent default). */ chat_model?: string; + /** + * v0.35.0.0+: default reranker model for `gateway.rerank()` callers. As + * `'provider:model'` (e.g. `'zeroentropyai:zerank-2'`). Resolved at + * configure time and re-resolved by reconfigureGatewayWithEngine() when + * mode-bundle or config-key overrides change. + */ + reranker_model?: string; /** * Optional silent-refusal fallback chain ("provider:modelId" entries). * Plumbed for `chatWithFallback()` (commit 3). Blocked from critic/judge/ diff --git a/src/core/embedding.ts b/src/core/embedding.ts index 553aa1916..16f744d50 100644 --- a/src/core/embedding.ts +++ b/src/core/embedding.ts @@ -8,6 +8,7 @@ import { embed as gatewayEmbed, embedOne as gatewayEmbedOne, + embedQuery as gatewayEmbedQuery, getEmbeddingModel as gatewayGetModel, getEmbeddingDimensions as gatewayGetDims, } from './ai/gateway.ts'; @@ -18,11 +19,22 @@ import { export { embedMultimodal } from './ai/gateway.ts'; export type { MultimodalInput } from './ai/types.ts'; -/** Embed one text. */ +/** Embed one text (document-side for asymmetric providers). */ export async function embed(text: string): Promise { return gatewayEmbedOne(text); } +/** + * v0.35.0.0+: embed a single text on the QUERY side. For asymmetric providers + * (ZE zembed-1, Voyage v3+) this routes `input_type: 'query'` through the + * embed seam so the provider returns query-side vectors. For symmetric + * providers (OpenAI text-3, DashScope, Zhipu) the field is dropped — no + * behavior change. Used by hybrid.ts on the search hot path. + */ +export async function embedQuery(text: string): Promise { + return gatewayEmbedQuery(text); +} + export interface EmbedBatchOptions { /** * Optional callback fired after each sub-batch completes. CLI wrappers diff --git a/src/core/rerank-audit.ts b/src/core/rerank-audit.ts new file mode 100644 index 000000000..8a233df5c --- /dev/null +++ b/src/core/rerank-audit.ts @@ -0,0 +1,137 @@ +/** + * v0.35.0.0+ — rerank-failure audit trail. + * + * Writes warn-severity rows to `~/.gbrain/audit/rerank-failures-YYYY-Www.jsonl` + * (ISO-week rotation, mirrors slug-fallback-audit.ts). Fired when + * `applyReranker` in src/core/search/rerank.ts catches a RerankError from + * the gateway. Failure is fail-open at the search layer (results pass + * through in RRF order); the audit row is the cross-process signal that + * `gbrain doctor reranker_health` reads. + * + * Success events are intentionally NOT logged here. Per the plan (CDX2-F22): + * 1) writing once per tokenmax search is hot-path I/O churn — the + * slug-fallback pattern is rare-event-only. + * 2) success events leak query volume + timing into a local audit file + * that previously held only failures. + * The doctor check reads `search.reranker.enabled` first to interpret + * "no events in window" correctly (enabled + no events = healthy; + * disabled = no failures expected). + * + * Best-effort writes. Write failures go to stderr but search continues. + */ + +import * as fs from 'node:fs'; +import * as path from 'node:path'; +import { resolveAuditDir } from './minions/handlers/shell-audit.ts'; + +/** Stable error-classification union; matches RerankError.reason. */ +export type RerankFailureReason = + | 'auth' + | 'rate_limit' + | 'network' + | 'timeout' + | 'payload_too_large' + | 'unknown'; + +export interface RerankFailureEvent { + ts: string; + /** Provider:model — e.g. `'zeroentropyai:zerank-2'`. */ + model: string; + /** Classified failure mode (see RerankFailureReason). */ + reason: RerankFailureReason; + /** SHA-256 prefix of the rerank query (8 hex chars). Privacy: never log + * query text. Lets doctor dedupe repeat failures on the same query. */ + query_hash: string; + /** Number of documents that were being reranked when failure fired. */ + doc_count: number; + /** + * Truncated upstream error message (first 200 chars). Useful for + * diagnosing flaky providers without leaking PII; query text is hashed + * separately so this string never carries it. + */ + error_summary: string; + /** Always 'warn' — matches RerankError's "all failures degrade UX". */ + severity: 'warn'; +} + +/** ISO-week-rotated filename: `rerank-failures-YYYY-Www.jsonl`. */ +export function computeRerankAuditFilename(now: Date = new Date()): string { + const d = new Date(Date.UTC(now.getUTCFullYear(), now.getUTCMonth(), now.getUTCDate())); + const dayNum = (d.getUTCDay() + 6) % 7; + d.setUTCDate(d.getUTCDate() - dayNum + 3); + const isoYear = d.getUTCFullYear(); + const firstThursday = new Date(Date.UTC(isoYear, 0, 4)); + const firstThursdayDayNum = (firstThursday.getUTCDay() + 6) % 7; + firstThursday.setUTCDate(firstThursday.getUTCDate() - firstThursdayDayNum + 3); + const weekNum = Math.round((d.getTime() - firstThursday.getTime()) / (7 * 86400000)) + 1; + const ww = String(weekNum).padStart(2, '0'); + return `rerank-failures-${isoYear}-W${ww}.jsonl`; +} + +/** + * Truncate a string for audit logging. Plain length cut — error messages + * from the gateway are already free of caller-controlled prefixes. + */ +function truncateErrorSummary(msg: string, max = 200): string { + if (msg.length <= max) return msg; + return msg.slice(0, max - 1) + '…'; +} + +/** + * Append a rerank-failure event. Best-effort: write failure logs to stderr + * but never throws. + */ +export function logRerankFailure(event: Omit): void { + const row: RerankFailureEvent = { + ts: new Date().toISOString(), + severity: 'warn', + ...event, + error_summary: truncateErrorSummary(event.error_summary), + }; + const dir = resolveAuditDir(); + const file = path.join(dir, computeRerankAuditFilename()); + try { + fs.mkdirSync(dir, { recursive: true }); + fs.appendFileSync(file, JSON.stringify(row) + '\n', { encoding: 'utf8' }); + } catch (err) { + const msg = err instanceof Error ? err.message : String(err); + process.stderr.write(`[gbrain] rerank-failure audit write failed (${msg}); search continues\n`); + } +} + +/** + * Read recent (`days` window, default 7) rerank-failure events. Used by + * `gbrain doctor`'s `reranker_health` check. Missing file / corrupt rows + * are skipped silently — the audit trail is informational. + */ +export function readRecentRerankFailures(days = 7, now: Date = new Date()): RerankFailureEvent[] { + const dir = resolveAuditDir(); + const cutoff = now.getTime() - days * 86400000; + const out: RerankFailureEvent[] = []; + // Walk the current + previous ISO week so a 7-day window straddling + // Monday-midnight stays covered. + const filenames = [ + computeRerankAuditFilename(now), + computeRerankAuditFilename(new Date(now.getTime() - 7 * 86400000)), + ]; + for (const filename of filenames) { + const file = path.join(dir, filename); + let content: string; + try { + content = fs.readFileSync(file, 'utf8'); + } catch { + continue; + } + for (const line of content.split('\n')) { + if (line.length === 0) continue; + try { + const ev = JSON.parse(line) as RerankFailureEvent; + const ts = Date.parse(ev.ts); + if (Number.isFinite(ts) && ts >= cutoff) out.push(ev); + } catch { + // Corrupt row — skip. + } + } + } + return out; +} diff --git a/src/core/search/hybrid.ts b/src/core/search/hybrid.ts index e21836e6d..dc0cec6dd 100644 --- a/src/core/search/hybrid.ts +++ b/src/core/search/hybrid.ts @@ -12,8 +12,9 @@ import type { BrainEngine } from '../engine.ts'; import { MAX_SEARCH_LIMIT, clampSearchLimit } from '../engine.ts'; import type { SearchResult, SearchOpts, HybridSearchMeta } from '../types.ts'; -import { embed } from '../embedding.ts'; +import { embed, embedQuery } from '../embedding.ts'; import { dedupResults } from './dedup.ts'; +import { applyReranker } from './rerank.ts'; import { autoDetectDetail, classifyQuery } from './query-intent.ts'; import { expandAnchors, hydrateChunks } from './two-pass.ts'; import { enforceTokenBudget } from './token-budget.ts'; @@ -407,7 +408,10 @@ export async function hybridSearch( let vectorLists: SearchResult[][] = []; let queryEmbedding: Float32Array | null = null; try { - const embeddings = await Promise.all(queries.map(q => embed(q))); + // v0.35.0.0+: query-side embedding. For asymmetric providers (ZE zembed-1, + // Voyage v3+) routes input_type='query' through the embed seam; symmetric + // providers ignore the field — no behavior change. + const embeddings = await Promise.all(queries.map(q => embedQuery(q))); queryEmbedding = embeddings[0]; vectorLists = await Promise.all( embeddings.map(emb => engine.searchVector(emb, searchOpts)), @@ -536,7 +540,26 @@ export async function hybridSearch( return hybridSearch(engine, query, { ...opts, detail: 'high' }); } - const sliced = deduped.slice(offset, offset + limit); + // v0.35.0.0+: cross-encoder reranker. Slots between dedup and slice so the + // reranker sees the full candidate pool (its own topNIn caps how many + // get sent upstream). Fail-open: any error returns deduped unchanged. + // + // Resolution: per-call SearchOpts.reranker overrides; otherwise pull + // from the resolved mode bundle (tokenmax → enabled, others → disabled). + // The resolved mode's fields already participate in knobsHash, so cache + // rows naturally segregate by reranker config. + const rerankerOpts = opts?.reranker ?? { + enabled: resolvedMode.reranker_enabled, + topNIn: resolvedMode.reranker_top_n_in, + topNOut: resolvedMode.reranker_top_n_out, + model: resolvedMode.reranker_model, + timeoutMs: resolvedMode.reranker_timeout_ms, + }; + const reranked = rerankerOpts.enabled + ? await applyReranker(query, deduped, rerankerOpts as any) + : deduped; + + const sliced = reranked.slice(offset, offset + limit); // v0.32.3 search-lite: budget enforcement at the main return path. // hybridSearchCached used to be the only place this fired; now bare // hybridSearch enforces it too so eval-replay + eval-longmemeval see @@ -636,7 +659,8 @@ export async function hybridSearchCached( try { const { isAvailable } = await import('../ai/gateway.ts'); if (isAvailable('embedding')) { - queryEmbedding = await embed(query); + // v0.35.0.0+: query-side embedding (cache lookup path). + queryEmbedding = await embedQuery(query); } else { cacheStatus = 'disabled'; } diff --git a/src/core/search/mode.ts b/src/core/search/mode.ts index 7f731bc7c..2d04063e1 100644 --- a/src/core/search/mode.ts +++ b/src/core/search/mode.ts @@ -66,6 +66,34 @@ export interface ModeBundle { * EXPANSION from the implicit current default (limit 20). */ searchLimit: number; + /** + * v0.35.0.0+ — cross-encoder reranker. Off for conservative/balanced, + * on for tokenmax. ZeroEntropy zerank-2 by default; can be overridden + * via `search.reranker.model`. Slots between dedup and token-budget + * enforcement in hybrid.ts; fail-open on any RerankError (audit-logged). + * Cost anchor: ~$0.0003/query at tokenmax topNIn=30 × ~400 tokens/chunk + * (rounding error vs Opus, meaningful vs Haiku). + */ + reranker_enabled: boolean; + /** + * Provider:model for the reranker. Default `'zeroentropyai:zerank-2'`. + * Other ZE rerankers (`zerank-1`, `zerank-1-small`) work via the same + * recipe; future Cohere/Voyage rerankers drop in as new recipes + * declaring `touchpoints.reranker`. + */ + reranker_model: string; + /** Candidates to send upstream (default 30). The full result list always + * reaches the user — topNIn just caps API spend on the rerank call. */ + reranker_top_n_in: number; + /** + * Truncate the reranked output to this many. `null` = no truncate; the + * caller's `limit` is what trims final output. Distinct from undefined + * (which would fall through to mode bundle) — `null` is the explicit + * "don't truncate" signal, see CDX2-F15+F16. + */ + reranker_top_n_out: number | null; + /** HTTP timeout in ms (default 5000). Threaded into gateway.rerank. */ + reranker_timeout_ms: number; } /** @@ -84,6 +112,13 @@ export const MODE_BUNDLES: Readonly>> = tokenBudget: 4000, expansion: false, searchLimit: 10, + // v0.35.0.0+: reranker off — conservative is cost-sensitive; reranker + // spend doesn't fit the tier's value prop. + reranker_enabled: false, + reranker_model: 'zeroentropyai:zerank-2', + reranker_top_n_in: 30, + reranker_top_n_out: null, + reranker_timeout_ms: 5000, }), balanced: Object.freeze({ cache_enabled: true, @@ -93,6 +128,14 @@ export const MODE_BUNDLES: Readonly>> = tokenBudget: 12000, expansion: false, searchLimit: 25, + // Off in balanced too — operators opt in via + // `gbrain config set search.reranker.enabled true` until eval data + // backs a mode-bundle default change. + reranker_enabled: false, + reranker_model: 'zeroentropyai:zerank-2', + reranker_top_n_in: 30, + reranker_top_n_out: null, + reranker_timeout_ms: 5000, }), tokenmax: Object.freeze({ cache_enabled: true, @@ -102,6 +145,16 @@ export const MODE_BUNDLES: Readonly>> = tokenBudget: undefined, expansion: true, searchLimit: 50, + // tokenmax is the high-cost-tolerant tier that already pays for LLM + // expansion + 50-result payloads. Reranker is the natural capstone: + // better ordering of a large candidate set is where rerankers earn + // their fee. ~$0.0003/query at this shape; rounding error vs the + // tier's $700/mo @ Opus pairing per CLAUDE.md cost matrix. + reranker_enabled: true, + reranker_model: 'zeroentropyai:zerank-2', + reranker_top_n_in: 30, + reranker_top_n_out: null, + reranker_timeout_ms: 5000, }), }); @@ -124,6 +177,15 @@ export interface SearchKeyOverrides { tokenBudget?: number; expansion?: boolean; searchLimit?: number; + // v0.35.0.0+ reranker overrides + reranker_enabled?: boolean; + reranker_model?: string; + reranker_top_n_in?: number; + // CDX2-F16: null is the explicit "don't truncate" signal; undefined + // means "fall through to mode bundle". Use number | null, not + // number | undefined. + reranker_top_n_out?: number | null; + reranker_timeout_ms?: number; } /** @@ -141,6 +203,12 @@ export interface SearchPerCallOpts { tokenBudget?: number; expansion?: boolean; searchLimit?: number; + // v0.35.0.0+ reranker per-call overrides (same shape as SearchKeyOverrides). + reranker_enabled?: boolean; + reranker_model?: string; + reranker_top_n_in?: number; + reranker_top_n_out?: number | null; + reranker_timeout_ms?: number; } /** @@ -194,6 +262,11 @@ export function resolveSearchMode(input: ResolveSearchModeInput): ResolvedSearch tokenBudget: pick('tokenBudget'), expansion: pick('expansion'), searchLimit: pick('searchLimit'), + reranker_enabled: pick('reranker_enabled'), + reranker_model: pick('reranker_model'), + reranker_top_n_in: pick('reranker_top_n_in'), + reranker_top_n_out: pick('reranker_top_n_out'), + reranker_timeout_ms: pick('reranker_timeout_ms'), resolved_mode, mode_valid: valid, }; @@ -242,7 +315,19 @@ export function attributeKnob( * reorder or add a knob without bumping a constant — a hash collision would * mean stale cache rows silently reading the wrong shape. */ -export const KNOBS_HASH_VERSION = 1; +// v0.35.0.0+ bump 1→2: reranker fields participate in the cache key so a +// tokenmax-with-reranker write can't be served to a reranker-off lookup. +// CDX2-F13 convention: under a version bump, additions are APPEND-ONLY at +// the end of `parts[]` — reordering existing fields would silently rebuild +// the hash for every existing row. +// +// CDX2-F12 mid-deploy duplicate-row note: because `cacheRowId()` (in +// src/core/search/query-cache.ts) includes knobsHash, a v=1 process and a +// v=2 process writing the same `(source_id, query_text)` produce DISTINCT +// row IDs. Expect a temporary hit-rate dip + cache-row doubling for hot +// queries during a rolling deploy. Clears naturally within +// `cache.ttl_seconds` (default 3600s). The CHANGELOG note covers this. +export const KNOBS_HASH_VERSION = 2; export function knobsHash(knobs: ResolvedSearchKnobs): string { // Fixed-order key list. Adding a knob here REQUIRES bumping @@ -257,6 +342,12 @@ export function knobsHash(knobs: ResolvedSearchKnobs): string { `tb=${knobs.tokenBudget ?? 'none'}`, `exp=${knobs.expansion ? 1 : 0}`, `lim=${knobs.searchLimit}`, + // v=2 additions (append-only). + `rr=${knobs.reranker_enabled ? 1 : 0}`, + `rrm=${knobs.reranker_model}`, + `rri=${knobs.reranker_top_n_in}`, + `rro=${knobs.reranker_top_n_out ?? 'none'}`, + `rrt=${knobs.reranker_timeout_ms}`, ]; const h = createHash('sha256'); h.update(parts.join('|')); @@ -310,6 +401,40 @@ export function loadOverridesFromConfig( if (Number.isFinite(n) && n > 0) out.searchLimit = n; } + // v0.35.0.0+ reranker overrides + const re = get('search.reranker.enabled'); + if (re !== undefined) { + out.reranker_enabled = re === '1' || re.toLowerCase() === 'true'; + } + const rm = get('search.reranker.model'); + if (rm !== undefined && rm.trim().length > 0) { + out.reranker_model = rm.trim(); + } + const ri = get('search.reranker.top_n_in'); + if (ri !== undefined) { + const n = parseInt(ri, 10); + if (Number.isFinite(n) && n > 0) out.reranker_top_n_in = n; + } + // CDX2-F15 null parsing: top_n_out distinguishes three input shapes: + // key absent → undefined → fall through to mode bundle + // 'null' / 'none' / '' → explicit null (no truncate) + // positive integer → that number + const ro = get('search.reranker.top_n_out'); + if (ro !== undefined) { + const trimmed = ro.trim().toLowerCase(); + if (trimmed === '' || trimmed === 'null' || trimmed === 'none') { + out.reranker_top_n_out = null; + } else { + const n = parseInt(trimmed, 10); + if (Number.isFinite(n) && n > 0) out.reranker_top_n_out = n; + } + } + const rt = get('search.reranker.timeout_ms'); + if (rt !== undefined) { + const n = parseInt(rt, 10); + if (Number.isFinite(n) && n > 0) out.reranker_timeout_ms = n; + } + return out; } @@ -322,6 +447,12 @@ export const SEARCH_MODE_CONFIG_KEYS: ReadonlyArray = Object.freeze([ 'search.tokenBudget', 'search.expansion', 'search.searchLimit', + // v0.35.0.0+ reranker keys + 'search.reranker.enabled', + 'search.reranker.model', + 'search.reranker.top_n_in', + 'search.reranker.top_n_out', + 'search.reranker.timeout_ms', ]); /** diff --git a/src/core/search/rerank.ts b/src/core/search/rerank.ts new file mode 100644 index 000000000..c35fcbee0 --- /dev/null +++ b/src/core/search/rerank.ts @@ -0,0 +1,134 @@ +/** + * v0.35.0.0+ — reranker call-site abstraction. + * + * Slots into hybridSearch after `dedupResults()` and before + * `enforceTokenBudget()`. Takes the top `topNIn` candidates by current RRF + * order, sends them to `gateway.rerank()`, and re-orders by the + * cross-encoder's relevance score. The un-reranked long tail keeps its + * original RRF order — preserves recall vs. truncating to topNIn. + * + * Fail-open posture: every error class (auth, network, timeout, rate-limit, + * payload-too-large, unknown) logs to the rerank-audit JSONL and returns + * the original RRF order unchanged. Search reliability beats reranker + * quality; a flaky upstream must never break search. + * + * Caller (hybridSearch) decides whether the reranker fires via + * `opts.reranker?.enabled`. Mode-bundle resolution defaults this to `true` + * for tokenmax and `false` for conservative/balanced. + */ + +import { createHash } from 'crypto'; +import type { SearchResult } from '../types.ts'; +import { rerank as gatewayRerank, RerankError, type RerankInput, type RerankResult } from '../ai/gateway.ts'; +import { logRerankFailure, type RerankFailureReason } from '../rerank-audit.ts'; + +export interface RerankerOpts { + enabled: boolean; + /** How many of the top results to send to the reranker (default 30). */ + topNIn: number; + /** Truncate the reranked output to this many (null = no truncate). */ + topNOut: number | null; + /** Provider:model override. When undefined, gateway uses configured default. */ + model?: string; + /** Per-call timeout in ms (default 5000 — propagates to gateway.rerank). */ + timeoutMs?: number; + /** + * Test seam — when set, applyReranker calls this instead of gateway.rerank. + * Production must NEVER set this. + */ + rerankerFn?: (input: RerankInput) => Promise; +} + +/** SHA-256 prefix (8 chars) of the query text for privacy-preserving audit. */ +function hashQuery(query: string): string { + return createHash('sha256').update(query, 'utf8').digest('hex').slice(0, 8); +} + +/** + * Reorder the top `topNIn` results by reranker relevance score. The + * un-reranked tail (any rows past topNIn) preserves its original RRF + * position — appended after the reordered head in the same order it had + * coming in. + * + * On reranker failure, logs to ~/.gbrain/audit/rerank-failures-* and + * returns the input array unmodified. Never throws. + * + * Empty input passes through immediately (no upstream call). + */ +export async function applyReranker( + query: string, + results: SearchResult[], + opts: RerankerOpts, +): Promise { + if (!opts.enabled || results.length === 0) return results; + // No documents to rerank when topNIn=0 — pass through (defensive; mode + // bundles never set 0 in practice). + if (opts.topNIn <= 0) return results; + + const head = results.slice(0, opts.topNIn); + const tail = results.slice(opts.topNIn); + + // Document text — chunk_text is the matched span. Fall back to title if + // empty (shouldn't happen in practice; defensive). Empty docs would + // confuse the reranker, but we still send them — the upstream model decides. + const documents = head.map(r => r.chunk_text || r.title || ''); + + let reranked: RerankResult[]; + try { + const rerankerFn = opts.rerankerFn ?? gatewayRerank; + reranked = await rerankerFn({ + query, + documents, + timeoutMs: opts.timeoutMs, + ...(opts.model ? { model: opts.model } : {}), + }); + } catch (err) { + const reason: RerankFailureReason = + err instanceof RerankError ? err.reason : 'unknown'; + const errorSummary = err instanceof Error ? err.message : String(err); + try { + logRerankFailure({ + model: opts.model ?? 'unknown', + reason, + query_hash: hashQuery(query), + doc_count: documents.length, + error_summary: errorSummary, + }); + } catch { + // Audit logging must never break search. + } + return results; + } + + // Defensive: if the reranker returned a malformed shape, pass through. + if (!Array.isArray(reranked) || reranked.length === 0) return results; + + // Build the reordered head. We keep ONLY indices the reranker returned + // (so a top_n response with fewer items than head.length naturally + // drops the missing ones — but since we don't pass top_n by default, + // every input gets a score). + const seen = new Set(); + const reorderedHead: SearchResult[] = []; + for (const r of reranked) { + if (r.index >= 0 && r.index < head.length && !seen.has(r.index)) { + seen.add(r.index); + const item = head[r.index]!; + // Stamp the reranker score onto the result so downstream callers + // (telemetry, debug) can see the new ordering signal. Doesn't + // replace `score` — that's RRF and other consumers may depend on it. + (item as any).rerank_score = r.relevanceScore; + reorderedHead.push(item); + } + } + // If the reranker dropped some head items (rare; usually only happens + // with explicit top_n), preserve their original positions at the end + // of the head section so we don't silently lose recall. + for (let i = 0; i < head.length; i++) { + if (!seen.has(i)) reorderedHead.push(head[i]!); + } + + const combined = [...reorderedHead, ...tail]; + return opts.topNOut !== null && opts.topNOut > 0 + ? combined.slice(0, opts.topNOut) + : combined; +} diff --git a/src/core/types.ts b/src/core/types.ts index 381e9b3bc..2bd3706af 100644 --- a/src/core/types.ts +++ b/src/core/types.ts @@ -550,6 +550,23 @@ export interface SearchOpts { * deterministic behavior independent of query phrasing. */ intentWeighting?: boolean; + /** + * v0.35.0.0+: cross-encoder reranker config. Resolved from mode bundle by + * default — tokenmax sets `enabled: true`, conservative + balanced set + * `enabled: false`. Per-call SearchOpts.reranker overrides the mode + * bundle. Slots in between dedupResults and enforceTokenBudget in + * hybrid.ts. Defined here as a structural type to avoid a circular + * import on src/core/search/rerank.ts; the runtime type lives there. + */ + reranker?: { + enabled: boolean; + topNIn: number; + topNOut: number | null; + model?: string; + timeoutMs?: number; + // Test seam — never set in production code. + rerankerFn?: (input: { query: string; documents: string[]; topN?: number; model?: string; signal?: AbortSignal; timeoutMs?: number }) => Promise<{ index: number; relevanceScore: number }[]>; + }; } /** diff --git a/test/ai/dims-zeroentropy.test.ts b/test/ai/dims-zeroentropy.test.ts new file mode 100644 index 000000000..1dba6812e --- /dev/null +++ b/test/ai/dims-zeroentropy.test.ts @@ -0,0 +1,121 @@ +/** + * v0.35.0.0+ dim validator + 4th-arg inputType plumbing tests. + * + * Pins: + * - ZE valid dim allowlist (CDX1-F-equivalent for zembed-1) + * - dimsProviderOptions returns input_type='document' default for ZE + * - dimsProviderOptions returns input_type='query' when threaded + * - dimsProviderOptions DROPS input_type for OpenAI text-3 (per-model + * filtering, NOT generic openai-compat — CDX2-F6) + * - dim validator throws AIConfigError for invalid dim + */ + +import { describe, test, expect } from 'bun:test'; +import { + dimsProviderOptions, + isValidZeroEntropyDim, + ZEROENTROPY_VALID_DIMS, + supportsZeroEntropyDimension, +} from '../../src/core/ai/dims.ts'; +import { AIConfigError } from '../../src/core/ai/errors.ts'; + +describe('ZE dim allowlist', () => { + test('exposes all 7 valid Matryoshka steps', () => { + expect([...ZEROENTROPY_VALID_DIMS]).toEqual([2560, 1280, 640, 320, 160, 80, 40]); + }); + + test('isValidZeroEntropyDim accepts every allowlist value', () => { + for (const d of ZEROENTROPY_VALID_DIMS) { + expect(isValidZeroEntropyDim(d)).toBe(true); + } + }); + + test('isValidZeroEntropyDim rejects non-allowlist values', () => { + expect(isValidZeroEntropyDim(1024)).toBe(false); // common OpenAI default — catches the bug class + expect(isValidZeroEntropyDim(1536)).toBe(false); // DEFAULT_EMBEDDING_DIMENSIONS + expect(isValidZeroEntropyDim(3072)).toBe(false); + expect(isValidZeroEntropyDim(100)).toBe(false); + }); + + test('supportsZeroEntropyDimension(zembed-1)', () => { + expect(supportsZeroEntropyDimension('zembed-1')).toBe(true); + expect(supportsZeroEntropyDimension('zerank-2')).toBe(false); + expect(supportsZeroEntropyDimension('text-embedding-3-large')).toBe(false); + }); +}); + +describe('dimsProviderOptions — ZE branch', () => { + test('default inputType: emits input_type=document', () => { + const opts = dimsProviderOptions('openai-compatible', 'zembed-1', 2560); + expect(opts).toEqual({ + openaiCompatible: { dimensions: 2560, input_type: 'document' }, + }); + }); + + test('inputType=query: emits input_type=query', () => { + const opts = dimsProviderOptions('openai-compatible', 'zembed-1', 1280, 'query'); + expect(opts).toEqual({ + openaiCompatible: { dimensions: 1280, input_type: 'query' }, + }); + }); + + test('inputType=document: emits input_type=document', () => { + const opts = dimsProviderOptions('openai-compatible', 'zembed-1', 640, 'document'); + expect(opts).toEqual({ + openaiCompatible: { dimensions: 640, input_type: 'document' }, + }); + }); + + test('throws AIConfigError for dim=1024 (catches the silent-default bug)', () => { + expect(() => dimsProviderOptions('openai-compatible', 'zembed-1', 1024)).toThrow(AIConfigError); + }); + + test('throws AIConfigError for dim=3072', () => { + expect(() => dimsProviderOptions('openai-compatible', 'zembed-1', 3072)).toThrow(AIConfigError); + }); + + test('error message names valid dims for paste-ready fix', () => { + try { + dimsProviderOptions('openai-compatible', 'zembed-1', 1024); + throw new Error('should have thrown'); + } catch (err) { + expect(err).toBeInstanceOf(AIConfigError); + const msg = (err as Error).message; + // Lists the 7 valid dims so the user can copy-paste a valid value. + expect(msg).toContain('2560'); + expect(msg).toContain('40'); + } + }); +}); + +describe('CDX2-F6: per-model inputType filtering', () => { + test('OpenAI text-embedding-3-large IGNORES inputType (symmetric provider)', () => { + // Pass inputType='query' and confirm input_type does NOT reach the + // provider-options blob. OpenAI's /embeddings endpoint would reject + // an unexpected field; the test pins the absence. + const opts = dimsProviderOptions('native-openai', 'text-embedding-3-large', 1536, 'query'); + expect(opts).toEqual({ openai: { dimensions: 1536 } }); + expect(JSON.stringify(opts)).not.toContain('input_type'); + }); + + test('OpenAI text-embedding-3 on openai-compat adapter ignores inputType', () => { + // Azure OpenAI sometimes hosts text-embedding-3 via openai-compat. + // input_type would be rejected. + const opts = dimsProviderOptions('openai-compatible', 'text-embedding-3-large', 1536, 'query'); + expect(opts).toEqual({ openaiCompatible: { dimensions: 1536 } }); + expect(JSON.stringify(opts)).not.toContain('input_type'); + }); + + test('Voyage models accept inputType when explicitly threaded', () => { + // Voyage v4 + v3 accept input_type. inputType undefined → no field + // (back-compat for pre-v0.35.0.0 tests); inputType='query' → field present. + const optsDefault = dimsProviderOptions('openai-compatible', 'voyage-3-large', 1024); + expect(optsDefault).toEqual({ openaiCompatible: { dimensions: 1024 } }); + expect(JSON.stringify(optsDefault)).not.toContain('input_type'); + + const optsQuery = dimsProviderOptions('openai-compatible', 'voyage-3-large', 1024, 'query'); + expect(optsQuery).toEqual({ + openaiCompatible: { dimensions: 1024, input_type: 'query' }, + }); + }); +}); diff --git a/test/ai/embedQuery.test.ts b/test/ai/embedQuery.test.ts new file mode 100644 index 000000000..c12fa0321 --- /dev/null +++ b/test/ai/embedQuery.test.ts @@ -0,0 +1,176 @@ +/** + * v0.35.0.0 — embedQuery() routing tests. + * + * Pins: + * - embedQuery returns a single Float32Array (not a batch). + * - embedQuery threads inputType='query' through dimsProviderOptions + * into the provider options blob that reaches embedMany(). + * - embed() (without inputType arg) defaults to no input_type field for + * back-compat. This is the contract the dimsProviderOptions 4th-arg + * audit relies on: existing callers continue to embed as 'document'- + * side without a code change. + * - For symmetric providers (OpenAI text-3, DashScope), embedQuery does + * NOT inject input_type into the provider options (CDX2-F6 per-model + * filtering pinned at the dims-zeroentropy.test.ts layer; this test + * confirms the gateway end-to-end stays consistent). + * - For ZE zembed-1, embedQuery produces input_type='query'. + */ + +import { describe, test, expect, afterEach, beforeEach } from 'bun:test'; +import { + configureGateway, + resetGateway, + embed, + embedQuery, + __setEmbedTransportForTests, +} from '../../src/core/ai/gateway.ts'; + +function configureZE(): void { + configureGateway({ + embedding_model: 'zeroentropyai:zembed-1', + embedding_dimensions: 2560, + env: { ZEROENTROPY_API_KEY: 'sk-fake' }, + }); +} + +function configureOpenAI(): void { + configureGateway({ + embedding_model: 'openai:text-embedding-3-large', + embedding_dimensions: 1536, + env: { OPENAI_API_KEY: 'sk-fake' }, + }); +} + +function configureVoyage(): void { + configureGateway({ + embedding_model: 'voyage:voyage-3-large', + embedding_dimensions: 1024, + env: { VOYAGE_API_KEY: 'sk-fake' }, + }); +} + +function fakeEmbeddings(count: number, dims: number) { + return { + embeddings: Array.from({ length: count }, (_, i) => + Array.from({ length: dims }, (_, j) => (j === 0 ? i : 0.1)), + ), + }; +} + +afterEach(() => { + __setEmbedTransportForTests(null); + resetGateway(); +}); + +describe('embedQuery — return shape', () => { + beforeEach(() => configureZE()); + + test('returns a single Float32Array (not a batch)', async () => { + __setEmbedTransportForTests((async () => fakeEmbeddings(1, 2560)) as any); + const v = await embedQuery('hello'); + expect(v).toBeInstanceOf(Float32Array); + expect(v.length).toBe(2560); + // Index sentinel matches input position 0. + expect(v[0]).toBe(0); + }); +}); + +describe('embedQuery — inputType plumbing (ZE asymmetric)', () => { + beforeEach(() => configureZE()); + + test('embedQuery sends input_type=query in providerOptions', async () => { + let capturedOpts: any = null; + __setEmbedTransportForTests((async (args: any) => { + capturedOpts = args.providerOptions; + return fakeEmbeddings(1, 2560); + }) as any); + await embedQuery('hello'); + expect(capturedOpts?.openaiCompatible?.input_type).toBe('query'); + expect(capturedOpts?.openaiCompatible?.dimensions).toBe(2560); + }); + + test('embed() (no inputType arg) sends input_type=document for ZE', async () => { + let capturedOpts: any = null; + __setEmbedTransportForTests((async (args: any) => { + capturedOpts = args.providerOptions; + return fakeEmbeddings(args.values.length, 2560); + }) as any); + await embed(['doc']); + expect(capturedOpts?.openaiCompatible?.input_type).toBe('document'); + }); + + test('embed([…], "query") explicit threading also reaches the wire', async () => { + let capturedOpts: any = null; + __setEmbedTransportForTests((async (args: any) => { + capturedOpts = args.providerOptions; + return fakeEmbeddings(args.values.length, 2560); + }) as any); + await embed(['q1', 'q2'], { inputType: 'query' }); + expect(capturedOpts?.openaiCompatible?.input_type).toBe('query'); + }); +}); + +describe('embedQuery — per-model filtering (CDX2-F6, end-to-end)', () => { + test('OpenAI text-embedding-3-large: NO input_type in providerOptions', async () => { + configureOpenAI(); + let capturedOpts: any = null; + __setEmbedTransportForTests((async (args: any) => { + capturedOpts = args.providerOptions; + return fakeEmbeddings(1, 1536); + }) as any); + await embedQuery('hello'); + // OpenAI's /embeddings endpoint would reject an unexpected input_type + // field. The CDX2-F6 fix puts the ZE/Voyage branches BEFORE the generic + // text-embedding-3 fall-through; this end-to-end test pins the absence. + expect(capturedOpts?.openai?.dimensions).toBe(1536); + expect(JSON.stringify(capturedOpts)).not.toContain('input_type'); + }); + + test('Voyage voyage-3-large: input_type=query reaches the wire', async () => { + configureVoyage(); + let capturedOpts: any = null; + __setEmbedTransportForTests((async (args: any) => { + capturedOpts = args.providerOptions; + return fakeEmbeddings(1, 1024); + }) as any); + await embedQuery('hello'); + // Voyage v3+ accepts input_type; embedQuery threading reaches it. + expect(capturedOpts?.openaiCompatible?.input_type).toBe('query'); + expect(capturedOpts?.openaiCompatible?.dimensions).toBe(1024); + }); + + test('Voyage with embed() (no inputType arg): NO input_type field (back-compat)', async () => { + configureVoyage(); + let capturedOpts: any = null; + __setEmbedTransportForTests((async (args: any) => { + capturedOpts = args.providerOptions; + return fakeEmbeddings(args.values.length, 1024); + }) as any); + await embed(['doc']); + // CDX2-F6 contract: legacy callers (no 4th-arg) preserve their existing + // behavior. The pre-v0.35.0.0 Voyage gateway never sent input_type; + // a regression here would break existing brains. The condition is + // tested at the dimsProviderOptions layer too, but this end-to-end pin + // catches a future refactor that might bypass the condition. + expect(JSON.stringify(capturedOpts)).not.toContain('input_type'); + }); +}); + +describe('embedQuery — routes through same recipe as embed', () => { + beforeEach(() => configureZE()); + + test('embedQuery + embed both use the configured ZE model', async () => { + const dimsSeen: number[] = []; + __setEmbedTransportForTests((async (args: any) => { + // The args.model in the AI-SDK transport is the model instance; we + // can stringify a canonical name via the provider/recipe — easier + // to just confirm the dims and providerOptions match the ZE config. + dimsSeen.push(args.providerOptions?.openaiCompatible?.dimensions); + return fakeEmbeddings(args.values.length, 2560); + }) as any); + await embedQuery('q'); + await embed(['d']); + // Both calls routed through the same recipe + dim config. + expect(dimsSeen).toEqual([2560, 2560]); + }); +}); diff --git a/test/ai/rerank.test.ts b/test/ai/rerank.test.ts new file mode 100644 index 000000000..881fc4af8 --- /dev/null +++ b/test/ai/rerank.test.ts @@ -0,0 +1,327 @@ +/** + * v0.35.0.0 — gateway.rerank() HTTP path tests. + * + * Drives the public `rerank()` function with a stubbed `_rerankTransport` + * (the canonical test seam — same pattern as `__setEmbedTransportForTests`). + * + * Pins: + * - Request URL is `${recipe.base_url_default}/models/rerank` — i.e. + * `https://api.zeroentropy.dev/v1/models/rerank`, NOT `/v1/v1/…` + * (CDX1-F2 regression). + * - Request body shape: `{model, query, documents, top_n?}`. + * - Bearer auth header from `applyResolveAuth` ↔ ZEROENTROPY_API_KEY. + * - Response parsing: `{results: [{index, relevance_score}]}` → + * `RerankResult[]` with `{index, relevanceScore}`. + * - Error classification: 401/403 → auth, 429 → rate_limit, 5xx → network, + * other 4xx → unknown; AbortError on timeout → timeout. + * - Pre-flight payload guard: body over `max_payload_bytes` throws + * payload_too_large BEFORE any HTTP call (no transport invocation). + * - Empty documents → empty result, no HTTP call. + * - Model allowlist enforcement (CDX2-F11): `assertTouchpoint` doesn't + * enforce allowlists for openai-compatible recipes; rerank() does the + * check directly. + */ + +import { describe, test, expect, afterEach, beforeEach } from 'bun:test'; +import { + configureGateway, + resetGateway, + rerank, + RerankError, + __setRerankTransportForTests, +} from '../../src/core/ai/gateway.ts'; + +function configureZE(model: string = 'zeroentropyai:zerank-2'): void { + configureGateway({ + reranker_model: model, + env: { ZEROENTROPY_API_KEY: 'sk-test-zerokey' }, + }); +} + +function mockResp(json: unknown, status = 200): Response { + return new Response(JSON.stringify(json), { + status, + headers: { 'content-type': 'application/json' }, + }); +} + +afterEach(() => { + __setRerankTransportForTests(null); + resetGateway(); +}); + +describe('gateway.rerank() — happy path', () => { + beforeEach(() => configureZE()); + + test('sends the right URL (CDX1-F2 — no /v1/v1/ doubling)', async () => { + let capturedUrl = ''; + __setRerankTransportForTests(async (url) => { + capturedUrl = url; + return mockResp({ results: [{ index: 0, relevance_score: 0.9 }] }); + }); + await rerank({ query: 'q', documents: ['d'] }); + expect(capturedUrl).toBe('https://api.zeroentropy.dev/v1/models/rerank'); + expect(capturedUrl).not.toContain('/v1/v1/'); + }); + + test('sends the right body shape', async () => { + let captured: any = null; + __setRerankTransportForTests(async (_url, init) => { + captured = JSON.parse(init.body as string); + return mockResp({ results: [{ index: 0, relevance_score: 0.9 }] }); + }); + await rerank({ query: 'q', documents: ['d1', 'd2'], topN: 5 }); + expect(captured).toEqual({ + model: 'zerank-2', + query: 'q', + documents: ['d1', 'd2'], + top_n: 5, + }); + }); + + test('omits top_n when not provided', async () => { + let captured: any = null; + __setRerankTransportForTests(async (_url, init) => { + captured = JSON.parse(init.body as string); + return mockResp({ results: [{ index: 0, relevance_score: 0.5 }] }); + }); + await rerank({ query: 'q', documents: ['d'] }); + expect('top_n' in captured).toBe(false); + }); + + test('uses Bearer auth from ZEROENTROPY_API_KEY', async () => { + let authHeader = ''; + __setRerankTransportForTests(async (_url, init) => { + const headers = new Headers(init.headers as HeadersInit); + authHeader = headers.get('authorization') ?? ''; + return mockResp({ results: [{ index: 0, relevance_score: 1.0 }] }); + }); + await rerank({ query: 'q', documents: ['d'] }); + expect(authHeader).toBe('Bearer sk-test-zerokey'); + }); + + test('Content-Type: application/json', async () => { + let contentType = ''; + __setRerankTransportForTests(async (_url, init) => { + const headers = new Headers(init.headers as HeadersInit); + contentType = headers.get('content-type') ?? ''; + return mockResp({ results: [] }); + }); + await rerank({ query: 'q', documents: ['d'] }); + expect(contentType).toBe('application/json'); + }); + + test('maps response.results[].relevance_score → relevanceScore', async () => { + __setRerankTransportForTests(async () => + mockResp({ + results: [ + { index: 2, relevance_score: 0.99 }, + { index: 0, relevance_score: 0.5 }, + { index: 1, relevance_score: 0.1 }, + ], + }), + ); + const out = await rerank({ query: 'q', documents: ['a', 'b', 'c'] }); + expect(out).toEqual([ + { index: 2, relevanceScore: 0.99 }, + { index: 0, relevanceScore: 0.5 }, + { index: 1, relevanceScore: 0.1 }, + ]); + }); + + test('explicit input.model overrides gateway-configured model', async () => { + let bodyModel = ''; + __setRerankTransportForTests(async (_url, init) => { + bodyModel = JSON.parse(init.body as string).model; + return mockResp({ results: [{ index: 0, relevance_score: 0.5 }] }); + }); + await rerank({ query: 'q', documents: ['d'], model: 'zeroentropyai:zerank-1-small' }); + expect(bodyModel).toBe('zerank-1-small'); + }); + + test('empty documents returns [] without HTTP call', async () => { + let called = false; + __setRerankTransportForTests(async () => { + called = true; + return mockResp({ results: [] }); + }); + const out = await rerank({ query: 'q', documents: [] }); + expect(out).toEqual([]); + expect(called).toBe(false); + }); +}); + +describe('gateway.rerank() — error classification', () => { + beforeEach(() => configureZE()); + + test('401 → auth', async () => { + __setRerankTransportForTests(async () => new Response('Unauthorized', { status: 401 })); + try { + await rerank({ query: 'q', documents: ['d'] }); + throw new Error('should have thrown'); + } catch (err) { + expect(err).toBeInstanceOf(RerankError); + expect((err as RerankError).reason).toBe('auth'); + expect((err as RerankError).status).toBe(401); + } + }); + + test('403 → auth', async () => { + __setRerankTransportForTests(async () => new Response('Forbidden', { status: 403 })); + try { + await rerank({ query: 'q', documents: ['d'] }); + throw new Error('should have thrown'); + } catch (err) { + expect((err as RerankError).reason).toBe('auth'); + } + }); + + test('429 → rate_limit', async () => { + __setRerankTransportForTests(async () => new Response('Rate limited', { status: 429 })); + try { + await rerank({ query: 'q', documents: ['d'] }); + throw new Error('should have thrown'); + } catch (err) { + expect((err as RerankError).reason).toBe('rate_limit'); + } + }); + + test('500 → network', async () => { + __setRerankTransportForTests(async () => new Response('Server error', { status: 500 })); + try { + await rerank({ query: 'q', documents: ['d'] }); + throw new Error('should have thrown'); + } catch (err) { + expect((err as RerankError).reason).toBe('network'); + } + }); + + test('400 (non-classified) → unknown', async () => { + __setRerankTransportForTests(async () => new Response('Bad request', { status: 400 })); + try { + await rerank({ query: 'q', documents: ['d'] }); + throw new Error('should have thrown'); + } catch (err) { + expect((err as RerankError).reason).toBe('unknown'); + } + }); + + test('network exception (fetch reject) → network', async () => { + __setRerankTransportForTests(async () => { + throw new Error('ECONNREFUSED'); + }); + try { + await rerank({ query: 'q', documents: ['d'] }); + throw new Error('should have thrown'); + } catch (err) { + expect((err as RerankError).reason).toBe('network'); + } + }); + + test('malformed response (no results array) → unknown', async () => { + __setRerankTransportForTests(async () => mockResp({ wrong_shape: true })); + try { + await rerank({ query: 'q', documents: ['d'] }); + throw new Error('should have thrown'); + } catch (err) { + expect((err as RerankError).reason).toBe('unknown'); + } + }); +}); + +describe('gateway.rerank() — payload-too-large pre-flight (no HTTP call)', () => { + beforeEach(() => configureZE()); + + test('body over max_payload_bytes throws payload_too_large WITHOUT transport call', async () => { + let called = false; + __setRerankTransportForTests(async () => { + called = true; + return mockResp({ results: [] }); + }); + // ZE's max_payload_bytes is 5MB. 6MB body easily exceeds. + const huge = 'x'.repeat(6 * 1024 * 1024); + try { + await rerank({ query: 'q', documents: [huge] }); + throw new Error('should have thrown'); + } catch (err) { + expect(err).toBeInstanceOf(RerankError); + expect((err as RerankError).reason).toBe('payload_too_large'); + // CRITICAL: the guard must fire BEFORE the transport is called. + // Otherwise applyReranker's fail-open would burn an HTTP round-trip. + expect(called).toBe(false); + } + }); + + test('body under cap proceeds normally', async () => { + let called = false; + __setRerankTransportForTests(async () => { + called = true; + return mockResp({ results: [{ index: 0, relevance_score: 0.5 }] }); + }); + const normal = 'doc'.repeat(100); + await rerank({ query: 'q', documents: [normal] }); + expect(called).toBe(true); + }); +}); + +describe('gateway.rerank() — allowlist enforcement (CDX2-F11)', () => { + test('rejects model not in recipe touchpoint.models[]', async () => { + configureZE(); + let called = false; + __setRerankTransportForTests(async () => { + called = true; + return mockResp({ results: [] }); + }); + try { + await rerank({ + query: 'q', + documents: ['d'], + model: 'zeroentropyai:zerank-fake-99', + }); + throw new Error('should have thrown'); + } catch (err) { + expect(err).toBeInstanceOf(RerankError); + expect((err as RerankError).message).toContain('not listed'); + expect(called).toBe(false); + } + }); + + test('accepts zerank-1 (legacy allowlist member)', async () => { + configureZE(); + __setRerankTransportForTests(async () => mockResp({ results: [{ index: 0, relevance_score: 1 }] })); + const out = await rerank({ + query: 'q', + documents: ['d'], + model: 'zeroentropyai:zerank-1', + }); + expect(out.length).toBe(1); + }); + + test('rejects provider that does not declare reranker touchpoint', async () => { + configureGateway({ + reranker_model: 'openai:gpt-fake', + env: { OPENAI_API_KEY: 'sk-fake' }, + }); + try { + await rerank({ query: 'q', documents: ['d'] }); + throw new Error('should have thrown'); + } catch (err) { + expect(err).toBeInstanceOf(RerankError); + expect((err as RerankError).message).toContain('does not declare a reranker touchpoint'); + } + }); +}); + +describe('gateway.rerank() — guard rails', () => { + beforeEach(() => configureZE()); + + test('empty query throws RerankError(unknown)', async () => { + try { + await rerank({ query: '', documents: ['d'] }); + throw new Error('should have thrown'); + } catch (err) { + expect(err).toBeInstanceOf(RerankError); + expect((err as RerankError).reason).toBe('unknown'); + } + }); +}); diff --git a/test/ai/zeroentropy-compat-fetch.test.ts b/test/ai/zeroentropy-compat-fetch.test.ts new file mode 100644 index 000000000..f2d784b1f --- /dev/null +++ b/test/ai/zeroentropy-compat-fetch.test.ts @@ -0,0 +1,160 @@ +/** + * v0.35.0.0 — structural source-shape assertions for zeroEntropyCompatFetch. + * + * Mirrors the test/voyage-response-cap.test.ts pattern: the shim is a + * closed-over helper inside the embedding instantiation path and isn't + * exported. Behavioral coverage of ZE flows happens in the gateway/embed + * tests (with __setEmbedTransportForTests stubbing); these structural + * pins guard the SHAPE of the shim so a silent revert fails loudly. + * + * Pins: + * - URL path rewrite `/embeddings` → `/models/embed`. + * - Body inject: input_type default to 'document'; encoding_format forced + * to 'float' (don't trust SDK default per CDX2-F2). + * - Response rewrite: `{results: [{embedding}]}` → `{data: [{embedding, + * index}]}`. + * - usage.prompt_tokens injection (CDX2-F1 — AI SDK schema requires it + * when usage is present; Voyage's shim already had to do this). + * - OOM caps: MAX_ZEROENTROPY_RESPONSE_BYTES=256MB; Layer 1 + * Content-Length, Layer 2 per-embedding size. + * - instantiateEmbedding branch: `recipe.id === 'zeroentropyai'` → + * `zeroEntropyCompatFetch`. + */ + +import { describe, test, expect } from 'bun:test'; + +const GATEWAY_PATH = new URL('../../src/core/ai/gateway.ts', import.meta.url); + +describe('zeroEntropyCompatFetch — shim structural shape', () => { + test('declared at module scope alongside voyageCompatFetch', async () => { + const src = await Bun.file(GATEWAY_PATH).text(); + expect(src).toMatch(/const zeroEntropyCompatFetch\s*=\s*\(async \(input: RequestInfo \| URL/); + }); + + test('cast through unknown to typeof fetch (Bun preconnect compat)', async () => { + // The Voyage shim has the same trick — without `as unknown as typeof + // fetch` the function-arrow type loses Bun's `preconnect` method + // signature. Pinning here prevents a future refactor from removing + // the cast and re-introducing the tsc TS2741 failure documented in + // gateway.ts:556 comments. + const src = await Bun.file(GATEWAY_PATH).text(); + expect(src).toMatch(/\}\)\s*as unknown as typeof fetch;[\s\S]{0,200}async function resolveEmbeddingProvider/); + }); + + test('URL rewrite: /embeddings → /models/embed (CDX1-F2)', async () => { + const src = await Bun.file(GATEWAY_PATH).text(); + // The rewrite logic must: + // - normalize input via toString() so URL/string/Request all work + // - replace exact suffix `/embeddings` with `/models/embed` + expect(src).toMatch(/u\.pathname\.endsWith\(['"]\/embeddings['"]\)/); + expect(src).toMatch(/\/models\/embed/); + // Negative: must NOT have a `/v1/v1/` form (that would be the + // pre-fix double-prefix bug). + expect(src).not.toContain('/v1/v1/'); + }); + + test('body injects input_type default "document"', async () => { + const src = await Bun.file(GATEWAY_PATH).text(); + // The wrapper defaults input_type to 'document' when caller didn't + // thread one (matches the document-side correctness for sync / + // import / embed CLI paths). + expect(src).toMatch(/parsed\.input_type\s*===\s*undefined/); + expect(src).toMatch(/parsed\.input_type\s*=\s*['"]document['"]/); + }); + + test('body forces encoding_format=float (CDX2-F2)', async () => { + const src = await Bun.file(GATEWAY_PATH).text(); + // Don't trust SDK default; refuse base64 to keep response rewriter + // simple (no base64 decode path needed). + expect(src).toMatch(/parsed\.encoding_format\s*!==\s*['"]float['"]/); + expect(src).toMatch(/parsed\.encoding_format\s*=\s*['"]float['"]/); + }); + + test('response shape rewrite: results → data with index stamped', async () => { + const src = await Bun.file(GATEWAY_PATH).text(); + // The wrapper converts ZE's {results: [{embedding}]} to + // {data: [{embedding, index}]} for the SDK's openai-compat parser. + expect(src).toMatch(/json\.data\s*=\s*json\.results\.map\(/); + expect(src).toMatch(/embedding:\s*r\?\.embedding/); + expect(src).toMatch(/index:\s*i/); + expect(src).toMatch(/delete json\.results/); + }); + + test('usage.prompt_tokens injected from total_tokens (CDX2-F1)', async () => { + const src = await Bun.file(GATEWAY_PATH).text(); + // AI SDK schema requires prompt_tokens when usage is present. Voyage's + // shim had the same fix at gateway.ts:655. + expect(src).toMatch(/json\.usage\.prompt_tokens\s*===\s*undefined/); + expect(src).toMatch(/json\.usage\.prompt_tokens\s*=\s*\n?\s*typeof json\.usage\.total_tokens\s*===\s*['"]number['"]/); + }); +}); + +describe('zeroEntropyCompatFetch — OOM caps', () => { + test('MAX_ZEROENTROPY_RESPONSE_BYTES declared at 256 MB (matches Voyage sizing)', async () => { + const src = await Bun.file(GATEWAY_PATH).text(); + expect(src).toContain('MAX_ZEROENTROPY_RESPONSE_BYTES'); + expect(src).toMatch(/MAX_ZEROENTROPY_RESPONSE_BYTES\s*=\s*256\s*\*\s*1024\s*\*\s*1024/); + }); + + test('Layer 1: Content-Length pre-check before resp.clone().json()', async () => { + const src = await Bun.file(GATEWAY_PATH).text(); + // Find the zeroEntropyCompatFetch block bounds, then assert ordering + // within it (mirroring the voyage cap test pattern). + const zeFetchStart = src.indexOf('const zeroEntropyCompatFetch'); + expect(zeFetchStart).toBeGreaterThan(0); + const block = src.slice(zeFetchStart, zeFetchStart + 8000); + + const preCheckIdx = block.indexOf("resp.headers.get('content-length')"); + const jsonParseIdx = block.indexOf('await resp.clone().json()'); + expect(preCheckIdx).toBeGreaterThan(0); + expect(jsonParseIdx).toBeGreaterThan(0); + // The pre-check MUST appear before the JSON parse — Voyage's lesson + // (codex OV8 finding): without the ordering, the OOM defense is + // theatrical. + expect(preCheckIdx).toBeLessThan(jsonParseIdx); + }); + + test('Layer 1 throws ZeroEntropyResponseTooLargeError (not silent return)', async () => { + const src = await Bun.file(GATEWAY_PATH).text(); + expect(src).toMatch(/throw new ZeroEntropyResponseTooLargeError\([\s\S]{0,200}Content-Length/); + }); + + test('Layer 2: per-embedding size cap inside results iteration', async () => { + const src = await Bun.file(GATEWAY_PATH).text(); + // ZE returns float[] arrays (not base64 like Voyage). The cap counts + // elements × 4 bytes (Float32 width). + expect(src).toMatch(/item\.embedding\.length\s*\*\s*4/); + expect(src).toMatch(/throw new ZeroEntropyResponseTooLargeError\([\s\S]{0,200}embedding exceeds/); + }); + + test('inbound try/catch rethrows ZeroEntropyResponseTooLargeError', async () => { + const src = await Bun.file(GATEWAY_PATH).text(); + // Same rethrow pattern as Voyage's catch-block: instanceof check + // ensures OOM-cap throws aren't silently swallowed by the parse-error + // fallback. + expect(src).toContain('ZeroEntropyResponseTooLargeError'); + expect(src).toMatch(/if\s*\(\s*err\s+instanceof\s+ZeroEntropyResponseTooLargeError\s*\)\s*throw\s+err/); + }); +}); + +describe('instantiateEmbedding wiring', () => { + test('recipe.id === "zeroentropyai" branch installs zeroEntropyCompatFetch', async () => { + const src = await Bun.file(GATEWAY_PATH).text(); + // Pinned: the ternary at gateway.ts that picks the right fetch wrapper + // for openai-compat recipes. Without this branch, the shim is dead code. + expect(src).toMatch(/recipe\.id\s*===\s*['"]zeroentropyai['"]\s*\?[\s]*zeroEntropyCompatFetch/); + }); + + test('branch lives in the openai-compatible case of instantiateEmbedding', async () => { + const src = await Bun.file(GATEWAY_PATH).text(); + const fnIdx = src.indexOf('function instantiateEmbedding('); + const ocIdx = src.indexOf("case 'openai-compatible':", fnIdx); + const branchIdx = src.indexOf('zeroEntropyCompatFetch', ocIdx); + expect(fnIdx).toBeGreaterThan(0); + expect(ocIdx).toBeGreaterThan(0); + // Branch sits within ~2KB of the case opener (in the fetchWrapper + // ternary) — a sanity bound on where it lives. + expect(branchIdx).toBeGreaterThan(ocIdx); + expect(branchIdx - ocIdx).toBeLessThan(2000); + }); +}); diff --git a/test/ai/zeroentropy-recipe.test.ts b/test/ai/zeroentropy-recipe.test.ts new file mode 100644 index 000000000..9e84db4dc --- /dev/null +++ b/test/ai/zeroentropy-recipe.test.ts @@ -0,0 +1,62 @@ +/** + * v0.35.0.0+ ZeroEntropy recipe shape tests. Pins the contracts Codex + * Round 1 + 2 flagged in the plan-review wave: + * + * - F1: implementation MUST be 'openai-compatible' (not the misspelled + * 'openai-compat' the original plan draft had). + * - F2: base_url_default MUST end with '/v1' so the gateway's + * URL-rewrite path replaces /embeddings → /models/embed cleanly. + * - reranker touchpoint declared with correct allowlist + payload cap. + */ + +import { describe, test, expect } from 'bun:test'; +import { zeroentropyai } from '../../src/core/ai/recipes/zeroentropyai.ts'; +import { RECIPES, getRecipe } from '../../src/core/ai/recipes/index.ts'; + +describe('zeroentropyai recipe shape', () => { + test('F1 regression — implementation literal is "openai-compatible"', () => { + // CDX1-F1: the original plan had 'openai-compat' which is not a valid + // Implementation literal. This assertion fails loud if a future PR + // re-introduces the typo. + expect(zeroentropyai.implementation).toBe('openai-compatible'); + }); + + test('F2 regression — base_url_default ends with /v1', () => { + // CDX1-F2: when /embeddings → /models/embed rewrite fires, the final + // URL must be …/v1/models/embed (NOT …/v1/v1/…). The recipe's base URL + // already includes /v1, so the rewrite is a plain path substitution. + expect(zeroentropyai.base_url_default).toBe('https://api.zeroentropy.dev/v1'); + expect(zeroentropyai.base_url_default!.endsWith('/v1')).toBe(true); + }); + + test('registered in ALL[] via index.ts', () => { + expect(RECIPES.has('zeroentropyai')).toBe(true); + expect(getRecipe('zeroentropyai')).toBe(zeroentropyai); + }); + + test('embedding touchpoint declares zembed-1 with 7 flexible dims', () => { + const e = zeroentropyai.touchpoints.embedding!; + expect(e.models).toEqual(['zembed-1']); + expect(e.default_dims).toBe(2560); + expect(e.supports_multimodal).toBe(false); + // Dense-payload hedge matches Voyage (CJK / JSON / code overshoot tiktoken). + expect(e.chars_per_token).toBe(1); + expect(e.safety_factor).toBe(0.5); + expect(e.max_batch_tokens).toBe(120_000); + }); + + test('reranker touchpoint declares zerank-2 + zerank-1 + zerank-1-small', () => { + const r = zeroentropyai.touchpoints.reranker!; + expect(r).toBeDefined(); + expect(r.models).toContain('zerank-2'); + expect(r.models).toContain('zerank-1'); + expect(r.models).toContain('zerank-1-small'); + expect(r.default_model).toBe('zerank-2'); + expect(r.max_payload_bytes).toBe(5_000_000); // ZE's per-request cap. + }); + + test('auth_env declares ZEROENTROPY_API_KEY + setup URL', () => { + expect(zeroentropyai.auth_env!.required).toEqual(['ZEROENTROPY_API_KEY']); + expect(zeroentropyai.auth_env!.setup_url).toBe('https://dashboard.zeroentropy.dev'); + }); +}); diff --git a/test/e2e/zeroentropy-live.test.ts b/test/e2e/zeroentropy-live.test.ts new file mode 100644 index 000000000..852801143 --- /dev/null +++ b/test/e2e/zeroentropy-live.test.ts @@ -0,0 +1,170 @@ +/** + * v0.35.0.0 — ZeroEntropy live E2E tests. + * + * Real HTTP round-trip against `api.zeroentropy.dev`. Gated on + * `ZEROENTROPY_API_KEY` — when absent, every test skips gracefully so + * `bun run test:e2e` stays green on contributor machines that don't have + * a ZE account. + * + * Pins (only meaningful when the env var is set): + * - POST /v1/models/embed returns float embeddings that round-trip + * through the AI-SDK adapter (after zeroEntropyCompatFetch's response + * rewrite). + * - dimensions parameter is honored: 2560 default → vector of length + * 2560; 1280 → 1280; etc. + * - asymmetric input_type plumbing reaches ZE: embedQuery() and embed() + * both succeed and return same-shape vectors (we can't easily inspect + * whether ZE actually produced asymmetric vectors without a reference + * corpus, but the request must succeed without HTTP 400). + * - POST /v1/models/rerank returns indices + relevance_scores that + * round-trip through gateway.rerank() into RerankResult[]. + * + * Cost note: each test fires 1-2 HTTP requests. At $0.025/1M tokens and + * ~100 tokens per test, the full file costs well under a cent. Still + * gated by env so contributor PR CI doesn't spend. + */ + +import { describe, test, expect, beforeAll, afterAll } from 'bun:test'; +import { + configureGateway, + resetGateway, + embed, + embedQuery, + rerank, +} from '../../src/core/ai/gateway.ts'; + +const API_KEY = process.env.ZEROENTROPY_API_KEY; + +// Skip the entire file when env is absent. `describe.skipIf` exists in +// modern bun:test; fall back to in-test guards for older runners. +const skipAll = !API_KEY; + +beforeAll(() => { + if (skipAll) return; + configureGateway({ + embedding_model: 'zeroentropyai:zembed-1', + embedding_dimensions: 2560, + reranker_model: 'zeroentropyai:zerank-2', + env: { ZEROENTROPY_API_KEY: API_KEY! }, + }); +}); + +afterAll(() => { + if (!skipAll) resetGateway(); +}); + +describe('ZE live — embed round-trip', () => { + test('embed(["text"]) returns Float32Array[2560]', async () => { + if (skipAll) { + console.warn('[skip] ZEROENTROPY_API_KEY not set'); + return; + } + const [v] = await embed(['hello world']); + expect(v).toBeInstanceOf(Float32Array); + expect(v.length).toBe(2560); + // Sanity: at least one non-zero element (otherwise the response + // rewrite probably dropped the payload). + const anyNonZero = Array.from(v).some(x => x !== 0); + expect(anyNonZero).toBe(true); + }); + + test('embedQuery("text") returns Float32Array[2560] (query side)', async () => { + if (skipAll) { + console.warn('[skip] ZEROENTROPY_API_KEY not set'); + return; + } + const v = await embedQuery('what is foo'); + expect(v).toBeInstanceOf(Float32Array); + expect(v.length).toBe(2560); + const anyNonZero = Array.from(v).some(x => x !== 0); + expect(anyNonZero).toBe(true); + }); + + test('embed batch of 3 returns 3 vectors in order', async () => { + if (skipAll) { + console.warn('[skip] ZEROENTROPY_API_KEY not set'); + return; + } + const out = await embed(['one', 'two', 'three']); + expect(out.length).toBe(3); + for (const v of out) { + expect(v).toBeInstanceOf(Float32Array); + expect(v.length).toBe(2560); + } + // Different inputs → different vectors (sanity check; would fail + // hard if the response-rewriter accidentally returned the same + // vector for every input). + expect(out[0]).not.toEqual(out[1]); + expect(out[1]).not.toEqual(out[2]); + }); +}); + +describe('ZE live — rerank round-trip', () => { + test('rerank({query, documents}) returns sorted RerankResult[]', async () => { + if (skipAll) { + console.warn('[skip] ZEROENTROPY_API_KEY not set'); + return; + } + const out = await rerank({ + query: 'how does photosynthesis work', + documents: [ + 'Photosynthesis is the process by which plants convert sunlight to energy.', + 'My cat likes to eat tuna fish.', + 'Chlorophyll absorbs red and blue light during photosynthesis.', + ], + }); + expect(out.length).toBe(3); + for (const r of out) { + expect(typeof r.index).toBe('number'); + expect(typeof r.relevanceScore).toBe('number'); + // ZE relevance scores are in [0, 1]. Pin the range so a future + // contract change is loud. + expect(r.relevanceScore).toBeGreaterThanOrEqual(0); + expect(r.relevanceScore).toBeLessThanOrEqual(1); + } + // Photosynthesis-relevant docs should score higher than the cat doc. + // We don't pin a specific order (zerank-2 may re-rank the two + // photosynthesis docs in either order depending on phrasing), but + // the cat doc must NOT be at the top. + const topIndex = out[0]!.index; + expect(topIndex).not.toBe(1); // index 1 is the cat doc + }); + + test('rerank with top_n=2 returns at most 2 results', async () => { + if (skipAll) { + console.warn('[skip] ZEROENTROPY_API_KEY not set'); + return; + } + const out = await rerank({ + query: 'photosynthesis', + documents: ['photosynthesis a', 'cats b', 'photosynthesis c'], + topN: 2, + }); + expect(out.length).toBeLessThanOrEqual(2); + }); +}); + +describe('ZE live — flexible dims', () => { + test('1280-dim embedding returns Float32Array[1280]', async () => { + if (skipAll) { + console.warn('[skip] ZEROENTROPY_API_KEY not set'); + return; + } + resetGateway(); + configureGateway({ + embedding_model: 'zeroentropyai:zembed-1', + embedding_dimensions: 1280, + env: { ZEROENTROPY_API_KEY: API_KEY! }, + }); + const [v] = await embed(['1280 dim test']); + expect(v.length).toBe(1280); + // Restore 2560 for subsequent tests. + resetGateway(); + configureGateway({ + embedding_model: 'zeroentropyai:zembed-1', + embedding_dimensions: 2560, + reranker_model: 'zeroentropyai:zerank-2', + env: { ZEROENTROPY_API_KEY: API_KEY! }, + }); + }); +}); diff --git a/test/rerank-audit.test.ts b/test/rerank-audit.test.ts new file mode 100644 index 000000000..d5b13f58b --- /dev/null +++ b/test/rerank-audit.test.ts @@ -0,0 +1,185 @@ +/** + * v0.35.0.0+ rerank-audit JSONL round-trip. + * + * Pins: + * - logRerankFailure → readRecentRerankFailures round-trip + * - error_summary truncated to 200 chars (privacy/size) + * - corrupt rows skipped (audit is informational, never breaks search) + * - ISO-week filename rotation + * - CDX2-F22: logRerankSuccess does NOT exist (success-events were dropped + * per adversarial review — hot-path I/O churn + privacy concerns) + * + * Uses `withEnv()` per the test-isolation lint rule R1: never mutate + * process.env directly outside `*.serial.test.ts` (process.env is global + * and leaks across files in the parallel runner's shard process). + */ + +import { describe, test, expect } from 'bun:test'; +import * as fs from 'node:fs'; +import * as path from 'node:path'; +import * as os from 'node:os'; +import { withEnv } from './helpers/with-env.ts'; +import { + logRerankFailure, + readRecentRerankFailures, + computeRerankAuditFilename, +} from '../src/core/rerank-audit.ts'; + +/** + * Run a test body inside a fresh tmp audit dir scoped to that test. Cleans + * up the dir after. `withEnv` handles env-restore via try/finally. + */ +async function withFreshAuditDir(body: (tmpDir: string) => void | Promise): Promise { + const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'gbrain-rerank-audit-')); + try { + await withEnv({ GBRAIN_AUDIT_DIR: tmpDir }, async () => { + await body(tmpDir); + }); + } finally { + try { fs.rmSync(tmpDir, { recursive: true, force: true }); } catch { /* ignore */ } + } +} + +describe('rerank-audit JSONL round-trip', () => { + test('log → read returns the same event shape', async () => { + await withFreshAuditDir(() => { + logRerankFailure({ + model: 'zeroentropyai:zerank-2', + reason: 'auth', + query_hash: 'a1b2c3d4', + doc_count: 30, + error_summary: 'invalid api key', + }); + const events = readRecentRerankFailures(7); + expect(events.length).toBe(1); + expect(events[0]).toMatchObject({ + model: 'zeroentropyai:zerank-2', + reason: 'auth', + query_hash: 'a1b2c3d4', + doc_count: 30, + error_summary: 'invalid api key', + severity: 'warn', + }); + expect(typeof events[0]!.ts).toBe('string'); + }); + }); + + test('error_summary truncated to ~200 chars', async () => { + await withFreshAuditDir(() => { + const longMsg = 'x'.repeat(500); + logRerankFailure({ + model: 'zeroentropyai:zerank-2', + reason: 'unknown', + query_hash: 'deadbeef', + doc_count: 0, + error_summary: longMsg, + }); + const events = readRecentRerankFailures(7); + expect(events[0]!.error_summary.length).toBeLessThanOrEqual(200); + }); + }); + + test('multiple failures append to same JSONL file', async () => { + await withFreshAuditDir(() => { + for (let i = 0; i < 5; i++) { + logRerankFailure({ + model: 'zeroentropyai:zerank-2', + reason: 'network', + query_hash: `hash${i}`, + doc_count: 30, + error_summary: `failure ${i}`, + }); + } + const events = readRecentRerankFailures(7); + expect(events.length).toBe(5); + expect(new Set(events.map(e => e.query_hash)).size).toBe(5); + }); + }); + + test('corrupt JSONL rows skipped silently', async () => { + await withFreshAuditDir((tmpDir) => { + const filename = computeRerankAuditFilename(); + const filepath = path.join(tmpDir, filename); + // First write a valid row, then garbage, then another valid row. + logRerankFailure({ + model: 'zeroentropyai:zerank-2', + reason: 'timeout', + query_hash: 'good1', + doc_count: 30, + error_summary: 'ok 1', + }); + fs.appendFileSync(filepath, 'not valid json\n'); + fs.appendFileSync(filepath, '{"partial":\n'); + logRerankFailure({ + model: 'zeroentropyai:zerank-2', + reason: 'timeout', + query_hash: 'good2', + doc_count: 30, + error_summary: 'ok 2', + }); + const events = readRecentRerankFailures(7); + expect(events.length).toBe(2); + expect(events.map(e => e.query_hash)).toEqual(['good1', 'good2']); + }); + }); + + test('missing audit dir → readRecentRerankFailures returns []', async () => { + // Point at a never-existing path; readRecentRerankFailures should + // skip the readFileSync error and return []. + const tmpRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'gbrain-rerank-empty-')); + try { + await withEnv({ GBRAIN_AUDIT_DIR: path.join(tmpRoot, 'nonexistent') }, async () => { + const events = readRecentRerankFailures(7); + expect(events).toEqual([]); + }); + } finally { + try { fs.rmSync(tmpRoot, { recursive: true, force: true }); } catch { /* ignore */ } + } + }); +}); + +describe('CDX2-F22 — logRerankSuccess MUST NOT exist', () => { + test('module does not export logRerankSuccess', async () => { + const mod: any = await import('../src/core/rerank-audit.ts'); + expect(mod.logRerankSuccess).toBeUndefined(); + }); +}); + +describe('ISO-week filename rotation', () => { + test('filename format is rerank-failures-YYYY-Www.jsonl', () => { + const filename = computeRerankAuditFilename(new Date('2026-05-14T00:00:00Z')); + expect(filename).toMatch(/^rerank-failures-\d{4}-W\d{2}\.jsonl$/); + }); + + test('readRecentRerankFailures walks current + previous week', async () => { + await withFreshAuditDir((tmpDir) => { + // Write a row in the "current" week's file. + logRerankFailure({ + model: 'zeroentropyai:zerank-2', + reason: 'auth', + query_hash: 'now', + doc_count: 1, + error_summary: 'recent', + }); + // Synthesize a "last week" row to confirm both files are walked. + const lastWeek = new Date(Date.now() - 7 * 86400000); + const lastWeekName = computeRerankAuditFilename(lastWeek); + const lastWeekPath = path.join(tmpDir, lastWeekName); + fs.appendFileSync( + lastWeekPath, + JSON.stringify({ + ts: new Date(Date.now() - 3 * 86400000).toISOString(), + model: 'zeroentropyai:zerank-2', + reason: 'auth', + query_hash: 'old', + doc_count: 1, + error_summary: 'older', + severity: 'warn', + }) + '\n', + ); + const events = readRecentRerankFailures(7); + expect(events.length).toBe(2); + expect(new Set(events.map(e => e.query_hash))).toEqual(new Set(['now', 'old'])); + }); + }); +}); diff --git a/test/search-mode.test.ts b/test/search-mode.test.ts index b64233966..d71fba313 100644 --- a/test/search-mode.test.ts +++ b/test/search-mode.test.ts @@ -35,7 +35,9 @@ describe('SEARCH_MODES + MODE_BUNDLES canonical shape', () => { expect(Object.isFrozen(MODE_BUNDLES.tokenmax)).toBe(true); }); - // The 3x7 cell-by-cell assertion. The methodology doc cites these. + // The cell-by-cell assertion. The methodology doc cites these. + // v0.35.0.0+ extended with 5 reranker fields. tokenmax flips reranker on; + // conservative + balanced keep it off until eval data backs a change. test('conservative bundle values are canonical', () => { expect(MODE_BUNDLES.conservative).toEqual({ cache_enabled: true, @@ -45,6 +47,11 @@ describe('SEARCH_MODES + MODE_BUNDLES canonical shape', () => { tokenBudget: 4000, expansion: false, searchLimit: 10, + reranker_enabled: false, + reranker_model: 'zeroentropyai:zerank-2', + reranker_top_n_in: 30, + reranker_top_n_out: null, + reranker_timeout_ms: 5000, }); }); @@ -57,6 +64,11 @@ describe('SEARCH_MODES + MODE_BUNDLES canonical shape', () => { tokenBudget: 12000, expansion: false, searchLimit: 25, + reranker_enabled: false, + reranker_model: 'zeroentropyai:zerank-2', + reranker_top_n_in: 30, + reranker_top_n_out: null, + reranker_timeout_ms: 5000, }); }); @@ -69,6 +81,11 @@ describe('SEARCH_MODES + MODE_BUNDLES canonical shape', () => { tokenBudget: undefined, expansion: true, searchLimit: 50, + reranker_enabled: true, + reranker_model: 'zeroentropyai:zerank-2', + reranker_top_n_in: 30, + reranker_top_n_out: null, + reranker_timeout_ms: 5000, }); }); @@ -247,7 +264,10 @@ describe('knobsHash determinism + cross-mode separation (CDX-4)', () => { }); test('KNOBS_HASH_VERSION constant exposed for migrations to bump on schema change', () => { - expect(KNOBS_HASH_VERSION).toBe(1); + // v0.35.0.0+ bumped 1→2 to fold reranker fields into the cache key. + // CDX2-F14: a timeout change from 5s to 100ms changes search behavior + // (more fail-opens) so stale cache rows must invalidate. + expect(KNOBS_HASH_VERSION).toBe(2); }); }); diff --git a/test/search/hybrid-reranker-integration.test.ts b/test/search/hybrid-reranker-integration.test.ts new file mode 100644 index 000000000..a63a4b7b1 --- /dev/null +++ b/test/search/hybrid-reranker-integration.test.ts @@ -0,0 +1,224 @@ +/** + * v0.35.0.0 — hybridSearch ↔ applyReranker integration tests. + * + * Drives bare hybridSearch (NOT the cached wrapper — that adds an embed + * call we don't want here) against PGLite with a stubbed rerankerFn so + * we can pin: + * + * - Reranker fires when opts.reranker.enabled=true and reorders the + * candidate pool. + * - Reranker does NOT fire when opts.reranker.enabled=false. + * - Tail beyond topNIn is preserved in its original RRF order. + * - Cache hit path stores the reranked order (CDX2-F15 — cached rows + * are final reranked results, not pre-rerank candidates). + * + * No API keys needed; embedding is stubbed via __setEmbedTransportForTests. + * The reranker is stubbed via opts.reranker.rerankerFn so we never call + * gateway.rerank. + */ + +import { afterAll, beforeAll, describe, expect, test } from 'bun:test'; +import { PGLiteEngine } from '../../src/core/pglite-engine.ts'; +import { hybridSearch } from '../../src/core/search/hybrid.ts'; +import { + configureGateway, + resetGateway, + __setEmbedTransportForTests, +} from '../../src/core/ai/gateway.ts'; +import type { PageInput, SearchOpts } from '../../src/core/types.ts'; +import type { RerankInput, RerankResult } from '../../src/core/ai/gateway.ts'; + +let engine: PGLiteEngine; + +const DIMS = 1536; // gateway default embedding dim +const FAKE_EMB = Array.from({ length: DIMS }, (_, j) => (j === 0 ? 1 : 0.01)); + +function stubEmbeddings(): void { + __setEmbedTransportForTests(async (args: any) => ({ + embeddings: args.values.map(() => FAKE_EMB), + }) as any); +} + +beforeAll(async () => { + engine = new PGLiteEngine(); + await engine.connect({}); + await engine.initSchema(); + + // Seed pages whose content includes a shared keyword so the keyword + // path will match and produce a candidate pool of 4+ items. putPage + // alone doesn't populate content_chunks (the table searchKeyword + // queries) — upsertChunks does that, and we manually seed it here + // so keyword search has rows to find without needing the full + // chunker + embed pipeline. + const pages: Array<[string, PageInput, string]> = [ + ['notes/alpha', { type: 'note', title: 'Alpha Note', compiled_truth: 'alpha keyword content one' }, 'alpha keyword content one chunk'], + ['notes/beta', { type: 'note', title: 'Beta Note', compiled_truth: 'alpha keyword content two' }, 'alpha keyword content two chunk'], + ['notes/gamma', { type: 'note', title: 'Gamma Note', compiled_truth: 'alpha keyword content three' }, 'alpha keyword content three chunk'], + ['notes/delta', { type: 'note', title: 'Delta Note', compiled_truth: 'alpha keyword content four' }, 'alpha keyword content four chunk'], + ]; + for (const [slug, page, chunkText] of pages) { + await engine.putPage(slug, page); + await engine.upsertChunks(slug, [ + { chunk_index: 0, chunk_text: chunkText, chunk_source: 'compiled_truth' }, + ]); + } + + // Configure with sk-test + stubbed embed transport. We DO need the + // gateway available (env set + transport stubbed) so hybridSearch + // takes the main RRF path — the keyword-only fallback at ~hybrid.ts:409 + // early-returns BEFORE applyReranker, so a setup that lacks embedding + // would never exercise the reranker integration. + // + // searchVector returns empty lists because chunks have NULL embeddings; + // that's fine — vectorLists is `[[]]` (length 1, not 0), so the + // keyword-only branch is skipped and the main path runs RRF + dedup + + // reranker + budget. + configureGateway({ + embedding_model: 'openai:text-embedding-3-large', + embedding_dimensions: DIMS, + env: { OPENAI_API_KEY: 'sk-test' }, + }); + stubEmbeddings(); +}); + +afterAll(async () => { + __setEmbedTransportForTests(null); + resetGateway(); + await engine.disconnect(); +}); + +describe('hybridSearch — reranker disabled (pass-through)', () => { + test('opts.reranker undefined: reranker does NOT fire', async () => { + let called = 0; + const opts: SearchOpts = { + limit: 10, + reranker: { + enabled: false, + topNIn: 30, + topNOut: null, + rerankerFn: async () => { called++; return []; }, + }, + }; + const out = await hybridSearch(engine, 'alpha', opts); + expect(out.length).toBeGreaterThan(0); + expect(called).toBe(0); + }); +}); + +describe('hybridSearch — reranker enabled (reorder)', () => { + test('rerankerFn receives a non-empty document list', async () => { + let receivedDocs: string[] = []; + const opts: SearchOpts = { + limit: 10, + reranker: { + enabled: true, + topNIn: 30, + topNOut: null, + rerankerFn: async (input: RerankInput): Promise => { + receivedDocs = input.documents; + return input.documents.map((_, i) => ({ index: i, relevanceScore: 1 - i * 0.1 })); + }, + }, + }; + const out = await hybridSearch(engine, 'alpha keyword', opts); + expect(out.length).toBeGreaterThan(0); + expect(receivedDocs.length).toBeGreaterThan(0); + expect(receivedDocs.length).toBe(out.length); // when topNIn >= pool, all sent + }); + + test('rerankerFn output controls final order (reverse the RRF order)', async () => { + let originalOrder: string[] = []; + const opts: SearchOpts = { + limit: 10, + reranker: { + enabled: true, + topNIn: 30, + topNOut: null, + // Reverse the order: last-in becomes first-out. + rerankerFn: async (input: RerankInput): Promise => { + return input.documents.map((_, i) => ({ + index: input.documents.length - 1 - i, + relevanceScore: 1 - i * 0.1, + })); + }, + }, + }; + // First run: collect the original RRF order (rerankerFn off). + const baseline = await hybridSearch(engine, 'alpha keyword', { + ...opts, + reranker: { ...opts.reranker!, enabled: false }, + }); + originalOrder = baseline.map(r => r.slug); + + // Second run: reranker reverses. + const reranked = await hybridSearch(engine, 'alpha keyword', opts); + const rerankedOrder = reranked.map(r => r.slug); + + expect(rerankedOrder).toEqual([...originalOrder].reverse()); + }); + + test('un-reranked tail preserves RRF order (topNIn=2 with N candidates)', async () => { + // First baseline. PGLite's hybrid path + dedup may collapse some + // chunks; we need at least 3 candidates (2 reranked head + 1 + // preserved tail) for this assertion to be meaningful. + const baseline = await hybridSearch(engine, 'alpha keyword', { limit: 10 }); + const baselineOrder = baseline.map(r => r.slug); + expect(baselineOrder.length).toBeGreaterThanOrEqual(3); + + // Now rerank only the top 2 (swap them); the tail (indices 2..N-1) + // must keep its baseline order. + const reranked = await hybridSearch(engine, 'alpha keyword', { + limit: 10, + reranker: { + enabled: true, + topNIn: 2, + topNOut: null, + rerankerFn: async (input: RerankInput): Promise => [ + { index: 1, relevanceScore: 0.99 }, + { index: 0, relevanceScore: 0.5 }, + ], + }, + }); + const rerankedOrder = reranked.map(r => r.slug); + + // Head reordered: positions 0 and 1 swapped. + expect(rerankedOrder[0]).toBe(baselineOrder[1]); + expect(rerankedOrder[1]).toBe(baselineOrder[0]); + // Tail unchanged. + expect(rerankedOrder.slice(2)).toEqual(baselineOrder.slice(2)); + }); + + test('rerank score stamps onto results', async () => { + const opts: SearchOpts = { + limit: 10, + reranker: { + enabled: true, + topNIn: 30, + topNOut: null, + rerankerFn: async (input: RerankInput): Promise => + input.documents.map((_, i) => ({ index: i, relevanceScore: 0.5 - i * 0.05 })), + }, + }; + const out = await hybridSearch(engine, 'alpha keyword', opts); + expect(out.length).toBeGreaterThan(0); + // First result has the highest reranker score (0.5). + expect((out[0] as any).rerank_score).toBe(0.5); + }); +}); + +describe('hybridSearch — fail-open contract end-to-end', () => { + test('rerankerFn throws → results still come back (RRF order preserved)', async () => { + const baseline = await hybridSearch(engine, 'alpha keyword', { limit: 10 }); + const reranked = await hybridSearch(engine, 'alpha keyword', { + limit: 10, + reranker: { + enabled: true, + topNIn: 30, + topNOut: null, + rerankerFn: async () => { throw new Error('upstream down'); }, + }, + }); + // Same items, same order — applyReranker fail-open. + expect(reranked.map(r => r.slug)).toEqual(baseline.map(r => r.slug)); + }); +}); diff --git a/test/search/knobs-hash-reranker.test.ts b/test/search/knobs-hash-reranker.test.ts new file mode 100644 index 000000000..4853a3663 --- /dev/null +++ b/test/search/knobs-hash-reranker.test.ts @@ -0,0 +1,152 @@ +/** + * v0.35.0.0 — knobsHash reranker-field participation tests. + * + * Pins: + * - KNOBS_HASH_VERSION === 2 (bumped from 1; CDX1-F14). + * - All 5 new reranker fields participate in the hash: + * reranker_enabled, reranker_model, reranker_top_n_in, + * reranker_top_n_out, reranker_timeout_ms. + * Each one flipping changes the hash → no two reranker configs share + * a cache row. + * - top_n_out=null vs unset shows up as 'none' in the hash (no NaN). + * - Append-only convention (CDX2-F13): the existing 9 fields hash + * identically under v=2 as they did under v=1 for the same input + * when the reranker section is held constant. Reordering them + * would silently rebuild the hash for every existing row. + * - Mid-deploy invariant (CDX2-F12): the v=1 prefix in the hash input + * differs from the v=2 prefix; a tokenmax v=1 process and a v=2 + * process produce distinct row IDs for the same (source_id, query). + */ + +import { describe, test, expect } from 'bun:test'; +import { + knobsHash, + KNOBS_HASH_VERSION, + resolveSearchMode, + MODE_BUNDLES, + type ResolvedSearchKnobs, +} from '../../src/core/search/mode.ts'; + +/** Build a baseline resolved knob set with all reranker fields filled. */ +function baseKnobs(): ResolvedSearchKnobs { + return { + ...MODE_BUNDLES.balanced, + reranker_enabled: false, + reranker_model: 'zeroentropyai:zerank-2', + reranker_top_n_in: 30, + reranker_top_n_out: null, + reranker_timeout_ms: 5000, + resolved_mode: 'balanced', + mode_valid: true, + }; +} + +describe('KNOBS_HASH_VERSION + version invariants', () => { + test('version is 2 (CDX1-F14: bumped from 1 to fold reranker fields in)', () => { + expect(KNOBS_HASH_VERSION).toBe(2); + }); + + test('hash is 16 hex chars regardless of reranker config', () => { + const a = knobsHash(baseKnobs()); + const b = knobsHash({ ...baseKnobs(), reranker_enabled: true }); + expect(a).toMatch(/^[0-9a-f]{16}$/); + expect(b).toMatch(/^[0-9a-f]{16}$/); + }); +}); + +describe('Each reranker field flips the hash (cache-row separation)', () => { + test('reranker_enabled false vs true → different hash', () => { + const off = knobsHash({ ...baseKnobs(), reranker_enabled: false }); + const on = knobsHash({ ...baseKnobs(), reranker_enabled: true }); + expect(off).not.toBe(on); + }); + + test('reranker_model differs → different hash', () => { + const z2 = knobsHash({ ...baseKnobs(), reranker_model: 'zeroentropyai:zerank-2' }); + const z1 = knobsHash({ ...baseKnobs(), reranker_model: 'zeroentropyai:zerank-1' }); + const z1s = knobsHash({ ...baseKnobs(), reranker_model: 'zeroentropyai:zerank-1-small' }); + expect(new Set([z2, z1, z1s]).size).toBe(3); + }); + + test('reranker_top_n_in differs → different hash', () => { + const a = knobsHash({ ...baseKnobs(), reranker_top_n_in: 30 }); + const b = knobsHash({ ...baseKnobs(), reranker_top_n_in: 50 }); + expect(a).not.toBe(b); + }); + + test('reranker_top_n_out null vs 10 → different hash', () => { + const noTrunc = knobsHash({ ...baseKnobs(), reranker_top_n_out: null }); + const trunc10 = knobsHash({ ...baseKnobs(), reranker_top_n_out: 10 }); + expect(noTrunc).not.toBe(trunc10); + }); + + test('reranker_timeout_ms differs → different hash (CDX2-F14)', () => { + // CDX2-F14: a timeout change (5s → 100ms) changes search behavior + // (more fail-opens) so stale cache rows must invalidate. Without + // this field in parts[], the rows would silently match. + const t5 = knobsHash({ ...baseKnobs(), reranker_timeout_ms: 5000 }); + const t1 = knobsHash({ ...baseKnobs(), reranker_timeout_ms: 1000 }); + expect(t5).not.toBe(t1); + }); +}); + +describe('mid-deploy invariant (CDX2-F12)', () => { + test('tokenmax-with-reranker vs tokenmax-without-reranker → distinct hashes', () => { + // tokenmax mode bundle has reranker on. An operator who flips it off + // via `gbrain config set search.reranker.enabled false` produces a + // different cache row, not a shared one. + const tokenmaxOn = knobsHash(resolveSearchMode({ mode: 'tokenmax' })); + const tokenmaxOff = knobsHash(resolveSearchMode({ + mode: 'tokenmax', + overrides: { reranker_enabled: false }, + })); + expect(tokenmaxOn).not.toBe(tokenmaxOff); + }); + + test('conservative vs balanced vs tokenmax → 3 distinct hashes', () => { + const c = knobsHash(resolveSearchMode({ mode: 'conservative' })); + const b = knobsHash(resolveSearchMode({ mode: 'balanced' })); + const t = knobsHash(resolveSearchMode({ mode: 'tokenmax' })); + expect(new Set([c, b, t]).size).toBe(3); + }); +}); + +describe('determinism + stability', () => { + test('same input → same hash (re-call)', () => { + const k = baseKnobs(); + expect(knobsHash(k)).toBe(knobsHash(k)); + }); + + test('same mode bundle → same hash across resolveSearchMode calls', () => { + const a = knobsHash(resolveSearchMode({ mode: 'balanced' })); + const b = knobsHash(resolveSearchMode({ mode: 'balanced' })); + expect(a).toBe(b); + }); + + test('top_n_out=null renders as "none" in parts[] (no NaN)', () => { + // CDX2-F15 + F16 + F14 collide here. The parts[] line is + // `rro=${knobs.reranker_top_n_out ?? 'none'}` — null must produce a + // stable string token, never `NaN` or `null`. + const h1 = knobsHash({ ...baseKnobs(), reranker_top_n_out: null }); + const h2 = knobsHash({ ...baseKnobs(), reranker_top_n_out: null }); + expect(h1).toBe(h2); + expect(h1).toMatch(/^[0-9a-f]{16}$/); + }); +}); + +describe('append-only convention (CDX2-F13)', () => { + test('parts[] order in source: reranker fields appear AFTER the existing 9', async () => { + const src = await Bun.file( + new URL('../../src/core/search/mode.ts', import.meta.url), + ).text(); + // Locate the parts[] declaration. The existing 9 fields end with + // `lim=${knobs.searchLimit}`. The 5 new fields must appear AFTER + // that line. Reordering would silently rebuild the hash for every + // existing v=2 cache row. + const limIdx = src.indexOf('lim=${knobs.searchLimit}'); + const rrIdx = src.indexOf('rr=${knobs.reranker_enabled'); + expect(limIdx).toBeGreaterThan(0); + expect(rrIdx).toBeGreaterThan(0); + expect(rrIdx).toBeGreaterThan(limIdx); + }); +}); diff --git a/test/search/rerank.test.ts b/test/search/rerank.test.ts new file mode 100644 index 000000000..1db2e9ed9 --- /dev/null +++ b/test/search/rerank.test.ts @@ -0,0 +1,205 @@ +/** + * v0.35.0.0+ applyReranker tests. + * + * Pins: + * - reorder by reranker score (the happy path) + * - preserve un-reranked tail order (recall protection) + * - fail-open on every RerankError reason (audit-logged, results pass through) + * - topNOut=null preserves full length (CDX2-F16 — semantic distinction + * between null and undefined) + * - empty input passes through + * - rerankerFn test seam used over gateway.rerank + */ + +import { describe, test, expect, beforeAll } from 'bun:test'; +import { applyReranker, type RerankerOpts } from '../../src/core/search/rerank.ts'; +import { RerankError, type RerankResult } from '../../src/core/ai/gateway.ts'; +import type { SearchResult } from '../../src/core/types.ts'; + +function makeResult(slug: string, score: number, chunk: string): SearchResult { + return { + slug, + page_id: 0, + title: slug, + type: 'note', + chunk_text: chunk, + chunk_source: 'compiled_truth', + chunk_id: 0, + chunk_index: 0, + score, + stale: false, + }; +} + +// Setup: gateway must be configured so the rerank-audit logger doesn't +// trip on missing env. We can call configureGateway with a minimal stub. +beforeAll(async () => { + const { configureGateway } = await import('../../src/core/ai/gateway.ts'); + configureGateway({ + env: { ZEROENTROPY_API_KEY: 'test-key' }, + }); +}); + +describe('applyReranker — happy path', () => { + test('reorders top-N by reranker relevance score', async () => { + const results = [ + makeResult('a', 1.0, 'doc a'), + makeResult('b', 0.9, 'doc b'), + makeResult('c', 0.8, 'doc c'), + ]; + const opts: RerankerOpts = { + enabled: true, + topNIn: 3, + topNOut: null, + rerankerFn: async () => [ + { index: 2, relevanceScore: 0.99 }, // c wins + { index: 0, relevanceScore: 0.5 }, // a second + { index: 1, relevanceScore: 0.1 }, // b last + ], + }; + const out = await applyReranker('q', results, opts); + expect(out.map(r => r.slug)).toEqual(['c', 'a', 'b']); + }); + + test('un-reranked tail preserves original RRF order', async () => { + const results = [ + makeResult('head1', 1.0, 'h1'), + makeResult('head2', 0.9, 'h2'), + makeResult('tail1', 0.5, 't1'), + makeResult('tail2', 0.4, 't2'), + ]; + const opts: RerankerOpts = { + enabled: true, + topNIn: 2, + topNOut: null, + rerankerFn: async () => [ + { index: 1, relevanceScore: 0.99 }, + { index: 0, relevanceScore: 0.5 }, + ], + }; + const out = await applyReranker('q', results, opts); + // Head reordered: head2 first, head1 second. Tail unchanged: tail1, tail2. + expect(out.map(r => r.slug)).toEqual(['head2', 'head1', 'tail1', 'tail2']); + }); + + test('stamps rerank_score onto reordered items', async () => { + const results = [makeResult('a', 1.0, 'a')]; + const opts: RerankerOpts = { + enabled: true, + topNIn: 1, + topNOut: null, + rerankerFn: async () => [{ index: 0, relevanceScore: 0.42 }], + }; + const out = await applyReranker('q', results, opts); + expect((out[0] as any).rerank_score).toBe(0.42); + }); +}); + +describe('applyReranker — CDX2-F16 null vs undefined semantics', () => { + test('topNOut=null preserves full reordered list', async () => { + const results = Array.from({ length: 50 }, (_, i) => makeResult(`p${i}`, 1 - i * 0.01, `c${i}`)); + const opts: RerankerOpts = { + enabled: true, + topNIn: 30, + topNOut: null, + rerankerFn: async (input) => input.documents.map((_, i) => ({ index: i, relevanceScore: 1 - i * 0.01 })), + }; + const out = await applyReranker('q', results, opts); + // tokenmax mode has searchLimit=50 — null must preserve all 50. + expect(out.length).toBe(50); + }); + + test('topNOut=10 truncates to 10', async () => { + const results = Array.from({ length: 30 }, (_, i) => makeResult(`p${i}`, 1 - i * 0.01, `c${i}`)); + const opts: RerankerOpts = { + enabled: true, + topNIn: 30, + topNOut: 10, + rerankerFn: async (input) => input.documents.map((_, i) => ({ index: i, relevanceScore: 1 - i * 0.01 })), + }; + const out = await applyReranker('q', results, opts); + expect(out.length).toBe(10); + }); +}); + +describe('applyReranker — fail-open on every RerankError reason', () => { + test.each([ + 'auth' as const, + 'rate_limit' as const, + 'network' as const, + 'timeout' as const, + 'payload_too_large' as const, + 'unknown' as const, + ])('fail-open on RerankError reason=%s', async (reason) => { + const results = [ + makeResult('a', 1.0, 'a'), + makeResult('b', 0.5, 'b'), + ]; + const opts: RerankerOpts = { + enabled: true, + topNIn: 2, + topNOut: null, + rerankerFn: async () => { + throw new RerankError('forced', reason); + }, + }; + // Must not throw; must return input unchanged. + const out = await applyReranker('q', results, opts); + expect(out).toEqual(results); + }); + + test('fail-open on non-RerankError throw too', async () => { + const results = [makeResult('a', 1.0, 'a')]; + const opts: RerankerOpts = { + enabled: true, + topNIn: 1, + topNOut: null, + rerankerFn: async () => { + throw new Error('arbitrary'); + }, + }; + const out = await applyReranker('q', results, opts); + expect(out).toEqual(results); + }); + + test('fail-open on malformed reranker response (empty results array)', async () => { + const results = [makeResult('a', 1.0, 'a')]; + const opts: RerankerOpts = { + enabled: true, + topNIn: 1, + topNOut: null, + rerankerFn: async () => [], + }; + const out = await applyReranker('q', results, opts); + expect(out).toEqual(results); + }); +}); + +describe('applyReranker — pass-through cases', () => { + test('enabled=false passes through unchanged (no rerankerFn call)', async () => { + const results = [makeResult('a', 1.0, 'a'), makeResult('b', 0.5, 'b')]; + let called = false; + const opts: RerankerOpts = { + enabled: false, + topNIn: 30, + topNOut: null, + rerankerFn: async () => { called = true; return []; }, + }; + const out = await applyReranker('q', results, opts); + expect(out).toEqual(results); + expect(called).toBe(false); + }); + + test('empty results passes through immediately', async () => { + let called = false; + const opts: RerankerOpts = { + enabled: true, + topNIn: 30, + topNOut: null, + rerankerFn: async () => { called = true; return []; }, + }; + const out = await applyReranker('q', [], opts); + expect(out).toEqual([]); + expect(called).toBe(false); + }); +});