mirror of
https://github.com/garrytan/gbrain.git
synced 2026-07-30 11:22:34 +00:00
* docs(designs): 2026-05 embedder shootout eval plan Adds docs/designs/2026_05_EVAL_PLAN.md — the approved plan + 6 Conductor session briefs for the OpenAI vs Voyage vs ZeroEntropy embedder comparison. Why: produce a publishable comparison report for v0.35.x release notes pinning "which embedder wins, and does zerank-2 carry the win for ZeroEntropy" against public LongMemEval + in-house BrainBench. Each session brief is self-contained — repo, branch, commits, verify, ship, deliverable, hand-off. Stewardable one section per Conductor session. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com> * feat(pricing): add voyage-4-large + zembed-1 to EMBEDDING_PRICING v0.35.0.0 shipped ZeroEntropy zembed-1 + zerank-2 reranker support and expanded the Voyage allow-list to include voyage-4-large. The pricing table missed both, so `gbrain upgrade`'s post-upgrade reembed prompt silently fell back to "estimate unavailable" for users on these models. - voyage:voyage-4-large @ $0.18/MTok (same as voyage-3-large) - zeroentropyai:zembed-1 @ $0.05/MTok New test file pins both entries plus the openai/voyage-3-large baselines, case-insensitive provider matching, bare-model openai-default fallback, table integrity (lowercase providers, finite non-negative prices), and the estimateCostFromChars approximation. 11 cases, 46 expect() calls. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com> * feat(exports): expose gbrain/ai/gateway with canary test Adds ./ai/gateway to the package.json exports map so external eval consumers (notably gbrain-evals, the sibling repo running the embedder shootout in docs/designs/2026_05_EVAL_PLAN.md) can call configureGateway directly to swap embedding providers per cell. Why: pre-v0.35.1.0, gbrain-evals adapters hardcoded gbrain/embedding, which means every retrieval adapter was OpenAI-only. The newly-exposed gateway lets adapters route through Voyage and ZeroEntropy without forking gbrain or duplicating the recipe wiring. - package.json: add "./ai/gateway" -> "./src/core/ai/gateway.ts" - scripts/check-exports-count.sh: bump expected count 17 -> 18 - test/public-exports.test.ts: add canary pinning configureGateway + embed, bump expected count assertion Pre-existing import-resolution failures in this test file (16 on master) are unrelated to this change — they're a longstanding Bun package self-import behavior. The count + EXPECTED_EXPORTS list-match assertions both pass cleanly. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com> * feat(eval): add --resume-from <jsonl> to gbrain eval longmemeval Multi-cell embedder shootouts spend $50+/cell on the gpt-4o judge after gbrain emits hypotheses. A mid-run abort (rate-limit, cost-cap, OS interrupt, SIGKILL) previously meant re-paying the full cell. This flag makes those aborts cheap: re-invoke with --resume-from pointed at the partial JSONL and only the unanswered question_ids re-run. Behavior: - Read question_ids from the file; skip them on this run. - Rows with non-empty hypothesis count as done. - Rows with hypothesis="" AND an error field are NOT skipped (retry case for per-question failures recorded by the existing try/catch). - Corrupt trailing lines (SIGKILL'd writer mid-line) are silently skipped with a stderr warn. - When --resume-from path == --output path, the output emitter opens the file in append mode instead of truncating, so the existing rows survive. - Empty resume case (all questions already done) returns immediately without spinning up the brain or calling the client. New exported helper loadResumeSet() makes the parser unit-testable. 6 new test cases pinning: - File-not-found returns empty set - Well-formed JSONL load - Error-row retry semantics (empty hypothesis + error -> not in set) - Truncated final line recovery - End-to-end resume against the 5-question mini fixture - All-done early-return (stub client must NOT be invoked) All 18 cases in test/eval-longmemeval.test.ts green; bun run typecheck clean. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com> * chore: v0.35.1.0 Bumps VERSION + package.json + CHANGELOG entry for the embedder-shootout prereq release. Three additive changes from the prior 4 commits: - pricing: voyage-4-large + zembed-1 entries - exports: gbrain/ai/gateway is now public - eval: gbrain eval longmemeval --resume-from <jsonl> Each commit on this branch is independently bisect-friendly and CI-green; the CHANGELOG entry is the user-facing rollup. No migrations, no breaking changes — the gateway export expands the surface, the resume-from flag is additive, the pricing patch only changes "estimate unavailable" -> a real dollar figure for two specific models. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com> --------- Co-authored-by: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
72 lines
3.0 KiB
TypeScript
72 lines
3.0 KiB
TypeScript
/**
|
|
* v0.32.7 CJK wave — embedding model pricing lookup table.
|
|
*
|
|
* Sibling to `anthropic-pricing.ts`. Used by `gbrain upgrade`'s post-upgrade
|
|
* cost-estimate prompt so users with large brains see a dollar figure
|
|
* before the chunker-version sweep re-embeds.
|
|
*
|
|
* Prices in USD per 1M tokens. Numbers as of 2026-05-11. Verify alongside
|
|
* the Anthropic-pricing refresh cycle; drift here produces estimates
|
|
* that mislead operators.
|
|
*
|
|
* Codex outside-voice C3 fold: non-OpenAI embedding providers (Voyage,
|
|
* Hunyuan, Dashscope, etc.) return UNKNOWN_PROVIDER from `lookupPrice`
|
|
* so the cost-estimate prompt can fall back to a "estimate unavailable
|
|
* for <provider>; press Ctrl-C in 10s to abort" message rather than
|
|
* fabricate numbers.
|
|
*/
|
|
|
|
export interface EmbeddingPricing {
|
|
/** USD per 1M tokens (embedding cost; embeddings have no separate output rate). */
|
|
pricePerMTok: number;
|
|
}
|
|
|
|
/**
|
|
* `provider:model` keyed pricing. The colon-separated key matches
|
|
* gateway model strings (e.g. 'openai:text-embedding-3-large').
|
|
*/
|
|
export const EMBEDDING_PRICING: Record<string, EmbeddingPricing> = {
|
|
// OpenAI (https://openai.com/api/pricing/, verified 2026-05-11)
|
|
'openai:text-embedding-3-large': { pricePerMTok: 0.13 },
|
|
'openai:text-embedding-3-small': { pricePerMTok: 0.02 },
|
|
// Legacy OpenAI ada (still common in older brains)
|
|
'openai:text-embedding-ada-002': { pricePerMTok: 0.10 },
|
|
// Voyage (https://www.voyageai.com/pricing)
|
|
'voyage:voyage-3-large': { pricePerMTok: 0.18 },
|
|
'voyage:voyage-3': { pricePerMTok: 0.06 },
|
|
'voyage:voyage-4-large': { pricePerMTok: 0.18 },
|
|
// ZeroEntropy (https://zeroentropy.dev/pricing — zembed-1)
|
|
'zeroentropyai:zembed-1': { pricePerMTok: 0.05 },
|
|
};
|
|
|
|
export type PriceLookupResult =
|
|
| { kind: 'known'; pricePerMTok: number; key: string }
|
|
| { kind: 'unknown'; provider: string; model: string };
|
|
|
|
/**
|
|
* Resolve a model string into a price-per-1M-tokens. Accepts both
|
|
* `provider:model` and bare `model` forms (bare assumes openai).
|
|
*/
|
|
export function lookupEmbeddingPrice(modelString: string): PriceLookupResult {
|
|
const [providerRaw, modelRaw] = modelString.includes(':')
|
|
? modelString.split(':', 2)
|
|
: ['openai', modelString];
|
|
const provider = providerRaw.trim().toLowerCase();
|
|
const model = (modelRaw ?? '').trim();
|
|
const key = `${provider}:${model}`;
|
|
const hit = EMBEDDING_PRICING[key];
|
|
if (hit) return { kind: 'known', pricePerMTok: hit.pricePerMTok, key };
|
|
return { kind: 'unknown', provider, model };
|
|
}
|
|
|
|
/**
|
|
* Estimate USD cost for embedding `charCount` characters. Uses
|
|
* 3.5 chars/token as the OpenAI tiktoken-shaped approximation for English;
|
|
* CJK-heavy brains will under-estimate by ~2x (one char ≈ one token), but
|
|
* we'd rather under-estimate than spook users with a 10x worst-case figure.
|
|
*/
|
|
export function estimateCostFromChars(charCount: number, pricePerMTok: number): number {
|
|
const tokens = Math.ceil(charCount / 3.5);
|
|
return (tokens / 1_000_000) * pricePerMTok;
|
|
}
|