mirror of
https://github.com/garrytan/gbrain.git
synced 2026-07-27 22:15:33 +00:00
fix(cost): embedding cost preview uses configured model rate, not hardcoded OpenAI
The sync --all cost gate computed spend from a hardcoded EMBEDDING_COST_PER_1K_TOKENS = 0.00013 (OpenAI text-embedding-3-large) and labeled the preview with the back-compat EMBEDDING_MODEL constant, regardless of the actually-configured embedding model. A brain running a cheaper model (e.g. zeroentropyai:zembed-1 @ $0.05/Mtok) saw a preview that named the wrong provider and over-stated spend ~2.6x ($337 vs $130 on a 2.6B-token corpus). estimateEmbeddingCostUsd now resolves the live model via the gateway and prices it through embedding-pricing.ts (the existing per-provider:model table), falling back to the OpenAI rate only when the gateway is unconfigured (unit-test context) or the model is unknown. sync.ts surfaces the real model name in the preview message and JSON. Regression test pins model-aware pricing: openai 3-large vs zembed-1 must produce materially different previews; collapsing both to the OpenAI number fails the assertion.
This commit is contained in:
@@ -17,7 +17,7 @@ import {
|
|||||||
formatCodeBreakdown,
|
formatCodeBreakdown,
|
||||||
} from '../core/sync.ts';
|
} from '../core/sync.ts';
|
||||||
import { estimateTokens, CHUNKER_VERSION } from '../core/chunkers/code.ts';
|
import { estimateTokens, CHUNKER_VERSION } from '../core/chunkers/code.ts';
|
||||||
import { EMBEDDING_MODEL, estimateEmbeddingCostUsd } from '../core/embedding.ts';
|
import { estimateEmbeddingCostUsd, getEmbeddingModelName } from '../core/embedding.ts';
|
||||||
import { errorFor, serializeError } from '../core/errors.ts';
|
import { errorFor, serializeError } from '../core/errors.ts';
|
||||||
import type { SyncManifest } from '../core/sync.ts';
|
import type { SyncManifest } from '../core/sync.ts';
|
||||||
import { createProgress } from '../core/progress.ts';
|
import { createProgress } from '../core/progress.ts';
|
||||||
@@ -2253,13 +2253,14 @@ See also:
|
|||||||
if (!noEmbed) {
|
if (!noEmbed) {
|
||||||
const preview = estimateSyncAllCost(sources);
|
const preview = estimateSyncAllCost(sources);
|
||||||
const costUsd = estimateEmbeddingCostUsd(preview.totalTokens);
|
const costUsd = estimateEmbeddingCostUsd(preview.totalTokens);
|
||||||
|
const embeddingModelName = getEmbeddingModelName();
|
||||||
const previewMsg =
|
const previewMsg =
|
||||||
`sync --all preview: ${preview.totalFiles} files across ${preview.activeSources} source(s), ` +
|
`sync --all preview: ${preview.totalFiles} files across ${preview.activeSources} source(s), ` +
|
||||||
`~${preview.totalTokens.toLocaleString()} tokens, est. $${costUsd.toFixed(2)} on ${EMBEDDING_MODEL}.`;
|
`~${preview.totalTokens.toLocaleString()} tokens, est. $${costUsd.toFixed(2)} on ${embeddingModelName}.`;
|
||||||
|
|
||||||
if (dryRun) {
|
if (dryRun) {
|
||||||
if (jsonOut) {
|
if (jsonOut) {
|
||||||
console.log(JSON.stringify({ status: 'dry_run', preview, costUsd, model: EMBEDDING_MODEL }));
|
console.log(JSON.stringify({ status: 'dry_run', preview, costUsd, model: getEmbeddingModelName() }));
|
||||||
} else {
|
} else {
|
||||||
console.log(previewMsg);
|
console.log(previewMsg);
|
||||||
console.log('--dry-run: exit without syncing.');
|
console.log('--dry-run: exit without syncing.');
|
||||||
@@ -2277,7 +2278,7 @@ See also:
|
|||||||
message: previewMsg,
|
message: previewMsg,
|
||||||
hint: 'Pass --yes to proceed, or --dry-run to see the preview and exit 0.',
|
hint: 'Pass --yes to proceed, or --dry-run to see the preview and exit 0.',
|
||||||
}));
|
}));
|
||||||
console.log(JSON.stringify({ error: envelope, preview, costUsd, model: EMBEDDING_MODEL }));
|
console.log(JSON.stringify({ error: envelope, preview, costUsd, model: getEmbeddingModelName() }));
|
||||||
process.exit(2);
|
process.exit(2);
|
||||||
}
|
}
|
||||||
// Interactive TTY path: prompt [y/N].
|
// Interactive TTY path: prompt [y/N].
|
||||||
|
|||||||
+29
-5
@@ -12,6 +12,7 @@ import {
|
|||||||
getEmbeddingModel as gatewayGetModel,
|
getEmbeddingModel as gatewayGetModel,
|
||||||
getEmbeddingDimensions as gatewayGetDims,
|
getEmbeddingDimensions as gatewayGetDims,
|
||||||
} from './ai/gateway.ts';
|
} from './ai/gateway.ts';
|
||||||
|
import { lookupEmbeddingPrice } from './embedding-pricing.ts';
|
||||||
|
|
||||||
// v0.27.1: re-export multimodal embedding so callers can pull both text and
|
// v0.27.1: re-export multimodal embedding so callers can pull both text and
|
||||||
// image embedding APIs from `src/core/embedding`. import-image-file consumes
|
// image embedding APIs from `src/core/embedding`. import-image-file consumes
|
||||||
@@ -127,13 +128,36 @@ export const EMBEDDING_MODEL = 'text-embedding-3-large';
|
|||||||
export const EMBEDDING_DIMENSIONS = 1536;
|
export const EMBEDDING_DIMENSIONS = 1536;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* USD cost per 1k tokens for text-embedding-3-large. Used by
|
* USD cost per 1k tokens for text-embedding-3-large. Retained for back-compat
|
||||||
* `gbrain sync --all` cost preview and `reindex-code` to surface
|
* with callers/tests that import it directly; new cost math resolves the
|
||||||
* expected spend before accepting expensive operations.
|
* ACTUAL configured model's rate via embedding-pricing.ts instead of assuming
|
||||||
|
* OpenAI. (Hardcoding this rate produced cost previews that named the wrong
|
||||||
|
* provider and over-stated spend ~2.6x when the brain ran on a cheaper model.)
|
||||||
*/
|
*/
|
||||||
export const EMBEDDING_COST_PER_1K_TOKENS = 0.00013;
|
export const EMBEDDING_COST_PER_1K_TOKENS = 0.00013;
|
||||||
|
|
||||||
/** Compute USD cost estimate for embedding `tokens` at current model rate. */
|
/**
|
||||||
|
* Resolve the price-per-1M-tokens for the currently-configured embedding
|
||||||
|
* model. Falls back to the OpenAI text-embedding-3-large rate only when the
|
||||||
|
* model is unknown to the pricing table.
|
||||||
|
*/
|
||||||
|
export function currentEmbeddingPricePerMTok(): number {
|
||||||
|
let modelString: string;
|
||||||
|
try {
|
||||||
|
modelString = gatewayGetModel(); // e.g. 'zeroentropyai:zembed-1'
|
||||||
|
} catch {
|
||||||
|
// Gateway not configured (e.g. unit tests, cost preview before connect).
|
||||||
|
// Fall back to the OpenAI text-embedding-3-large default rate.
|
||||||
|
return 0.13;
|
||||||
|
}
|
||||||
|
const hit = lookupEmbeddingPrice(modelString);
|
||||||
|
return hit.kind === 'known' ? hit.pricePerMTok : 0.13;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Compute USD cost estimate for embedding `tokens` at the CURRENT configured
|
||||||
|
* model's rate (not a hardcoded OpenAI rate).
|
||||||
|
*/
|
||||||
export function estimateEmbeddingCostUsd(tokens: number): number {
|
export function estimateEmbeddingCostUsd(tokens: number): number {
|
||||||
return (tokens / 1000) * EMBEDDING_COST_PER_1K_TOKENS;
|
return (tokens / 1_000_000) * currentEmbeddingPricePerMTok();
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -15,22 +15,48 @@
|
|||||||
|
|
||||||
import { describe, test, expect } from 'bun:test';
|
import { describe, test, expect } from 'bun:test';
|
||||||
import { EMBEDDING_COST_PER_1K_TOKENS, estimateEmbeddingCostUsd } from '../src/core/embedding.ts';
|
import { EMBEDDING_COST_PER_1K_TOKENS, estimateEmbeddingCostUsd } from '../src/core/embedding.ts';
|
||||||
|
import { lookupEmbeddingPrice } from '../src/core/embedding-pricing.ts';
|
||||||
import { estimateTokens } from '../src/core/chunkers/code.ts';
|
import { estimateTokens } from '../src/core/chunkers/code.ts';
|
||||||
|
|
||||||
describe('Layer 8 D1 — embedding cost model', () => {
|
describe('Layer 8 D1 — embedding cost model', () => {
|
||||||
test('EMBEDDING_COST_PER_1K_TOKENS is text-embedding-3-large pricing', () => {
|
test('EMBEDDING_COST_PER_1K_TOKENS back-compat constant is the OpenAI 3-large rate', () => {
|
||||||
// Update this when OpenAI changes text-embedding-3-large pricing.
|
// Retained only for back-compat imports. Live cost math now resolves the
|
||||||
// As of 2026-04-24: $0.00013 / 1k tokens.
|
// CONFIGURED model's rate via embedding-pricing.ts (see model-aware test
|
||||||
|
// below). As of 2026-04-24: $0.00013 / 1k tokens.
|
||||||
expect(EMBEDDING_COST_PER_1K_TOKENS).toBe(0.00013);
|
expect(EMBEDDING_COST_PER_1K_TOKENS).toBe(0.00013);
|
||||||
});
|
});
|
||||||
|
|
||||||
test('estimateEmbeddingCostUsd scales linearly with tokens', () => {
|
test('estimateEmbeddingCostUsd scales linearly (gateway-unconfigured fallback = OpenAI rate)', () => {
|
||||||
|
// With no gateway configured (unit-test context) the estimator falls back
|
||||||
|
// to the OpenAI text-embedding-3-large rate ($0.13/Mtok = $0.00013/1k).
|
||||||
expect(estimateEmbeddingCostUsd(0)).toBe(0);
|
expect(estimateEmbeddingCostUsd(0)).toBe(0);
|
||||||
expect(estimateEmbeddingCostUsd(1000)).toBeCloseTo(0.00013, 5);
|
expect(estimateEmbeddingCostUsd(1000)).toBeCloseTo(0.00013, 5);
|
||||||
expect(estimateEmbeddingCostUsd(10_000)).toBeCloseTo(0.0013, 4);
|
expect(estimateEmbeddingCostUsd(10_000)).toBeCloseTo(0.0013, 4);
|
||||||
expect(estimateEmbeddingCostUsd(1_000_000)).toBeCloseTo(0.13, 4);
|
expect(estimateEmbeddingCostUsd(1_000_000)).toBeCloseTo(0.13, 4);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
test('cost preview uses the CONFIGURED model rate, not a hardcoded OpenAI rate', () => {
|
||||||
|
// Regression: the cost gate previously hardcoded $0.00013/1k (OpenAI
|
||||||
|
// text-embedding-3-large) regardless of the configured embedding model,
|
||||||
|
// so a brain on a cheaper model (e.g. zeroentropyai:zembed-1 @ $0.05/Mtok)
|
||||||
|
// saw a preview that named the wrong provider and over-stated spend ~2.6x.
|
||||||
|
// The pricing table is the single source of truth per provider:model.
|
||||||
|
const TOKENS = 2_590_710_262; // a real large-brain sync preview
|
||||||
|
const openai = lookupEmbeddingPrice('openai:text-embedding-3-large');
|
||||||
|
const zeroentropy = lookupEmbeddingPrice('zeroentropyai:zembed-1');
|
||||||
|
expect(openai.kind).toBe('known');
|
||||||
|
expect(zeroentropy.kind).toBe('known');
|
||||||
|
if (openai.kind === 'known' && zeroentropy.kind === 'known') {
|
||||||
|
const openaiCost = (TOKENS / 1_000_000) * openai.pricePerMTok;
|
||||||
|
const zeCost = (TOKENS / 1_000_000) * zeroentropy.pricePerMTok;
|
||||||
|
// The two models must produce materially different previews; a fix that
|
||||||
|
// collapses both to the OpenAI number would regress this assertion.
|
||||||
|
expect(openaiCost).toBeCloseTo(336.79, 1);
|
||||||
|
expect(zeCost).toBeCloseTo(129.54, 1);
|
||||||
|
expect(zeCost).toBeLessThan(openaiCost);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
test('5K-file TS repo sanity check: ~$5 at ~400k tokens', () => {
|
test('5K-file TS repo sanity check: ~$5 at ~400k tokens', () => {
|
||||||
// A 5K-file TS repo at ~80 tokens/file averages ~400k tokens. Cost:
|
// A 5K-file TS repo at ~80 tokens/file averages ~400k tokens. Cost:
|
||||||
// 400_000 / 1000 * 0.00013 = $0.052 ≈ $0.05. Not $5. The CHANGELOG
|
// 400_000 / 1000 * 0.00013 = $0.052 ≈ $0.05. Not $5. The CHANGELOG
|
||||||
|
|||||||
Reference in New Issue
Block a user