mirror of
https://github.com/garrytan/gbrain.git
synced 2026-07-30 11:22:34 +00:00
Merge origin/master into garrytan/managua-v3
Resolves v0.34.0.0 (W1-W8 code intelligence) with master's v0.33.2.1 + search-lite work (query cache + intent weighting + token budget + drift watch + metric glossary + search modes). Conflict resolutions: - VERSION / package.json: kept 0.34.0.0 (mine; higher than master's 0.33.2.1) - CHANGELOG.md: both entries preserved; reordered so v0.33.2.1 sits above v0.33.2.0 (semver order) - src/cli.ts CLI_ONLY: union of both — `edges-backfill` (mine) + `cache` (master) - src/core/migrate.ts: renumbered my migrations to avoid collision with master's query_cache_search_lite (v55), query_cache_knobs_hash (v56), search_telemetry_rollup (v57). My `edges_backfilled_at_v0_33_2` moves v55 → v58; my `code_traversal_cache_v0_34` moves v56 → v59. Code refs in `src/core/code-intel/traversal-cache.ts` and the paired test updated to match. - src/core/operations.ts query op: kept master's `hybridSearchCached` routing (search-lite cache integration) AND my `sourceId` resolution block (D4 source-routing fix from v0.34 STEP 0). Both apply. Verification: - `bun run typecheck` clean - `bun run verify` clean (includes check-cli-executable, check-jsonb, check-system-of-record, check-eval-glossary-fresh, etc.) - Migration v50→v59 apply cleanly on PGLite in isolated test runs - Individual test files pass (e.g. test/search-lang-symbol-kind.test.ts: 9 pass / 0 fail in 913ms) Known follow-up: the parallel test shard runner times out some beforeAll hooks at the default 7s budget. Tests pass when run sequentially (`--max-concurrency=1`); 27/0 confirmed across 3 sample files in 2.4s sequential vs timeouts under parallel-shard contention. Master added 4 new migrations (v55-v57 + search-lite related) increasing per-test-file PGLite init cost; on 8 shards racing for OS resources, some shards hit the 7s ceiling. This is a test-infrastructure issue (shard isolation under heavier migrations), not a code-correctness issue. Fix is a follow-up: either raise shard test timeout, reduce shard count, or migrate to fixture-based engine setup for hot tests. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
Garry Tan
+216
-4
@@ -3,12 +3,18 @@ import {
|
||||
configureGateway,
|
||||
resetGateway,
|
||||
isAvailable,
|
||||
embed,
|
||||
getEmbeddingModel,
|
||||
getEmbeddingDimensions,
|
||||
getExpansionModel,
|
||||
VoyageResponseTooLargeError,
|
||||
} from '../../src/core/ai/gateway.ts';
|
||||
import { parseModelId, resolveRecipe } from '../../src/core/ai/model-resolver.ts';
|
||||
import { dimsProviderOptions } from '../../src/core/ai/dims.ts';
|
||||
import {
|
||||
dimsProviderOptions,
|
||||
VOYAGE_VALID_OUTPUT_DIMS,
|
||||
isValidVoyageOutputDim,
|
||||
} from '../../src/core/ai/dims.ts';
|
||||
import { AIConfigError } from '../../src/core/ai/errors.ts';
|
||||
|
||||
describe('gateway configuration', () => {
|
||||
@@ -156,13 +162,219 @@ describe('dims.dimsProviderOptions', () => {
|
||||
expect(opts).toBeUndefined();
|
||||
});
|
||||
|
||||
test('Voyage openai-compatible returns output_dimension', () => {
|
||||
const opts = dimsProviderOptions('openai-compatible', 'voyage-4-large', 2048);
|
||||
expect(opts).toEqual({ openaiCompatible: { output_dimension: 2048 } });
|
||||
test('Voyage flexible-dim models return dimensions for the SDK shim', () => {
|
||||
const opts = dimsProviderOptions('openai-compatible', 'voyage-3-large', 1024);
|
||||
expect(opts).toEqual({ openaiCompatible: { dimensions: 1024 } });
|
||||
const v4Opts = dimsProviderOptions('openai-compatible', 'voyage-4-large', 2048);
|
||||
expect(v4Opts).toEqual({ openaiCompatible: { dimensions: 2048 } });
|
||||
});
|
||||
|
||||
test('Voyage model without flexible dimensions returns undefined', () => {
|
||||
const opts = dimsProviderOptions('openai-compatible', 'voyage-3-lite', 1024);
|
||||
expect(opts).toBeUndefined();
|
||||
});
|
||||
|
||||
// Negative regression pin: voyage-4-nano is an open-weight variant that
|
||||
// Voyage's hosted API rejects `output_dimension` on (fixed 1024-dim).
|
||||
// Don't re-add it to VOYAGE_OUTPUT_DIMENSION_MODELS without cross-checking
|
||||
// Voyage's docs. See src/core/ai/dims.ts for the rationale.
|
||||
test('voyage-4-nano returns undefined (open-weight, fixed-dim)', () => {
|
||||
const opts = dimsProviderOptions('openai-compatible', 'voyage-4-nano', 512);
|
||||
expect(opts).toBeUndefined();
|
||||
});
|
||||
});
|
||||
|
||||
describe('Voyage openai-compatible request shim', () => {
|
||||
beforeEach(() => resetGateway());
|
||||
|
||||
test('sends output_dimension on the actual Voyage embedding request body', async () => {
|
||||
const originalFetch = globalThis.fetch;
|
||||
let requestBody: Record<string, unknown> | undefined;
|
||||
globalThis.fetch = (async (_url: string | URL | Request, init?: RequestInit) => {
|
||||
requestBody = JSON.parse(String(init?.body ?? '{}'));
|
||||
return new Response(JSON.stringify({
|
||||
object: 'list',
|
||||
data: [
|
||||
{
|
||||
object: 'embedding',
|
||||
index: 0,
|
||||
embedding: new Array(2048).fill(0.01),
|
||||
},
|
||||
],
|
||||
model: 'voyage-4-large',
|
||||
usage: { total_tokens: 3 },
|
||||
}), {
|
||||
status: 200,
|
||||
headers: { 'content-type': 'application/json' },
|
||||
});
|
||||
}) as unknown as typeof fetch;
|
||||
|
||||
try {
|
||||
configureGateway({
|
||||
embedding_model: 'voyage:voyage-4-large',
|
||||
embedding_dimensions: 2048,
|
||||
env: { VOYAGE_API_KEY: 'voyage-fake' },
|
||||
});
|
||||
|
||||
const vectors = await embed(['dimension probe']);
|
||||
|
||||
expect(vectors[0].length).toBe(2048);
|
||||
expect(requestBody?.output_dimension).toBe(2048);
|
||||
expect(requestBody?.encoding_format).toBe('base64');
|
||||
} finally {
|
||||
globalThis.fetch = originalFetch;
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
// ─────────────────────────────────────────────────────────────────────
|
||||
// Voyage OOM-cap rethrow regression (Codex P3 follow-up after PR #962).
|
||||
// Pins the contract that VoyageResponseTooLargeError thrown from the
|
||||
// inbound rewriter is NOT swallowed by the surrounding try/catch.
|
||||
// ─────────────────────────────────────────────────────────────────────
|
||||
describe('Voyage OOM-cap: too-large response throws (Codex P3 follow-up)', () => {
|
||||
beforeEach(() => resetGateway());
|
||||
|
||||
test('Layer 1 — Content-Length above cap propagates as VoyageResponseTooLargeError', async () => {
|
||||
const originalFetch = globalThis.fetch;
|
||||
// 257 MB > 256 MB cap.
|
||||
const oversized = String(257 * 1024 * 1024);
|
||||
globalThis.fetch = (async () => {
|
||||
return new Response('{"data": []}', {
|
||||
status: 200,
|
||||
headers: {
|
||||
'content-type': 'application/json',
|
||||
'content-length': oversized,
|
||||
},
|
||||
});
|
||||
}) as unknown as typeof fetch;
|
||||
try {
|
||||
configureGateway({
|
||||
embedding_model: 'voyage:voyage-4-large',
|
||||
embedding_dimensions: 1024,
|
||||
env: { VOYAGE_API_KEY: 'voyage-fake' },
|
||||
});
|
||||
let caught: unknown;
|
||||
try {
|
||||
await embed(['probe']);
|
||||
} catch (e) {
|
||||
caught = e;
|
||||
}
|
||||
// The OOM throw propagates. Provider plumbing may wrap it, but the
|
||||
// VoyageResponseTooLargeError class name + characteristic message
|
||||
// must survive.
|
||||
const msg = caught instanceof Error ? caught.message : String(caught);
|
||||
expect(msg).toContain('Content-Length=');
|
||||
expect(msg).toContain('exceeds');
|
||||
} finally {
|
||||
globalThis.fetch = originalFetch;
|
||||
}
|
||||
});
|
||||
|
||||
test('Layer 2 — oversized base64 embedding string propagates (not swallowed)', async () => {
|
||||
const originalFetch = globalThis.fetch;
|
||||
// Build a JSON response with an `embedding` base64 string that decodes
|
||||
// to > 256 MB. base64 ratio is ~0.75; 360 MB of base64 chars ≈ 270 MB
|
||||
// decoded.
|
||||
const oversizedBase64 = 'A'.repeat(360 * 1024 * 1024);
|
||||
const respBody = `{"object":"list","data":[{"object":"embedding","index":0,"embedding":"${oversizedBase64}"}],"model":"voyage-4-large","usage":{"total_tokens":1}}`;
|
||||
globalThis.fetch = (async () => {
|
||||
// No Content-Length header → Layer 1 skipped, Layer 2 must fire.
|
||||
return new Response(respBody, {
|
||||
status: 200,
|
||||
headers: { 'content-type': 'application/json' },
|
||||
});
|
||||
}) as unknown as typeof fetch;
|
||||
try {
|
||||
configureGateway({
|
||||
embedding_model: 'voyage:voyage-4-large',
|
||||
embedding_dimensions: 1024,
|
||||
env: { VOYAGE_API_KEY: 'voyage-fake' },
|
||||
});
|
||||
let caught: unknown;
|
||||
try {
|
||||
await embed(['probe']);
|
||||
} catch (e) {
|
||||
caught = e;
|
||||
}
|
||||
const msg = caught instanceof Error ? caught.message : String(caught);
|
||||
// The Layer 2 throw fired and was not swallowed by the inbound
|
||||
// try/catch (pre-fix bug: bare `catch {}` returned the original
|
||||
// response and let the AI SDK OOM trying to parse it).
|
||||
expect(msg).toContain('Voyage embedding base64 exceeds');
|
||||
} finally {
|
||||
globalThis.fetch = originalFetch;
|
||||
}
|
||||
}, 15000);
|
||||
|
||||
test('VoyageResponseTooLargeError is exported as a tagged class', () => {
|
||||
expect(VoyageResponseTooLargeError).toBeDefined();
|
||||
const err = new VoyageResponseTooLargeError('test');
|
||||
expect(err).toBeInstanceOf(Error);
|
||||
expect(err).toBeInstanceOf(VoyageResponseTooLargeError);
|
||||
expect(err.name).toBe('VoyageResponseTooLargeError');
|
||||
});
|
||||
});
|
||||
|
||||
// ─────────────────────────────────────────────────────────────────────
|
||||
// Voyage flexible-dim runtime validation (Codex P3 follow-up after PR #962).
|
||||
// The bug class: brain configured for Voyage flexible-dim model without
|
||||
// `embedding_dimensions` → gateway falls back to DEFAULT 1536 → Voyage
|
||||
// HTTP 400. Catch it at the embed-call boundary with a clear AIConfigError.
|
||||
// ─────────────────────────────────────────────────────────────────────
|
||||
describe('Voyage flexible-dim runtime validation', () => {
|
||||
test('rejects 1536 (the default that bites Voyage-first users) with AIConfigError', () => {
|
||||
expect(() => dimsProviderOptions('openai-compatible', 'voyage-4-large', 1536))
|
||||
.toThrow(AIConfigError);
|
||||
expect(() => dimsProviderOptions('openai-compatible', 'voyage-4-large', 1536))
|
||||
.toThrow(/embedding_dimensions|256.*512.*1024.*2048/);
|
||||
});
|
||||
|
||||
test('rejects 3072 with AIConfigError', () => {
|
||||
expect(() => dimsProviderOptions('openai-compatible', 'voyage-3-large', 3072))
|
||||
.toThrow(AIConfigError);
|
||||
});
|
||||
|
||||
test('accepts every Voyage-allowed flexible dim', () => {
|
||||
for (const dim of VOYAGE_VALID_OUTPUT_DIMS) {
|
||||
const opts = dimsProviderOptions('openai-compatible', 'voyage-4-large', dim);
|
||||
expect(opts).toEqual({ openaiCompatible: { dimensions: dim } });
|
||||
}
|
||||
});
|
||||
|
||||
test('VOYAGE_VALID_OUTPUT_DIMS pins exactly the four Voyage values', () => {
|
||||
expect([...VOYAGE_VALID_OUTPUT_DIMS]).toEqual([256, 512, 1024, 2048]);
|
||||
});
|
||||
|
||||
test('isValidVoyageOutputDim returns true only for the four valid sizes', () => {
|
||||
expect(isValidVoyageOutputDim(256)).toBe(true);
|
||||
expect(isValidVoyageOutputDim(512)).toBe(true);
|
||||
expect(isValidVoyageOutputDim(1024)).toBe(true);
|
||||
expect(isValidVoyageOutputDim(2048)).toBe(true);
|
||||
expect(isValidVoyageOutputDim(1536)).toBe(false);
|
||||
expect(isValidVoyageOutputDim(3072)).toBe(false);
|
||||
expect(isValidVoyageOutputDim(0)).toBe(false);
|
||||
expect(isValidVoyageOutputDim(-1)).toBe(false);
|
||||
});
|
||||
|
||||
test('voyage-3-lite (non-flexible-dim) bypasses the validator — still returns undefined', () => {
|
||||
// Sanity: the validator only fires inside the flexible-dim branch, so
|
||||
// a fixed-dim Voyage model with any dim value goes straight through to
|
||||
// the `undefined` return path (no error, no providerOptions).
|
||||
expect(dimsProviderOptions('openai-compatible', 'voyage-3-lite', 1536)).toBeUndefined();
|
||||
expect(dimsProviderOptions('openai-compatible', 'voyage-4-nano', 1536)).toBeUndefined();
|
||||
});
|
||||
|
||||
test('AIConfigError fix hint names the canonical recovery commands', () => {
|
||||
let caught: AIConfigError | undefined;
|
||||
try {
|
||||
dimsProviderOptions('openai-compatible', 'voyage-4-large', 1536);
|
||||
} catch (e) {
|
||||
caught = e as AIConfigError;
|
||||
}
|
||||
expect(caught).toBeInstanceOf(AIConfigError);
|
||||
expect(caught?.fix).toContain('embedding_dimensions');
|
||||
expect(caught?.fix).toContain('256');
|
||||
expect(caught?.fix).toContain('2048');
|
||||
});
|
||||
});
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
* v0.34 W3b — code_traversal_cache module tests.
|
||||
*
|
||||
* Hermetic PGLite test suite covering:
|
||||
* - cache hit returns memoized response (after migration v56)
|
||||
* - cache hit returns memoized response (after migration v59)
|
||||
* - cache miss triggers compute
|
||||
* - D3: cluster_generation bump invalidates cached rows
|
||||
* - clearTraversalCache: source-scoped clear deletes the right rows
|
||||
|
||||
@@ -0,0 +1,257 @@
|
||||
/**
|
||||
* v0.32.3 — `gbrain search modes/stats/tune` CLI tests.
|
||||
*
|
||||
* Covers dispatch + JSON output shape + idempotent --reset + recommendation
|
||||
* generation. Pure unit-level: bypasses the cli.ts entrypoint and calls
|
||||
* runSearch directly against a fresh PGLite engine.
|
||||
*/
|
||||
import { afterAll, beforeAll, beforeEach, describe, expect, test } from 'bun:test';
|
||||
import { PGLiteEngine } from '../src/core/pglite-engine.ts';
|
||||
import { runSearch } from '../src/commands/search.ts';
|
||||
import { recordSearchTelemetry, _resetTelemetryWriterForTest, getTelemetryWriter } from '../src/core/search/telemetry.ts';
|
||||
import type { HybridSearchMeta } from '../src/core/types.ts';
|
||||
|
||||
let engine: PGLiteEngine;
|
||||
|
||||
beforeAll(async () => {
|
||||
engine = new PGLiteEngine();
|
||||
await engine.connect({});
|
||||
await engine.initSchema();
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
await engine.disconnect();
|
||||
});
|
||||
|
||||
beforeEach(async () => {
|
||||
_resetTelemetryWriterForTest();
|
||||
await engine.executeRaw(`DELETE FROM config WHERE key LIKE 'search.%' OR key LIKE 'models.%'`);
|
||||
await engine.executeRaw('DELETE FROM search_telemetry');
|
||||
});
|
||||
|
||||
// Capture-stdout helper so we can assert command output without exec'ing.
|
||||
async function captureRun(fn: () => Promise<void>): Promise<string> {
|
||||
const original = console.log;
|
||||
const captured: string[] = [];
|
||||
console.log = (...args: unknown[]) => { captured.push(args.map(String).join(' ')); };
|
||||
try {
|
||||
await fn();
|
||||
} finally {
|
||||
console.log = original;
|
||||
}
|
||||
return captured.join('\n');
|
||||
}
|
||||
|
||||
const makeMeta = (overrides: Partial<HybridSearchMeta> = {}): HybridSearchMeta => ({
|
||||
vector_enabled: true,
|
||||
detail_resolved: null,
|
||||
expansion_applied: false,
|
||||
intent: 'general',
|
||||
mode: 'balanced',
|
||||
...overrides,
|
||||
});
|
||||
|
||||
describe('gbrain search modes (read-only dashboard)', () => {
|
||||
test('--json emits structured report with all 3 bundles and active mode', async () => {
|
||||
await engine.setConfig('search.mode', 'tokenmax');
|
||||
const out = await captureRun(() => runSearch(engine, ['modes', '--json']));
|
||||
const report = JSON.parse(out);
|
||||
expect(report.schema_version).toBe(2);
|
||||
expect(report.active_mode).toBe('tokenmax');
|
||||
expect(report.active_mode_valid).toBe(true);
|
||||
expect(report.bundles.conservative.searchLimit).toBe(10);
|
||||
expect(report.bundles.balanced.searchLimit).toBe(25);
|
||||
expect(report.bundles.tokenmax.searchLimit).toBe(50);
|
||||
expect(report.resolved.tokenBudget.source).toBe('mode');
|
||||
});
|
||||
|
||||
test('unset mode → balanced fallback with mode_valid=false', async () => {
|
||||
const out = await captureRun(() => runSearch(engine, ['modes', '--json']));
|
||||
const report = JSON.parse(out);
|
||||
expect(report.active_mode).toBe('balanced');
|
||||
expect(report.active_mode_valid).toBe(false);
|
||||
expect(report.resolved.searchLimit.source).toBe('fallback');
|
||||
});
|
||||
|
||||
test('per-key override shows up with source=override', async () => {
|
||||
await engine.setConfig('search.mode', 'conservative');
|
||||
await engine.setConfig('search.cache.enabled', 'false');
|
||||
const out = await captureRun(() => runSearch(engine, ['modes', '--json']));
|
||||
const report = JSON.parse(out);
|
||||
expect(report.resolved.cache_enabled.value).toBe(false);
|
||||
expect(report.resolved.cache_enabled.source).toBe('override');
|
||||
// Other knobs still come from the mode bundle.
|
||||
expect(report.resolved.searchLimit.source).toBe('mode');
|
||||
});
|
||||
|
||||
test('default text output names the active mode', async () => {
|
||||
await engine.setConfig('search.mode', 'tokenmax');
|
||||
const out = await captureRun(() => runSearch(engine, ['modes']));
|
||||
expect(out).toContain('tokenmax');
|
||||
expect(out).toContain('conservative');
|
||||
expect(out).toContain('balanced');
|
||||
});
|
||||
});
|
||||
|
||||
describe('gbrain search modes --reset', () => {
|
||||
test('--source <mode> is a dry-run (no writes)', async () => {
|
||||
await engine.setConfig('search.cache.enabled', 'false');
|
||||
await engine.setConfig('search.tokenBudget', '4000');
|
||||
const out = await captureRun(() => runSearch(engine, ['modes', '--source', 'balanced']));
|
||||
expect(out).toContain('dry run');
|
||||
expect(out).toContain('search.cache.enabled');
|
||||
expect(out).toContain('search.tokenBudget');
|
||||
// Verify nothing was deleted.
|
||||
expect(await engine.getConfig('search.cache.enabled')).toBe('false');
|
||||
expect(await engine.getConfig('search.tokenBudget')).toBe('4000');
|
||||
});
|
||||
|
||||
test('--reset clears every search.* override (but NOT search.mode itself)', async () => {
|
||||
await engine.setConfig('search.mode', 'conservative');
|
||||
await engine.setConfig('search.cache.enabled', 'false');
|
||||
await engine.setConfig('search.tokenBudget', '8000');
|
||||
await engine.setConfig('search.searchLimit', '15');
|
||||
await captureRun(() => runSearch(engine, ['modes', '--reset']));
|
||||
// Mode preserved; overrides gone.
|
||||
expect(await engine.getConfig('search.mode')).toBe('conservative');
|
||||
expect(await engine.getConfig('search.cache.enabled')).toBeNull();
|
||||
expect(await engine.getConfig('search.tokenBudget')).toBeNull();
|
||||
expect(await engine.getConfig('search.searchLimit')).toBeNull();
|
||||
});
|
||||
|
||||
test('--reset on a clean install reports "no overrides"', async () => {
|
||||
const out = await captureRun(() => runSearch(engine, ['modes', '--reset']));
|
||||
expect(out).toContain('No search.* overrides set');
|
||||
});
|
||||
|
||||
test('--reset preserves the upgrade-notice state key', async () => {
|
||||
await engine.setConfig('search.mode_upgrade_notice_shown', 'true');
|
||||
await engine.setConfig('search.tokenBudget', '4000');
|
||||
await captureRun(() => runSearch(engine, ['modes', '--reset']));
|
||||
// Notice key preserved (it's not an "override"); tokenBudget gone.
|
||||
expect(await engine.getConfig('search.mode_upgrade_notice_shown')).toBe('true');
|
||||
expect(await engine.getConfig('search.tokenBudget')).toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
describe('gbrain search stats', () => {
|
||||
test('empty table → total_calls 0, message about no data', async () => {
|
||||
const out = await captureRun(() => runSearch(engine, ['stats']));
|
||||
expect(out).toContain('Total searches:');
|
||||
expect(out).toContain('0');
|
||||
});
|
||||
|
||||
test('after telemetry writes → hit rate + intent mix surfaced', async () => {
|
||||
const w = getTelemetryWriter();
|
||||
w.setEngine(engine);
|
||||
recordSearchTelemetry(engine, makeMeta({ cache: { status: 'hit' } }), { results_count: 5 });
|
||||
recordSearchTelemetry(engine, makeMeta({ cache: { status: 'hit' } }), { results_count: 7 });
|
||||
recordSearchTelemetry(engine, makeMeta({ cache: { status: 'miss' } }), { results_count: 9 });
|
||||
recordSearchTelemetry(engine, makeMeta({ intent: 'entity' }), { results_count: 3 });
|
||||
await w.flush();
|
||||
|
||||
const out = await captureRun(() => runSearch(engine, ['stats', '--json']));
|
||||
const stats = JSON.parse(out);
|
||||
expect(stats.total_calls).toBe(4);
|
||||
expect(stats.cache_hits).toBe(2);
|
||||
expect(stats.cache_misses).toBe(1);
|
||||
expect(stats.cache_hit_rate).toBeCloseTo(2 / 3, 3);
|
||||
expect(stats._meta.metric_glossary.cache_hit_rate).toBeDefined();
|
||||
});
|
||||
|
||||
test('--days N clamps to [1, 365]', async () => {
|
||||
const out0 = await captureRun(() => runSearch(engine, ['stats', '--days', '0', '--json']));
|
||||
expect(JSON.parse(out0).window_days).toBe(1);
|
||||
const outBig = await captureRun(() => runSearch(engine, ['stats', '--days', '9999', '--json']));
|
||||
expect(JSON.parse(outBig).window_days).toBe(365);
|
||||
});
|
||||
});
|
||||
|
||||
describe('gbrain search tune (recommendations)', () => {
|
||||
test('insufficient data → no_recommendations status', async () => {
|
||||
const out = await captureRun(() => runSearch(engine, ['tune', '--json']));
|
||||
const r = JSON.parse(out);
|
||||
expect(r.status).toBe('insufficient_data');
|
||||
expect(r.recommendations).toEqual([]);
|
||||
});
|
||||
|
||||
test('conservative + high budget drop rate → recommends balanced', async () => {
|
||||
await engine.setConfig('search.mode', 'conservative');
|
||||
const w = getTelemetryWriter();
|
||||
w.setEngine(engine);
|
||||
// 30 calls, each dropping 5 results — strong signal.
|
||||
for (let i = 0; i < 30; i++) {
|
||||
recordSearchTelemetry(engine, makeMeta({
|
||||
mode: 'conservative',
|
||||
token_budget: { budget: 4000, used: 4000, kept: 5, dropped: 5 },
|
||||
}), { results_count: 5 });
|
||||
}
|
||||
await w.flush();
|
||||
|
||||
const out = await captureRun(() => runSearch(engine, ['tune', '--json']));
|
||||
const r = JSON.parse(out);
|
||||
expect(r.status).toBe('has_recommendations');
|
||||
const modeRec = r.recommendations.find((x: { knob: string }) => x.knob === 'search.mode');
|
||||
expect(modeRec).toBeDefined();
|
||||
expect(modeRec.suggested).toBe('balanced');
|
||||
});
|
||||
|
||||
test('tokenmax + Haiku subagent → recommends balanced', async () => {
|
||||
await engine.setConfig('search.mode', 'tokenmax');
|
||||
await engine.setConfig('models.tier.subagent', 'anthropic:claude-haiku-4-5');
|
||||
const w = getTelemetryWriter();
|
||||
w.setEngine(engine);
|
||||
for (let i = 0; i < 25; i++) {
|
||||
recordSearchTelemetry(engine, makeMeta({ mode: 'tokenmax' }), { results_count: 30 });
|
||||
}
|
||||
await w.flush();
|
||||
|
||||
const out = await captureRun(() => runSearch(engine, ['tune', '--json']));
|
||||
const r = JSON.parse(out);
|
||||
const rec = r.recommendations.find((x: { knob: string; suggested: string }) =>
|
||||
x.knob === 'search.mode' && x.suggested === 'balanced'
|
||||
);
|
||||
expect(rec).toBeDefined();
|
||||
expect(rec.reason).toMatch(/Haiku/);
|
||||
});
|
||||
|
||||
test('--apply mutates config', async () => {
|
||||
await engine.setConfig('search.mode', 'conservative');
|
||||
const w = getTelemetryWriter();
|
||||
w.setEngine(engine);
|
||||
for (let i = 0; i < 30; i++) {
|
||||
recordSearchTelemetry(engine, makeMeta({
|
||||
mode: 'conservative',
|
||||
token_budget: { budget: 4000, used: 4000, kept: 5, dropped: 5 },
|
||||
}), { results_count: 5 });
|
||||
}
|
||||
await w.flush();
|
||||
|
||||
await captureRun(() => runSearch(engine, ['tune', '--apply']));
|
||||
expect(await engine.getConfig('search.mode')).toBe('balanced');
|
||||
});
|
||||
});
|
||||
|
||||
describe('gbrain search dispatch', () => {
|
||||
test('--help shows usage', async () => {
|
||||
const out = await captureRun(() => runSearch(engine, ['--help']));
|
||||
expect(out).toContain('Usage:');
|
||||
expect(out).toContain('modes');
|
||||
expect(out).toContain('stats');
|
||||
expect(out).toContain('tune');
|
||||
});
|
||||
|
||||
test('unknown subcommand exits 1', async () => {
|
||||
let exitCode = 0;
|
||||
const originalExit = process.exit;
|
||||
(process.exit as unknown as (code?: number) => void) = ((code?: number) => { exitCode = code ?? 0; throw new Error('exit-' + code); }) as never;
|
||||
const originalErr = console.error;
|
||||
console.error = () => { /* swallow */ };
|
||||
try {
|
||||
await runSearch(engine, ['nonsense']);
|
||||
} catch { /* expected */ }
|
||||
expect(exitCode).toBe(1);
|
||||
process.exit = originalExit;
|
||||
console.error = originalErr;
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,101 @@
|
||||
/**
|
||||
* Pins the v0.32.3 `gbrain config unset` + `listConfigKeys` engine surface.
|
||||
* Required by `gbrain search modes --reset` [CDX-8].
|
||||
*/
|
||||
import { afterAll, beforeAll, beforeEach, describe, expect, test } from 'bun:test';
|
||||
import { PGLiteEngine } from '../src/core/pglite-engine.ts';
|
||||
|
||||
let engine: PGLiteEngine;
|
||||
|
||||
beforeAll(async () => {
|
||||
engine = new PGLiteEngine();
|
||||
await engine.connect({});
|
||||
await engine.initSchema();
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
await engine.disconnect();
|
||||
});
|
||||
|
||||
beforeEach(async () => {
|
||||
await engine.executeRaw(`DELETE FROM config WHERE key LIKE 'test.%' OR key LIKE 'search.%'`);
|
||||
});
|
||||
|
||||
describe('engine.unsetConfig', () => {
|
||||
test('removes an existing key, returns 1', async () => {
|
||||
await engine.setConfig('test.k1', 'v1');
|
||||
const n = await engine.unsetConfig('test.k1');
|
||||
expect(n).toBe(1);
|
||||
const after = await engine.getConfig('test.k1');
|
||||
expect(after).toBeNull();
|
||||
});
|
||||
|
||||
test('returns 0 for a missing key (no error)', async () => {
|
||||
const n = await engine.unsetConfig('test.never-existed');
|
||||
expect(n).toBe(0);
|
||||
});
|
||||
|
||||
test('does not affect other keys', async () => {
|
||||
await engine.setConfig('test.keep', 'keep');
|
||||
await engine.setConfig('test.remove', 'gone');
|
||||
await engine.unsetConfig('test.remove');
|
||||
expect(await engine.getConfig('test.keep')).toBe('keep');
|
||||
expect(await engine.getConfig('test.remove')).toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
describe('engine.listConfigKeys prefix-matcher', () => {
|
||||
test('empty when no key matches', async () => {
|
||||
expect(await engine.listConfigKeys('test.')).toEqual([]);
|
||||
});
|
||||
|
||||
test('returns matching keys sorted ascending', async () => {
|
||||
await engine.setConfig('test.b', '2');
|
||||
await engine.setConfig('test.a', '1');
|
||||
await engine.setConfig('test.c', '3');
|
||||
await engine.setConfig('other', 'no');
|
||||
const keys = await engine.listConfigKeys('test.');
|
||||
expect(keys).toEqual(['test.a', 'test.b', 'test.c']);
|
||||
});
|
||||
|
||||
test('prefix is an EXACT literal match — no glob wildcards', async () => {
|
||||
await engine.setConfig('test.matchme', 'yes');
|
||||
// % in user input is escaped — does NOT act as wildcard.
|
||||
expect(await engine.listConfigKeys('test%')).toEqual([]);
|
||||
expect(await engine.listConfigKeys('test.match')).toEqual(['test.matchme']);
|
||||
});
|
||||
|
||||
test('underscore in prefix is escaped (literal _)', async () => {
|
||||
await engine.setConfig('test_underscore', 'val');
|
||||
await engine.setConfig('testXunderscore', 'no');
|
||||
const keys = await engine.listConfigKeys('test_');
|
||||
// test_underscore matches because we asked for prefix "test_"; the
|
||||
// testXunderscore does NOT because the _ is escaped to literal.
|
||||
expect(keys).toEqual(['test_underscore']);
|
||||
});
|
||||
|
||||
test('search.* prefix sweep returns every search-mode override key set', async () => {
|
||||
await engine.setConfig('search.cache.enabled', 'true');
|
||||
await engine.setConfig('search.cache.ttl_seconds', '7200');
|
||||
await engine.setConfig('search.tokenBudget', '8000');
|
||||
await engine.setConfig('search.mode', 'tokenmax'); // mode key itself
|
||||
const keys = await engine.listConfigKeys('search.');
|
||||
expect(keys.length).toBe(4);
|
||||
expect(keys).toContain('search.cache.enabled');
|
||||
expect(keys).toContain('search.cache.ttl_seconds');
|
||||
expect(keys).toContain('search.tokenBudget');
|
||||
expect(keys).toContain('search.mode');
|
||||
});
|
||||
});
|
||||
|
||||
describe('round-trip: set → unset → get', () => {
|
||||
test('setting and unsetting in a tight loop is idempotent', async () => {
|
||||
for (let i = 0; i < 5; i++) {
|
||||
await engine.setConfig('test.loopkey', `iteration-${i}`);
|
||||
expect(await engine.getConfig('test.loopkey')).toBe(`iteration-${i}`);
|
||||
const n = await engine.unsetConfig('test.loopkey');
|
||||
expect(n).toBe(1);
|
||||
expect(await engine.getConfig('test.loopkey')).toBeNull();
|
||||
}
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,84 @@
|
||||
/**
|
||||
* v0.32.3 — doctor search_mode + eval_drift check tests.
|
||||
* Pins [CDX-20]: status stays 'ok', no health-score docking; hint lives
|
||||
* in `message`. Tests the two exported helpers directly to avoid the
|
||||
* expensive full runDoctor walk.
|
||||
*/
|
||||
import { afterAll, beforeAll, beforeEach, describe, expect, test } from 'bun:test';
|
||||
import { PGLiteEngine } from '../src/core/pglite-engine.ts';
|
||||
import { checkSearchMode, checkEvalDrift } from '../src/commands/doctor.ts';
|
||||
|
||||
let engine: PGLiteEngine;
|
||||
|
||||
beforeAll(async () => {
|
||||
engine = new PGLiteEngine();
|
||||
await engine.connect({});
|
||||
await engine.initSchema();
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
await engine.disconnect();
|
||||
});
|
||||
|
||||
beforeEach(async () => {
|
||||
await engine.executeRaw(`DELETE FROM config WHERE key LIKE 'search.%'`);
|
||||
});
|
||||
|
||||
describe('checkSearchMode [CDX-20]', () => {
|
||||
test('unset mode → ok with hint to pick a mode', async () => {
|
||||
const c = await checkSearchMode(engine);
|
||||
expect(c.name).toBe('search_mode');
|
||||
expect(c.status).toBe('ok'); // never warn, never dock score
|
||||
expect(c.message).toMatch(/unset/i);
|
||||
expect(c.message).toContain('gbrain search modes');
|
||||
});
|
||||
|
||||
test('mode set, no overrides → ok with "canonical" message', async () => {
|
||||
await engine.setConfig('search.mode', 'balanced');
|
||||
const c = await checkSearchMode(engine);
|
||||
expect(c.status).toBe('ok');
|
||||
expect(c.message).toContain('balanced');
|
||||
expect(c.message).toContain('canonical');
|
||||
});
|
||||
|
||||
test('mode set + overrides → ok with reset hint + override list', async () => {
|
||||
await engine.setConfig('search.mode', 'conservative');
|
||||
await engine.setConfig('search.cache.enabled', 'false');
|
||||
await engine.setConfig('search.tokenBudget', '8000');
|
||||
const c = await checkSearchMode(engine);
|
||||
expect(c.status).toBe('ok'); // [CDX-20]: still ok, never warn
|
||||
expect(c.message).toContain('conservative');
|
||||
expect(c.message).toContain('search.cache.enabled');
|
||||
expect(c.message).toContain('search.tokenBudget');
|
||||
expect(c.message).toContain('gbrain search modes --reset');
|
||||
});
|
||||
|
||||
test('upgrade-notice state key is excluded from override count', async () => {
|
||||
await engine.setConfig('search.mode', 'balanced');
|
||||
await engine.setConfig('search.mode_upgrade_notice_shown', 'true');
|
||||
const c = await checkSearchMode(engine);
|
||||
expect(c.message).toContain('no per-key overrides');
|
||||
});
|
||||
|
||||
test('tokenmax mode is recognized without any override warning', async () => {
|
||||
await engine.setConfig('search.mode', 'tokenmax');
|
||||
const c = await checkSearchMode(engine);
|
||||
expect(c.status).toBe('ok');
|
||||
expect(c.message).toContain('tokenmax');
|
||||
expect(c.message).toContain('canonical');
|
||||
});
|
||||
});
|
||||
|
||||
describe('checkEvalDrift [CDX-6]', () => {
|
||||
test('returns ok status (never warn — per [CDX-20])', async () => {
|
||||
const c = await checkEvalDrift(engine);
|
||||
expect(c.name).toBe('eval_drift');
|
||||
expect(c.status).toBe('ok');
|
||||
});
|
||||
|
||||
test('message is non-empty (either no-drift or drift summary)', async () => {
|
||||
const c = await checkEvalDrift(engine);
|
||||
expect(c.message).toBeTruthy();
|
||||
expect(c.message.length).toBeGreaterThan(0);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,82 @@
|
||||
/**
|
||||
* v0.32.3 — drift-watch module unit tests.
|
||||
* Pins the curated watch-list + matchesWatchPattern semantics.
|
||||
*/
|
||||
import { describe, expect, test } from 'bun:test';
|
||||
import {
|
||||
RETRIEVAL_WATCH_PATTERNS,
|
||||
matchesWatchPattern,
|
||||
watchedFilesDrifted,
|
||||
filesDriftedSince,
|
||||
} from '../src/core/eval/drift-watch.ts';
|
||||
|
||||
describe('RETRIEVAL_WATCH_PATTERNS canonical list', () => {
|
||||
test('includes src/core/search/ prefix', () => {
|
||||
expect(RETRIEVAL_WATCH_PATTERNS).toContain('src/core/search/');
|
||||
});
|
||||
|
||||
test('includes the embedding file', () => {
|
||||
expect(RETRIEVAL_WATCH_PATTERNS).toContain('src/core/embedding.ts');
|
||||
});
|
||||
|
||||
test('includes chunkers/ directory', () => {
|
||||
expect(RETRIEVAL_WATCH_PATTERNS).toContain('src/core/chunkers/');
|
||||
});
|
||||
|
||||
test('includes the query operation definition', () => {
|
||||
expect(RETRIEVAL_WATCH_PATTERNS).toContain('src/core/operations.ts');
|
||||
});
|
||||
|
||||
test('is frozen at module load', () => {
|
||||
expect(Object.isFrozen(RETRIEVAL_WATCH_PATTERNS)).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
describe('matchesWatchPattern semantics', () => {
|
||||
test('directory pattern matches any descendant', () => {
|
||||
expect(matchesWatchPattern('src/core/search/hybrid.ts')).toBe(true);
|
||||
expect(matchesWatchPattern('src/core/search/mode.ts')).toBe(true);
|
||||
expect(matchesWatchPattern('src/core/search/deep/nested/file.ts')).toBe(true);
|
||||
});
|
||||
|
||||
test('directory pattern does NOT match a sibling with the same prefix', () => {
|
||||
// src/core/search-related-but-different/foo.ts should NOT match
|
||||
// src/core/search/ because the pattern ends with a slash.
|
||||
expect(matchesWatchPattern('src/core/searchengine.ts')).toBe(false);
|
||||
expect(matchesWatchPattern('src/core/searches/file.ts')).toBe(false);
|
||||
});
|
||||
|
||||
test('bare file pattern requires exact equality', () => {
|
||||
expect(matchesWatchPattern('src/core/embedding.ts')).toBe(true);
|
||||
expect(matchesWatchPattern('src/core/embedding.test.ts')).toBe(false);
|
||||
expect(matchesWatchPattern('src/core/embedding')).toBe(false);
|
||||
});
|
||||
|
||||
test('custom patterns work', () => {
|
||||
const custom = ['foo/', 'bar.ts'];
|
||||
expect(matchesWatchPattern('foo/x.ts', custom)).toBe(true);
|
||||
expect(matchesWatchPattern('bar.ts', custom)).toBe(true);
|
||||
expect(matchesWatchPattern('baz.ts', custom)).toBe(false);
|
||||
});
|
||||
|
||||
test('non-matching path returns false', () => {
|
||||
expect(matchesWatchPattern('docs/eval/METRIC_GLOSSARY.md')).toBe(false);
|
||||
expect(matchesWatchPattern('test/foo.test.ts')).toBe(false);
|
||||
expect(matchesWatchPattern('README.md')).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe('filesDriftedSince + watchedFilesDrifted graceful failure', () => {
|
||||
test('missing repo root returns empty array', () => {
|
||||
expect(filesDriftedSince('/does/not/exist')).toEqual([]);
|
||||
});
|
||||
|
||||
test('watchedFilesDrifted filters through the same matcher', () => {
|
||||
// Smoke test: should not throw on this repo. Could be empty.
|
||||
const out = watchedFilesDrifted(process.cwd());
|
||||
expect(Array.isArray(out)).toBe(true);
|
||||
for (const p of out) {
|
||||
expect(matchesWatchPattern(p)).toBe(true);
|
||||
}
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,135 @@
|
||||
/**
|
||||
* v0.32.3 — eval-compare report tests.
|
||||
* Pins the markdown + JSON shape and the metric-glossary integration.
|
||||
*/
|
||||
import { afterAll, beforeAll, beforeEach, describe, expect, test } from 'bun:test';
|
||||
import { mkdtempSync, rmSync, writeFileSync } from 'fs';
|
||||
import { tmpdir } from 'os';
|
||||
import { join } from 'path';
|
||||
import { runEvalCompare } from '../src/commands/eval-compare.ts';
|
||||
|
||||
let tmp: string;
|
||||
|
||||
beforeAll(() => {
|
||||
tmp = mkdtempSync(join(tmpdir(), 'gbrain-eval-compare-'));
|
||||
});
|
||||
|
||||
afterAll(() => {
|
||||
rmSync(tmp, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
const SAMPLE_RECORDS = [
|
||||
{
|
||||
schema_version: 2,
|
||||
run_id: 'a-longmemeval-conservative-42',
|
||||
ran_at: '2026-05-12T10:00:00Z',
|
||||
suite: 'longmemeval',
|
||||
mode: 'conservative',
|
||||
commit: 'a',
|
||||
seed: 42,
|
||||
status: 'completed',
|
||||
duration_ms: 1000,
|
||||
metrics: { 'recall@10': 0.71, 'ndcg@10': 0.682 },
|
||||
},
|
||||
{
|
||||
schema_version: 2,
|
||||
run_id: 'a-longmemeval-balanced-42',
|
||||
ran_at: '2026-05-12T10:05:00Z',
|
||||
suite: 'longmemeval',
|
||||
mode: 'balanced',
|
||||
commit: 'a',
|
||||
seed: 42,
|
||||
status: 'completed',
|
||||
duration_ms: 1000,
|
||||
metrics: { 'recall@10': 0.78, 'ndcg@10': 0.741 },
|
||||
},
|
||||
{
|
||||
schema_version: 2,
|
||||
run_id: 'a-longmemeval-tokenmax-42',
|
||||
ran_at: '2026-05-12T10:10:00Z',
|
||||
suite: 'longmemeval',
|
||||
mode: 'tokenmax',
|
||||
commit: 'a',
|
||||
seed: 42,
|
||||
status: 'completed',
|
||||
duration_ms: 1000,
|
||||
metrics: { 'recall@10': 0.81, 'ndcg@10': 0.762 },
|
||||
},
|
||||
];
|
||||
|
||||
function writeJsonl(records: object[]): string {
|
||||
const path = join(tmp, 'eval-results.jsonl');
|
||||
writeFileSync(path, records.map(r => JSON.stringify(r)).join('\n') + '\n', 'utf-8');
|
||||
return path;
|
||||
}
|
||||
|
||||
async function captureRun(fn: () => Promise<void>): Promise<string> {
|
||||
const originalWrite = process.stdout.write.bind(process.stdout);
|
||||
const captured: string[] = [];
|
||||
(process.stdout.write as unknown as (s: string) => boolean) = ((s: string) => { captured.push(s); return true; }) as never;
|
||||
try {
|
||||
await fn();
|
||||
} finally {
|
||||
process.stdout.write = originalWrite;
|
||||
}
|
||||
return captured.join('');
|
||||
}
|
||||
|
||||
beforeEach(() => {
|
||||
rmSync(join(tmp, 'eval-results.jsonl'), { force: true });
|
||||
});
|
||||
|
||||
describe('runEvalCompare', () => {
|
||||
test('--json output has schema_version + grouped + _meta.metric_glossary', async () => {
|
||||
const path = writeJsonl(SAMPLE_RECORDS);
|
||||
const out = await captureRun(() => runEvalCompare(['--json', '--input', path]));
|
||||
const report = JSON.parse(out);
|
||||
expect(report.schema_version).toBe(2);
|
||||
expect(report.grouped.longmemeval).toBeDefined();
|
||||
expect(report.grouped.longmemeval.conservative.metrics['recall@10']).toBe(0.71);
|
||||
expect(report.grouped.longmemeval.tokenmax.metrics['ndcg@10']).toBe(0.762);
|
||||
expect(report._meta.metric_glossary['recall@10']).toBeDefined();
|
||||
expect(report._meta.metric_glossary['ndcg@10']).toBeDefined();
|
||||
expect(report._meta.methodology).toContain('bootstrap');
|
||||
});
|
||||
|
||||
test('--md output names every mode + metric', async () => {
|
||||
const path = writeJsonl(SAMPLE_RECORDS);
|
||||
const out = await captureRun(() => runEvalCompare(['--md', '--input', path]));
|
||||
expect(out).toContain('# Search Mode Comparison');
|
||||
expect(out).toContain('## longmemeval');
|
||||
expect(out).toContain('conservative');
|
||||
expect(out).toContain('balanced');
|
||||
expect(out).toContain('tokenmax');
|
||||
expect(out).toContain('### recall@10');
|
||||
expect(out).toContain('### ndcg@10');
|
||||
expect(out).toContain('Plain English:'); // Glossary line surfaced
|
||||
});
|
||||
|
||||
test('missing file → friendly hint, no crash', async () => {
|
||||
const path = join(tmp, 'does-not-exist.jsonl');
|
||||
const out = await captureRun(() => runEvalCompare(['--md', '--input', path]));
|
||||
expect(out).toContain('No eval-results.jsonl found');
|
||||
expect(out).toContain('gbrain eval run-all');
|
||||
});
|
||||
|
||||
test('--modes filter narrows the table', async () => {
|
||||
const path = writeJsonl(SAMPLE_RECORDS);
|
||||
const out = await captureRun(() => runEvalCompare(['--json', '--modes', 'conservative,tokenmax', '--input', path]));
|
||||
const report = JSON.parse(out);
|
||||
// Records list is filtered.
|
||||
expect(report.records.length).toBe(2);
|
||||
// The grouped table preserves the SEARCH_MODES order (conservative/balanced/tokenmax).
|
||||
// Filtered balanced → null in grouped output.
|
||||
expect(report.grouped.longmemeval.balanced).toBeNull();
|
||||
});
|
||||
|
||||
test('multiple runs for same (suite, mode) → most-recent wins', async () => {
|
||||
const oldRun = { ...SAMPLE_RECORDS[0], ran_at: '2026-05-12T09:00:00Z', metrics: { 'recall@10': 0.5 } };
|
||||
const newRun = { ...SAMPLE_RECORDS[0], ran_at: '2026-05-12T11:00:00Z', metrics: { 'recall@10': 0.9 } };
|
||||
const path = writeJsonl([oldRun, newRun]);
|
||||
const out = await captureRun(() => runEvalCompare(['--json', '--input', path]));
|
||||
const report = JSON.parse(out);
|
||||
expect(report.grouped.longmemeval.conservative.metrics['recall@10']).toBe(0.9);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,197 @@
|
||||
/**
|
||||
* v0.32.3 — eval-run-all orchestrator unit tests.
|
||||
* Pins arg parsing + cost-guard semantics + persist hook + audit trail.
|
||||
*/
|
||||
import { afterAll, beforeAll, beforeEach, describe, expect, test } from 'bun:test';
|
||||
import { mkdtempSync, rmSync, readFileSync, existsSync } from 'fs';
|
||||
import { tmpdir } from 'os';
|
||||
import { join } from 'path';
|
||||
import {
|
||||
parseRunAllArgs,
|
||||
estimateRunCost,
|
||||
evaluateCostGuard,
|
||||
persistRunRecord,
|
||||
type EvalRunRecord,
|
||||
} from '../src/commands/eval-run-all.ts';
|
||||
|
||||
let tmp: string;
|
||||
|
||||
beforeAll(() => {
|
||||
tmp = mkdtempSync(join(tmpdir(), 'gbrain-eval-runall-'));
|
||||
});
|
||||
|
||||
afterAll(() => {
|
||||
rmSync(tmp, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
describe('parseRunAllArgs', () => {
|
||||
test('defaults: all modes, longmemeval+replay suites, seed=42', () => {
|
||||
const opts = parseRunAllArgs([]);
|
||||
expect(opts.modes).toEqual(['conservative', 'balanced', 'tokenmax']);
|
||||
expect(opts.suites).toEqual(['longmemeval', 'replay']);
|
||||
expect(opts.seed).toBe(42);
|
||||
expect(opts.parallel).toBe(1);
|
||||
expect(opts.budgetUsdRetrieval).toBe(5);
|
||||
expect(opts.budgetUsdAnswer).toBe(20);
|
||||
expect(opts.yes).toBe(false);
|
||||
});
|
||||
|
||||
test('--modes filters to a subset', () => {
|
||||
const opts = parseRunAllArgs(['--modes', 'conservative,tokenmax']);
|
||||
expect(opts.modes).toEqual(['conservative', 'tokenmax']);
|
||||
});
|
||||
|
||||
test('--modes rejects invalid mode', () => {
|
||||
expect(() => parseRunAllArgs(['--modes', 'frontier'])).toThrow(/not a valid mode/);
|
||||
});
|
||||
|
||||
test('--suites filters; rejects unknown', () => {
|
||||
const opts = parseRunAllArgs(['--suites', 'longmemeval']);
|
||||
expect(opts.suites).toEqual(['longmemeval']);
|
||||
expect(() => parseRunAllArgs(['--suites', 'foo'])).toThrow(/not a recognized suite/);
|
||||
});
|
||||
|
||||
test('--budget-usd-retrieval + --budget-usd-answer override defaults', () => {
|
||||
const opts = parseRunAllArgs(['--budget-usd-retrieval', '15', '--budget-usd-answer', '50']);
|
||||
expect(opts.budgetUsdRetrieval).toBe(15);
|
||||
expect(opts.budgetUsdAnswer).toBe(50);
|
||||
});
|
||||
|
||||
test('--parallel clamps to mode count', () => {
|
||||
const opts = parseRunAllArgs(['--parallel', '10']);
|
||||
expect(opts.parallel).toBe(3); // clamped to SEARCH_MODES.length
|
||||
});
|
||||
|
||||
test('--parallel rejects 0 / negative', () => {
|
||||
expect(() => parseRunAllArgs(['--parallel', '0'])).toThrow(/must be >= 1/);
|
||||
expect(() => parseRunAllArgs(['--parallel', '-1'])).toThrow(/must be >= 1/);
|
||||
});
|
||||
|
||||
test('--yes flag toggles', () => {
|
||||
expect(parseRunAllArgs(['--yes']).yes).toBe(true);
|
||||
expect(parseRunAllArgs(['-y']).yes).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
describe('estimateRunCost', () => {
|
||||
test('returns retrieval/answer/total breakdown', () => {
|
||||
const est = estimateRunCost({ suites: ['longmemeval'], modes: ['balanced'], limit: 100 });
|
||||
expect(est.total_usd).toBe(est.retrieval_usd + est.answer_usd);
|
||||
expect(est.answer_usd).toBeGreaterThan(0); // longmemeval has answer-gen
|
||||
expect(est.retrieval_usd).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
test('tokenmax incurs expansion cost (Haiku per query)', () => {
|
||||
const bal = estimateRunCost({ suites: ['replay'], modes: ['balanced'], limit: 100 });
|
||||
const tok = estimateRunCost({ suites: ['replay'], modes: ['tokenmax'], limit: 100 });
|
||||
expect(tok.retrieval_usd).toBeGreaterThan(bal.retrieval_usd);
|
||||
});
|
||||
|
||||
test('replay suite has zero answer cost', () => {
|
||||
const est = estimateRunCost({ suites: ['replay'], modes: ['balanced'], limit: 100 });
|
||||
expect(est.answer_usd).toBe(0);
|
||||
});
|
||||
|
||||
test('multi-suite multi-mode estimate sums correctly', () => {
|
||||
const est = estimateRunCost({ suites: ['longmemeval', 'replay'], modes: ['conservative', 'balanced'], limit: 100 });
|
||||
expect(Object.keys(est.per_suite).length).toBe(4); // 2 × 2
|
||||
});
|
||||
});
|
||||
|
||||
describe('evaluateCostGuard', () => {
|
||||
test('within-budget proceeds without --yes', () => {
|
||||
const g = evaluateCostGuard(
|
||||
{ retrieval_usd: 1, answer_usd: 10, total_usd: 11 },
|
||||
{ budgetUsdRetrieval: 5, budgetUsdAnswer: 20, yes: false, isTty: true },
|
||||
);
|
||||
expect(g.proceed).toBe(true);
|
||||
});
|
||||
|
||||
test('over-cap TTY without --yes refuses', () => {
|
||||
const g = evaluateCostGuard(
|
||||
{ retrieval_usd: 30, answer_usd: 30, total_usd: 60 },
|
||||
{ budgetUsdRetrieval: 5, budgetUsdAnswer: 20, yes: false, isTty: true },
|
||||
);
|
||||
expect(g.proceed).toBe(false);
|
||||
expect(g.reason).toContain('--yes');
|
||||
});
|
||||
|
||||
test('over-cap TTY with --yes proceeds', () => {
|
||||
const g = evaluateCostGuard(
|
||||
{ retrieval_usd: 30, answer_usd: 30, total_usd: 60 },
|
||||
{ budgetUsdRetrieval: 5, budgetUsdAnswer: 20, yes: true, isTty: true },
|
||||
);
|
||||
expect(g.proceed).toBe(true);
|
||||
});
|
||||
|
||||
test('over-cap non-TTY without --yes refuses (exit 2 path)', () => {
|
||||
const g = evaluateCostGuard(
|
||||
{ retrieval_usd: 30, answer_usd: 30, total_usd: 60 },
|
||||
{ budgetUsdRetrieval: 5, budgetUsdAnswer: 20, yes: false, isTty: false },
|
||||
);
|
||||
expect(g.proceed).toBe(false);
|
||||
expect(g.reason).toContain('Non-TTY requires --yes');
|
||||
});
|
||||
|
||||
test('only-retrieval-over OR only-answer-over both trigger guard', () => {
|
||||
const a = evaluateCostGuard(
|
||||
{ retrieval_usd: 100, answer_usd: 1, total_usd: 101 },
|
||||
{ budgetUsdRetrieval: 5, budgetUsdAnswer: 20, yes: false, isTty: false },
|
||||
);
|
||||
expect(a.proceed).toBe(false);
|
||||
const b = evaluateCostGuard(
|
||||
{ retrieval_usd: 1, answer_usd: 100, total_usd: 101 },
|
||||
{ budgetUsdRetrieval: 5, budgetUsdAnswer: 20, yes: false, isTty: false },
|
||||
);
|
||||
expect(b.proceed).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe('persistRunRecord audit trail', () => {
|
||||
beforeEach(() => {
|
||||
rmSync(join(tmp, '.gbrain-evals'), { recursive: true, force: true });
|
||||
rmSync(join(tmp, 'eval-results.jsonl'), { force: true });
|
||||
});
|
||||
|
||||
test('appends to eval-results.jsonl, creates dir if missing', () => {
|
||||
const record: EvalRunRecord = {
|
||||
schema_version: 2,
|
||||
run_id: 'abc123-longmemeval-conservative-42',
|
||||
ran_at: '2026-05-12T12:00:00Z',
|
||||
suite: 'longmemeval',
|
||||
mode: 'conservative',
|
||||
commit: 'abc123',
|
||||
seed: 42,
|
||||
params: {},
|
||||
status: 'completed',
|
||||
duration_ms: 12_345,
|
||||
};
|
||||
persistRunRecord(tmp, record, tmp);
|
||||
const path = join(tmp, 'eval-results.jsonl');
|
||||
expect(existsSync(path)).toBe(true);
|
||||
const content = readFileSync(path, 'utf-8');
|
||||
expect(content).toContain('abc123-longmemeval-conservative-42');
|
||||
const parsed = JSON.parse(content.trim());
|
||||
expect(parsed.mode).toBe('conservative');
|
||||
expect(parsed.schema_version).toBe(2);
|
||||
});
|
||||
|
||||
test('appends multiple records (NDJSON)', () => {
|
||||
const base = {
|
||||
schema_version: 2 as const,
|
||||
ran_at: '2026-05-12T12:00:00Z',
|
||||
suite: 'longmemeval' as const,
|
||||
commit: 'abc',
|
||||
seed: 42,
|
||||
params: {},
|
||||
status: 'completed' as const,
|
||||
duration_ms: 1,
|
||||
};
|
||||
persistRunRecord(tmp, { ...base, run_id: 'a', mode: 'conservative' }, tmp);
|
||||
persistRunRecord(tmp, { ...base, run_id: 'b', mode: 'balanced' }, tmp);
|
||||
persistRunRecord(tmp, { ...base, run_id: 'c', mode: 'tokenmax' }, tmp);
|
||||
const content = readFileSync(join(tmp, 'eval-results.jsonl'), 'utf-8');
|
||||
const lines = content.trim().split('\n');
|
||||
expect(lines.length).toBe(3);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,185 @@
|
||||
/**
|
||||
* v0.32.x search-lite \u2014 hybridSearchCached integration.
|
||||
*
|
||||
* End-to-end PGLite test that confirms the three search-lite features
|
||||
* fire through the actual hybrid pipeline (not the units in isolation):
|
||||
*
|
||||
* 1. Token budget: results are capped after search.
|
||||
* 2. Cache: meta surfaces hit/miss; disabled mode is a clean pass-through.
|
||||
* 3. Intent classifier: meta.intent matches the classifier output.
|
||||
*
|
||||
* Vector search isn't enabled (no embedding provider in test), so we
|
||||
* exercise the keyword-only path \u2014 which still surfaces intent and
|
||||
* budget. The cache path is exercised separately in query-cache.test.ts
|
||||
* because it needs a real embedding to key on.
|
||||
*/
|
||||
|
||||
import { describe, test, expect, beforeAll, afterAll } from 'bun:test';
|
||||
import { PGLiteEngine } from '../src/core/pglite-engine.ts';
|
||||
import { hybridSearchCached } from '../src/core/search/hybrid.ts';
|
||||
import type { PageInput, HybridSearchMeta } from '../src/core/types.ts';
|
||||
|
||||
let engine: PGLiteEngine;
|
||||
const savedKey = process.env.OPENAI_API_KEY;
|
||||
|
||||
beforeAll(async () => {
|
||||
engine = new PGLiteEngine();
|
||||
await engine.connect({});
|
||||
await engine.initSchema();
|
||||
// Insert a small fixture set so keyword search has something to find.
|
||||
// Use long chunk_texts so token budget cuts have observable effect.
|
||||
const longText = 'x'.repeat(800); // ~200 tokens of body text
|
||||
const pages: Array<{ slug: string; page: PageInput }> = [
|
||||
{
|
||||
slug: 'alice-foo',
|
||||
page: {
|
||||
type: 'person',
|
||||
title: 'Alice Foo',
|
||||
compiled_truth: `Alice Foo is a builder. ${longText}`,
|
||||
},
|
||||
},
|
||||
{
|
||||
slug: 'bob-bar',
|
||||
page: {
|
||||
type: 'person',
|
||||
title: 'Bob Bar',
|
||||
compiled_truth: `Bob Bar is a builder. ${longText}`,
|
||||
},
|
||||
},
|
||||
{
|
||||
slug: 'carol-baz',
|
||||
page: {
|
||||
type: 'person',
|
||||
title: 'Carol Baz',
|
||||
compiled_truth: `Carol Baz is a builder. ${longText}`,
|
||||
},
|
||||
},
|
||||
];
|
||||
for (const p of pages) {
|
||||
await engine.putPage(p.slug, p.page);
|
||||
}
|
||||
// Force keyword-only fallback by unsetting the embedding provider key.
|
||||
delete process.env.OPENAI_API_KEY;
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
if (savedKey) process.env.OPENAI_API_KEY = savedKey;
|
||||
try { await engine.disconnect(); } catch { /* ignore */ }
|
||||
});
|
||||
|
||||
describe('hybridSearchCached \u2014 meta surfaces intent', () => {
|
||||
test('entity query classifies as entity', async () => {
|
||||
let meta: HybridSearchMeta | undefined;
|
||||
await hybridSearchCached(engine, 'who is alice-foo', {
|
||||
limit: 5,
|
||||
onMeta: (m) => { meta = m; },
|
||||
});
|
||||
expect(meta?.intent).toBe('entity');
|
||||
});
|
||||
|
||||
test('temporal query classifies as temporal', async () => {
|
||||
let meta: HybridSearchMeta | undefined;
|
||||
await hybridSearchCached(engine, 'what happened last week', {
|
||||
limit: 5,
|
||||
onMeta: (m) => { meta = m; },
|
||||
});
|
||||
expect(meta?.intent).toBe('temporal');
|
||||
});
|
||||
|
||||
test('event query classifies as event', async () => {
|
||||
let meta: HybridSearchMeta | undefined;
|
||||
await hybridSearchCached(engine, 'who raised $10M', {
|
||||
limit: 5,
|
||||
onMeta: (m) => { meta = m; },
|
||||
});
|
||||
expect(meta?.intent).toBe('event');
|
||||
});
|
||||
});
|
||||
|
||||
describe('hybridSearchCached \u2014 token budget', () => {
|
||||
test('budget undefined returns no token_budget meta (no cut)', async () => {
|
||||
let meta: HybridSearchMeta | undefined;
|
||||
const results = await hybridSearchCached(engine, 'alice', {
|
||||
limit: 10,
|
||||
onMeta: (m) => { meta = m; },
|
||||
});
|
||||
// Don't assert non-empty here — keyword tokenization depends on the
|
||||
// pglite analyzer config. What matters: meta is shaped right and
|
||||
// budget metadata is absent when budget isn't set.
|
||||
expect(results).toBeDefined();
|
||||
expect(meta?.token_budget).toBeUndefined();
|
||||
});
|
||||
|
||||
test('budget meta is always emitted when budget is set', async () => {
|
||||
let meta: HybridSearchMeta | undefined;
|
||||
const results = await hybridSearchCached(engine, 'alice', {
|
||||
limit: 10,
|
||||
tokenBudget: 250,
|
||||
onMeta: (m) => { meta = m; },
|
||||
});
|
||||
expect(meta?.token_budget).toBeDefined();
|
||||
expect(meta?.token_budget?.budget).toBe(250);
|
||||
expect(meta?.token_budget?.kept).toBe(results.length);
|
||||
});
|
||||
|
||||
test('tight budget cuts the result set', async () => {
|
||||
// First find out the result count without a budget so the assertion
|
||||
// is robust to the fixture’s actual chunking.
|
||||
const unbounded = await hybridSearchCached(engine, 'builder', { limit: 10 });
|
||||
// Skip the cut test if the fixture happens to return only one row
|
||||
// (keyword search may dedupe by page); the budget enforcement itself
|
||||
// is exhaustively unit-tested in test/token-budget.test.ts.
|
||||
if (unbounded.length < 2) return;
|
||||
|
||||
let meta: HybridSearchMeta | undefined;
|
||||
const results = await hybridSearchCached(engine, 'builder', {
|
||||
limit: 10,
|
||||
tokenBudget: 250, // enough for ~1 row of fixture data
|
||||
onMeta: (m) => { meta = m; },
|
||||
});
|
||||
expect(meta?.token_budget?.budget).toBe(250);
|
||||
expect(meta?.token_budget?.kept).toBe(results.length);
|
||||
expect(meta?.token_budget?.dropped).toBeGreaterThan(0);
|
||||
// The budget must hold: cumulative cost <= budget.
|
||||
expect(meta?.token_budget?.used).toBeLessThanOrEqual(250);
|
||||
});
|
||||
});
|
||||
|
||||
describe('hybridSearchCached \u2014 cache disabled fallback', () => {
|
||||
test('keyword-only path emits cache.status=disabled', async () => {
|
||||
// No embedding available \u2192 cache decision degrades to disabled.
|
||||
let meta: HybridSearchMeta | undefined;
|
||||
await hybridSearchCached(engine, 'who is alice-foo', {
|
||||
limit: 5,
|
||||
onMeta: (m) => { meta = m; },
|
||||
});
|
||||
// cache may be 'disabled' (no embedding provider) or 'miss'.
|
||||
// Either way the field exists.
|
||||
expect(meta?.cache).toBeDefined();
|
||||
expect(['disabled', 'miss']).toContain(meta?.cache?.status ?? '');
|
||||
});
|
||||
|
||||
test('useCache=false explicitly disables', async () => {
|
||||
let meta: HybridSearchMeta | undefined;
|
||||
await hybridSearchCached(engine, 'who is bob-bar', {
|
||||
limit: 5,
|
||||
useCache: false,
|
||||
onMeta: (m) => { meta = m; },
|
||||
});
|
||||
expect(meta?.cache?.status).toBe('disabled');
|
||||
});
|
||||
});
|
||||
|
||||
describe('hybridSearchCached \u2014 intent weighting toggle', () => {
|
||||
test('intentWeighting=false still emits intent in meta (for visibility)', async () => {
|
||||
let meta: HybridSearchMeta | undefined;
|
||||
await hybridSearchCached(engine, 'who is alice-foo', {
|
||||
limit: 5,
|
||||
intentWeighting: false,
|
||||
onMeta: (m) => { meta = m; },
|
||||
});
|
||||
// Intent classification itself still runs (cheap regex); only the
|
||||
// weight adjustment is disabled. So meta.intent stays populated.
|
||||
expect(meta?.intent).toBe('entity');
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,251 @@
|
||||
/**
|
||||
* v0.32.3 search-lite install-time mode picker tests.
|
||||
*
|
||||
* Pure-function coverage (recommendModeFor + parseModeInput) plus the
|
||||
* idempotent runModePicker behavior. The interactive TTY branch is
|
||||
* exercised indirectly via the non-TTY path here; full TTY simulation
|
||||
* lives in the e2e suite (test/e2e/...).
|
||||
*/
|
||||
import { afterAll, beforeAll, beforeEach, describe, expect, test } from 'bun:test';
|
||||
import { PGLiteEngine } from '../src/core/pglite-engine.ts';
|
||||
import {
|
||||
recommendModeFor,
|
||||
parseModeInput,
|
||||
runModePicker,
|
||||
} from '../src/commands/init-mode-picker.ts';
|
||||
import { withEnv } from './helpers/with-env.ts';
|
||||
|
||||
let engine: PGLiteEngine;
|
||||
|
||||
beforeAll(async () => {
|
||||
engine = new PGLiteEngine();
|
||||
await engine.connect({});
|
||||
await engine.initSchema();
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
await engine.disconnect();
|
||||
});
|
||||
|
||||
beforeEach(async () => {
|
||||
await engine.executeRaw(`DELETE FROM config WHERE key LIKE 'search.%' OR key LIKE 'models.%'`);
|
||||
});
|
||||
|
||||
describe('recommendModeFor — auto-suggestion heuristic', () => {
|
||||
test('Opus default → tokenmax', () => {
|
||||
const r = recommendModeFor({ defaultModel: 'anthropic:claude-opus-4-7' });
|
||||
expect(r.mode).toBe('tokenmax');
|
||||
expect(r.reason).toMatch(/Opus/);
|
||||
});
|
||||
|
||||
test('Opus subagent → tokenmax', () => {
|
||||
const r = recommendModeFor({ subagentModel: 'anthropic:claude-opus-4-7' });
|
||||
expect(r.mode).toBe('tokenmax');
|
||||
});
|
||||
|
||||
test('Haiku subagent → conservative', () => {
|
||||
const r = recommendModeFor({ subagentModel: 'anthropic:claude-haiku-4-5' });
|
||||
expect(r.mode).toBe('conservative');
|
||||
expect(r.reason).toMatch(/Haiku/);
|
||||
});
|
||||
|
||||
test('No OpenAI key → conservative (no LLM expansion possible)', () => {
|
||||
const r = recommendModeFor({ hasOpenAIKey: false });
|
||||
expect(r.mode).toBe('conservative');
|
||||
expect(r.reason).toMatch(/No OpenAI/);
|
||||
});
|
||||
|
||||
test('Sonnet / unknown → tokenmax (preserve-v0.31.x default)', () => {
|
||||
const r = recommendModeFor({ subagentModel: 'anthropic:claude-sonnet-4-6', hasOpenAIKey: true });
|
||||
expect(r.mode).toBe('tokenmax');
|
||||
expect(r.reason).toMatch(/v0\.31\.x|preserve/i);
|
||||
});
|
||||
|
||||
test('Empty inputs → tokenmax (preserve-v0.31.x default)', () => {
|
||||
const r = recommendModeFor({});
|
||||
expect(r.mode).toBe('tokenmax');
|
||||
});
|
||||
|
||||
|
||||
test('Haiku subagent wins over Opus default (cost-sensitive takes precedence)', () => {
|
||||
// Reordered in the install-picker DX pass: Haiku check fires BEFORE Opus
|
||||
// because a user running a Haiku subagent loop is signalling cost
|
||||
// sensitivity. Tokenmax over Haiku would silently dump 50-chunk payloads
|
||||
// into a model that struggles past 5-10 chunks. The Haiku floor wins.
|
||||
const r = recommendModeFor({
|
||||
defaultModel: 'anthropic:claude-opus-4-7',
|
||||
subagentModel: 'anthropic:claude-haiku-4-5',
|
||||
hasOpenAIKey: true,
|
||||
});
|
||||
expect(r.mode).toBe('conservative');
|
||||
});
|
||||
});
|
||||
|
||||
describe('MENU_TEXT cost-matrix anchors (must match CLAUDE.md + methodology doc)', () => {
|
||||
test('25x corner-to-corner spread framing is named', async () => {
|
||||
const { MODE_PICKER_MENU } = await import('../src/commands/init-mode-picker.ts');
|
||||
// Updating these REQUIRES bumping CLAUDE.md ## Search Mode + the
|
||||
// methodology doc + README in lockstep.
|
||||
expect(MODE_PICKER_MENU).toContain('25x');
|
||||
expect(MODE_PICKER_MENU).toContain('corner-to-corner');
|
||||
});
|
||||
|
||||
test('cost matrix lists every cell at the natural diagonal and corners', async () => {
|
||||
const { MODE_PICKER_MENU } = await import('../src/commands/init-mode-picker.ts');
|
||||
// Three anchor cells from the natural diagonal at 10K/mo volume. Scales
|
||||
// linearly; multiplying by 10 gives the 100K/mo numbers the methodology
|
||||
// doc + CLAUDE.md cite ("multiply by 10 for 100K/mo" prose anchor).
|
||||
expect(MODE_PICKER_MENU).toContain('$40/mo'); // conservative + Haiku
|
||||
expect(MODE_PICKER_MENU).toContain('$300/mo'); // balanced + Sonnet (default natural)
|
||||
expect(MODE_PICKER_MENU).toContain('$1,000/mo'); // tokenmax + Opus
|
||||
});
|
||||
|
||||
test('volume frame is 10K queries/month with linear-scale callout', async () => {
|
||||
const { MODE_PICKER_MENU } = await import('../src/commands/init-mode-picker.ts');
|
||||
expect(MODE_PICKER_MENU).toContain('10K queries/mo');
|
||||
expect(MODE_PICKER_MENU).toContain('scales linearly');
|
||||
});
|
||||
|
||||
test('all three downstream model rates are explicit', async () => {
|
||||
const { MODE_PICKER_MENU } = await import('../src/commands/init-mode-picker.ts');
|
||||
expect(MODE_PICKER_MENU).toContain('Haiku 4.5');
|
||||
expect(MODE_PICKER_MENU).toContain('Sonnet 4.6');
|
||||
expect(MODE_PICKER_MENU).toContain('Opus 4.7');
|
||||
expect(MODE_PICKER_MENU).toContain('$1/M');
|
||||
expect(MODE_PICKER_MENU).toContain('$3/M');
|
||||
expect(MODE_PICKER_MENU).toContain('$5/M');
|
||||
});
|
||||
|
||||
test('tokenmax Haiku-expansion surcharge is named explicitly', async () => {
|
||||
const { MODE_PICKER_MENU } = await import('../src/commands/init-mode-picker.ts');
|
||||
// Cross-line match — the surcharge phrase can wrap.
|
||||
expect(MODE_PICKER_MENU.replace(/\s+/g, ' ')).toContain('~$1.50 per 1K queries');
|
||||
expect(MODE_PICKER_MENU).toContain('Haiku expansion call');
|
||||
});
|
||||
|
||||
test('cache-hit discount framing is named', async () => {
|
||||
const { MODE_PICKER_MENU } = await import('../src/commands/init-mode-picker.ts');
|
||||
expect(MODE_PICKER_MENU).toContain('cache');
|
||||
// The numbers below are full-payload pre-cache; real loops see 50-80% discount.
|
||||
expect(MODE_PICKER_MENU).toMatch(/50-80%|discount/);
|
||||
});
|
||||
|
||||
test('tune command is surfaced as the next step', async () => {
|
||||
const { MODE_PICKER_MENU } = await import('../src/commands/init-mode-picker.ts');
|
||||
expect(MODE_PICKER_MENU).toContain('gbrain search tune');
|
||||
});
|
||||
});
|
||||
|
||||
describe('parseModeInput — menu choice mapper', () => {
|
||||
test('numeric 1/2/3 → conservative/balanced/tokenmax', () => {
|
||||
expect(parseModeInput('1')).toBe('conservative');
|
||||
expect(parseModeInput('2')).toBe('balanced');
|
||||
expect(parseModeInput('3')).toBe('tokenmax');
|
||||
});
|
||||
|
||||
test('mode names (case-insensitive)', () => {
|
||||
expect(parseModeInput('conservative')).toBe('conservative');
|
||||
expect(parseModeInput('CONSERVATIVE')).toBe('conservative');
|
||||
expect(parseModeInput('TokenMax')).toBe('tokenmax');
|
||||
expect(parseModeInput(' balanced ')).toBe('balanced');
|
||||
});
|
||||
|
||||
test('empty / unrecognized → null', () => {
|
||||
expect(parseModeInput('')).toBeNull();
|
||||
expect(parseModeInput(' ')).toBeNull();
|
||||
expect(parseModeInput('foo')).toBeNull();
|
||||
expect(parseModeInput('4')).toBeNull();
|
||||
expect(parseModeInput('0')).toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
describe('runModePicker non-TTY surfaces full matrix + [AGENT] directive', () => {
|
||||
test('non-TTY output includes the cost matrix and the [AGENT] directive', async () => {
|
||||
const originalLog = console.log;
|
||||
const captured: string[] = [];
|
||||
console.log = (msg: string) => { captured.push(String(msg)); };
|
||||
try {
|
||||
await runModePicker(engine);
|
||||
} finally {
|
||||
console.log = originalLog;
|
||||
}
|
||||
const out = captured.join('\n');
|
||||
// Matrix corners + diagonal mid
|
||||
expect(out).toContain('$40/mo');
|
||||
expect(out).toContain('$300/mo');
|
||||
expect(out).toContain('$1,000/mo');
|
||||
// 25x spread framing
|
||||
expect(out).toContain('25x corner-to-corner');
|
||||
// Explicit agent directive — load-bearing for agent-platform install paths
|
||||
expect(out).toContain('[AGENT]');
|
||||
expect(out.toLowerCase()).toContain('show this matrix');
|
||||
expect(out).toContain('INSTALL_FOR_AGENTS.md');
|
||||
});
|
||||
});
|
||||
|
||||
describe('runModePicker — non-TTY auto-select + idempotent', () => {
|
||||
test('non-TTY auto-selects + writes config + emits operator hint', async () => {
|
||||
// Bun test runs non-TTY by default.
|
||||
const picked = await runModePicker(engine);
|
||||
// Default model unset → balanced. Should write search.mode.
|
||||
const stored = await engine.getConfig('search.mode');
|
||||
expect(stored).toBe(picked);
|
||||
expect(['conservative', 'balanced', 'tokenmax']).toContain(picked);
|
||||
});
|
||||
|
||||
test('idempotent: second call returns existing mode without overwrite', async () => {
|
||||
await engine.setConfig('search.mode', 'tokenmax');
|
||||
const picked = await runModePicker(engine);
|
||||
expect(picked).toBe('tokenmax');
|
||||
// No accidental overwrite.
|
||||
const stored = await engine.getConfig('search.mode');
|
||||
expect(stored).toBe('tokenmax');
|
||||
});
|
||||
|
||||
test('--force re-prompts even if mode is already set', async () => {
|
||||
await engine.setConfig('search.mode', 'conservative');
|
||||
// Non-TTY + force → re-runs auto-suggest with current inputs.
|
||||
const picked = await runModePicker(engine, { force: true });
|
||||
// With no model hints + no API key state, default is balanced. The picker
|
||||
// will overwrite the existing mode.
|
||||
expect(['conservative', 'balanced', 'tokenmax']).toContain(picked);
|
||||
const stored = await engine.getConfig('search.mode');
|
||||
expect(stored).toBe(picked);
|
||||
});
|
||||
|
||||
test('jsonOutput mode emits a structured event and writes config', async () => {
|
||||
// Capture console.log output.
|
||||
const originalLog = console.log;
|
||||
const captured: string[] = [];
|
||||
console.log = (msg: string) => { captured.push(msg); };
|
||||
try {
|
||||
const picked = await runModePicker(engine, { jsonOutput: true });
|
||||
const stored = await engine.getConfig('search.mode');
|
||||
expect(stored).toBe(picked);
|
||||
const jsonLine = captured.find(l => l.startsWith('{'));
|
||||
expect(jsonLine).toBeDefined();
|
||||
const obj = JSON.parse(jsonLine!);
|
||||
expect(obj.phase).toBe('search_mode_picker');
|
||||
expect(obj.auto).toBe(true);
|
||||
expect(['conservative', 'balanced', 'tokenmax']).toContain(obj.mode);
|
||||
} finally {
|
||||
console.log = originalLog;
|
||||
}
|
||||
});
|
||||
|
||||
test('Opus default model + OpenAI key → picker auto-recommends tokenmax', async () => {
|
||||
// OPENAI_API_KEY must be present — the no-key short-circuit fires before
|
||||
// the Opus check (gbrain can't do vector search without embeddings).
|
||||
await withEnv({ OPENAI_API_KEY: 'sk-test-stub' }, async () => {
|
||||
await engine.setConfig('models.default', 'anthropic:claude-opus-4-7');
|
||||
const picked = await runModePicker(engine);
|
||||
expect(picked).toBe('tokenmax');
|
||||
});
|
||||
});
|
||||
|
||||
test('Haiku subagent → picker auto-recommends conservative', async () => {
|
||||
await engine.setConfig('models.tier.subagent', 'anthropic:claude-haiku-4-5');
|
||||
const picked = await runModePicker(engine);
|
||||
expect(picked).toBe('conservative');
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,120 @@
|
||||
/**
|
||||
* v0.32.x search-lite \u2014 intent \u2192 weight adjustment tests.
|
||||
*
|
||||
* Pure module under test (no DB). Confirms:
|
||||
* - weightsForIntent returns the expected per-intent factors
|
||||
* - effectiveRrfK scales correctly with the weight
|
||||
* - applyExactMatchBoost only fires on exact slug/title matches
|
||||
* - general intent is a no-op (preserves pre-v0.32 behavior)
|
||||
*/
|
||||
|
||||
import { describe, test, expect } from 'bun:test';
|
||||
import {
|
||||
weightsForIntent,
|
||||
effectiveRrfK,
|
||||
applyExactMatchBoost,
|
||||
} from '../src/core/search/intent-weights.ts';
|
||||
import type { SearchResult } from '../src/core/types.ts';
|
||||
|
||||
function makeResult(overrides: Partial<SearchResult> = {}): SearchResult {
|
||||
return {
|
||||
slug: 'test-page',
|
||||
page_id: 1,
|
||||
title: 'Test Title',
|
||||
type: 'concept',
|
||||
chunk_text: 'test chunk text',
|
||||
chunk_source: 'compiled_truth',
|
||||
chunk_id: 1,
|
||||
chunk_index: 0,
|
||||
score: 1.0,
|
||||
stale: false,
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
describe('weightsForIntent', () => {
|
||||
test('general intent is the identity', () => {
|
||||
const w = weightsForIntent('general');
|
||||
expect(w.keywordWeight).toBe(1.0);
|
||||
expect(w.vectorWeight).toBe(1.0);
|
||||
expect(w.suggestedRecency).toBe(null);
|
||||
expect(w.exactMatchBoost).toBe(1.0);
|
||||
});
|
||||
|
||||
test('entity intent boosts keyword + exact match', () => {
|
||||
const w = weightsForIntent('entity');
|
||||
expect(w.keywordWeight).toBeGreaterThan(1.0);
|
||||
expect(w.exactMatchBoost).toBeGreaterThan(1.0);
|
||||
expect(w.suggestedRecency).toBe(null);
|
||||
});
|
||||
|
||||
test('temporal intent suggests recency=on', () => {
|
||||
const w = weightsForIntent('temporal');
|
||||
expect(w.suggestedRecency).toBe('on');
|
||||
});
|
||||
|
||||
test('event intent boosts keyword (rare named entities)', () => {
|
||||
const w = weightsForIntent('event');
|
||||
expect(w.keywordWeight).toBeGreaterThan(1.0);
|
||||
expect(w.suggestedRecency).toBe('on');
|
||||
});
|
||||
});
|
||||
|
||||
describe('effectiveRrfK', () => {
|
||||
test('weight=1.0 returns base k unchanged', () => {
|
||||
expect(effectiveRrfK(60, 1.0)).toBe(60);
|
||||
});
|
||||
|
||||
test('weight > 1 lowers k (stronger top-rank contribution)', () => {
|
||||
expect(effectiveRrfK(60, 1.2)).toBeLessThan(60);
|
||||
expect(effectiveRrfK(60, 1.2)).toBe(50);
|
||||
});
|
||||
|
||||
test('weight <= 0 returns base k (safety)', () => {
|
||||
expect(effectiveRrfK(60, 0)).toBe(60);
|
||||
expect(effectiveRrfK(60, -1)).toBe(60);
|
||||
});
|
||||
});
|
||||
|
||||
describe('applyExactMatchBoost', () => {
|
||||
test('boost=1.0 is a no-op', () => {
|
||||
const results = [makeResult({ slug: 'foo', score: 1.0 })];
|
||||
applyExactMatchBoost(results, 'foo', weightsForIntent('general'));
|
||||
expect(results[0].score).toBe(1.0);
|
||||
});
|
||||
|
||||
test('exact slug match gets boosted (entity intent)', () => {
|
||||
const results = [
|
||||
makeResult({ slug: 'garry-tan', score: 1.0, title: 'Garry Tan' }),
|
||||
makeResult({ slug: 'someone-else', score: 1.0, title: 'Someone Else' }),
|
||||
];
|
||||
applyExactMatchBoost(results, 'garry-tan', weightsForIntent('entity'));
|
||||
// First result has slug=query \u2192 boosted; second is unchanged.
|
||||
expect(results[0].score).toBeGreaterThan(1.0);
|
||||
expect(results[1].score).toBe(1.0);
|
||||
});
|
||||
|
||||
test('query with spaces matches kebab slug ("garry tan" \u2192 "garry-tan")', () => {
|
||||
const results = [makeResult({ slug: 'garry-tan', score: 1.0 })];
|
||||
applyExactMatchBoost(results, 'garry tan', weightsForIntent('entity'));
|
||||
expect(results[0].score).toBeGreaterThan(1.0);
|
||||
});
|
||||
|
||||
test('exact title match (case-insensitive) gets boosted', () => {
|
||||
const results = [makeResult({ slug: 'random-slug', title: 'Garry Tan', score: 1.0 })];
|
||||
applyExactMatchBoost(results, 'GARRY TAN', weightsForIntent('entity'));
|
||||
expect(results[0].score).toBeGreaterThan(1.0);
|
||||
});
|
||||
|
||||
test('partial / substring match does NOT trigger boost', () => {
|
||||
const results = [makeResult({ slug: 'garry-tan', title: 'Garry Tan', score: 1.0 })];
|
||||
applyExactMatchBoost(results, 'garry', weightsForIntent('entity'));
|
||||
expect(results[0].score).toBe(1.0);
|
||||
});
|
||||
|
||||
test('namespaced slug matches via suffix ("people/garry-tan" + query "garry-tan")', () => {
|
||||
const results = [makeResult({ slug: 'people/garry-tan', score: 1.0 })];
|
||||
applyExactMatchBoost(results, 'garry-tan', weightsForIntent('entity'));
|
||||
expect(results[0].score).toBeGreaterThan(1.0);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,128 @@
|
||||
/**
|
||||
* v0.32.3 — metric glossary module tests.
|
||||
* Pins the public surface that drives gbrain search stats / eval compare
|
||||
* output AND the auto-generated METRIC_GLOSSARY.md doc.
|
||||
*/
|
||||
import { describe, expect, test } from 'bun:test';
|
||||
import {
|
||||
METRIC_GLOSSARY,
|
||||
ALL_METRICS,
|
||||
getMetricGloss,
|
||||
eli10For,
|
||||
buildMetricGlossaryMeta,
|
||||
renderMetricGlossaryMarkdown,
|
||||
} from '../src/core/eval/metric-glossary.ts';
|
||||
|
||||
describe('METRIC_GLOSSARY canonical entries', () => {
|
||||
test('every retrieval-IR metric is present', () => {
|
||||
for (const m of ['precision@k', 'recall@k', 'mrr', 'ndcg@k']) {
|
||||
expect(METRIC_GLOSSARY[m]).toBeDefined();
|
||||
}
|
||||
});
|
||||
|
||||
test('every stability metric is present', () => {
|
||||
for (const m of ['jaccard@k', 'top1_stability']) {
|
||||
expect(METRIC_GLOSSARY[m]).toBeDefined();
|
||||
}
|
||||
});
|
||||
|
||||
test('every stat-significance metric is present', () => {
|
||||
for (const m of ['p_value', 'confidence_interval']) {
|
||||
expect(METRIC_GLOSSARY[m]).toBeDefined();
|
||||
}
|
||||
});
|
||||
|
||||
test('every operational metric is present', () => {
|
||||
for (const m of ['cache_hit_rate', 'avg_results', 'avg_tokens', 'cost_per_query_usd', 'p99_latency_ms']) {
|
||||
expect(METRIC_GLOSSARY[m]).toBeDefined();
|
||||
}
|
||||
});
|
||||
|
||||
test('every entry has the three required fields', () => {
|
||||
for (const [key, entry] of Object.entries(METRIC_GLOSSARY)) {
|
||||
expect(entry.industry_term, `${key}.industry_term`).toBeTruthy();
|
||||
expect(entry.eli10, `${key}.eli10`).toBeTruthy();
|
||||
expect(entry.range, `${key}.range`).toBeTruthy();
|
||||
}
|
||||
});
|
||||
|
||||
test('industry_term preserved verbatim (case-sensitive)', () => {
|
||||
expect(METRIC_GLOSSARY['ndcg@k'].industry_term).toBe('Normalized Discounted Cumulative Gain at k (nDCG@k)');
|
||||
expect(METRIC_GLOSSARY['mrr'].industry_term).toBe('Mean Reciprocal Rank (MRR)');
|
||||
});
|
||||
});
|
||||
|
||||
describe('getMetricGloss + eli10For accessors', () => {
|
||||
test('getMetricGloss returns the full entry', () => {
|
||||
const g = getMetricGloss('recall@k');
|
||||
expect(g).not.toBeNull();
|
||||
expect(g!.industry_term).toContain('Recall at k');
|
||||
});
|
||||
|
||||
test('getMetricGloss returns null for unknown metrics', () => {
|
||||
expect(getMetricGloss('made_up_metric')).toBeNull();
|
||||
});
|
||||
|
||||
test('eli10For returns just the plain-English line', () => {
|
||||
const text = eli10For('cache_hit_rate');
|
||||
expect(text).toContain('Fraction of searches');
|
||||
});
|
||||
|
||||
test('eli10For returns null for unknown metric', () => {
|
||||
expect(eli10For('nope')).toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
describe('buildMetricGlossaryMeta', () => {
|
||||
test('returns a flat record keyed by metric name → eli10 string', () => {
|
||||
const meta = buildMetricGlossaryMeta(['recall@k', 'mrr']);
|
||||
expect(Object.keys(meta).sort()).toEqual(['mrr', 'recall@k']);
|
||||
expect(meta['recall@k']).toContain('relevant');
|
||||
expect(meta['mrr']).toContain('FIRST relevant result');
|
||||
});
|
||||
|
||||
test('unknown metrics silently dropped (no error)', () => {
|
||||
const meta = buildMetricGlossaryMeta(['recall@k', 'made_up']);
|
||||
expect(meta['recall@k']).toBeDefined();
|
||||
expect(meta['made_up']).toBeUndefined();
|
||||
});
|
||||
|
||||
test('empty array → empty object', () => {
|
||||
expect(buildMetricGlossaryMeta([])).toEqual({});
|
||||
});
|
||||
});
|
||||
|
||||
describe('ALL_METRICS roster', () => {
|
||||
test('has every glossary key', () => {
|
||||
expect([...ALL_METRICS].sort()).toEqual(Object.keys(METRIC_GLOSSARY).sort());
|
||||
});
|
||||
|
||||
test('matches the renderer output (no orphans)', () => {
|
||||
const md = renderMetricGlossaryMarkdown();
|
||||
for (const m of ALL_METRICS) {
|
||||
const entry = METRIC_GLOSSARY[m];
|
||||
expect(md, `Markdown should mention ${m}`).toContain(entry.industry_term);
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe('renderMetricGlossaryMarkdown determinism', () => {
|
||||
test('repeated calls produce identical output (deterministic)', () => {
|
||||
const a = renderMetricGlossaryMarkdown();
|
||||
const b = renderMetricGlossaryMarkdown();
|
||||
expect(a).toBe(b);
|
||||
});
|
||||
|
||||
test('output includes the auto-generated marker', () => {
|
||||
const md = renderMetricGlossaryMarkdown();
|
||||
expect(md).toContain('Auto-generated from `src/core/eval/metric-glossary.ts`');
|
||||
});
|
||||
|
||||
test('output groups metrics into 4 sections', () => {
|
||||
const md = renderMetricGlossaryMarkdown();
|
||||
expect(md).toContain('## Retrieval Metrics');
|
||||
expect(md).toContain('## Set-Similarity / Stability Metrics');
|
||||
expect(md).toContain('## Statistical-Significance Metrics');
|
||||
expect(md).toContain('## Operational / Cost Metrics');
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,217 @@
|
||||
/**
|
||||
* Regression test for [CDX-4] cross-mode cache contamination.
|
||||
*
|
||||
* Before v0.32.3 (PR #897 as merged), the query_cache primary key was
|
||||
* sha256(source_id::query_text) — a tokenmax search (expansion=on, limit=50)
|
||||
* would populate a row that a subsequent conservative call (no expansion,
|
||||
* limit=10) read back, serving the wrong-shape results.
|
||||
*
|
||||
* After v0.32.3:
|
||||
* - cacheRowId(query, source, knobsHash) — knobsHash is part of the PK
|
||||
* - SemanticQueryCache.lookup({knobsHash}) filters WHERE knobs_hash = $
|
||||
* - SemanticQueryCache.store({knobsHash}) writes the resolved hash
|
||||
*
|
||||
* This test exercises the cache class directly on a fresh PGLite brain
|
||||
* to verify cross-mode writes don't collide.
|
||||
*/
|
||||
import { afterAll, beforeAll, beforeEach, describe, expect, test } from 'bun:test';
|
||||
import { PGLiteEngine } from '../src/core/pglite-engine.ts';
|
||||
import { SemanticQueryCache, cacheRowId } from '../src/core/search/query-cache.ts';
|
||||
import type { SearchResult } from '../src/core/types.ts';
|
||||
import { knobsHash, resolveSearchMode } from '../src/core/search/mode.ts';
|
||||
|
||||
let engine: PGLiteEngine;
|
||||
|
||||
const conservativeHash = knobsHash(resolveSearchMode({ mode: 'conservative' }));
|
||||
const balancedHash = knobsHash(resolveSearchMode({ mode: 'balanced' }));
|
||||
const tokenmaxHash = knobsHash(resolveSearchMode({ mode: 'tokenmax' }));
|
||||
|
||||
beforeAll(async () => {
|
||||
engine = new PGLiteEngine();
|
||||
await engine.connect({});
|
||||
await engine.initSchema();
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
await engine.disconnect();
|
||||
});
|
||||
|
||||
beforeEach(async () => {
|
||||
// Clear cache before each test so writes start from empty state.
|
||||
await engine.executeRaw('DELETE FROM query_cache');
|
||||
});
|
||||
|
||||
const makeEmbedding = (seed: number): Float32Array => {
|
||||
const arr = new Float32Array(1536);
|
||||
for (let i = 0; i < 1536; i++) {
|
||||
arr[i] = Math.sin(seed + i * 0.001);
|
||||
}
|
||||
// Normalize to unit length so cosine similarity is well-defined.
|
||||
let norm = 0;
|
||||
for (let i = 0; i < 1536; i++) norm += arr[i] * arr[i];
|
||||
norm = Math.sqrt(norm);
|
||||
if (norm > 0) for (let i = 0; i < 1536; i++) arr[i] /= norm;
|
||||
return arr;
|
||||
};
|
||||
|
||||
const makeResults = (label: string, n: number): SearchResult[] =>
|
||||
Array.from({ length: n }, (_, i) => ({
|
||||
slug: `${label}/result-${i}`,
|
||||
title: `${label} result ${i}`,
|
||||
chunk_text: `chunk-${label}-${i}`,
|
||||
chunk_id: (i + 1) * 1000,
|
||||
score: 1 / (i + 1),
|
||||
chunk_index: i,
|
||||
type: 'note' as const,
|
||||
chunk_source: 'compiled_truth' as const,
|
||||
page_id: i + 1,
|
||||
stale: false,
|
||||
}));
|
||||
|
||||
describe('cacheRowId is bifurcated by knobsHash', () => {
|
||||
test('same (query, source) but different knobs → different row IDs', () => {
|
||||
const id1 = cacheRowId('what is the meaning of life', 'default', conservativeHash);
|
||||
const id2 = cacheRowId('what is the meaning of life', 'default', tokenmaxHash);
|
||||
expect(id1).not.toBe(id2);
|
||||
});
|
||||
|
||||
test('same (query, source, knobs) → same row ID (idempotent)', () => {
|
||||
const id1 = cacheRowId('what is the meaning of life', 'default', balancedHash);
|
||||
const id2 = cacheRowId('what is the meaning of life', 'default', balancedHash);
|
||||
expect(id1).toBe(id2);
|
||||
});
|
||||
|
||||
test('empty knobsHash still produces a valid ID (test-fixture compatibility)', () => {
|
||||
const id = cacheRowId('q', 'default', '');
|
||||
expect(id).toMatch(/^[0-9a-f]{32}$/);
|
||||
});
|
||||
|
||||
test('all three mode hashes are distinct', () => {
|
||||
expect(conservativeHash).not.toBe(balancedHash);
|
||||
expect(balancedHash).not.toBe(tokenmaxHash);
|
||||
expect(conservativeHash).not.toBe(tokenmaxHash);
|
||||
});
|
||||
});
|
||||
|
||||
describe('SemanticQueryCache cross-mode isolation (CDX-4 hotfix)', () => {
|
||||
test('tokenmax write does NOT contaminate conservative lookup', async () => {
|
||||
const cache = new SemanticQueryCache(engine);
|
||||
const emb = makeEmbedding(1);
|
||||
const tokenmaxResults = makeResults('tokenmax', 50);
|
||||
|
||||
// Write under tokenmax knobs.
|
||||
await cache.store('what is the meaning of life', emb, tokenmaxResults, {
|
||||
vector_enabled: true,
|
||||
detail_resolved: null,
|
||||
expansion_applied: true,
|
||||
}, { knobsHash: tokenmaxHash });
|
||||
|
||||
// Lookup under conservative knobs with the same embedding → MISS.
|
||||
const conservativeHit = await cache.lookup(emb, { knobsHash: conservativeHash });
|
||||
expect(conservativeHit.hit).toBe(false);
|
||||
|
||||
// Lookup under tokenmax knobs with the same embedding → HIT.
|
||||
const tokenmaxHit = await cache.lookup(emb, { knobsHash: tokenmaxHash });
|
||||
expect(tokenmaxHit.hit).toBe(true);
|
||||
expect(tokenmaxHit.results?.length).toBe(50);
|
||||
expect(tokenmaxHit.results?.[0].slug).toBe('tokenmax/result-0');
|
||||
});
|
||||
|
||||
test('three modes coexist as distinct rows for the same query', async () => {
|
||||
const cache = new SemanticQueryCache(engine);
|
||||
const emb = makeEmbedding(2);
|
||||
|
||||
await cache.store('q', emb, makeResults('conservative', 10), {
|
||||
vector_enabled: true, detail_resolved: null, expansion_applied: false,
|
||||
}, { knobsHash: conservativeHash });
|
||||
await cache.store('q', emb, makeResults('balanced', 25), {
|
||||
vector_enabled: true, detail_resolved: null, expansion_applied: false,
|
||||
}, { knobsHash: balancedHash });
|
||||
await cache.store('q', emb, makeResults('tokenmax', 50), {
|
||||
vector_enabled: true, detail_resolved: null, expansion_applied: true,
|
||||
}, { knobsHash: tokenmaxHash });
|
||||
|
||||
const rows = await engine.executeRaw<{ n: number }>(
|
||||
`SELECT COUNT(*)::int AS n FROM query_cache WHERE query_text = 'q'`,
|
||||
);
|
||||
expect(rows[0].n).toBe(3);
|
||||
|
||||
const cHit = await cache.lookup(emb, { knobsHash: conservativeHash });
|
||||
expect(cHit.hit).toBe(true);
|
||||
expect(cHit.results?.length).toBe(10);
|
||||
expect(cHit.results?.[0].slug).toBe('conservative/result-0');
|
||||
|
||||
const bHit = await cache.lookup(emb, { knobsHash: balancedHash });
|
||||
expect(bHit.hit).toBe(true);
|
||||
expect(bHit.results?.length).toBe(25);
|
||||
|
||||
const tHit = await cache.lookup(emb, { knobsHash: tokenmaxHash });
|
||||
expect(tHit.hit).toBe(true);
|
||||
expect(tHit.results?.length).toBe(50);
|
||||
});
|
||||
|
||||
test('legacy rows (NULL knobs_hash) are excluded from lookup', async () => {
|
||||
// Manually insert a row with NULL knobs_hash (simulating pre-v0.32.3 state).
|
||||
const emb = makeEmbedding(3);
|
||||
const vecStr = `[${Array.from(emb).map(v => v.toFixed(6)).join(',')}]`;
|
||||
await engine.executeRaw(
|
||||
`INSERT INTO query_cache (id, query_text, source_id, knobs_hash, embedding, results, meta, ttl_seconds, created_at)
|
||||
VALUES ($1, $2, $3, NULL, $4::vector, $5::jsonb, $6::jsonb, 3600, now())`,
|
||||
[
|
||||
'legacy-row-id',
|
||||
'legacy-query',
|
||||
'default',
|
||||
vecStr,
|
||||
JSON.stringify(makeResults('legacy', 5)),
|
||||
JSON.stringify({ vector_enabled: true, detail_resolved: null, expansion_applied: false }),
|
||||
],
|
||||
);
|
||||
|
||||
const cache = new SemanticQueryCache(engine);
|
||||
// Any mode's lookup → MISS (NULL row is excluded by the knobs_hash filter).
|
||||
expect((await cache.lookup(emb, { knobsHash: conservativeHash })).hit).toBe(false);
|
||||
expect((await cache.lookup(emb, { knobsHash: balancedHash })).hit).toBe(false);
|
||||
expect((await cache.lookup(emb, { knobsHash: tokenmaxHash })).hit).toBe(false);
|
||||
});
|
||||
|
||||
test('same mode written twice updates in place (no duplicate rows)', async () => {
|
||||
const cache = new SemanticQueryCache(engine);
|
||||
const emb = makeEmbedding(4);
|
||||
|
||||
await cache.store('q', emb, makeResults('first', 5), {
|
||||
vector_enabled: true, detail_resolved: null, expansion_applied: false,
|
||||
}, { knobsHash: balancedHash });
|
||||
|
||||
await cache.store('q', emb, makeResults('second', 7), {
|
||||
vector_enabled: true, detail_resolved: null, expansion_applied: false,
|
||||
}, { knobsHash: balancedHash });
|
||||
|
||||
const rows = await engine.executeRaw<{ n: number }>(
|
||||
`SELECT COUNT(*)::int AS n FROM query_cache WHERE query_text = 'q'`,
|
||||
);
|
||||
expect(rows[0].n).toBe(1);
|
||||
|
||||
const hit = await cache.lookup(emb, { knobsHash: balancedHash });
|
||||
expect(hit.hit).toBe(true);
|
||||
expect(hit.results?.length).toBe(7);
|
||||
expect(hit.results?.[0].slug).toBe('second/result-0');
|
||||
});
|
||||
|
||||
test('empty knobsHash arg writes a row but does not collide with mode-hash rows', async () => {
|
||||
const cache = new SemanticQueryCache(engine);
|
||||
const emb = makeEmbedding(5);
|
||||
|
||||
await cache.store('q', emb, makeResults('no-mode', 3), {
|
||||
vector_enabled: true, detail_resolved: null, expansion_applied: false,
|
||||
});
|
||||
|
||||
// No-mode lookup hits its own row.
|
||||
const noModeHit = await cache.lookup(emb);
|
||||
expect(noModeHit.hit).toBe(true);
|
||||
expect(noModeHit.results?.length).toBe(3);
|
||||
|
||||
// Conservative-hash lookup misses (the no-mode row had empty hash).
|
||||
const conservativeHit = await cache.lookup(emb, { knobsHash: conservativeHash });
|
||||
expect(conservativeHit.hit).toBe(false);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,246 @@
|
||||
/**
|
||||
* v0.32.x search-lite \u2014 semantic query cache.
|
||||
*
|
||||
* PGLite-backed test. Confirms:
|
||||
* - migration v51 creates the query_cache table
|
||||
* - store + lookup roundtrip with EXACT same embedding \u2192 hit
|
||||
* - lookup with a similar embedding (cosine > 0.92) \u2192 hit
|
||||
* - lookup with a far embedding \u2192 miss
|
||||
* - TTL expiration: a stale row is skipped at read time
|
||||
* - clear / prune / stats work as advertised
|
||||
* - source_id isolation: brain A's cache doesn't leak to brain B
|
||||
* - disabled cache is a pure no-op
|
||||
*
|
||||
* Uses synthetic Float32Array embeddings so the test doesn't depend on
|
||||
* any external embedding provider.
|
||||
*/
|
||||
|
||||
import { describe, test, expect, beforeAll, afterAll, beforeEach } from 'bun:test';
|
||||
import { PGLiteEngine } from '../src/core/pglite-engine.ts';
|
||||
import { SemanticQueryCache, cacheRowId } from '../src/core/search/query-cache.ts';
|
||||
import type { SearchResult, HybridSearchMeta } from '../src/core/types.ts';
|
||||
|
||||
let engine: PGLiteEngine;
|
||||
|
||||
// Build a stable, normalized embedding. PGLite ships pgvector with 1536-dim
|
||||
// support (the default); a smaller test dim won't match the column. We
|
||||
// truncate / pad to 1536 to match the migration's resolved dim.
|
||||
const DIM = 1536;
|
||||
|
||||
function makeEmbedding(seed: number, dim = DIM): Float32Array {
|
||||
const e = new Float32Array(dim);
|
||||
// Simple deterministic generator with a unique fingerprint per seed
|
||||
// so similar seeds produce similar (cosine > 0.95) vectors and distinct
|
||||
// seeds produce orthogonal-ish ones.
|
||||
for (let i = 0; i < dim; i++) {
|
||||
e[i] = Math.sin(seed * 0.001 + i * 0.01);
|
||||
}
|
||||
// L2-normalize so cosine = dot product.
|
||||
let mag = 0;
|
||||
for (let i = 0; i < dim; i++) mag += e[i] * e[i];
|
||||
mag = Math.sqrt(mag);
|
||||
if (mag > 0) for (let i = 0; i < dim; i++) e[i] /= mag;
|
||||
return e;
|
||||
}
|
||||
|
||||
function makeOrthogonalEmbedding(seed: number, dim = DIM): Float32Array {
|
||||
// Use a totally different basis so cosine is near-zero.
|
||||
const e = new Float32Array(dim);
|
||||
for (let i = 0; i < dim; i++) {
|
||||
e[i] = Math.cos(seed * 13.7 + i * 0.97);
|
||||
}
|
||||
let mag = 0;
|
||||
for (let i = 0; i < dim; i++) mag += e[i] * e[i];
|
||||
mag = Math.sqrt(mag);
|
||||
if (mag > 0) for (let i = 0; i < dim; i++) e[i] /= mag;
|
||||
return e;
|
||||
}
|
||||
|
||||
function makeResult(slug: string): SearchResult {
|
||||
return {
|
||||
slug,
|
||||
page_id: 1,
|
||||
title: `Title for ${slug}`,
|
||||
type: 'concept',
|
||||
chunk_text: `chunk text for ${slug}`,
|
||||
chunk_source: 'compiled_truth',
|
||||
chunk_id: 1,
|
||||
chunk_index: 0,
|
||||
score: 1.0,
|
||||
stale: false,
|
||||
};
|
||||
}
|
||||
|
||||
const META: HybridSearchMeta = {
|
||||
vector_enabled: true,
|
||||
detail_resolved: 'medium',
|
||||
expansion_applied: false,
|
||||
intent: 'general',
|
||||
};
|
||||
|
||||
beforeAll(async () => {
|
||||
engine = new PGLiteEngine();
|
||||
await engine.connect({});
|
||||
await engine.initSchema();
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
try { await engine.disconnect(); } catch { /* ignore */ }
|
||||
});
|
||||
|
||||
beforeEach(async () => {
|
||||
// Wipe the cache between tests so ordering doesn't matter.
|
||||
await engine.executeRaw(`DELETE FROM query_cache`);
|
||||
});
|
||||
|
||||
describe('migration v51 \u2014 query_cache table exists', () => {
|
||||
test('table is present and has expected columns', async () => {
|
||||
const rows = await engine.executeRaw<{ column_name: string }>(
|
||||
`SELECT column_name FROM information_schema.columns
|
||||
WHERE table_name = 'query_cache'`,
|
||||
);
|
||||
const names = rows.map(r => r.column_name);
|
||||
expect(names).toContain('id');
|
||||
expect(names).toContain('query_text');
|
||||
expect(names).toContain('source_id');
|
||||
expect(names).toContain('embedding');
|
||||
expect(names).toContain('results');
|
||||
expect(names).toContain('meta');
|
||||
expect(names).toContain('ttl_seconds');
|
||||
expect(names).toContain('created_at');
|
||||
expect(names).toContain('hit_count');
|
||||
});
|
||||
});
|
||||
|
||||
describe('cacheRowId', () => {
|
||||
test('is deterministic across same input', () => {
|
||||
expect(cacheRowId('hello', 'default')).toBe(cacheRowId('hello', 'default'));
|
||||
});
|
||||
test('differs across source_id', () => {
|
||||
expect(cacheRowId('hello', 'a')).not.toBe(cacheRowId('hello', 'b'));
|
||||
});
|
||||
});
|
||||
|
||||
describe('SemanticQueryCache \u2014 store + lookup', () => {
|
||||
test('roundtrip: exact embedding match returns a hit', async () => {
|
||||
const cache = new SemanticQueryCache(engine);
|
||||
const emb = makeEmbedding(1);
|
||||
const results = [makeResult('a'), makeResult('b')];
|
||||
|
||||
await cache.store('what is foo', emb, results, META);
|
||||
const hit = await cache.lookup(emb);
|
||||
|
||||
expect(hit.hit).toBe(true);
|
||||
expect(hit.results).toHaveLength(2);
|
||||
expect(hit.results?.[0].slug).toBe('a');
|
||||
expect(hit.similarity).toBeGreaterThan(0.99);
|
||||
});
|
||||
|
||||
test('similar embedding (cosine > 0.92) is a hit', async () => {
|
||||
const cache = new SemanticQueryCache(engine);
|
||||
const base = makeEmbedding(100);
|
||||
|
||||
// Construct a near-neighbor: tweak a few dims so cosine stays > 0.92.
|
||||
const near = new Float32Array(base);
|
||||
for (let i = 0; i < 10; i++) near[i] += 0.005;
|
||||
// Re-normalize.
|
||||
let mag = 0;
|
||||
for (let i = 0; i < DIM; i++) mag += near[i] * near[i];
|
||||
mag = Math.sqrt(mag);
|
||||
for (let i = 0; i < DIM; i++) near[i] /= mag;
|
||||
|
||||
await cache.store('what is foo', base, [makeResult('a')], META);
|
||||
const hit = await cache.lookup(near);
|
||||
|
||||
expect(hit.hit).toBe(true);
|
||||
expect(hit.similarity).toBeGreaterThan(0.92);
|
||||
});
|
||||
|
||||
test('orthogonal embedding is a miss', async () => {
|
||||
const cache = new SemanticQueryCache(engine);
|
||||
const a = makeEmbedding(1);
|
||||
const b = makeOrthogonalEmbedding(2);
|
||||
await cache.store('q1', a, [makeResult('a')], META);
|
||||
const hit = await cache.lookup(b);
|
||||
expect(hit.hit).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe('SemanticQueryCache \u2014 TTL', () => {
|
||||
test('stale row (past TTL) is not returned', async () => {
|
||||
const cache = new SemanticQueryCache(engine, { ttlSeconds: 1 });
|
||||
const emb = makeEmbedding(42);
|
||||
await cache.store('q', emb, [makeResult('a')], META, { ttlSeconds: 1 });
|
||||
|
||||
// Manually rewind created_at to simulate expiration.
|
||||
await engine.executeRaw(
|
||||
`UPDATE query_cache SET created_at = now() - interval '10 seconds'`,
|
||||
);
|
||||
const hit = await cache.lookup(emb);
|
||||
expect(hit.hit).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe('SemanticQueryCache \u2014 source isolation', () => {
|
||||
test('different source_id cannot read each other\u2019s rows', async () => {
|
||||
const cache = new SemanticQueryCache(engine);
|
||||
const emb = makeEmbedding(7);
|
||||
await cache.store('q', emb, [makeResult('a')], META, { sourceId: 'src-A' });
|
||||
const hitB = await cache.lookup(emb, { sourceId: 'src-B' });
|
||||
expect(hitB.hit).toBe(false);
|
||||
const hitA = await cache.lookup(emb, { sourceId: 'src-A' });
|
||||
expect(hitA.hit).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
describe('SemanticQueryCache \u2014 management', () => {
|
||||
test('clear() wipes all rows', async () => {
|
||||
const cache = new SemanticQueryCache(engine);
|
||||
const emb = makeEmbedding(9);
|
||||
await cache.store('q1', emb, [makeResult('a')], META);
|
||||
await cache.store('q2', makeEmbedding(10), [makeResult('b')], META);
|
||||
const removed = await cache.clear();
|
||||
expect(removed).toBeGreaterThanOrEqual(2);
|
||||
const stats = await cache.stats();
|
||||
expect(stats.total_rows).toBe(0);
|
||||
});
|
||||
|
||||
test('prune() deletes only stale rows', async () => {
|
||||
const cache = new SemanticQueryCache(engine);
|
||||
await cache.store('fresh', makeEmbedding(11), [makeResult('a')], META);
|
||||
await cache.store('stale', makeEmbedding(12), [makeResult('b')], META, { ttlSeconds: 1 });
|
||||
await engine.executeRaw(
|
||||
`UPDATE query_cache SET created_at = now() - interval '10 seconds' WHERE query_text = 'stale'`,
|
||||
);
|
||||
const removed = await cache.prune();
|
||||
expect(removed).toBe(1);
|
||||
const stats = await cache.stats();
|
||||
expect(stats.total_rows).toBe(1);
|
||||
expect(stats.fresh_rows).toBe(1);
|
||||
});
|
||||
|
||||
test('stats() reports fresh / stale / total / hit counters', async () => {
|
||||
const cache = new SemanticQueryCache(engine);
|
||||
const emb = makeEmbedding(13);
|
||||
await cache.store('q', emb, [makeResult('a')], META);
|
||||
await cache.lookup(emb); // bump hit
|
||||
// Hit bump is async/fire-and-forget; give it a moment to land.
|
||||
await new Promise(r => setTimeout(r, 50));
|
||||
const stats = await cache.stats();
|
||||
expect(stats.total_rows).toBe(1);
|
||||
expect(stats.fresh_rows).toBe(1);
|
||||
expect(stats.stale_rows).toBe(0);
|
||||
expect(stats.total_hits).toBeGreaterThanOrEqual(1);
|
||||
});
|
||||
});
|
||||
|
||||
describe('SemanticQueryCache \u2014 disabled', () => {
|
||||
test('disabled cache is a pure no-op on lookup', async () => {
|
||||
const cache = new SemanticQueryCache(engine, { enabled: false });
|
||||
const emb = makeEmbedding(99);
|
||||
await cache.store('q', emb, [makeResult('a')], META);
|
||||
// Even after a store call, lookup must miss because enabled=false.
|
||||
const hit = await cache.lookup(emb);
|
||||
expect(hit.hit).toBe(false);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,317 @@
|
||||
/**
|
||||
* Pins the v0.32.3 search-lite mode core: MODE_BUNDLES + resolveSearchMode
|
||||
* + knobsHash. The 3x7 mode table is asserted cell-by-cell because the
|
||||
* public eval methodology doc cites these values verbatim — drift here is
|
||||
* a documentation-honesty bug, not a refactor.
|
||||
*/
|
||||
import { describe, expect, test } from 'bun:test';
|
||||
import {
|
||||
MODE_BUNDLES,
|
||||
SEARCH_MODES,
|
||||
DEFAULT_SEARCH_MODE,
|
||||
isSearchMode,
|
||||
resolveSearchMode,
|
||||
attributeKnob,
|
||||
knobsHash,
|
||||
loadOverridesFromConfig,
|
||||
KNOBS_HASH_VERSION,
|
||||
SEARCH_MODE_CONFIG_KEYS,
|
||||
type SearchMode,
|
||||
} from '../src/core/search/mode.ts';
|
||||
|
||||
describe('SEARCH_MODES + MODE_BUNDLES canonical shape', () => {
|
||||
test('SEARCH_MODES is exactly the 3 expected values', () => {
|
||||
expect([...SEARCH_MODES]).toEqual(['conservative', 'balanced', 'tokenmax']);
|
||||
});
|
||||
|
||||
test('DEFAULT_SEARCH_MODE is balanced (matches v0.31.x current default surface)', () => {
|
||||
expect(DEFAULT_SEARCH_MODE).toBe('balanced');
|
||||
});
|
||||
|
||||
test('MODE_BUNDLES is frozen (cannot be mutated)', () => {
|
||||
expect(Object.isFrozen(MODE_BUNDLES)).toBe(true);
|
||||
expect(Object.isFrozen(MODE_BUNDLES.conservative)).toBe(true);
|
||||
expect(Object.isFrozen(MODE_BUNDLES.balanced)).toBe(true);
|
||||
expect(Object.isFrozen(MODE_BUNDLES.tokenmax)).toBe(true);
|
||||
});
|
||||
|
||||
// The 3x7 cell-by-cell assertion. The methodology doc cites these.
|
||||
test('conservative bundle values are canonical', () => {
|
||||
expect(MODE_BUNDLES.conservative).toEqual({
|
||||
cache_enabled: true,
|
||||
cache_similarity_threshold: 0.92,
|
||||
cache_ttl_seconds: 3600,
|
||||
intentWeighting: true,
|
||||
tokenBudget: 4000,
|
||||
expansion: false,
|
||||
searchLimit: 10,
|
||||
});
|
||||
});
|
||||
|
||||
test('balanced bundle values are canonical', () => {
|
||||
expect(MODE_BUNDLES.balanced).toEqual({
|
||||
cache_enabled: true,
|
||||
cache_similarity_threshold: 0.92,
|
||||
cache_ttl_seconds: 3600,
|
||||
intentWeighting: true,
|
||||
tokenBudget: 12000,
|
||||
expansion: false,
|
||||
searchLimit: 25,
|
||||
});
|
||||
});
|
||||
|
||||
test('tokenmax bundle values are canonical (NOTE: limit=50, NOT current=20)', () => {
|
||||
expect(MODE_BUNDLES.tokenmax).toEqual({
|
||||
cache_enabled: true,
|
||||
cache_similarity_threshold: 0.92,
|
||||
cache_ttl_seconds: 3600,
|
||||
intentWeighting: true,
|
||||
tokenBudget: undefined,
|
||||
expansion: true,
|
||||
searchLimit: 50,
|
||||
});
|
||||
});
|
||||
|
||||
test('cache_enabled is true in every mode (free win)', () => {
|
||||
for (const m of SEARCH_MODES) {
|
||||
expect(MODE_BUNDLES[m].cache_enabled).toBe(true);
|
||||
}
|
||||
});
|
||||
|
||||
test('intentWeighting is true in every mode (zero-LLM cost)', () => {
|
||||
for (const m of SEARCH_MODES) {
|
||||
expect(MODE_BUNDLES[m].intentWeighting).toBe(true);
|
||||
}
|
||||
});
|
||||
|
||||
test('tokenBudget escalates: 4000 → 12000 → undefined', () => {
|
||||
expect(MODE_BUNDLES.conservative.tokenBudget).toBe(4000);
|
||||
expect(MODE_BUNDLES.balanced.tokenBudget).toBe(12000);
|
||||
expect(MODE_BUNDLES.tokenmax.tokenBudget).toBeUndefined();
|
||||
});
|
||||
|
||||
test('searchLimit escalates: 10 → 25 → 50', () => {
|
||||
expect(MODE_BUNDLES.conservative.searchLimit).toBe(10);
|
||||
expect(MODE_BUNDLES.balanced.searchLimit).toBe(25);
|
||||
expect(MODE_BUNDLES.tokenmax.searchLimit).toBe(50);
|
||||
});
|
||||
});
|
||||
|
||||
describe('isSearchMode', () => {
|
||||
test('accepts every documented mode', () => {
|
||||
for (const m of SEARCH_MODES) {
|
||||
expect(isSearchMode(m)).toBe(true);
|
||||
}
|
||||
});
|
||||
test('rejects unknown strings, numbers, null, undefined', () => {
|
||||
expect(isSearchMode('conservativeX')).toBe(false);
|
||||
expect(isSearchMode('')).toBe(false);
|
||||
expect(isSearchMode('CONSERVATIVE')).toBe(false); // case-sensitive at the type guard layer
|
||||
expect(isSearchMode(42)).toBe(false);
|
||||
expect(isSearchMode(null)).toBe(false);
|
||||
expect(isSearchMode(undefined)).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe('resolveSearchMode resolution chain', () => {
|
||||
test('no inputs → balanced bundle (fallback)', () => {
|
||||
const r = resolveSearchMode({});
|
||||
expect(r.resolved_mode).toBe('balanced');
|
||||
expect(r.mode_valid).toBe(false);
|
||||
expect(r.searchLimit).toBe(25);
|
||||
expect(r.tokenBudget).toBe(12000);
|
||||
expect(r.expansion).toBe(false);
|
||||
});
|
||||
|
||||
test('valid mode picked, no overrides → bundle values pass through', () => {
|
||||
const r = resolveSearchMode({ mode: 'conservative' });
|
||||
expect(r.resolved_mode).toBe('conservative');
|
||||
expect(r.mode_valid).toBe(true);
|
||||
expect(r.searchLimit).toBe(10);
|
||||
expect(r.tokenBudget).toBe(4000);
|
||||
});
|
||||
|
||||
test('invalid mode string → balanced fallback (mode_valid=false)', () => {
|
||||
const r = resolveSearchMode({ mode: 'NUKE_MODE' });
|
||||
expect(r.resolved_mode).toBe('balanced');
|
||||
expect(r.mode_valid).toBe(false);
|
||||
expect(r.searchLimit).toBe(25);
|
||||
});
|
||||
|
||||
test('mode string case-normalized (TokenMax → tokenmax)', () => {
|
||||
const r = resolveSearchMode({ mode: 'TokenMax' });
|
||||
expect(r.resolved_mode).toBe('tokenmax');
|
||||
expect(r.mode_valid).toBe(true);
|
||||
});
|
||||
|
||||
test('per-key override wins over mode bundle (CDX-5 chain)', () => {
|
||||
const r = resolveSearchMode({
|
||||
mode: 'conservative',
|
||||
overrides: { tokenBudget: 99999, cache_enabled: false },
|
||||
});
|
||||
expect(r.resolved_mode).toBe('conservative');
|
||||
expect(r.tokenBudget).toBe(99999);
|
||||
expect(r.cache_enabled).toBe(false);
|
||||
expect(r.searchLimit).toBe(10); // not overridden, still from bundle
|
||||
});
|
||||
|
||||
test('per-call override wins over per-key override', () => {
|
||||
const r = resolveSearchMode({
|
||||
mode: 'conservative',
|
||||
overrides: { tokenBudget: 99999 },
|
||||
perCall: { tokenBudget: 77 },
|
||||
});
|
||||
expect(r.tokenBudget).toBe(77);
|
||||
});
|
||||
|
||||
test('per-call false-y values (false / 0) still beat fallback', () => {
|
||||
const r = resolveSearchMode({
|
||||
mode: 'tokenmax',
|
||||
perCall: { expansion: false, cache_enabled: false },
|
||||
});
|
||||
expect(r.expansion).toBe(false); // beat tokenmax's true
|
||||
expect(r.cache_enabled).toBe(false); // beat tokenmax's true
|
||||
});
|
||||
|
||||
test('undefined fields in perCall fall through (not coerced to false)', () => {
|
||||
const r = resolveSearchMode({
|
||||
mode: 'tokenmax',
|
||||
perCall: { tokenBudget: undefined, expansion: undefined },
|
||||
});
|
||||
expect(r.tokenBudget).toBeUndefined(); // from tokenmax bundle
|
||||
expect(r.expansion).toBe(true); // from tokenmax bundle, NOT overridden
|
||||
});
|
||||
});
|
||||
|
||||
describe('attributeKnob source attribution', () => {
|
||||
test('per-call source labeled correctly', () => {
|
||||
const input = { mode: 'conservative', perCall: { tokenBudget: 999 } };
|
||||
const resolved = resolveSearchMode(input);
|
||||
const a = attributeKnob('tokenBudget', input, resolved);
|
||||
expect(a.source).toBe('per-call');
|
||||
expect(a.value).toBe(999);
|
||||
});
|
||||
|
||||
test('override source labels the config key path', () => {
|
||||
const input = { mode: 'conservative', overrides: { cache_enabled: false } };
|
||||
const resolved = resolveSearchMode(input);
|
||||
const a = attributeKnob('cache_enabled', input, resolved);
|
||||
expect(a.source).toBe('override');
|
||||
expect(a.source_detail).toContain('search.cache_enabled');
|
||||
});
|
||||
|
||||
test('mode source labels the mode name', () => {
|
||||
const input = { mode: 'conservative' };
|
||||
const resolved = resolveSearchMode(input);
|
||||
const a = attributeKnob('searchLimit', input, resolved);
|
||||
expect(a.source).toBe('mode');
|
||||
expect(a.source_detail).toContain('conservative');
|
||||
});
|
||||
|
||||
test('fallback source labels the unset state explicitly', () => {
|
||||
const input = {}; // no mode set
|
||||
const resolved = resolveSearchMode(input);
|
||||
const a = attributeKnob('searchLimit', input, resolved);
|
||||
expect(a.source).toBe('fallback');
|
||||
expect(a.source_detail).toContain('balanced');
|
||||
expect(a.source_detail).toContain('unset');
|
||||
});
|
||||
});
|
||||
|
||||
describe('knobsHash determinism + cross-mode separation (CDX-4)', () => {
|
||||
test('hash is deterministic across calls', () => {
|
||||
const knobs = resolveSearchMode({ mode: 'conservative' });
|
||||
const h1 = knobsHash(knobs);
|
||||
const h2 = knobsHash(knobs);
|
||||
expect(h1).toBe(h2);
|
||||
});
|
||||
|
||||
test('different modes produce different hashes', () => {
|
||||
const c = knobsHash(resolveSearchMode({ mode: 'conservative' }));
|
||||
const b = knobsHash(resolveSearchMode({ mode: 'balanced' }));
|
||||
const t = knobsHash(resolveSearchMode({ mode: 'tokenmax' }));
|
||||
expect(c).not.toBe(b);
|
||||
expect(b).not.toBe(t);
|
||||
expect(c).not.toBe(t);
|
||||
});
|
||||
|
||||
test('per-call override changes the hash (cache key bifurcates)', () => {
|
||||
const a = knobsHash(resolveSearchMode({ mode: 'conservative' }));
|
||||
const b = knobsHash(resolveSearchMode({ mode: 'conservative', perCall: { tokenBudget: 999 } }));
|
||||
expect(a).not.toBe(b);
|
||||
});
|
||||
|
||||
test('hash is short (16 hex chars) and stable shape', () => {
|
||||
const h = knobsHash(resolveSearchMode({ mode: 'balanced' }));
|
||||
expect(h).toMatch(/^[0-9a-f]{16}$/);
|
||||
});
|
||||
|
||||
test('KNOBS_HASH_VERSION constant exposed for migrations to bump on schema change', () => {
|
||||
expect(KNOBS_HASH_VERSION).toBe(1);
|
||||
});
|
||||
});
|
||||
|
||||
describe('loadOverridesFromConfig flat-map parser', () => {
|
||||
test('empty config map → empty overrides', () => {
|
||||
const ov = loadOverridesFromConfig({});
|
||||
expect(ov).toEqual({});
|
||||
});
|
||||
|
||||
test('cache.enabled accepts 1 / 0 / true / false strings', () => {
|
||||
expect(loadOverridesFromConfig({ 'search.cache.enabled': '1' }).cache_enabled).toBe(true);
|
||||
expect(loadOverridesFromConfig({ 'search.cache.enabled': '0' }).cache_enabled).toBe(false);
|
||||
expect(loadOverridesFromConfig({ 'search.cache.enabled': 'true' }).cache_enabled).toBe(true);
|
||||
expect(loadOverridesFromConfig({ 'search.cache.enabled': 'false' }).cache_enabled).toBe(false);
|
||||
expect(loadOverridesFromConfig({ 'search.cache.enabled': 'TRUE' }).cache_enabled).toBe(true);
|
||||
});
|
||||
|
||||
test('numeric keys parse and clamp', () => {
|
||||
expect(loadOverridesFromConfig({ 'search.cache.similarity_threshold': '0.95' }).cache_similarity_threshold).toBe(0.95);
|
||||
expect(loadOverridesFromConfig({ 'search.cache.ttl_seconds': '7200' }).cache_ttl_seconds).toBe(7200);
|
||||
expect(loadOverridesFromConfig({ 'search.tokenBudget': '8000' }).tokenBudget).toBe(8000);
|
||||
expect(loadOverridesFromConfig({ 'search.searchLimit': '30' }).searchLimit).toBe(30);
|
||||
});
|
||||
|
||||
test('invalid numerics are ignored (not coerced to NaN/0)', () => {
|
||||
const ov = loadOverridesFromConfig({
|
||||
'search.cache.similarity_threshold': 'NaN',
|
||||
'search.tokenBudget': 'cheese',
|
||||
'search.searchLimit': '-1',
|
||||
'search.cache.ttl_seconds': '0',
|
||||
});
|
||||
expect(ov.cache_similarity_threshold).toBeUndefined();
|
||||
expect(ov.tokenBudget).toBeUndefined();
|
||||
expect(ov.searchLimit).toBeUndefined();
|
||||
expect(ov.cache_ttl_seconds).toBeUndefined();
|
||||
});
|
||||
|
||||
test('similarity_threshold rejects values outside (0, 1]', () => {
|
||||
expect(loadOverridesFromConfig({ 'search.cache.similarity_threshold': '1.5' }).cache_similarity_threshold).toBeUndefined();
|
||||
expect(loadOverridesFromConfig({ 'search.cache.similarity_threshold': '0' }).cache_similarity_threshold).toBeUndefined();
|
||||
expect(loadOverridesFromConfig({ 'search.cache.similarity_threshold': '-0.1' }).cache_similarity_threshold).toBeUndefined();
|
||||
});
|
||||
});
|
||||
|
||||
describe('SEARCH_MODE_CONFIG_KEYS is the full reset surface', () => {
|
||||
test('every key starts with search. prefix (gbrain config unset --pattern search.* compatibility)', () => {
|
||||
for (const k of SEARCH_MODE_CONFIG_KEYS) {
|
||||
expect(k.startsWith('search.')).toBe(true);
|
||||
}
|
||||
});
|
||||
|
||||
test('every ModeBundle field has a config key (consistency check)', () => {
|
||||
// If a new knob is added to ModeBundle, this test fails until the operator
|
||||
// adds the corresponding config key to SEARCH_MODE_CONFIG_KEYS. That's the
|
||||
// intentional regression guard: `gbrain search modes --reset` must clear
|
||||
// every knob.
|
||||
const knobs = Object.keys(MODE_BUNDLES.balanced);
|
||||
expect(SEARCH_MODE_CONFIG_KEYS.length).toBeGreaterThanOrEqual(knobs.length);
|
||||
});
|
||||
});
|
||||
|
||||
describe('Type-only smoke test (compiler sees SearchMode union)', () => {
|
||||
test('SearchMode union is exactly 3 modes (compile-time)', () => {
|
||||
const valid: SearchMode[] = ['conservative', 'balanced', 'tokenmax'];
|
||||
expect(valid.length).toBe(3);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,243 @@
|
||||
/**
|
||||
* v0.32.3 search-lite telemetry rollup writer tests.
|
||||
*
|
||||
* Pins the architecture decisions from D2 + [CDX-17] + [CDX-18]:
|
||||
* - In-memory bucket flushed periodically (NOT per-call DB write)
|
||||
* - Sums + counts, NEVER pre-averaged columns
|
||||
* - Date-bucketed cache hit/miss derivable over --days N window
|
||||
* - ON CONFLICT DO UPDATE adds raw values (concurrent flushes accumulate)
|
||||
* - Per-bucket isolation: one bad row doesn't lose the others
|
||||
*/
|
||||
import { afterAll, beforeAll, beforeEach, describe, expect, test } from 'bun:test';
|
||||
import { PGLiteEngine } from '../src/core/pglite-engine.ts';
|
||||
import {
|
||||
recordSearchTelemetry,
|
||||
readSearchStats,
|
||||
getTelemetryWriter,
|
||||
_resetTelemetryWriterForTest,
|
||||
} from '../src/core/search/telemetry.ts';
|
||||
import type { HybridSearchMeta } from '../src/core/types.ts';
|
||||
|
||||
let engine: PGLiteEngine;
|
||||
|
||||
beforeAll(async () => {
|
||||
engine = new PGLiteEngine();
|
||||
await engine.connect({});
|
||||
await engine.initSchema();
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
await engine.disconnect();
|
||||
});
|
||||
|
||||
beforeEach(async () => {
|
||||
_resetTelemetryWriterForTest();
|
||||
await engine.executeRaw('DELETE FROM search_telemetry');
|
||||
});
|
||||
|
||||
const makeMeta = (overrides: Partial<HybridSearchMeta> = {}): HybridSearchMeta => ({
|
||||
vector_enabled: true,
|
||||
detail_resolved: null,
|
||||
expansion_applied: false,
|
||||
intent: 'general',
|
||||
mode: 'balanced',
|
||||
...overrides,
|
||||
});
|
||||
|
||||
describe('recordSearchTelemetry — in-memory bucket', () => {
|
||||
test('first record creates a bucket; record() never blocks the caller', () => {
|
||||
const w = getTelemetryWriter();
|
||||
w.setEngine(engine);
|
||||
expect(w.bucketCountForTest()).toBe(0);
|
||||
recordSearchTelemetry(engine, makeMeta(), { results_count: 5 });
|
||||
expect(w.bucketCountForTest()).toBe(1);
|
||||
});
|
||||
|
||||
test('same (date, mode, intent) accumulates into one bucket', () => {
|
||||
const w = getTelemetryWriter();
|
||||
w.setEngine(engine);
|
||||
recordSearchTelemetry(engine, makeMeta(), { results_count: 5 });
|
||||
recordSearchTelemetry(engine, makeMeta(), { results_count: 7 });
|
||||
recordSearchTelemetry(engine, makeMeta(), { results_count: 3 });
|
||||
expect(w.bucketCountForTest()).toBe(1);
|
||||
const today = new Date().toISOString().slice(0, 10);
|
||||
const b = w.bucketForTest(today, 'balanced', 'general');
|
||||
expect(b?.count).toBe(3);
|
||||
expect(b?.sum_results).toBe(15); // 5 + 7 + 3
|
||||
});
|
||||
|
||||
test('different modes / intents create distinct buckets', () => {
|
||||
const w = getTelemetryWriter();
|
||||
w.setEngine(engine);
|
||||
recordSearchTelemetry(engine, makeMeta({ mode: 'conservative', intent: 'entity' }), { results_count: 2 });
|
||||
recordSearchTelemetry(engine, makeMeta({ mode: 'tokenmax', intent: 'temporal' }), { results_count: 9 });
|
||||
recordSearchTelemetry(engine, makeMeta({ mode: 'balanced', intent: 'event' }), { results_count: 4 });
|
||||
expect(w.bucketCountForTest()).toBe(3);
|
||||
});
|
||||
|
||||
test('cache_hit / cache_miss counters fire from meta.cache.status', () => {
|
||||
const w = getTelemetryWriter();
|
||||
w.setEngine(engine);
|
||||
recordSearchTelemetry(engine, makeMeta({ cache: { status: 'hit' } }));
|
||||
recordSearchTelemetry(engine, makeMeta({ cache: { status: 'hit' } }));
|
||||
recordSearchTelemetry(engine, makeMeta({ cache: { status: 'miss' } }));
|
||||
recordSearchTelemetry(engine, makeMeta({ cache: { status: 'disabled' } }));
|
||||
const today = new Date().toISOString().slice(0, 10);
|
||||
const b = w.bucketForTest(today, 'balanced', 'general')!;
|
||||
expect(b.cache_hit).toBe(2);
|
||||
expect(b.cache_miss).toBe(1);
|
||||
expect(b.count).toBe(4);
|
||||
});
|
||||
|
||||
test('sum_budget_dropped accumulates from meta.token_budget.dropped', () => {
|
||||
const w = getTelemetryWriter();
|
||||
w.setEngine(engine);
|
||||
recordSearchTelemetry(engine, makeMeta({ token_budget: { budget: 4000, used: 3800, kept: 8, dropped: 12 } }));
|
||||
recordSearchTelemetry(engine, makeMeta({ token_budget: { budget: 4000, used: 4000, kept: 10, dropped: 7 } }));
|
||||
const today = new Date().toISOString().slice(0, 10);
|
||||
const b = w.bucketForTest(today, 'balanced', 'general')!;
|
||||
expect(b.sum_budget_dropped).toBe(19); // 12 + 7
|
||||
});
|
||||
|
||||
test('missing mode / intent fall back to "unset" — telemetry is non-blocking', () => {
|
||||
const w = getTelemetryWriter();
|
||||
w.setEngine(engine);
|
||||
recordSearchTelemetry(engine, { vector_enabled: true, detail_resolved: null, expansion_applied: false });
|
||||
const today = new Date().toISOString().slice(0, 10);
|
||||
expect(w.bucketForTest(today, 'unset', 'unset')).not.toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
describe('flush() writes to search_telemetry', () => {
|
||||
test('flush drains the bucket map atomically', async () => {
|
||||
const w = getTelemetryWriter();
|
||||
w.setEngine(engine);
|
||||
recordSearchTelemetry(engine, makeMeta(), { results_count: 5 });
|
||||
recordSearchTelemetry(engine, makeMeta(), { results_count: 7 });
|
||||
expect(w.bucketCountForTest()).toBe(1);
|
||||
await w.flush();
|
||||
expect(w.bucketCountForTest()).toBe(0);
|
||||
|
||||
const rows = await engine.executeRaw<{ count: number; sum_results: number }>(
|
||||
'SELECT count, sum_results FROM search_telemetry',
|
||||
);
|
||||
expect(rows.length).toBe(1);
|
||||
expect(rows[0].count).toBe(2);
|
||||
expect(rows[0].sum_results).toBe(12);
|
||||
});
|
||||
|
||||
test('ON CONFLICT DO UPDATE adds raw values (concurrent-flush semantics)', async () => {
|
||||
const w = getTelemetryWriter();
|
||||
w.setEngine(engine);
|
||||
|
||||
// First flush: 3 calls under balanced/general.
|
||||
recordSearchTelemetry(engine, makeMeta(), { results_count: 5 });
|
||||
recordSearchTelemetry(engine, makeMeta(), { results_count: 5 });
|
||||
recordSearchTelemetry(engine, makeMeta(), { results_count: 5 });
|
||||
await w.flush();
|
||||
|
||||
// Second flush: 2 more calls under same (date, mode, intent).
|
||||
recordSearchTelemetry(engine, makeMeta(), { results_count: 10 });
|
||||
recordSearchTelemetry(engine, makeMeta(), { results_count: 10 });
|
||||
await w.flush();
|
||||
|
||||
const rows = await engine.executeRaw<{ count: number; sum_results: number }>(
|
||||
'SELECT count, sum_results FROM search_telemetry',
|
||||
);
|
||||
expect(rows.length).toBe(1);
|
||||
expect(rows[0].count).toBe(5); // 3 + 2
|
||||
expect(rows[0].sum_results).toBe(35); // (5+5+5) + (10+10)
|
||||
});
|
||||
|
||||
test('flush is no-op when bucket map is empty', async () => {
|
||||
const w = getTelemetryWriter();
|
||||
w.setEngine(engine);
|
||||
await w.flush(); // no records, no rows
|
||||
const rows = await engine.executeRaw<{ n: number }>('SELECT COUNT(*)::int AS n FROM search_telemetry');
|
||||
expect(rows[0].n).toBe(0);
|
||||
});
|
||||
|
||||
test('concurrent flush() calls coalesce (flushInFlight reuse)', async () => {
|
||||
const w = getTelemetryWriter();
|
||||
w.setEngine(engine);
|
||||
recordSearchTelemetry(engine, makeMeta(), { results_count: 1 });
|
||||
// Two simultaneous flush() awaits → both observe the same underlying drain.
|
||||
const [a, b] = await Promise.all([w.flush(), w.flush()]);
|
||||
expect(a).toBeUndefined();
|
||||
expect(b).toBeUndefined();
|
||||
const rows = await engine.executeRaw<{ count: number }>('SELECT count FROM search_telemetry');
|
||||
expect(rows[0].count).toBe(1); // not doubled
|
||||
});
|
||||
});
|
||||
|
||||
describe('readSearchStats — read-time derived averages', () => {
|
||||
test('empty table → all-zero stats', async () => {
|
||||
const s = await readSearchStats(engine, { days: 7 });
|
||||
expect(s.total_calls).toBe(0);
|
||||
expect(s.cache_hit_rate).toBe(0);
|
||||
expect(s.avg_results).toBe(0);
|
||||
});
|
||||
|
||||
test('one bucket flushed → stats derive averages from sums/counts', async () => {
|
||||
const w = getTelemetryWriter();
|
||||
w.setEngine(engine);
|
||||
recordSearchTelemetry(engine, makeMeta(), { results_count: 10 });
|
||||
recordSearchTelemetry(engine, makeMeta(), { results_count: 20 });
|
||||
recordSearchTelemetry(engine, makeMeta(), { results_count: 30 });
|
||||
await w.flush();
|
||||
|
||||
const s = await readSearchStats(engine, { days: 7 });
|
||||
expect(s.total_calls).toBe(3);
|
||||
expect(s.avg_results).toBe(20); // (10 + 20 + 30) / 3 = 20
|
||||
});
|
||||
|
||||
test('cache_hit_rate computed from hits + misses (excludes disabled)', async () => {
|
||||
const w = getTelemetryWriter();
|
||||
w.setEngine(engine);
|
||||
recordSearchTelemetry(engine, makeMeta({ cache: { status: 'hit' } }));
|
||||
recordSearchTelemetry(engine, makeMeta({ cache: { status: 'hit' } }));
|
||||
recordSearchTelemetry(engine, makeMeta({ cache: { status: 'hit' } }));
|
||||
recordSearchTelemetry(engine, makeMeta({ cache: { status: 'miss' } }));
|
||||
recordSearchTelemetry(engine, makeMeta({ cache: { status: 'disabled' } }));
|
||||
await w.flush();
|
||||
|
||||
const s = await readSearchStats(engine, { days: 7 });
|
||||
expect(s.cache_hits).toBe(3);
|
||||
expect(s.cache_misses).toBe(1);
|
||||
expect(s.cache_hit_rate).toBeCloseTo(0.75, 5); // 3 / (3 + 1) = 0.75
|
||||
});
|
||||
|
||||
test('intent_distribution and mode_distribution surface counts', async () => {
|
||||
const w = getTelemetryWriter();
|
||||
w.setEngine(engine);
|
||||
recordSearchTelemetry(engine, makeMeta({ mode: 'conservative', intent: 'entity' }));
|
||||
recordSearchTelemetry(engine, makeMeta({ mode: 'conservative', intent: 'entity' }));
|
||||
recordSearchTelemetry(engine, makeMeta({ mode: 'tokenmax', intent: 'temporal' }));
|
||||
await w.flush();
|
||||
|
||||
const s = await readSearchStats(engine, { days: 7 });
|
||||
expect(s.intent_distribution.entity).toBe(2);
|
||||
expect(s.intent_distribution.temporal).toBe(1);
|
||||
expect(s.mode_distribution.conservative).toBe(2);
|
||||
expect(s.mode_distribution.tokenmax).toBe(1);
|
||||
});
|
||||
|
||||
test('days window clamps to [1, 365]', async () => {
|
||||
const a = await readSearchStats(engine, { days: 0 });
|
||||
expect(a.window_days).toBe(1);
|
||||
const b = await readSearchStats(engine, { days: 9999 });
|
||||
expect(b.window_days).toBe(365);
|
||||
const c = await readSearchStats(engine, {});
|
||||
expect(c.window_days).toBe(7); // default
|
||||
});
|
||||
|
||||
test('missing search_telemetry table → empty stats (graceful)', async () => {
|
||||
// Drop the table to simulate a pre-v0.32.3 brain.
|
||||
await engine.executeRaw('DROP TABLE IF EXISTS search_telemetry');
|
||||
const s = await readSearchStats(engine, { days: 7 });
|
||||
expect(s.total_calls).toBe(0);
|
||||
expect(s.cache_hit_rate).toBe(0);
|
||||
// Restore for subsequent tests in this describe block.
|
||||
await engine.initSchema();
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,123 @@
|
||||
/**
|
||||
* v0.32.x search-lite \u2014 token budget enforcement.
|
||||
*
|
||||
* Pure module under test (no DB, no LLM). Confirms:
|
||||
* - char/4 heuristic estimates correctly
|
||||
* - greedy walk preserves caller ordering
|
||||
* - meta accounting (used / kept / dropped) matches the actual cut
|
||||
* - undefined / <=0 budget is a no-op
|
||||
* - first-result-too-big returns empty list
|
||||
*
|
||||
* Lives in test/token-budget.test.ts to mirror existing search/* test naming.
|
||||
*/
|
||||
|
||||
import { describe, test, expect } from 'bun:test';
|
||||
import { enforceTokenBudget, estimateTokens, resultTokens } from '../src/core/search/token-budget.ts';
|
||||
import type { SearchResult } from '../src/core/types.ts';
|
||||
|
||||
function makeResult(overrides: Partial<SearchResult> = {}): SearchResult {
|
||||
return {
|
||||
slug: 'test-page',
|
||||
page_id: 1,
|
||||
title: 'Test Title',
|
||||
type: 'concept',
|
||||
chunk_text: 'test chunk text',
|
||||
chunk_source: 'compiled_truth',
|
||||
chunk_id: 1,
|
||||
chunk_index: 0,
|
||||
score: 1.0,
|
||||
stale: false,
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
describe('estimateTokens', () => {
|
||||
test('empty / nullish strings cost 0 tokens', () => {
|
||||
expect(estimateTokens('')).toBe(0);
|
||||
expect(estimateTokens(undefined)).toBe(0);
|
||||
expect(estimateTokens(null)).toBe(0);
|
||||
});
|
||||
|
||||
test('rounds up so a single char still costs 1 token', () => {
|
||||
expect(estimateTokens('a')).toBe(1);
|
||||
expect(estimateTokens('abc')).toBe(1);
|
||||
expect(estimateTokens('abcd')).toBe(1);
|
||||
expect(estimateTokens('abcde')).toBe(2);
|
||||
});
|
||||
|
||||
test('scales linearly at ~4 chars / token', () => {
|
||||
// 100 chars \u2192 25 tokens
|
||||
expect(estimateTokens('x'.repeat(100))).toBe(25);
|
||||
// 401 chars \u2192 ceil(401/4) = 101
|
||||
expect(estimateTokens('x'.repeat(401))).toBe(101);
|
||||
});
|
||||
});
|
||||
|
||||
describe('resultTokens', () => {
|
||||
test('sums title + chunk_text', () => {
|
||||
const r = makeResult({ title: 'abcd', chunk_text: 'efgh' }); // 1 + 1 = 2
|
||||
expect(resultTokens(r)).toBe(2);
|
||||
});
|
||||
});
|
||||
|
||||
describe('enforceTokenBudget', () => {
|
||||
test('undefined budget is a no-op', () => {
|
||||
const results = [makeResult({ slug: 'a' }), makeResult({ slug: 'b' })];
|
||||
const { results: kept, meta } = enforceTokenBudget(results, undefined);
|
||||
expect(kept).toHaveLength(2);
|
||||
expect(meta.dropped).toBe(0);
|
||||
expect(meta.kept).toBe(2);
|
||||
expect(meta.budget).toBe(0); // safe-budget normalization
|
||||
});
|
||||
|
||||
test('zero / negative budget is a no-op', () => {
|
||||
const results = [makeResult({ slug: 'a' })];
|
||||
expect(enforceTokenBudget(results, 0).results).toHaveLength(1);
|
||||
expect(enforceTokenBudget(results, -5).results).toHaveLength(1);
|
||||
});
|
||||
|
||||
test('empty input returns empty', () => {
|
||||
const { results, meta } = enforceTokenBudget([], 100);
|
||||
expect(results).toHaveLength(0);
|
||||
expect(meta.kept).toBe(0);
|
||||
expect(meta.dropped).toBe(0);
|
||||
});
|
||||
|
||||
test('greedy top-down: stops as soon as cumulative cost would exceed budget', () => {
|
||||
// Each result: title 'a' (1 tok) + chunk_text 'xxxx' (1 tok) = 2 tokens each
|
||||
const results = [
|
||||
makeResult({ slug: 'a', title: 'a', chunk_text: 'xxxx' }),
|
||||
makeResult({ slug: 'b', title: 'a', chunk_text: 'xxxx' }),
|
||||
makeResult({ slug: 'c', title: 'a', chunk_text: 'xxxx' }),
|
||||
makeResult({ slug: 'd', title: 'a', chunk_text: 'xxxx' }),
|
||||
];
|
||||
// Budget 5 \u2192 fits 2 results (cost 2+2=4); a 3rd would push to 6.
|
||||
const { results: kept, meta } = enforceTokenBudget(results, 5);
|
||||
expect(kept).toHaveLength(2);
|
||||
expect(kept.map(r => r.slug)).toEqual(['a', 'b']);
|
||||
expect(meta.used).toBe(4);
|
||||
expect(meta.dropped).toBe(2);
|
||||
expect(meta.kept).toBe(2);
|
||||
expect(meta.budget).toBe(5);
|
||||
});
|
||||
|
||||
test('preserves caller ordering (never re-ranks)', () => {
|
||||
const results = [
|
||||
makeResult({ slug: 'low-score', score: 0.1, title: 'a', chunk_text: 'xxxx' }),
|
||||
makeResult({ slug: 'high-score', score: 0.9, title: 'a', chunk_text: 'xxxx' }),
|
||||
];
|
||||
const { results: kept } = enforceTokenBudget(results, 2);
|
||||
// 2 tokens fits exactly one result; budget pass should keep the FIRST,
|
||||
// even though it has a worse score \u2014 ordering is caller's contract.
|
||||
expect(kept).toHaveLength(1);
|
||||
expect(kept[0].slug).toBe('low-score');
|
||||
});
|
||||
|
||||
test('first result exceeds budget alone \u2192 returns empty', () => {
|
||||
const big = makeResult({ slug: 'big', title: 'a', chunk_text: 'x'.repeat(1000) });
|
||||
const small = makeResult({ slug: 'small', title: 'a', chunk_text: 'xxxx' });
|
||||
const { results: kept, meta } = enforceTokenBudget([big, small], 5);
|
||||
expect(kept).toHaveLength(0);
|
||||
expect(meta.dropped).toBe(2);
|
||||
});
|
||||
});
|
||||
@@ -59,10 +59,12 @@ describe('v0.31.8 — voyage Content-Length pre-check + per-item cap', () => {
|
||||
|
||||
test('Layer 1 throws on Content-Length over the cap (not silent return)', async () => {
|
||||
const source = await Bun.file(new URL('../src/core/ai/gateway.ts', import.meta.url)).text();
|
||||
// The cap check must use `throw new Error(...)` so the embed flow's
|
||||
// retry/backoff sees a failure, NOT a silent pass-through.
|
||||
// The cap check must throw a VoyageResponseTooLargeError so the inbound
|
||||
// try/catch at the bottom of voyageCompatFetch rethrows it (instead of
|
||||
// the pre-fix bare `catch {}` that swallowed `throw new Error(...)` and
|
||||
// returned the original response — making the cap theatrical).
|
||||
expect(source).toMatch(/exceeds[^`]*MAX_VOYAGE_RESPONSE_BYTES[^`]*bytes/);
|
||||
expect(source).toMatch(/throw new Error\([\s\S]{0,200}Voyage response Content-Length/);
|
||||
expect(source).toMatch(/throw new VoyageResponseTooLargeError\([\s\S]{0,200}Voyage response Content-Length/);
|
||||
});
|
||||
|
||||
test('Layer 2: per-embedding base64 cap fires inside the json.data iteration', async () => {
|
||||
@@ -70,7 +72,18 @@ describe('v0.31.8 — voyage Content-Length pre-check + per-item cap', () => {
|
||||
// Defense-in-depth: even when Content-Length header is missing
|
||||
// (chunked encoding), each embedding string is bounded.
|
||||
expect(source).toMatch(/item\.embedding\.length\s*\*\s*0\.75/);
|
||||
expect(source).toMatch(/throw new Error\([\s\S]{0,200}Voyage embedding base64/);
|
||||
expect(source).toMatch(/throw new VoyageResponseTooLargeError\([\s\S]{0,200}Voyage embedding base64/);
|
||||
});
|
||||
|
||||
test('inbound try/catch rethrows VoyageResponseTooLargeError (Codex P3 follow-up)', async () => {
|
||||
const source = await Bun.file(new URL('../src/core/ai/gateway.ts', import.meta.url)).text();
|
||||
// Critical structural invariant after the Codex P3 fix: the catch
|
||||
// around the inbound JSON-rewrite block MUST rethrow OOM-cap errors.
|
||||
// Pre-fix, a bare `catch {}` swallowed the throw and returned the
|
||||
// original (oversized) response, making Layer 2 ineffective. The
|
||||
// tagged class + instanceof check restores fail-loud behavior.
|
||||
expect(source).toContain('VoyageResponseTooLargeError');
|
||||
expect(source).toMatch(/if\s*\(\s*err\s+instanceof\s+VoyageResponseTooLargeError\s*\)\s*throw\s+err/);
|
||||
});
|
||||
|
||||
test('comment thread documents both layers + the cap-sizing decision', async () => {
|
||||
|
||||
Reference in New Issue
Block a user