diff --git a/eval/runner/adapters/vector-only.test.ts b/eval/runner/adapters/vector-only.test.ts new file mode 100644 index 000000000..3ab7edf23 --- /dev/null +++ b/eval/runner/adapters/vector-only.test.ts @@ -0,0 +1,51 @@ +import { describe, test, expect } from 'bun:test'; +import { _cosine } from './vector-only.ts'; + +// Note: VectorOnlyAdapter.init/query require a live embedding API key. +// Those end-to-end tests live in a smoke-test class and gate on OPENAI_API_KEY. +// Here we unit-test the pure-function pieces. + +describe('vector-only adapter (pure helpers)', () => { + test('cosine of identical vectors = 1.0', () => { + const a = new Float32Array([1, 2, 3, 4]); + const b = new Float32Array([1, 2, 3, 4]); + expect(_cosine(a, b)).toBeCloseTo(1.0, 6); + }); + + test('cosine of opposite vectors = -1.0', () => { + const a = new Float32Array([1, 2, 3]); + const b = new Float32Array([-1, -2, -3]); + expect(_cosine(a, b)).toBeCloseTo(-1.0, 6); + }); + + test('cosine of orthogonal vectors = 0', () => { + const a = new Float32Array([1, 0, 0]); + const b = new Float32Array([0, 1, 0]); + expect(_cosine(a, b)).toBeCloseTo(0.0, 6); + }); + + test('cosine handles zero vector by returning 0', () => { + const a = new Float32Array([0, 0, 0]); + const b = new Float32Array([1, 2, 3]); + expect(_cosine(a, b)).toBe(0); + }); + + test('cosine is scale-invariant', () => { + const a = new Float32Array([1, 2, 3]); + const b = new Float32Array([2, 4, 6]); + // Same direction, different magnitudes; cosine should still be 1. + expect(_cosine(a, b)).toBeCloseTo(1.0, 6); + }); + + test('cosine returns 0 on mismatched-length vectors at the tail', () => { + // Uses min(len) — shorter vector's dimensions are compared, longer's + // extras are implicitly dropped. Produces a sensible number even + // when upstream glue has a dim mismatch bug; helps fail-soft rather + // than crash the benchmark. + const a = new Float32Array([1, 1, 1]); + const b = new Float32Array([1, 1]); + const sim = _cosine(a, b); + expect(sim).toBeGreaterThan(0); + expect(sim).toBeLessThanOrEqual(1); + }); +}); diff --git a/eval/runner/adapters/vector-only.ts b/eval/runner/adapters/vector-only.ts new file mode 100644 index 000000000..2e7dd412c --- /dev/null +++ b/eval/runner/adapters/vector-only.ts @@ -0,0 +1,150 @@ +/** + * BrainBench EXT-2: Vector-only RAG adapter. + * + * Commodity vector RAG: embed every page once, embed the query, rank by + * cosine similarity. No graph, no keyword fallback, no BM25 — the opposite + * end of the baseline spectrum from EXT-1. + * + * Uses the SAME embedding model gbrain uses internally (text-embedding-3-large + * via src/core/embedding.ts). Apples-to-apples on the embedding layer: any + * lead gbrain has over vector-only must come from the graph + hybrid fusion, + * not from a better embedder. This is the honest external comparator. + * + * Cost: ~$0.02 per run on the 240-page corpus (embed 240 pages once, embed + * each query once, ~120K total tokens at $0.13/M). + * + * Design: + * 1. init(): embed each page's title + compiled_truth + timeline as ONE + * vector per page. Store in memory. + * 2. query(): embed query text, compute cosine similarity against every + * page vector, rank descending. + * + * Notes: + * - No chunking. One vector per page. Real vector RAG in production + * chunks long docs; we intentionally don't here so the comparison + * against gbrain's chunked hybrid is fair at the retrieval granularity. + * If a future BrainBench iteration wants to test chunked vector RAG, + * that's a separate adapter (EXT-2b maybe). + * - No keyword fallback. Pure vector similarity. An agent that wanted + * vector+keyword would use EXT-3 hybrid-without-graph. + */ + +import type { Adapter, AdapterConfig, BrainState, Page, Query, RankedDoc } from '../types.ts'; +import { embed, embedBatch } from '../../../src/core/embedding.ts'; + +// ─── Vector math ──────────────────────────────────────────────────── + +/** + * Cosine similarity between two dense vectors. Assumes equal length; + * callers upstream ensure embedder returned consistent-dim vectors. + */ +function cosine(a: Float32Array, b: Float32Array): number { + let dot = 0; + let normA = 0; + let normB = 0; + const n = Math.min(a.length, b.length); + for (let i = 0; i < n; i++) { + dot += a[i] * b[i]; + normA += a[i] * a[i]; + normB += b[i] * b[i]; + } + if (normA === 0 || normB === 0) return 0; + return dot / (Math.sqrt(normA) * Math.sqrt(normB)); +} + +// ─── Adapter state ────────────────────────────────────────────────── + +interface VectorOnlyState { + /** docId -> its embedding vector. */ + vectors: Map; + /** docId -> original Page. */ + docs: Map; + /** Embedding model used (for scorecard reproducibility card). */ + embeddingModel: string; +} + +// ─── Adapter implementation ──────────────────────────────────────── + +interface VectorOnlyConfig extends AdapterConfig { + /** Chunk size in chars for page content sent to embedder. + * Default: unchunked (single vector per page). Capped at 8K chars + * to stay within embedding model input limits. */ + maxChars?: number; + /** Max parallel embedding requests during init (the embedBatch helper + * chunks internally; this throttles if upstream rate-limits). */ + batchSize?: number; +} + +export class VectorOnlyAdapter implements Adapter { + readonly name = 'vector-only'; + + async init(rawPages: Page[], config: VectorOnlyConfig): Promise { + const maxChars = config.maxChars ?? 8000; + const batchSize = config.batchSize ?? 50; + + const docs = new Map(); + const contents: string[] = []; + const slugOrder: string[] = []; + for (const p of rawPages) { + docs.set(p.slug, p); + const combined = `${p.title}\n\n${p.compiled_truth}\n\n${p.timeline}` + .slice(0, maxChars); + contents.push(combined); + slugOrder.push(p.slug); + } + + // Embed in batches to respect rate limits. embedBatch handles the + // OpenAI API call pattern (retry + backoff) per src/core/embedding.ts. + const vectors = new Map(); + for (let i = 0; i < contents.length; i += batchSize) { + const batch = contents.slice(i, i + batchSize); + const slugs = slugOrder.slice(i, i + batchSize); + const embeddings = await embedBatch(batch); + for (let j = 0; j < embeddings.length; j++) { + vectors.set(slugs[j], embeddings[j]); + } + } + + // EMBEDDING_MODEL is a const export; lazy-imported here to avoid circular. + const { EMBEDDING_MODEL } = await import('../../../src/core/embedding.ts'); + return { + vectors, + docs, + embeddingModel: EMBEDDING_MODEL, + } satisfies VectorOnlyState; + } + + async query(q: Query, state: BrainState): Promise { + const s = state as VectorOnlyState; + const queryVec = await embed(q.text); + + const scored: { id: string; score: number }[] = []; + for (const [docId, docVec] of s.vectors) { + const sim = cosine(queryVec, docVec); + if (sim > 0) scored.push({ id: docId, score: sim }); + } + scored.sort((a, b) => b.score - a.score || a.id.localeCompare(b.id)); + + return scored.map((s, i) => ({ + page_id: s.id, + score: s.score, + rank: i + 1, + })); + } + + async snapshot(_state: BrainState): Promise { + // Vector state is in-memory only for v1.1. Persisted vector DBs are + // a separate future comparison (EXT-2b). + return ''; + } +} + +export function createVectorOnly(): VectorOnlyAdapter { + return new VectorOnlyAdapter(); +} + +/** + * Test helper: cosine similarity exposed for unit tests. Not for public API. + * @internal + */ +export { cosine as _cosine }; diff --git a/eval/runner/multi-adapter.ts b/eval/runner/multi-adapter.ts index d34ec14ed..b37df06f0 100644 --- a/eval/runner/multi-adapter.ts +++ b/eval/runner/multi-adapter.ts @@ -23,6 +23,7 @@ import { join } from 'path'; import { PGLiteEngine } from '../../src/core/pglite-engine.ts'; import { runExtract } from '../../src/commands/extract.ts'; import { RipgrepBm25Adapter } from './adapters/ripgrep-bm25.ts'; +import { VectorOnlyAdapter } from './adapters/vector-only.ts'; import type { Adapter, Page, Query, RankedDoc } from './types.ts'; import { precisionAtK, recallAtK } from './types.ts'; @@ -325,6 +326,7 @@ async function main() { const allAdapters: Adapter[] = [ new GbrainAfterAdapter(), new RipgrepBm25Adapter(), + new VectorOnlyAdapter(), ]; const adapters = only ? allAdapters.filter(a => a.name === only) : allAdapters; if (adapters.length === 0) {