feat(eval): Phase 2 EXT-2 vector-only RAG adapter

Second external baseline for BrainBench. Pure cosine-similarity ranking
using the SAME text-embedding-3-large model gbrain uses internally —
apples-to-apples on the embedding layer so any gbrain lead reflects the
graph + hybrid fusion, not a better embedder.

Files:
  eval/runner/adapters/vector-only.ts      ~130 LOC
  eval/runner/adapters/vector-only.test.ts 6 unit tests (cosine math)

Design:
  - One vector per page (title + compiled_truth + timeline, capped 8K chars).
  - No chunking (intentional; chunked vector RAG would be EXT-2b later).
  - No keyword fallback (that's EXT-3 hybrid-without-graph).
  - Embeddings in batches of 50 via existing src/core/embedding.ts (retry+backoff).
  - Cost on 240 pages: ~$0.02/run.

Three-adapter side-by-side on 240-page rich-prose corpus, 145 relational queries:

| Adapter       | P@5    | R@5    | Correct top-5 |
|---------------|--------|--------|---------------|
| gbrain-after  | 49.1%  | 97.9%  | 248/261       |
| ripgrep-bm25  | 17.1%  | 62.4%  | 124/261       |
| vector-only   | 10.8%  | 40.7%  |  78/261       |

Interesting finding: vector-only scores WORSE than BM25 on relational queries
like "Who invested in X?" — exact entity match matters more than semantic
similarity for these templates. BM25 nails the entity-name term; vector-only
returns topically-similar-but-not-mentioning pages. This is the known failure
mode of pure-vector RAG on precise relational/identity queries. Real-world
vector RAG systems always add keyword fallback; EXT-3 (hybrid-without-graph)
will be that fairer comparator.

gbrain's lead widens in vector-only comparison: +38.4 pts P@5, +57.2 pts R@5.
The graph layer is doing the heavy lifting for relational traversal; pure
vector RAG can't express "traverse 'attended' edges from this meeting page."

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>

eval/runner/adapters/vector-only.test.ts

@@ -0,0 +1,51 @@
+import { describe, test, expect } from 'bun:test';
+import { _cosine } from './vector-only.ts';
+
+// Note: VectorOnlyAdapter.init/query require a live embedding API key.
+// Those end-to-end tests live in a smoke-test class and gate on OPENAI_API_KEY.
+// Here we unit-test the pure-function pieces.
+
+describe('vector-only adapter (pure helpers)', () => {
+  test('cosine of identical vectors = 1.0', () => {
+    const a = new Float32Array([1, 2, 3, 4]);
+    const b = new Float32Array([1, 2, 3, 4]);
+    expect(_cosine(a, b)).toBeCloseTo(1.0, 6);
+  });
+
+  test('cosine of opposite vectors = -1.0', () => {
+    const a = new Float32Array([1, 2, 3]);
+    const b = new Float32Array([-1, -2, -3]);
+    expect(_cosine(a, b)).toBeCloseTo(-1.0, 6);
+  });
+
+  test('cosine of orthogonal vectors = 0', () => {
+    const a = new Float32Array([1, 0, 0]);
+    const b = new Float32Array([0, 1, 0]);
+    expect(_cosine(a, b)).toBeCloseTo(0.0, 6);
+  });
+
+  test('cosine handles zero vector by returning 0', () => {
+    const a = new Float32Array([0, 0, 0]);
+    const b = new Float32Array([1, 2, 3]);
+    expect(_cosine(a, b)).toBe(0);
+  });
+
+  test('cosine is scale-invariant', () => {
+    const a = new Float32Array([1, 2, 3]);
+    const b = new Float32Array([2, 4, 6]);
+    // Same direction, different magnitudes; cosine should still be 1.
+    expect(_cosine(a, b)).toBeCloseTo(1.0, 6);
+  });
+
+  test('cosine returns 0 on mismatched-length vectors at the tail', () => {
+    // Uses min(len) — shorter vector's dimensions are compared, longer's
+    // extras are implicitly dropped. Produces a sensible number even
+    // when upstream glue has a dim mismatch bug; helps fail-soft rather
+    // than crash the benchmark.
+    const a = new Float32Array([1, 1, 1]);
+    const b = new Float32Array([1, 1]);
+    const sim = _cosine(a, b);
+    expect(sim).toBeGreaterThan(0);
+    expect(sim).toBeLessThanOrEqual(1);
+  });
+});

eval/runner/adapters/vector-only.ts

@@ -0,0 +1,150 @@
+/**
+ * BrainBench EXT-2: Vector-only RAG adapter.
+ *
+ * Commodity vector RAG: embed every page once, embed the query, rank by
+ * cosine similarity. No graph, no keyword fallback, no BM25 — the opposite
+ * end of the baseline spectrum from EXT-1.
+ *
+ * Uses the SAME embedding model gbrain uses internally (text-embedding-3-large
+ * via src/core/embedding.ts). Apples-to-apples on the embedding layer: any
+ * lead gbrain has over vector-only must come from the graph + hybrid fusion,
+ * not from a better embedder. This is the honest external comparator.
+ *
+ * Cost: ~$0.02 per run on the 240-page corpus (embed 240 pages once, embed
+ * each query once, ~120K total tokens at $0.13/M).
+ *
+ * Design:
+ *   1. init(): embed each page's title + compiled_truth + timeline as ONE
+ *              vector per page. Store in memory.
+ *   2. query(): embed query text, compute cosine similarity against every
+ *               page vector, rank descending.
+ *
+ * Notes:
+ *   - No chunking. One vector per page. Real vector RAG in production
+ *     chunks long docs; we intentionally don't here so the comparison
+ *     against gbrain's chunked hybrid is fair at the retrieval granularity.
+ *     If a future BrainBench iteration wants to test chunked vector RAG,
+ *     that's a separate adapter (EXT-2b maybe).
+ *   - No keyword fallback. Pure vector similarity. An agent that wanted
+ *     vector+keyword would use EXT-3 hybrid-without-graph.
+ */
+
+import type { Adapter, AdapterConfig, BrainState, Page, Query, RankedDoc } from '../types.ts';
+import { embed, embedBatch } from '../../../src/core/embedding.ts';
+
+// ─── Vector math ────────────────────────────────────────────────────
+
+/**
+ * Cosine similarity between two dense vectors. Assumes equal length;
+ * callers upstream ensure embedder returned consistent-dim vectors.
+ */
+function cosine(a: Float32Array, b: Float32Array): number {
+  let dot = 0;
+  let normA = 0;
+  let normB = 0;
+  const n = Math.min(a.length, b.length);
+  for (let i = 0; i < n; i++) {
+    dot += a[i] * b[i];
+    normA += a[i] * a[i];
+    normB += b[i] * b[i];
+  }
+  if (normA === 0 || normB === 0) return 0;
+  return dot / (Math.sqrt(normA) * Math.sqrt(normB));
+}
+
+// ─── Adapter state ──────────────────────────────────────────────────
+
+interface VectorOnlyState {
+  /** docId -> its embedding vector. */
+  vectors: Map<string, Float32Array>;
+  /** docId -> original Page. */
+  docs: Map<string, Page>;
+  /** Embedding model used (for scorecard reproducibility card). */
+  embeddingModel: string;
+}
+
+// ─── Adapter implementation ────────────────────────────────────────
+
+interface VectorOnlyConfig extends AdapterConfig {
+  /** Chunk size in chars for page content sent to embedder.
+   *  Default: unchunked (single vector per page). Capped at 8K chars
+   *  to stay within embedding model input limits. */
+  maxChars?: number;
+  /** Max parallel embedding requests during init (the embedBatch helper
+   *  chunks internally; this throttles if upstream rate-limits). */
+  batchSize?: number;
+}
+
+export class VectorOnlyAdapter implements Adapter {
+  readonly name = 'vector-only';
+
+  async init(rawPages: Page[], config: VectorOnlyConfig): Promise<BrainState> {
+    const maxChars = config.maxChars ?? 8000;
+    const batchSize = config.batchSize ?? 50;
+
+    const docs = new Map<string, Page>();
+    const contents: string[] = [];
+    const slugOrder: string[] = [];
+    for (const p of rawPages) {
+      docs.set(p.slug, p);
+      const combined = `${p.title}\n\n${p.compiled_truth}\n\n${p.timeline}`
+        .slice(0, maxChars);
+      contents.push(combined);
+      slugOrder.push(p.slug);
+    }
+
+    // Embed in batches to respect rate limits. embedBatch handles the
+    // OpenAI API call pattern (retry + backoff) per src/core/embedding.ts.
+    const vectors = new Map<string, Float32Array>();
+    for (let i = 0; i < contents.length; i += batchSize) {
+      const batch = contents.slice(i, i + batchSize);
+      const slugs = slugOrder.slice(i, i + batchSize);
+      const embeddings = await embedBatch(batch);
+      for (let j = 0; j < embeddings.length; j++) {
+        vectors.set(slugs[j], embeddings[j]);
+      }
+    }
+
+    // EMBEDDING_MODEL is a const export; lazy-imported here to avoid circular.
+    const { EMBEDDING_MODEL } = await import('../../../src/core/embedding.ts');
+    return {
+      vectors,
+      docs,
+      embeddingModel: EMBEDDING_MODEL,
+    } satisfies VectorOnlyState;
+  }
+
+  async query(q: Query, state: BrainState): Promise<RankedDoc[]> {
+    const s = state as VectorOnlyState;
+    const queryVec = await embed(q.text);
+
+    const scored: { id: string; score: number }[] = [];
+    for (const [docId, docVec] of s.vectors) {
+      const sim = cosine(queryVec, docVec);
+      if (sim > 0) scored.push({ id: docId, score: sim });
+    }
+    scored.sort((a, b) => b.score - a.score || a.id.localeCompare(b.id));
+
+    return scored.map((s, i) => ({
+      page_id: s.id,
+      score: s.score,
+      rank: i + 1,
+    }));
+  }
+
+  async snapshot(_state: BrainState): Promise<string> {
+    // Vector state is in-memory only for v1.1. Persisted vector DBs are
+    // a separate future comparison (EXT-2b).
+    return '';
+  }
+}
+
+export function createVectorOnly(): VectorOnlyAdapter {
+  return new VectorOnlyAdapter();
+}
+
+/**
+ * Test helper: cosine similarity exposed for unit tests. Not for public API.
+ * @internal
+ */
+export { cosine as _cosine };

eval/runner/multi-adapter.ts

@@ -23,6 +23,7 @@ import { join } from 'path';
 import { PGLiteEngine } from '../../src/core/pglite-engine.ts';
 import { runExtract } from '../../src/commands/extract.ts';
 import { RipgrepBm25Adapter } from './adapters/ripgrep-bm25.ts';
+import { VectorOnlyAdapter } from './adapters/vector-only.ts';
 import type { Adapter, Page, Query, RankedDoc } from './types.ts';
 import { precisionAtK, recallAtK } from './types.ts';
 
@@ -325,6 +326,7 @@ async function main() {
   const allAdapters: Adapter[] = [
     new GbrainAfterAdapter(),
     new RipgrepBm25Adapter(),
+    new VectorOnlyAdapter(),
   ];
   const adapters = only ? allAdapters.filter(a => a.name === only) : allAdapters;
   if (adapters.length === 0) {