mirror of
https://github.com/garrytan/gbrain.git
synced 2026-07-31 04:07:52 +00:00
Garry TanandClaude Opus 4.6
934d7ea4b6
Three infrastructure modules for BrainBench v1 Complete Cats 5/8/9/11.
**eval/runner/loaders/pdf.ts** — Thin pdf-parse wrapper. Lazy import keeps
pdf-parse out of the module-load path (avoids library debug-mode side
effects). Size cap (50MB default), encryption detection, structured error
classes (PdfEncryptedError, PdfTooLargeError, PdfParseError). Only Cat 11
multimodal will import this; production bundle never sees pdf-parse.
**eval/runner/tool-bridge.ts** — Maps 12 read-only operations from
src/core/operations.ts to Anthropic tool definitions + adds 3 dry_run write
tools. Three structural invariants enforced:
1. No hidden LLM calls. `operations.query` defaults expand=true which
routes through expansion.ts → Haiku. Bridge strips `expand` from the
query tool's input schema AND executor hard-sets expand:false. Zero
nested Haiku calls in any agent trace.
2. Mutating ops throw ForbiddenOpError. put_page, add_link, delete_page,
etc. are rejected by name. Agents record intent via dry_run_put_page /
dry_run_add_link / dry_run_add_timeline_entry which persist to the
flight-recorder without mutating the engine. This is how Cat 8's
back_link_compliance + citation_format metrics measure anything with
a read-only tool surface.
3. Poison tagged by the bridge, not the judge. Every tool result is
scanned for slugs matching gold/poison.json fixtures. Matched
fixture_ids flow into tool_call_summary.saw_poison_items for the
structured-evidence judge contract. Judge never reads raw tool
output — Section-3 defense against paraphrased prompt injections
(poison payloads never reach the judge model at all).
32K-token cap (~128K chars) with "…[truncated]" suffix.
**eval/runner/recorder.ts** — Per-run flight-recorder bundle emitter. Full
6-artifact bundle (transcript.md, brain-export.json, entity-graph.json,
citations.json, scorecard.json, judge-notes.md) when the adapter provides
an AdapterExport; 3-artifact fallback (transcript + scorecard +
judge-notes) otherwise. Atomic writes via tmp+rename. Collision-safe:
duplicate directory names get incremental -2, -3 suffix. `safeStringify`
handles circular references without throwing and JSON-serializes
Float32Array embeddings.
**package.json:** adds pdf-parse@2.4.5 as a devDependency. Scoped to eval/
use only; production gbrain binary unaffected.
**Tests:** 63 new — 30 tool-bridge, 21 recorder, 12 pdf-loader. All pass.
Fake engine uses a Proxy with `__default__` fallback so poison-matching
tests don't have to mock the exact engine method name that each operation
calls (some route via searchKeyword, others via getPage — proxy handles
both uniformly).
Total eval suite now: 132 pass, 0 fail, 923 expect() calls.
Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
88 lines
2.6 KiB
TypeScript
88 lines
2.6 KiB
TypeScript
/**
|
|
* eval/runner/loaders/pdf.ts tests.
|
|
*
|
|
* Does NOT test real PDF parsing (that's Cat 11 integration territory with
|
|
* actual fixture PDFs via `bun run eval:fetch-multimodal`). Focuses on the
|
|
* guard behavior + error class shape that Cat 11 depends on.
|
|
*/
|
|
|
|
import { describe, test, expect } from 'bun:test';
|
|
import { writeFileSync, mkdirSync, rmSync } from 'fs';
|
|
import { join } from 'path';
|
|
import { tmpdir } from 'os';
|
|
import {
|
|
extractPdfText,
|
|
PdfEncryptedError,
|
|
PdfParseError,
|
|
PdfTooLargeError,
|
|
} from '../../eval/runner/loaders/pdf.ts';
|
|
|
|
function tmpFile(content: Uint8Array | string, name: string): string {
|
|
const dir = join(tmpdir(), `pdf-test-${process.pid}-${Date.now()}-${Math.random().toString(36).slice(2)}`);
|
|
mkdirSync(dir, { recursive: true });
|
|
const path = join(dir, name);
|
|
writeFileSync(path, content as any);
|
|
return path;
|
|
}
|
|
|
|
describe('extractPdfText — guards', () => {
|
|
test('throws PdfParseError for non-existent file', async () => {
|
|
let err: unknown = null;
|
|
try {
|
|
await extractPdfText('/nonexistent/path/to/file.pdf');
|
|
} catch (e) {
|
|
err = e;
|
|
}
|
|
expect(err).toBeInstanceOf(PdfParseError);
|
|
});
|
|
|
|
test('throws PdfTooLargeError when file exceeds maxBytes', async () => {
|
|
const big = new Uint8Array(2048);
|
|
const path = tmpFile(big, 'big.pdf');
|
|
let err: unknown = null;
|
|
try {
|
|
await extractPdfText(path, { maxBytes: 1024 });
|
|
} catch (e) {
|
|
err = e;
|
|
}
|
|
expect(err).toBeInstanceOf(PdfTooLargeError);
|
|
rmSync(path);
|
|
});
|
|
|
|
test('throws PdfParseError for non-PDF content', async () => {
|
|
const junk = 'this is not a PDF file';
|
|
const path = tmpFile(junk, 'not-a-pdf.pdf');
|
|
let err: unknown = null;
|
|
try {
|
|
await extractPdfText(path);
|
|
} catch (e) {
|
|
err = e;
|
|
}
|
|
expect(err).toBeInstanceOf(PdfParseError);
|
|
rmSync(path);
|
|
});
|
|
});
|
|
|
|
describe('error classes', () => {
|
|
test('PdfEncryptedError has kind = "encrypted"', () => {
|
|
const err = new PdfEncryptedError('/x.pdf');
|
|
expect(err.kind).toBe('encrypted');
|
|
expect(err.name).toBe('PdfEncryptedError');
|
|
expect(err.message).toContain('/x.pdf');
|
|
});
|
|
|
|
test('PdfTooLargeError reports size + max', () => {
|
|
const err = new PdfTooLargeError('/x.pdf', 5_000_000, 1_000_000);
|
|
expect(err.kind).toBe('too_large');
|
|
expect(err.message).toContain('5000000');
|
|
expect(err.message).toContain('1000000');
|
|
});
|
|
|
|
test('PdfParseError preserves cause chain', () => {
|
|
const inner = new Error('underlying failure');
|
|
const err = new PdfParseError('/x.pdf', inner);
|
|
expect(err.kind).toBe('parse');
|
|
expect((err as Error & { cause?: unknown }).cause).toBe(inner);
|
|
});
|
|
});
|