mirror of
https://github.com/garrytan/gbrain.git
synced 2026-07-31 04:07:52 +00:00
Three infrastructure modules for BrainBench v1 Complete Cats 5/8/9/11.
**eval/runner/loaders/pdf.ts** — Thin pdf-parse wrapper. Lazy import keeps
pdf-parse out of the module-load path (avoids library debug-mode side
effects). Size cap (50MB default), encryption detection, structured error
classes (PdfEncryptedError, PdfTooLargeError, PdfParseError). Only Cat 11
multimodal will import this; production bundle never sees pdf-parse.
**eval/runner/tool-bridge.ts** — Maps 12 read-only operations from
src/core/operations.ts to Anthropic tool definitions + adds 3 dry_run write
tools. Three structural invariants enforced:
1. No hidden LLM calls. `operations.query` defaults expand=true which
routes through expansion.ts → Haiku. Bridge strips `expand` from the
query tool's input schema AND executor hard-sets expand:false. Zero
nested Haiku calls in any agent trace.
2. Mutating ops throw ForbiddenOpError. put_page, add_link, delete_page,
etc. are rejected by name. Agents record intent via dry_run_put_page /
dry_run_add_link / dry_run_add_timeline_entry which persist to the
flight-recorder without mutating the engine. This is how Cat 8's
back_link_compliance + citation_format metrics measure anything with
a read-only tool surface.
3. Poison tagged by the bridge, not the judge. Every tool result is
scanned for slugs matching gold/poison.json fixtures. Matched
fixture_ids flow into tool_call_summary.saw_poison_items for the
structured-evidence judge contract. Judge never reads raw tool
output — Section-3 defense against paraphrased prompt injections
(poison payloads never reach the judge model at all).
32K-token cap (~128K chars) with "…[truncated]" suffix.
**eval/runner/recorder.ts** — Per-run flight-recorder bundle emitter. Full
6-artifact bundle (transcript.md, brain-export.json, entity-graph.json,
citations.json, scorecard.json, judge-notes.md) when the adapter provides
an AdapterExport; 3-artifact fallback (transcript + scorecard +
judge-notes) otherwise. Atomic writes via tmp+rename. Collision-safe:
duplicate directory names get incremental -2, -3 suffix. `safeStringify`
handles circular references without throwing and JSON-serializes
Float32Array embeddings.
**package.json:** adds pdf-parse@2.4.5 as a devDependency. Scoped to eval/
use only; production gbrain binary unaffected.
**Tests:** 63 new — 30 tool-bridge, 21 recorder, 12 pdf-loader. All pass.
Fake engine uses a Proxy with `__default__` fallback so poison-matching
tests don't have to mock the exact engine method name that each operation
calls (some route via searchKeyword, others via getPage — proxy handles
both uniformly).
Total eval suite now: 132 pass, 0 fail, 923 expect() calls.
Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
110 lines
3.5 KiB
TypeScript
110 lines
3.5 KiB
TypeScript
/**
|
|
* Thin pdf-parse wrapper for BrainBench v1 Cat 11 multimodal ingest.
|
|
*
|
|
* pdf-parse is loaded lazily (dynamic import) so module load does not
|
|
* trigger any debug-mode file reads that the library does on some versions.
|
|
* Lazy import also keeps the dep out of the production bundle path — only
|
|
* eval/runner/cat11-multimodal.ts imports this file.
|
|
*
|
|
* Guards:
|
|
* - Size cap (default 50MB) — prevents ingesting malicious/malformed pathological PDFs
|
|
* - Encrypted PDFs throw `PdfEncryptedError` with a clear message instead of
|
|
* hanging or returning garbage
|
|
* - Empty/corrupt PDFs throw `PdfParseError` with path + errno-style context
|
|
*
|
|
* No production code path consumes this loader. Cat 11 test fixtures only.
|
|
*/
|
|
|
|
import { statSync } from 'fs';
|
|
import { readFile } from 'fs/promises';
|
|
|
|
const DEFAULT_MAX_BYTES = 50 * 1024 * 1024; // 50MB
|
|
|
|
export class PdfEncryptedError extends Error {
|
|
readonly kind = 'encrypted' as const;
|
|
constructor(path: string) {
|
|
super(`PDF is encrypted: ${path}`);
|
|
this.name = 'PdfEncryptedError';
|
|
}
|
|
}
|
|
|
|
export class PdfParseError extends Error {
|
|
readonly kind = 'parse' as const;
|
|
constructor(path: string, cause?: unknown) {
|
|
super(`PDF parse failed: ${path}${cause ? ` (${String(cause)})` : ''}`);
|
|
this.name = 'PdfParseError';
|
|
if (cause instanceof Error) this.cause = cause;
|
|
}
|
|
}
|
|
|
|
export class PdfTooLargeError extends Error {
|
|
readonly kind = 'too_large' as const;
|
|
constructor(path: string, sizeBytes: number, maxBytes: number) {
|
|
super(`PDF too large: ${path} is ${sizeBytes} bytes, max ${maxBytes}`);
|
|
this.name = 'PdfTooLargeError';
|
|
}
|
|
}
|
|
|
|
export interface PdfExtractOptions {
|
|
/** Max PDF size in bytes. Default 50MB. */
|
|
maxBytes?: number;
|
|
}
|
|
|
|
export interface PdfExtractResult {
|
|
/** Full extracted text, newline-joined across pages. */
|
|
text: string;
|
|
/** Number of pages in the PDF. */
|
|
numPages: number;
|
|
/** Raw PDF metadata dictionary (title, author, etc.) if present. */
|
|
info?: Record<string, unknown>;
|
|
}
|
|
|
|
/**
|
|
* Extract text from a PDF file path. Size-guarded and encryption-aware.
|
|
*
|
|
* This is the ONLY entry point for PDF extraction in BrainBench. The
|
|
* pdf-parse package is lazy-loaded here so that (a) a module-load crash in
|
|
* pdf-parse cannot break unrelated eval runners and (b) the dep stays in
|
|
* devDependencies — production gbrain binary never sees it.
|
|
*/
|
|
export async function extractPdfText(
|
|
path: string,
|
|
opts: PdfExtractOptions = {},
|
|
): Promise<PdfExtractResult> {
|
|
const maxBytes = opts.maxBytes ?? DEFAULT_MAX_BYTES;
|
|
|
|
// Size guard BEFORE reading into memory.
|
|
let stat;
|
|
try {
|
|
stat = statSync(path);
|
|
} catch (err) {
|
|
throw new PdfParseError(path, err);
|
|
}
|
|
if (stat.size > maxBytes) {
|
|
throw new PdfTooLargeError(path, stat.size, maxBytes);
|
|
}
|
|
|
|
const buffer = await readFile(path);
|
|
|
|
// Lazy import — avoids triggering any init-time side-effects of pdf-parse
|
|
// unless this function is actually called.
|
|
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
|
const pdfParseModule: any = await import('pdf-parse');
|
|
const pdfParse = pdfParseModule.default ?? pdfParseModule;
|
|
|
|
try {
|
|
const result = await pdfParse(buffer);
|
|
return {
|
|
text: String(result.text ?? ''),
|
|
numPages: Number(result.numpages ?? 0),
|
|
info: result.info ? { ...result.info } : undefined,
|
|
};
|
|
} catch (err: unknown) {
|
|
const message = err instanceof Error ? err.message : String(err);
|
|
if (/encrypt/i.test(message) || /password/i.test(message)) {
|
|
throw new PdfEncryptedError(path);
|
|
}
|
|
throw new PdfParseError(path, err);
|
|
}
|
|
}
|