mirror of
https://github.com/garrytan/gbrain.git
synced 2026-07-31 04:07:52 +00:00
feat(eval): Day 8 — Cat 8 skill compliance + Cat 9 end-to-end workflows
**eval/runner/cat8-skill-compliance.ts** — Deterministic, judge-free Cat 8
scoring. Replays inbound signals through the agent adapter (Day 5) and
extracts four iron-law metrics directly from the tool-bridge state:
- brain_first_compliance: agent called search/get_page BEFORE producing
its final answer. Non-compliance = hallucinating from general knowledge.
- back_link_compliance: every dry_run_put_page intent has at least one
markdown [Name](slug) back-link in its compiled_truth.
- citation_format: timeline entries use canonical `- **YYYY-MM-DD** |
Source — Summary`; long final answers cite at least one slug.
- tier_escalation: simple probes use light tooling (≥1 brain call);
complex probes require ≥2 brain calls or a dry_run write when
expects_dry_run_write is set.
No judge call required — everything is computable from
`tool_bridge_state.made_dry_run_writes` + `count_by_tool` + final_answer
regex. Fast, deterministic, reproducible.
Bounded concurrency (p-limit style) worker pool at default 4 to keep
Sonnet rate limits comfortable across 100-probe batches.
**eval/runner/cat9-workflows.ts** — Rubric-graded Cat 9. 5 canonical
workflows (meeting_ingestion, email_to_brain, daily_task_prep, briefing,
sync) × ~10 scenarios each. Each scenario runs through the agent adapter,
then judge.ts scores the answer against a per-scenario rubric.
`buildEvidence(scenario, agentResult, pagesBySlug)` composes the
JudgeEvidence contract: resolves ground_truth_slugs to full
GroundTruthPage[] from a slug-map, pulls tool_call_summary directly from
tool_bridge_state (no raw tool_result content — Section-3 defense),
attaches rubric from the scenario.
Per-workflow rollup: each workflow gets its own pass_rate so the verdict
can fail one workflow without failing the whole Cat. Overall verdict
requires every populated workflow's pass_rate ≥ threshold (default 0.80)
when enableThreshold=true.
Both Cats default to verdict=baseline_only in v1 per codex fix #9: real
thresholds return after 10-probe Haiku-vs-hand-score calibration (κ > 0.7)
runs against the Day 3b amara-life-v1 corpus.
**Tests (23):** Cat 8 per-metric scorer unit tests covering every branch
(brain_first ordering, back-link compliance on mixed writes, long vs
short answer citation requirement, tier escalation for simple/complex/
writey probes, finalAnswerCiteCount dedups across syntaxes). Cat 9
buildEvidence contract shape — evidence_refs flow from agent, missing
slugs skip gracefully, no raw_transcript/tool_result leakage to judge.
Cat 9 runCat9 integration with stubbed agent + mixed-verdict judge
produces fractional pass rates correctly.
Total eval suite now: 273 pass, 0 fail.
Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
Garry Tan
co-authored by
Claude Opus 4.6
Claude Opus 4.6
parent
33e2e73940
commit
a4cdb41b07
@@ -0,0 +1,291 @@
|
||||
/**
|
||||
* Cat 8 — Skill Behavior Compliance.
|
||||
*
|
||||
* Replays inbound signals through the agent adapter (Sonnet + gbrain tools)
|
||||
* and measures four structural iron-laws:
|
||||
*
|
||||
* - **brain_first_compliance** — did the agent call search/get_page
|
||||
* BEFORE producing its final answer? Threshold (informational): >0.95.
|
||||
* - **back_link_compliance** — every `dry_run_put_page` intent has
|
||||
* at least one markdown back-link in its compiled_truth. Threshold: >0.90.
|
||||
* - **citation_format** — timeline entries in dry_run writes follow the
|
||||
* canonical `- **YYYY-MM-DD** | Source — Summary` pattern; final
|
||||
* answers cite slugs in markdown or backticks. Threshold: >0.95.
|
||||
* - **tier_escalation** — complex probes get tool calls (not just direct
|
||||
* answers from general knowledge); simple probes stay light. Threshold: >0.80.
|
||||
*
|
||||
* v1 verdict is `baseline_only` — the 10-probe calibration that would
|
||||
* set thresholds requires real Opus runs against the Day 3b amara-life-v1
|
||||
* corpus. Once κ > 0.7 calibration lands, enableThreshold can flip on.
|
||||
*
|
||||
* All metrics are computed from `tool_bridge_state` + `made_dry_run_writes`
|
||||
* + the final answer text. NO judge call needed for Cat 8 — everything is
|
||||
* deterministic from the transcript.
|
||||
*/
|
||||
|
||||
import type { Anthropic } from '@anthropic-ai/sdk';
|
||||
import {
|
||||
runAgentLoop,
|
||||
type AgentAdapterState,
|
||||
type AgentRunConfig,
|
||||
type AgentRunResult,
|
||||
} from './adapters/claude-sonnet-with-tools.ts';
|
||||
|
||||
// ─── Types ────────────────────────────────────────────────────────────
|
||||
|
||||
export type ProbeTier = 'simple' | 'complex';
|
||||
|
||||
export interface SkillComplianceProbe {
|
||||
/** Stable id, e.g. "sig-0001". */
|
||||
id: string;
|
||||
/** The inbound signal text — what a user/agent would paste to the brain. */
|
||||
text: string;
|
||||
/**
|
||||
* Tier hint for the tier_escalation metric.
|
||||
* - simple: direct fact lookup, expected to resolve in ≤2 tool calls
|
||||
* - complex: multi-hop, synthesis, or dry_run writes expected
|
||||
*/
|
||||
tier: ProbeTier;
|
||||
/**
|
||||
* If true, the probe expects the agent to perform at least one dry_run
|
||||
* write. Used by tier_escalation: a complex probe that never writes is a
|
||||
* failure.
|
||||
*/
|
||||
expects_dry_run_write?: boolean;
|
||||
}
|
||||
|
||||
export interface SkillCompliancePerProbe {
|
||||
probe_id: string;
|
||||
tier: ProbeTier;
|
||||
brain_first: AgentRunResult['brain_first_ordering'];
|
||||
brain_first_compliant: boolean;
|
||||
dry_run_writes: number;
|
||||
back_link_compliant: boolean;
|
||||
citation_format_compliant: boolean;
|
||||
tier_escalation_correct: boolean;
|
||||
stop_reason: AgentRunResult['stop_reason'];
|
||||
total_cost_usd: number;
|
||||
}
|
||||
|
||||
export interface Cat8Report {
|
||||
schema_version: 1;
|
||||
ran_at: string;
|
||||
total_probes: number;
|
||||
brain_first_compliance: number;
|
||||
back_link_compliance: number;
|
||||
citation_format: number;
|
||||
tier_escalation: number;
|
||||
per_probe: SkillCompliancePerProbe[];
|
||||
total_cost_usd: number;
|
||||
verdict: 'pass' | 'fail' | 'baseline_only';
|
||||
}
|
||||
|
||||
export interface Cat8Config extends Omit<AgentRunConfig, 'client'> {
|
||||
/** Agent Sonnet client. Default uses ANTHROPIC_API_KEY lazy singleton. */
|
||||
client?: Anthropic;
|
||||
/** Thresholds override. Default: design-doc METRICS.md numbers. */
|
||||
thresholds?: {
|
||||
brain_first_compliance?: number;
|
||||
back_link_compliance?: number;
|
||||
citation_format?: number;
|
||||
tier_escalation?: number;
|
||||
};
|
||||
/** When false (default in v1), verdict is always baseline_only. */
|
||||
enableThreshold?: boolean;
|
||||
/** Bounded concurrency across probes. Default 4. */
|
||||
concurrency?: number;
|
||||
}
|
||||
|
||||
export interface RunCat8Options extends Cat8Config {
|
||||
probes: SkillComplianceProbe[];
|
||||
state: AgentAdapterState;
|
||||
}
|
||||
|
||||
// ─── Compliance checks ───────────────────────────────────────────────
|
||||
|
||||
const CITATION_TIMELINE_RE = /^- \*\*\d{4}-\d{2}-\d{2}\*\*\s*\|\s*.+?\s*[—-]\s*.+$/m;
|
||||
const MD_LINK_OR_BACKTICK_SLUG_RE = /(\[[^\]]+\]\(([a-z][a-z0-9-]*\/[a-z0-9][a-z0-9-]*)\)|`([a-z][a-z0-9-]*\/[a-z0-9][a-z0-9-]*)`)/i;
|
||||
|
||||
function finalAnswerCiteCount(text: string): number {
|
||||
const md = /\[[^\]]+\]\(([a-z][a-z0-9-]*\/[a-z0-9][a-z0-9-]*)\)/g;
|
||||
const bt = /`([a-z][a-z0-9-]*\/[a-z0-9][a-z0-9-]*)`/g;
|
||||
const slugs = new Set<string>();
|
||||
let m: RegExpExecArray | null;
|
||||
while ((m = md.exec(text)) !== null) slugs.add(m[1]);
|
||||
while ((m = bt.exec(text)) !== null) slugs.add(m[1]);
|
||||
return slugs.size;
|
||||
}
|
||||
|
||||
function scoreBrainFirst(result: AgentRunResult): boolean {
|
||||
return result.brain_first_ordering === 'brain_before_answer';
|
||||
}
|
||||
|
||||
function scoreBackLinkCompliance(result: AgentRunResult): boolean {
|
||||
const writes = result.tool_bridge_state.made_dry_run_writes;
|
||||
if (writes.length === 0) return true; // vacuous
|
||||
// Every dry_run_put_page write must have back-links. dry_run_add_link
|
||||
// and dry_run_add_timeline_entry are link writes themselves and always
|
||||
// satisfy this metric.
|
||||
for (const w of writes) {
|
||||
if (w.tool_name === 'dry_run_put_page' && w.has_back_links === false) return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
function scoreCitationFormat(result: AgentRunResult): boolean {
|
||||
// 1. Timeline entries in any dry_run write must match the canonical format.
|
||||
for (const w of result.tool_bridge_state.made_dry_run_writes) {
|
||||
if (w.citation_format_ok === false) return false;
|
||||
}
|
||||
|
||||
// 2. If the final answer makes any factual claims (non-trivial length),
|
||||
// it must cite at least one slug. Keep this permissive — a terse
|
||||
// answer like "I don't know" shouldn't be penalized.
|
||||
const text = result.final_answer.trim();
|
||||
if (text.length >= 80) {
|
||||
if (!MD_LINK_OR_BACKTICK_SLUG_RE.test(text)) return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
function scoreTierEscalation(probe: SkillComplianceProbe, result: AgentRunResult): boolean {
|
||||
const brainCalls = Object.entries(result.tool_bridge_state.count_by_tool)
|
||||
.filter(([name]) => BRAIN_READ_TOOLS.has(name))
|
||||
.reduce((sum, [, n]) => sum + n, 0);
|
||||
const writes = result.tool_bridge_state.made_dry_run_writes.length;
|
||||
|
||||
if (probe.tier === 'simple') {
|
||||
// Simple probes should use LITTLE tooling. One brain read is enough.
|
||||
// We're lenient: any brain read at all passes. (A simple probe that
|
||||
// makes zero brain calls fails brain_first anyway.)
|
||||
return brainCalls >= 1;
|
||||
}
|
||||
|
||||
// Complex probes: must use brain + must satisfy expects_dry_run_write
|
||||
if (probe.expects_dry_run_write) {
|
||||
return brainCalls >= 1 && writes >= 1;
|
||||
}
|
||||
// Complex without explicit write expectation: ≥2 tool calls total
|
||||
// (multi-hop signal).
|
||||
return brainCalls >= 2;
|
||||
}
|
||||
|
||||
const BRAIN_READ_TOOLS = new Set([
|
||||
'search',
|
||||
'query',
|
||||
'get_page',
|
||||
'list_pages',
|
||||
'get_backlinks',
|
||||
'get_links',
|
||||
'get_timeline',
|
||||
'get_tags',
|
||||
'traverse_graph',
|
||||
'resolve_slugs',
|
||||
'get_chunks',
|
||||
'get_stats',
|
||||
]);
|
||||
|
||||
// ─── Runner ──────────────────────────────────────────────────────────
|
||||
|
||||
async function runConcurrently<T, R>(
|
||||
items: T[],
|
||||
concurrency: number,
|
||||
fn: (item: T) => Promise<R>,
|
||||
): Promise<R[]> {
|
||||
const results: R[] = new Array(items.length);
|
||||
let next = 0;
|
||||
async function worker() {
|
||||
while (true) {
|
||||
const idx = next++;
|
||||
if (idx >= items.length) return;
|
||||
results[idx] = await fn(items[idx]);
|
||||
}
|
||||
}
|
||||
const workerCount = Math.min(concurrency, items.length);
|
||||
const workers: Promise<void>[] = [];
|
||||
for (let i = 0; i < workerCount; i++) workers.push(worker());
|
||||
await Promise.all(workers);
|
||||
return results;
|
||||
}
|
||||
|
||||
export async function runCat8(opts: RunCat8Options): Promise<Cat8Report> {
|
||||
const concurrency = opts.concurrency ?? 4;
|
||||
const thresholds = {
|
||||
brain_first_compliance: 0.95,
|
||||
back_link_compliance: 0.9,
|
||||
citation_format: 0.95,
|
||||
tier_escalation: 0.8,
|
||||
...opts.thresholds,
|
||||
};
|
||||
|
||||
const perProbe = await runConcurrently(opts.probes, concurrency, async probe => {
|
||||
const result = await runAgentLoop(probe.id, probe.text, opts.state, {
|
||||
client: opts.client,
|
||||
model: opts.model,
|
||||
maxTokens: opts.maxTokens,
|
||||
turnCap: opts.turnCap,
|
||||
systemPrompt: opts.systemPrompt,
|
||||
maxRetries: opts.maxRetries,
|
||||
});
|
||||
const score: SkillCompliancePerProbe = {
|
||||
probe_id: probe.id,
|
||||
tier: probe.tier,
|
||||
brain_first: result.brain_first_ordering,
|
||||
brain_first_compliant: scoreBrainFirst(result),
|
||||
dry_run_writes: result.tool_bridge_state.made_dry_run_writes.length,
|
||||
back_link_compliant: scoreBackLinkCompliance(result),
|
||||
citation_format_compliant: scoreCitationFormat(result),
|
||||
tier_escalation_correct: scoreTierEscalation(probe, result),
|
||||
stop_reason: result.stop_reason,
|
||||
total_cost_usd: result.total_cost_usd,
|
||||
};
|
||||
void finalAnswerCiteCount; // exported below for test visibility
|
||||
return score;
|
||||
});
|
||||
|
||||
const total = perProbe.length;
|
||||
const brain = perProbe.filter(p => p.brain_first_compliant).length;
|
||||
const back = perProbe.filter(p => p.back_link_compliant).length;
|
||||
const cite = perProbe.filter(p => p.citation_format_compliant).length;
|
||||
const tier = perProbe.filter(p => p.tier_escalation_correct).length;
|
||||
const metrics = {
|
||||
brain_first_compliance: total === 0 ? 0 : brain / total,
|
||||
back_link_compliance: total === 0 ? 0 : back / total,
|
||||
citation_format: total === 0 ? 0 : cite / total,
|
||||
tier_escalation: total === 0 ? 0 : tier / total,
|
||||
};
|
||||
|
||||
let verdict: 'pass' | 'fail' | 'baseline_only';
|
||||
if (!opts.enableThreshold) {
|
||||
verdict = 'baseline_only';
|
||||
} else {
|
||||
const allPass =
|
||||
metrics.brain_first_compliance >= thresholds.brain_first_compliance &&
|
||||
metrics.back_link_compliance >= thresholds.back_link_compliance &&
|
||||
metrics.citation_format >= thresholds.citation_format &&
|
||||
metrics.tier_escalation >= thresholds.tier_escalation;
|
||||
verdict = allPass ? 'pass' : 'fail';
|
||||
}
|
||||
|
||||
return {
|
||||
schema_version: 1,
|
||||
ran_at: new Date().toISOString(),
|
||||
total_probes: total,
|
||||
brain_first_compliance: metrics.brain_first_compliance,
|
||||
back_link_compliance: metrics.back_link_compliance,
|
||||
citation_format: metrics.citation_format,
|
||||
tier_escalation: metrics.tier_escalation,
|
||||
per_probe: perProbe,
|
||||
total_cost_usd: perProbe.reduce((sum, p) => sum + p.total_cost_usd, 0),
|
||||
verdict,
|
||||
};
|
||||
}
|
||||
|
||||
// Exports for tests
|
||||
export {
|
||||
scoreBrainFirst,
|
||||
scoreBackLinkCompliance,
|
||||
scoreCitationFormat,
|
||||
scoreTierEscalation,
|
||||
finalAnswerCiteCount,
|
||||
};
|
||||
@@ -0,0 +1,249 @@
|
||||
/**
|
||||
* Cat 9 — End-to-End Workflows.
|
||||
*
|
||||
* Replays ~50 scripted scenarios across 5 workflows through the agent
|
||||
* adapter, then scores each answer via judge.ts's rubric. Threshold
|
||||
* (informational): >80% pass rate per workflow.
|
||||
*
|
||||
* Workflows (canonical per TODOS.md and design-doc):
|
||||
* - meeting_ingestion
|
||||
* - email_to_brain
|
||||
* - daily_task_prep
|
||||
* - briefing
|
||||
* - sync
|
||||
*
|
||||
* Each scenario carries its own rubric (3-5 criteria, weights 1-2). The
|
||||
* rubric lives in `eval/data/gold/personalization-rubric.json` alongside
|
||||
* ground-truth slugs. The runner resolves slugs to full
|
||||
* GroundTruthPage[] before handing evidence to the judge.
|
||||
*
|
||||
* v1 verdict is baseline_only. Thresholds flip on after the 10-probe
|
||||
* Haiku-vs-hand-score calibration (κ > 0.7) lands alongside Day 3b corpus.
|
||||
*/
|
||||
|
||||
import type { Anthropic } from '@anthropic-ai/sdk';
|
||||
import {
|
||||
runAgentLoop,
|
||||
type AgentAdapterState,
|
||||
type AgentRunConfig,
|
||||
type AgentRunResult,
|
||||
} from './adapters/claude-sonnet-with-tools.ts';
|
||||
import {
|
||||
scoreAnswer,
|
||||
type JudgeEvidence,
|
||||
type JudgeResult,
|
||||
type JudgeConfig,
|
||||
type RubricCriterion,
|
||||
type GroundTruthPage,
|
||||
} from './judge.ts';
|
||||
|
||||
// ─── Types ────────────────────────────────────────────────────────────
|
||||
|
||||
export type WorkflowId =
|
||||
| 'meeting_ingestion'
|
||||
| 'email_to_brain'
|
||||
| 'daily_task_prep'
|
||||
| 'briefing'
|
||||
| 'sync';
|
||||
|
||||
export const ALL_WORKFLOWS: readonly WorkflowId[] = [
|
||||
'meeting_ingestion',
|
||||
'email_to_brain',
|
||||
'daily_task_prep',
|
||||
'briefing',
|
||||
'sync',
|
||||
] as const;
|
||||
|
||||
export interface WorkflowScenario {
|
||||
id: string;
|
||||
workflow: WorkflowId;
|
||||
text: string;
|
||||
/** Slugs that resolve to GroundTruthPage objects for the judge. */
|
||||
ground_truth_slugs: string[];
|
||||
rubric: RubricCriterion[];
|
||||
}
|
||||
|
||||
export interface Cat9PerScenario {
|
||||
scenario_id: string;
|
||||
workflow: WorkflowId;
|
||||
judge_result: JudgeResult;
|
||||
pass: boolean;
|
||||
agent_stop_reason: AgentRunResult['stop_reason'];
|
||||
agent_cost_usd: number;
|
||||
}
|
||||
|
||||
export interface WorkflowRollup {
|
||||
workflow: WorkflowId;
|
||||
total: number;
|
||||
passed: number;
|
||||
pass_rate: number;
|
||||
}
|
||||
|
||||
export interface Cat9Report {
|
||||
schema_version: 1;
|
||||
ran_at: string;
|
||||
total_scenarios: number;
|
||||
overall_pass_rate: number;
|
||||
per_workflow: WorkflowRollup[];
|
||||
per_scenario: Cat9PerScenario[];
|
||||
total_cost_usd: number;
|
||||
verdict: 'pass' | 'fail' | 'baseline_only';
|
||||
}
|
||||
|
||||
export interface Cat9Config extends Omit<AgentRunConfig, 'client'> {
|
||||
agentClient?: Anthropic;
|
||||
judgeClient?: Anthropic;
|
||||
judge?: Omit<JudgeConfig, 'client'>;
|
||||
/** Pass-rate threshold per workflow. Default 0.80. */
|
||||
passRateThreshold?: number;
|
||||
/** When false (default in v1), verdict is baseline_only. */
|
||||
enableThreshold?: boolean;
|
||||
concurrency?: number;
|
||||
}
|
||||
|
||||
export interface RunCat9Options extends Cat9Config {
|
||||
scenarios: WorkflowScenario[];
|
||||
state: AgentAdapterState;
|
||||
pagesBySlug: Map<string, GroundTruthPage>;
|
||||
}
|
||||
|
||||
// ─── Runner ──────────────────────────────────────────────────────────
|
||||
|
||||
async function runConcurrently<T, R>(
|
||||
items: T[],
|
||||
concurrency: number,
|
||||
fn: (item: T) => Promise<R>,
|
||||
): Promise<R[]> {
|
||||
const results: R[] = new Array(items.length);
|
||||
let next = 0;
|
||||
async function worker() {
|
||||
while (true) {
|
||||
const idx = next++;
|
||||
if (idx >= items.length) return;
|
||||
results[idx] = await fn(items[idx]);
|
||||
}
|
||||
}
|
||||
const workerCount = Math.min(concurrency, items.length);
|
||||
const workers: Promise<void>[] = [];
|
||||
for (let i = 0; i < workerCount; i++) workers.push(worker());
|
||||
await Promise.all(workers);
|
||||
return results;
|
||||
}
|
||||
|
||||
/**
|
||||
* Build a JudgeEvidence object for a scenario given the agent's run result.
|
||||
* Exported for tests — they verify the contract assembly in isolation.
|
||||
*/
|
||||
export function buildEvidence(
|
||||
scenario: WorkflowScenario,
|
||||
runResult: AgentRunResult,
|
||||
pagesBySlug: Map<string, GroundTruthPage>,
|
||||
): JudgeEvidence {
|
||||
const ground_truth_pages: GroundTruthPage[] = [];
|
||||
for (const slug of scenario.ground_truth_slugs) {
|
||||
const page = pagesBySlug.get(slug);
|
||||
if (page) ground_truth_pages.push(page);
|
||||
}
|
||||
|
||||
return {
|
||||
schema_version: 1,
|
||||
probe: {
|
||||
id: scenario.id,
|
||||
query: scenario.text,
|
||||
category: 9,
|
||||
},
|
||||
final_answer_text: runResult.final_answer,
|
||||
evidence_refs: runResult.evidence_refs,
|
||||
tool_call_summary: {
|
||||
count_by_tool: runResult.tool_bridge_state.count_by_tool,
|
||||
saw_poison_items: runResult.tool_bridge_state.saw_poison_items,
|
||||
brain_first_ordering: runResult.brain_first_ordering,
|
||||
made_dry_run_writes: runResult.tool_bridge_state.made_dry_run_writes.map(w => ({
|
||||
slug: w.slug,
|
||||
has_back_links: w.has_back_links,
|
||||
citation_format_ok: w.citation_format_ok,
|
||||
tool_name: w.tool_name,
|
||||
})),
|
||||
},
|
||||
ground_truth_pages,
|
||||
rubric: scenario.rubric,
|
||||
};
|
||||
}
|
||||
|
||||
export async function runCat9(opts: RunCat9Options): Promise<Cat9Report> {
|
||||
const concurrency = opts.concurrency ?? 4;
|
||||
const passRateThreshold = opts.passRateThreshold ?? 0.8;
|
||||
|
||||
const perScenario = await runConcurrently(opts.scenarios, concurrency, async scenario => {
|
||||
// Step 1: run the agent loop.
|
||||
const agentResult = await runAgentLoop(scenario.id, scenario.text, opts.state, {
|
||||
client: opts.agentClient,
|
||||
model: opts.model,
|
||||
maxTokens: opts.maxTokens,
|
||||
turnCap: opts.turnCap,
|
||||
systemPrompt: opts.systemPrompt,
|
||||
maxRetries: opts.maxRetries,
|
||||
});
|
||||
|
||||
// Step 2: build evidence, score with judge.
|
||||
const evidence = buildEvidence(scenario, agentResult, opts.pagesBySlug);
|
||||
const judgeResult = await scoreAnswer(evidence, {
|
||||
...opts.judge,
|
||||
client: opts.judgeClient,
|
||||
});
|
||||
|
||||
const score: Cat9PerScenario = {
|
||||
scenario_id: scenario.id,
|
||||
workflow: scenario.workflow,
|
||||
judge_result: judgeResult,
|
||||
pass: judgeResult.verdict === 'pass',
|
||||
agent_stop_reason: agentResult.stop_reason,
|
||||
agent_cost_usd: agentResult.total_cost_usd,
|
||||
};
|
||||
return score;
|
||||
});
|
||||
|
||||
// Per-workflow rollup
|
||||
const byWorkflow = new Map<WorkflowId, Cat9PerScenario[]>();
|
||||
for (const w of ALL_WORKFLOWS) byWorkflow.set(w, []);
|
||||
for (const s of perScenario) byWorkflow.get(s.workflow)!.push(s);
|
||||
|
||||
const per_workflow: WorkflowRollup[] = [];
|
||||
for (const [workflow, scenarios] of byWorkflow) {
|
||||
const total = scenarios.length;
|
||||
const passed = scenarios.filter(s => s.pass).length;
|
||||
per_workflow.push({
|
||||
workflow,
|
||||
total,
|
||||
passed,
|
||||
pass_rate: total === 0 ? 0 : passed / total,
|
||||
});
|
||||
}
|
||||
|
||||
const totalPassed = perScenario.filter(s => s.pass).length;
|
||||
const overallPassRate = perScenario.length === 0 ? 0 : totalPassed / perScenario.length;
|
||||
|
||||
let verdict: 'pass' | 'fail' | 'baseline_only';
|
||||
if (!opts.enableThreshold) {
|
||||
verdict = 'baseline_only';
|
||||
} else {
|
||||
const allWorkflowsPass = per_workflow
|
||||
.filter(w => w.total > 0) // ignore empty workflows
|
||||
.every(w => w.pass_rate >= passRateThreshold);
|
||||
verdict = allWorkflowsPass ? 'pass' : 'fail';
|
||||
}
|
||||
|
||||
const totalCost =
|
||||
perScenario.reduce((sum, s) => sum + s.agent_cost_usd + s.judge_result.cost_usd, 0);
|
||||
|
||||
return {
|
||||
schema_version: 1,
|
||||
ran_at: new Date().toISOString(),
|
||||
total_scenarios: perScenario.length,
|
||||
overall_pass_rate: overallPassRate,
|
||||
per_workflow,
|
||||
per_scenario: perScenario,
|
||||
total_cost_usd: totalCost,
|
||||
verdict,
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,436 @@
|
||||
/**
|
||||
* Cat 8 + Cat 9 runner tests — Day 8 of BrainBench v1 Complete.
|
||||
*
|
||||
* Uses stubbed Sonnet (agent) + Haiku (judge) clients. No real LLM calls,
|
||||
* no PGLite engine initialization — tests the scoring and aggregation
|
||||
* layers over synthetic agent run results.
|
||||
*
|
||||
* Covers:
|
||||
* - Cat 8 per-metric scorers (brain_first, back_link, citation_format, tier_escalation)
|
||||
* - Cat 8 aggregate pass rates
|
||||
* - Cat 8 baseline_only verdict by default
|
||||
* - Cat 8 pass/fail verdict when enableThreshold=true
|
||||
* - Cat 9 buildEvidence contract shape (no raw tool_result text)
|
||||
* - Cat 9 runCat9 end-to-end with stubbed agent + judge
|
||||
* - Cat 9 per-workflow rollup
|
||||
*/
|
||||
|
||||
import { describe, test, expect } from 'bun:test';
|
||||
import Anthropic from '@anthropic-ai/sdk';
|
||||
import type { AgentRunResult } from '../../eval/runner/adapters/claude-sonnet-with-tools.ts';
|
||||
import type { ToolBridgeState } from '../../eval/runner/tool-bridge.ts';
|
||||
import {
|
||||
scoreBrainFirst,
|
||||
scoreBackLinkCompliance,
|
||||
scoreCitationFormat,
|
||||
scoreTierEscalation,
|
||||
finalAnswerCiteCount,
|
||||
type SkillComplianceProbe,
|
||||
} from '../../eval/runner/cat8-skill-compliance.ts';
|
||||
import {
|
||||
buildEvidence,
|
||||
type WorkflowScenario,
|
||||
} from '../../eval/runner/cat9-workflows.ts';
|
||||
import type { GroundTruthPage } from '../../eval/runner/judge.ts';
|
||||
|
||||
// ─── Mock AgentRunResult builder ──────────────────────────────────────
|
||||
|
||||
function mockRunResult(overrides: Partial<AgentRunResult> & {
|
||||
brainCalls?: number;
|
||||
writes?: Array<{
|
||||
tool_name: 'dry_run_put_page' | 'dry_run_add_link' | 'dry_run_add_timeline_entry';
|
||||
slug?: string;
|
||||
has_back_links?: boolean;
|
||||
citation_format_ok?: boolean;
|
||||
}>;
|
||||
poisonHits?: string[];
|
||||
finalAnswer?: string;
|
||||
evidenceRefs?: string[];
|
||||
}): AgentRunResult {
|
||||
const {
|
||||
brainCalls = 0,
|
||||
writes = [],
|
||||
poisonHits = [],
|
||||
finalAnswer = '',
|
||||
evidenceRefs = [],
|
||||
...rest
|
||||
} = overrides;
|
||||
|
||||
const count_by_tool: Record<string, number> = {};
|
||||
if (brainCalls > 0) count_by_tool.search = brainCalls;
|
||||
for (const w of writes) {
|
||||
count_by_tool[w.tool_name] = (count_by_tool[w.tool_name] ?? 0) + 1;
|
||||
}
|
||||
|
||||
const state: ToolBridgeState = {
|
||||
count_by_tool,
|
||||
call_order: [
|
||||
...Array(brainCalls).fill('search'),
|
||||
...writes.map(w => w.tool_name),
|
||||
],
|
||||
made_dry_run_writes: writes.map(w => ({
|
||||
tool_name: w.tool_name,
|
||||
input: {},
|
||||
ts: '2026-04-20T00:00:00Z',
|
||||
slug: w.slug,
|
||||
has_back_links: w.has_back_links,
|
||||
citation_format_ok: w.citation_format_ok,
|
||||
})),
|
||||
saw_poison_items: poisonHits,
|
||||
};
|
||||
|
||||
return {
|
||||
transcript: {
|
||||
schema_version: 1,
|
||||
probe_id: 'p1',
|
||||
adapter: { name: 'claude-sonnet-with-tools', stack_id: 'gbrain' },
|
||||
started_at: '2026-04-20T00:00:00Z',
|
||||
ended_at: '2026-04-20T00:00:01Z',
|
||||
turns: [],
|
||||
total_input_tokens: 100,
|
||||
total_output_tokens: 50,
|
||||
elapsed_ms: 1000,
|
||||
},
|
||||
final_answer: finalAnswer,
|
||||
evidence_refs: evidenceRefs,
|
||||
tool_bridge_state: state,
|
||||
brain_first_ordering:
|
||||
brainCalls > 0 ? 'brain_before_answer' : finalAnswer ? 'no_brain_calls' : 'no_brain_calls',
|
||||
stop_reason: 'end_turn',
|
||||
total_input_tokens: 100,
|
||||
total_output_tokens: 50,
|
||||
total_cost_usd: 0.01,
|
||||
...rest,
|
||||
};
|
||||
}
|
||||
|
||||
// ─── Cat 8 per-metric scorers ────────────────────────────────────────
|
||||
|
||||
describe('Cat 8 scoreBrainFirst', () => {
|
||||
test('compliant when brain_before_answer', () => {
|
||||
const r = mockRunResult({ brainCalls: 2 });
|
||||
r.brain_first_ordering = 'brain_before_answer';
|
||||
expect(scoreBrainFirst(r)).toBe(true);
|
||||
});
|
||||
|
||||
test('non-compliant when no_brain_calls', () => {
|
||||
const r = mockRunResult({});
|
||||
r.brain_first_ordering = 'no_brain_calls';
|
||||
expect(scoreBrainFirst(r)).toBe(false);
|
||||
});
|
||||
|
||||
test('non-compliant when answer_before_brain', () => {
|
||||
const r = mockRunResult({ brainCalls: 1 });
|
||||
r.brain_first_ordering = 'answer_before_brain';
|
||||
expect(scoreBrainFirst(r)).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe('Cat 8 scoreBackLinkCompliance', () => {
|
||||
test('vacuously true when no dry_run writes', () => {
|
||||
expect(scoreBackLinkCompliance(mockRunResult({}))).toBe(true);
|
||||
});
|
||||
|
||||
test('true when all put_page writes have back_links', () => {
|
||||
const r = mockRunResult({
|
||||
writes: [
|
||||
{ tool_name: 'dry_run_put_page', slug: 'people/x', has_back_links: true, citation_format_ok: true },
|
||||
],
|
||||
});
|
||||
expect(scoreBackLinkCompliance(r)).toBe(true);
|
||||
});
|
||||
|
||||
test('false when any put_page write has has_back_links=false', () => {
|
||||
const r = mockRunResult({
|
||||
writes: [
|
||||
{ tool_name: 'dry_run_put_page', slug: 'people/x', has_back_links: true, citation_format_ok: true },
|
||||
{ tool_name: 'dry_run_put_page', slug: 'people/y', has_back_links: false, citation_format_ok: true },
|
||||
],
|
||||
});
|
||||
expect(scoreBackLinkCompliance(r)).toBe(false);
|
||||
});
|
||||
|
||||
test('add_link writes are always back-link compliant (by definition)', () => {
|
||||
const r = mockRunResult({
|
||||
writes: [{ tool_name: 'dry_run_add_link', has_back_links: false, citation_format_ok: true }],
|
||||
});
|
||||
expect(scoreBackLinkCompliance(r)).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
describe('Cat 8 scoreCitationFormat', () => {
|
||||
test('short final answers skip the citation requirement', () => {
|
||||
const r = mockRunResult({ finalAnswer: 'I do not know.' });
|
||||
expect(scoreCitationFormat(r)).toBe(true);
|
||||
});
|
||||
|
||||
test('long final answer without any slug citation is non-compliant', () => {
|
||||
const text =
|
||||
'Amara Okafor is a Partner at Halfway Capital and has been working in venture for several years now. ' +
|
||||
'She focuses primarily on climate and AI infrastructure investments at the seed and Series A stages.';
|
||||
const r = mockRunResult({ finalAnswer: text });
|
||||
expect(scoreCitationFormat(r)).toBe(false);
|
||||
});
|
||||
|
||||
test('long final answer with markdown slug citation passes', () => {
|
||||
const text =
|
||||
'Amara Okafor is a Partner at Halfway Capital. ' +
|
||||
'She focuses on climate + AI infrastructure investments. See [Amara](people/amara-okafor) for background.';
|
||||
const r = mockRunResult({ finalAnswer: text });
|
||||
expect(scoreCitationFormat(r)).toBe(true);
|
||||
});
|
||||
|
||||
test('write with bad citation_format_ok=false flags non-compliant', () => {
|
||||
const r = mockRunResult({
|
||||
writes: [
|
||||
{ tool_name: 'dry_run_add_timeline_entry', citation_format_ok: false },
|
||||
],
|
||||
});
|
||||
expect(scoreCitationFormat(r)).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe('Cat 8 scoreTierEscalation', () => {
|
||||
const simpleProbe: SkillComplianceProbe = { id: 'p1', text: 'x', tier: 'simple' };
|
||||
const complexProbe: SkillComplianceProbe = { id: 'p2', text: 'x', tier: 'complex' };
|
||||
const writeyProbe: SkillComplianceProbe = {
|
||||
id: 'p3',
|
||||
text: 'x',
|
||||
tier: 'complex',
|
||||
expects_dry_run_write: true,
|
||||
};
|
||||
|
||||
test('simple probe passes with ≥1 brain call', () => {
|
||||
expect(scoreTierEscalation(simpleProbe, mockRunResult({ brainCalls: 1 }))).toBe(true);
|
||||
});
|
||||
|
||||
test('simple probe fails with 0 brain calls', () => {
|
||||
expect(scoreTierEscalation(simpleProbe, mockRunResult({}))).toBe(false);
|
||||
});
|
||||
|
||||
test('complex probe requires ≥2 brain calls when no write expected', () => {
|
||||
expect(scoreTierEscalation(complexProbe, mockRunResult({ brainCalls: 1 }))).toBe(false);
|
||||
expect(scoreTierEscalation(complexProbe, mockRunResult({ brainCalls: 2 }))).toBe(true);
|
||||
});
|
||||
|
||||
test('complex + expects_dry_run_write requires brain call + write', () => {
|
||||
expect(scoreTierEscalation(writeyProbe, mockRunResult({ brainCalls: 1 }))).toBe(false);
|
||||
expect(
|
||||
scoreTierEscalation(
|
||||
writeyProbe,
|
||||
mockRunResult({
|
||||
brainCalls: 1,
|
||||
writes: [{ tool_name: 'dry_run_put_page', has_back_links: true, citation_format_ok: true }],
|
||||
}),
|
||||
),
|
||||
).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
describe('Cat 8 finalAnswerCiteCount', () => {
|
||||
test('counts unique slugs across markdown + backtick syntax', () => {
|
||||
const text =
|
||||
'See [Amara](people/amara) and `people/amara` and [Halfway](companies/halfway).';
|
||||
expect(finalAnswerCiteCount(text)).toBe(2);
|
||||
});
|
||||
|
||||
test('returns 0 on text with no slug references', () => {
|
||||
expect(finalAnswerCiteCount('no slugs here at all.')).toBe(0);
|
||||
});
|
||||
});
|
||||
|
||||
// ─── Cat 9 buildEvidence ──────────────────────────────────────────────
|
||||
|
||||
describe('Cat 9 buildEvidence', () => {
|
||||
const SCENARIO: WorkflowScenario = {
|
||||
id: 's1',
|
||||
workflow: 'briefing',
|
||||
text: 'Give me a briefing',
|
||||
ground_truth_slugs: ['people/amara', 'companies/halfway'],
|
||||
rubric: [{ id: 'names_person', criterion: 'Names Amara', weight: 1 }],
|
||||
};
|
||||
|
||||
const PAGES = new Map<string, GroundTruthPage>([
|
||||
['people/amara', { slug: 'people/amara', title: 'Amara', content: 'Partner.' }],
|
||||
['companies/halfway', { slug: 'companies/halfway', title: 'Halfway', content: 'VC firm.' }],
|
||||
]);
|
||||
|
||||
test('resolves ground_truth_slugs to full pages', () => {
|
||||
const run = mockRunResult({ brainCalls: 1, finalAnswer: 'Amara is a Partner.' });
|
||||
run.brain_first_ordering = 'brain_before_answer';
|
||||
const evidence = buildEvidence(SCENARIO, run, PAGES);
|
||||
expect(evidence.ground_truth_pages.length).toBe(2);
|
||||
expect(evidence.ground_truth_pages[0].content).toBe('Partner.');
|
||||
});
|
||||
|
||||
test('skips slugs not in pagesBySlug (defensive)', () => {
|
||||
const scenarioWithGhost: WorkflowScenario = {
|
||||
...SCENARIO,
|
||||
ground_truth_slugs: ['people/amara', 'people/ghost'],
|
||||
};
|
||||
const evidence = buildEvidence(scenarioWithGhost, mockRunResult({}), PAGES);
|
||||
expect(evidence.ground_truth_pages.length).toBe(1);
|
||||
expect(evidence.ground_truth_pages[0].slug).toBe('people/amara');
|
||||
});
|
||||
|
||||
test('includes tool_call_summary without raw tool_result content', () => {
|
||||
const run = mockRunResult({
|
||||
brainCalls: 3,
|
||||
poisonHits: ['poison-001'],
|
||||
writes: [
|
||||
{ tool_name: 'dry_run_put_page', slug: 'people/jane', has_back_links: true, citation_format_ok: true },
|
||||
],
|
||||
});
|
||||
run.brain_first_ordering = 'brain_before_answer';
|
||||
const evidence = buildEvidence(SCENARIO, run, PAGES);
|
||||
expect(evidence.tool_call_summary.count_by_tool.search).toBe(3);
|
||||
expect(evidence.tool_call_summary.saw_poison_items).toEqual(['poison-001']);
|
||||
expect(evidence.tool_call_summary.brain_first_ordering).toBe('brain_before_answer');
|
||||
expect(evidence.tool_call_summary.made_dry_run_writes[0].slug).toBe('people/jane');
|
||||
// CRITICAL: the evidence contract must NOT carry a raw tool_result or
|
||||
// raw_content field. assertNoRawToolOutput from judge.ts is the strict
|
||||
// check; here we just spot-check.
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
expect('tool_result' in (evidence as any)).toBe(false);
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
expect('raw_transcript' in (evidence as any)).toBe(false);
|
||||
});
|
||||
|
||||
test('final_answer_text + evidence_refs flow from agent run', () => {
|
||||
const run = mockRunResult({
|
||||
finalAnswer: 'See [Amara](people/amara).',
|
||||
evidenceRefs: ['people/amara'],
|
||||
});
|
||||
const evidence = buildEvidence(SCENARIO, run, PAGES);
|
||||
expect(evidence.final_answer_text).toBe('See [Amara](people/amara).');
|
||||
expect(evidence.evidence_refs).toEqual(['people/amara']);
|
||||
});
|
||||
});
|
||||
|
||||
// ─── Cat 9 runCat9 end-to-end ─────────────────────────────────────────
|
||||
|
||||
describe('Cat 9 runCat9 integration', () => {
|
||||
// Stub clients for agent + judge.
|
||||
function makeAgentClient(): Anthropic {
|
||||
return {
|
||||
messages: {
|
||||
create: async () => ({
|
||||
content: [{ type: 'text', text: 'Amara Okafor is a Partner at [Halfway](companies/halfway).' }],
|
||||
usage: { input_tokens: 100, output_tokens: 40 },
|
||||
stop_reason: 'end_turn',
|
||||
}),
|
||||
},
|
||||
} as unknown as Anthropic;
|
||||
}
|
||||
|
||||
function makeJudgeClient(verdict: 'pass' | 'partial' | 'fail' = 'pass'): Anthropic {
|
||||
return {
|
||||
messages: {
|
||||
create: async () => ({
|
||||
content: [
|
||||
{
|
||||
type: 'tool_use',
|
||||
id: 'x',
|
||||
name: 'score_answer',
|
||||
input: {
|
||||
scores: [
|
||||
{ criterion_id: 'names_person', score: verdict === 'pass' ? 5 : 1, rationale: '.' },
|
||||
],
|
||||
verdict,
|
||||
overall_rationale: 'ok',
|
||||
},
|
||||
},
|
||||
],
|
||||
usage: { input_tokens: 500, output_tokens: 100 },
|
||||
}),
|
||||
},
|
||||
} as unknown as Anthropic;
|
||||
}
|
||||
|
||||
test('end-to-end agent+judge run produces per_workflow rollup', async () => {
|
||||
// We need an AgentAdapterState but the stub agent never hits the engine
|
||||
// since its response has stop_reason=end_turn and no tool_use blocks.
|
||||
// Provide a no-op state.
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
const state: any = { engine: {}, poisonFixtures: [] };
|
||||
|
||||
const scenarios: WorkflowScenario[] = [
|
||||
{
|
||||
id: 's-briefing-1',
|
||||
workflow: 'briefing',
|
||||
text: 'Give me a briefing',
|
||||
ground_truth_slugs: ['companies/halfway'],
|
||||
rubric: [{ id: 'names_person', criterion: 'Names Amara', weight: 1 }],
|
||||
},
|
||||
{
|
||||
id: 's-sync-1',
|
||||
workflow: 'sync',
|
||||
text: 'Sync my brain',
|
||||
ground_truth_slugs: ['companies/halfway'],
|
||||
rubric: [{ id: 'names_person', criterion: 'Names Amara', weight: 1 }],
|
||||
},
|
||||
];
|
||||
const pages = new Map<string, GroundTruthPage>([
|
||||
['companies/halfway', { slug: 'companies/halfway', title: 'Halfway', content: 'VC firm.' }],
|
||||
]);
|
||||
|
||||
const { runCat9 } = await import('../../eval/runner/cat9-workflows.ts');
|
||||
const report = await runCat9({
|
||||
scenarios,
|
||||
state,
|
||||
pagesBySlug: pages,
|
||||
agentClient: makeAgentClient(),
|
||||
judgeClient: makeJudgeClient('pass'),
|
||||
concurrency: 1,
|
||||
});
|
||||
|
||||
expect(report.total_scenarios).toBe(2);
|
||||
expect(report.overall_pass_rate).toBe(1);
|
||||
expect(report.per_workflow.find(w => w.workflow === 'briefing')!.pass_rate).toBe(1);
|
||||
expect(report.per_workflow.find(w => w.workflow === 'sync')!.pass_rate).toBe(1);
|
||||
expect(report.verdict).toBe('baseline_only');
|
||||
});
|
||||
|
||||
test('mixed verdicts produce a fractional pass rate', async () => {
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
const state: any = { engine: {}, poisonFixtures: [] };
|
||||
const scenarios: WorkflowScenario[] = [
|
||||
{ id: 's1', workflow: 'briefing', text: 't', ground_truth_slugs: [], rubric: [{ id: 'c', criterion: 'x', weight: 1 }] },
|
||||
{ id: 's2', workflow: 'briefing', text: 't', ground_truth_slugs: [], rubric: [{ id: 'c', criterion: 'x', weight: 1 }] },
|
||||
];
|
||||
// Alternating pass/fail verdicts — make judge client return different responses per call
|
||||
let call = 0;
|
||||
const mixedJudge = {
|
||||
messages: {
|
||||
create: async () => {
|
||||
const v = call++ === 0 ? 'pass' : 'fail';
|
||||
return {
|
||||
content: [
|
||||
{
|
||||
type: 'tool_use',
|
||||
id: 'x',
|
||||
name: 'score_answer',
|
||||
input: {
|
||||
scores: [{ criterion_id: 'c', score: v === 'pass' ? 5 : 0, rationale: '.' }],
|
||||
verdict: v,
|
||||
overall_rationale: '.',
|
||||
},
|
||||
},
|
||||
],
|
||||
usage: { input_tokens: 100, output_tokens: 20 },
|
||||
};
|
||||
},
|
||||
},
|
||||
} as unknown as Anthropic;
|
||||
const { runCat9 } = await import('../../eval/runner/cat9-workflows.ts');
|
||||
const report = await runCat9({
|
||||
scenarios,
|
||||
state,
|
||||
pagesBySlug: new Map(),
|
||||
agentClient: makeAgentClient(),
|
||||
judgeClient: mixedJudge,
|
||||
concurrency: 1, // sequential so call order is deterministic
|
||||
});
|
||||
expect(report.overall_pass_rate).toBe(0.5);
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user