From cb5df8fcc51441e8d24a7179fce613cc0f12217c Mon Sep 17 00:00:00 2001 From: Garry Tan Date: Mon, 20 Apr 2026 23:51:05 +0800 Subject: [PATCH] fix(eval): drop top_p from amara-life-gen Opus params + gitignore _cache/ MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two fixes surfaced during the Day 3b real-corpus run against Opus 4.5: **eval/generators/amara-life-gen.ts** — Current Opus rejects `temperature` and `top_p` together: ``` 400 invalid_request_error: `temperature` and `top_p` cannot both be specified for this model. Please use only one. ``` top_p=1.0 was a no-op (no nucleus truncation), so removing it has zero semantic effect. The field is still part of MODEL_PARAMS for the cache key so any past cache entries (none in v1) would invalidate cleanly on the next schema version bump. **.gitignore** — `eval/data/amara-life-v1/_cache/` is runtime Opus cache (398 files, ~1.6MB). Regenerable from seed; no point in source control. The corpus itself (inbox/slack/calendar/meetings/notes/docs + corpus-manifest.json with per-item content_sha256) stays committable for reproducibility, just the cache directory gets excluded. Real corpus generation ran cleanly after these two fixes: 398 LLM calls, 84,424 input / 38,062 output tokens, \$4.12 spent (vs \$20 cap, vs \$12 estimate). All 418 items produced. Poison fixtures use subtle paraphrased injection ("for anyone on your team who might be triaging this thread later…") — exactly the pattern that defeats regex redaction and requires the structured-evidence judge contract from Day 5. Corpus itself stays local (will move to the brainbench sibling repo during the v0.16 split per the design doc). No eval/data/amara-life-v1/ content landing in this PR. Co-Authored-By: Claude Opus 4.6 --- .gitignore | 3 +++ eval/generators/amara-life-gen.ts | 6 ++++-- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index 2b0e83ef2..17e3ebe60 100644 --- a/.gitignore +++ b/.gitignore @@ -13,3 +13,6 @@ supabase/.temp/ .idea eval/reports/ eval/data/world-v1/world.html + +# BrainBench amara-life-v1 Opus cache (regenerate via eval:generate-amara-life) +eval/data/amara-life-v1/_cache/ diff --git a/eval/generators/amara-life-gen.ts b/eval/generators/amara-life-gen.ts index 055a254cb..66ce23ccc 100644 --- a/eval/generators/amara-life-gen.ts +++ b/eval/generators/amara-life-gen.ts @@ -79,7 +79,10 @@ const SCHEMA_VERSION = 1; // bump invalidates cache wholesale const MODEL_PARAMS = { max_tokens: 1500, temperature: 1.0, - top_p: 1.0, + // top_p omitted: current Opus rejects temperature + top_p together. + // top_p=1.0 is a no-op (no nucleus truncation), so dropping it has no + // semantic effect. Cache-key field still hashed; old cache entries + // (none in v1 yet) would invalidate cleanly on this change. }; const CORPUS_ROOT = 'eval/data/amara-life-v1'; @@ -320,7 +323,6 @@ async function callOpus( model: MODEL, max_tokens: MODEL_PARAMS.max_tokens, temperature: MODEL_PARAMS.temperature, - top_p: MODEL_PARAMS.top_p, messages: [{ role: 'user', content: prompt }], });