mirror of
https://github.com/garrytan/gbrain.git
synced 2026-07-31 04:07:52 +00:00
b91350d778
* fix(autopilot,eval): nightly quality probe enable path works end-to-end + wire conversation-parser probe Takeover of #2629 and #2630 (rebased onto master; dropped the test/engine-find-trajectory.test.ts hunk both PRs carried — master already ships the equivalent gateway-dims fix). #2629 — nightly quality probe enable path: - autopilot + doctor read the probe flag dual-plane (DB config row from 'gbrain config set' wins, ~/.gbrain/config.json fallback) via new resolveProbeEnabled/resolveProbeMaxUsd helpers - resolveRepoRoot prefers the gbrain package root where the committed fixture lives, not the brain repoPath - rate_limited skips no longer write an audit row every autopilot cycle - eval-longmemeval strips 'provider:' recipe ids before raw Anthropic SDK calls and emits the gold answer for downstream judges - cross-modal batch folds the gold answer into the judge task; probe passes QA-shaped dimensions instead of the agent-response rubric - DEFAULT_SLOTS slot A moves to openai:gpt-5.2 (gpt-4o left the recipe); new consistency test pins every default slot to its recipe #2630 — conversation-parser nightly probe wire-up: - autopilot step 4.6 invokes runConversationParserNightlyProbe (dual-plane flag + D10 tokenmax mode-gate, package-root fixtures, 24h gate, audit trail via new src/core/audit-parser-probe.ts) - doctor's conversation_parser_probe_health replaces the hardcoded 'Skipped' stub with a real pure-function check over the audit trail Co-authored-by: p3ob7o <p3ob7o@users.noreply.github.com> Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> * fix(pricing): add openai:gpt-5.2 canonical entry for the new default slot A DEFAULT_SLOTS slot A moved to openai:gpt-5.2, which had no CANONICAL_PRICING entry — estimateCost silently dropped slot A from the --max-usd pre-flight and est_cost_usd audit rows (~1/3 under-count on the default panel). Rates from the OpenAI recipe chat touchpoint (verified 2026-04-20). Also refresh the --slot-a-model help text default and pin a pricing-presence assertion in the DEFAULT_SLOTS consistency test. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> --------- Co-authored-by: Sinabina <sinabina@Sinabinas-MacBook-Pro-4.local> Co-authored-by: p3ob7o <p3ob7o@users.noreply.github.com> Co-authored-by: Claude Fable 5 <noreply@anthropic.com> Co-authored-by: Garry Tan <garrytan@gmail.com>
52 lines
2.1 KiB
TypeScript
52 lines
2.1 KiB
TypeScript
/**
|
|
* Tests for computeConversationParserProbeHealthCheck — the pure function
|
|
* behind doctor's conversation_parser_probe_health check, which replaced
|
|
* the v0.41.13.0 hardcoded "Skipped" stub when the autopilot wiring
|
|
* landed. Mirrors the branch coverage style of the quality-probe check.
|
|
*/
|
|
import { describe, expect, test } from 'bun:test';
|
|
|
|
import { computeConversationParserProbeHealthCheck } from '../src/commands/doctor.ts';
|
|
|
|
const ev = (outcome: string, reason?: string, ts = new Date().toISOString()) => ({
|
|
outcome,
|
|
ts,
|
|
...(reason !== undefined ? { reason } : {}),
|
|
});
|
|
|
|
describe('computeConversationParserProbeHealthCheck', () => {
|
|
test('disabled + no events → ok with paste-ready enable hint', () => {
|
|
const check = computeConversationParserProbeHealthCheck(false, []);
|
|
expect(check.status).toBe('ok');
|
|
expect(check.message).toContain('gbrain config set autopilot.conversation_parser_probe.enabled true');
|
|
});
|
|
|
|
test('enabled + no events yet → ok, next run by autopilot', () => {
|
|
const check = computeConversationParserProbeHealthCheck(true, []);
|
|
expect(check.status).toBe('ok');
|
|
expect(check.message).toContain('no probe events');
|
|
});
|
|
|
|
test('disabled flag but events exist (tokenmax mode-gate ran it) → events win over the hint', () => {
|
|
const check = computeConversationParserProbeHealthCheck(false, [ev('pass')]);
|
|
expect(check.status).toBe('ok');
|
|
expect(check.message).toContain('all pass');
|
|
});
|
|
|
|
test('any non-pass outcome in the window → warn, latest surfaced with reason', () => {
|
|
const check = computeConversationParserProbeHealthCheck(true, [
|
|
ev('pass'),
|
|
ev('adversarial_false_positive', '1 adversarial fixture(s) parsed to non-empty'),
|
|
]);
|
|
expect(check.status).toBe('warn');
|
|
expect(check.message).toContain('adversarial_false_positive');
|
|
expect(check.message).toContain('parsed to non-empty');
|
|
});
|
|
|
|
test('all pass → ok with run count', () => {
|
|
const check = computeConversationParserProbeHealthCheck(true, [ev('pass'), ev('pass')]);
|
|
expect(check.status).toBe('ok');
|
|
expect(check.message).toContain('2 probe run(s)');
|
|
});
|
|
});
|