mirror of
https://github.com/open-jarvis/OpenJarvis.git
synced 2026-07-30 19:02:16 +00:00
Jon Saad-FalconandClaude Opus 4.6
d89bbcce52
Add 9 capabilities to match IPW pipeline: Eval Pipeline: - AgenticRunner for multi-turn agent execution with per-turn trace decomposition - QueryTrace/TurnTrace data model for agentic workload telemetry - EventRecorder for thread-safe agent event collection - TerminalBenchTaskEnv for Docker-based task execution - Cost computation via engine/cloud.py PRICING table - Rich export: JSONL, HF Arrow, summary JSON, artifacts manifest - CLI: --agentic, --concurrency, --query-timeout flags Telemetry: - TelemetrySession with background-sampling ring buffer (Python fallback) - Phase metrics: prefill/decode energy split at TTFT boundary - ITL percentile tracking (p50/p90/p95/p99) - FLOPs estimation and MFU computation - EnergyMonitor.snapshot() method Rust Performance Layer: - Ring buffer with binary search O(log n) window queries - Trapezoidal energy integration - Phase metrics, ITL stats, FLOPs estimation in Rust - PyO3 bindings for all new telemetry modules (50 Rust tests) Savings Meter & Benchmarks: - Use-case benchmark datasets (coding, email, research, knowledge, morning brief) - Savings dashboard component with cost comparison visualization - Cloud cost calculator and comparison server routes - Use-case eval configs for multiple agent/engine combinations Tests: 80 new tests (3779 total pass, 37 skipped, 0 failures) Lint: ruff check src/ tests/ — all checks passed Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
39 lines
1.2 KiB
Python
39 lines
1.2 KiB
Python
"""Tests for eval pricing module."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import pytest
|
|
|
|
from openjarvis.evals.core.pricing import PRICING, compute_turn_cost, estimate_cost
|
|
|
|
|
|
class TestPricing:
|
|
def test_pricing_dict_nonempty(self):
|
|
assert isinstance(PRICING, dict)
|
|
assert len(PRICING) > 0
|
|
|
|
def test_compute_turn_cost_known_model(self):
|
|
# Pick a model that's in PRICING (cloud models)
|
|
if not PRICING:
|
|
pytest.skip("No models in PRICING dict")
|
|
model = next(iter(PRICING))
|
|
cost = compute_turn_cost(model, 1000, 500)
|
|
assert isinstance(cost, (int, float))
|
|
assert cost >= 0
|
|
|
|
def test_compute_turn_cost_unknown_model(self):
|
|
cost = compute_turn_cost("totally-unknown-local-model", 1000, 500)
|
|
assert cost == 0.0
|
|
|
|
def test_compute_turn_cost_zero_tokens(self):
|
|
if not PRICING:
|
|
pytest.skip("No models in PRICING dict")
|
|
model = next(iter(PRICING))
|
|
cost = compute_turn_cost(model, 0, 0)
|
|
assert cost == 0.0
|
|
|
|
def test_estimate_cost_alias(self):
|
|
# estimate_cost is the same as engine/cloud.py estimate_cost
|
|
cost = estimate_cost("unknown-model", 100, 50)
|
|
assert cost == 0.0
|