mirror of
https://github.com/open-jarvis/OpenJarvis.git
synced 2026-07-31 03:12:16 +00:00
Jon Saad-FalconandClaude Opus 4.6
d89bbcce52
Add 9 capabilities to match IPW pipeline: Eval Pipeline: - AgenticRunner for multi-turn agent execution with per-turn trace decomposition - QueryTrace/TurnTrace data model for agentic workload telemetry - EventRecorder for thread-safe agent event collection - TerminalBenchTaskEnv for Docker-based task execution - Cost computation via engine/cloud.py PRICING table - Rich export: JSONL, HF Arrow, summary JSON, artifacts manifest - CLI: --agentic, --concurrency, --query-timeout flags Telemetry: - TelemetrySession with background-sampling ring buffer (Python fallback) - Phase metrics: prefill/decode energy split at TTFT boundary - ITL percentile tracking (p50/p90/p95/p99) - FLOPs estimation and MFU computation - EnergyMonitor.snapshot() method Rust Performance Layer: - Ring buffer with binary search O(log n) window queries - Trapezoidal energy integration - Phase metrics, ITL stats, FLOPs estimation in Rust - PyO3 bindings for all new telemetry modules (50 Rust tests) Savings Meter & Benchmarks: - Use-case benchmark datasets (coding, email, research, knowledge, morning brief) - Savings dashboard component with cost comparison visualization - Cloud cost calculator and comparison server routes - Use-case eval configs for multiple agent/engine combinations Tests: 80 new tests (3779 total pass, 37 skipped, 0 failures) Lint: ruff check src/ tests/ — all checks passed Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
49 lines
1.6 KiB
Python
49 lines
1.6 KiB
Python
"""Tests for TerminalBenchTaskEnv (mocked terminal_bench dependency)."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import pytest
|
|
|
|
from openjarvis.evals.execution.terminalbench_env import TerminalBenchTaskEnv
|
|
|
|
# terminal_bench is an optional dep — skip all tests if unavailable
|
|
terminal_bench = pytest.importorskip(
|
|
"terminal_bench", reason="terminal_bench not installed"
|
|
)
|
|
|
|
|
|
class TestTerminalBenchTaskEnv:
|
|
def test_init(self):
|
|
metadata = {"task_id": "test-1"}
|
|
env = TerminalBenchTaskEnv(metadata)
|
|
assert env._metadata is metadata
|
|
assert env._terminal is None
|
|
|
|
def test_enter_without_task_raises(self):
|
|
metadata = {"task_id": "test-1"}
|
|
env = TerminalBenchTaskEnv(metadata)
|
|
with pytest.raises(ValueError, match="Task metadata missing"):
|
|
env.__enter__()
|
|
|
|
def test_exit_cleans_metadata(self):
|
|
metadata = {
|
|
"task_id": "test-1",
|
|
"terminal": "fake_terminal",
|
|
"session": "fake_session",
|
|
"container": "fake_container",
|
|
}
|
|
env = TerminalBenchTaskEnv(metadata)
|
|
env.__exit__(None, None, None)
|
|
assert "terminal" not in metadata
|
|
assert "session" not in metadata
|
|
assert "container" not in metadata
|
|
|
|
def test_run_tests_without_terminal(self):
|
|
metadata = {"task": "mock_task", "task_paths": "mock_paths"}
|
|
env = TerminalBenchTaskEnv(metadata)
|
|
env._terminal = None
|
|
is_resolved, results = env.run_tests()
|
|
assert is_resolved is False
|
|
assert results["error"] == "terminal_not_running"
|
|
assert metadata["is_resolved"] is False
|