mirror of
https://github.com/open-jarvis/OpenJarvis.git
synced 2026-07-28 14:07:55 +00:00
383 lines
12 KiB
Python
383 lines
12 KiB
Python
"""Tests for SkillBenchmarkRunner (Plan 2B)."""
|
|
|
|
from __future__ import annotations
|
|
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
import pytest
|
|
|
|
|
|
class TestSkillBenchmarkConfigDefaults:
|
|
def test_defaults(self):
|
|
from openjarvis.evals.skill_benchmark import SkillBenchmarkConfig
|
|
|
|
cfg = SkillBenchmarkConfig()
|
|
assert cfg.benchmark == "pinchbench"
|
|
assert cfg.model == "qwen3.5:9b"
|
|
assert cfg.engine == "ollama"
|
|
assert cfg.agent == "native_react"
|
|
assert cfg.seeds == [42, 43, 44]
|
|
assert cfg.max_samples is None
|
|
assert "shell_exec" in cfg.tools
|
|
assert "web_search" in cfg.tools
|
|
|
|
def test_construct_with_overrides(self):
|
|
from openjarvis.evals.skill_benchmark import SkillBenchmarkConfig
|
|
|
|
cfg = SkillBenchmarkConfig(
|
|
benchmark="pinchbench",
|
|
model="other-model",
|
|
seeds=[1, 2],
|
|
max_samples=5,
|
|
)
|
|
assert cfg.model == "other-model"
|
|
assert cfg.seeds == [1, 2]
|
|
assert cfg.max_samples == 5
|
|
|
|
|
|
class TestConditionResult:
|
|
def test_create(self):
|
|
from openjarvis.evals.skill_benchmark import ConditionResult
|
|
|
|
r = ConditionResult(
|
|
condition="no_skills",
|
|
seeds=[42, 43, 44],
|
|
per_seed_pass_rate={42: 0.30, 43: 0.32, 44: 0.28},
|
|
mean_pass_rate=0.30,
|
|
stddev_pass_rate=0.02,
|
|
per_task_results={"task_001": [True, False, True]},
|
|
skill_invocation_counts={},
|
|
total_tokens=1000,
|
|
total_runtime_seconds=120.0,
|
|
)
|
|
assert r.condition == "no_skills"
|
|
assert r.mean_pass_rate == 0.30
|
|
assert r.skill_invocation_counts == {}
|
|
|
|
|
|
class TestConditionComparison:
|
|
def test_create(self):
|
|
from openjarvis.evals.skill_benchmark import (
|
|
ConditionComparison,
|
|
ConditionResult,
|
|
SkillBenchmarkConfig,
|
|
)
|
|
|
|
cfg = SkillBenchmarkConfig()
|
|
cmp = ConditionComparison(
|
|
config=cfg,
|
|
started_at="2026-04-08T00:00:00Z",
|
|
ended_at="2026-04-08T01:00:00Z",
|
|
results={
|
|
"no_skills": ConditionResult(
|
|
condition="no_skills",
|
|
seeds=[42],
|
|
per_seed_pass_rate={42: 0.30},
|
|
mean_pass_rate=0.30,
|
|
stddev_pass_rate=0.0,
|
|
per_task_results={},
|
|
skill_invocation_counts={},
|
|
total_tokens=0,
|
|
total_runtime_seconds=0.0,
|
|
),
|
|
},
|
|
deltas={},
|
|
)
|
|
assert "no_skills" in cmp.results
|
|
assert cmp.config.benchmark == "pinchbench"
|
|
|
|
|
|
class TestBuildBackendForCondition:
|
|
def _make_runner(self, tmp_path: Path):
|
|
from openjarvis.evals.skill_benchmark import (
|
|
SkillBenchmarkConfig,
|
|
SkillBenchmarkRunner,
|
|
)
|
|
|
|
cfg = SkillBenchmarkConfig(
|
|
engine="ollama",
|
|
model="qwen3.5:9b",
|
|
tools=[],
|
|
seeds=[42],
|
|
output_dir=tmp_path,
|
|
overlay_dir_dspy=tmp_path / "dspy",
|
|
overlay_dir_gepa=tmp_path / "gepa",
|
|
)
|
|
return SkillBenchmarkRunner(cfg)
|
|
|
|
def test_unknown_condition_raises(self, tmp_path: Path):
|
|
runner = self._make_runner(tmp_path)
|
|
with pytest.raises(ValueError, match="condition"):
|
|
runner._backend_kwargs_for_condition("not_a_condition")
|
|
|
|
def test_no_skills_kwargs(self, tmp_path: Path):
|
|
runner = self._make_runner(tmp_path)
|
|
kw = runner._backend_kwargs_for_condition("no_skills")
|
|
assert kw["skills_enabled"] is False
|
|
assert kw["overlay_dir"] is None
|
|
|
|
def test_skills_on_kwargs(self, tmp_path: Path):
|
|
runner = self._make_runner(tmp_path)
|
|
kw = runner._backend_kwargs_for_condition("skills_on")
|
|
assert kw["skills_enabled"] is True
|
|
# skills_on uses an empty/missing overlay dir so no overlays load
|
|
assert kw["overlay_dir"] is not None
|
|
assert "skills_on_empty_overlays" in str(kw["overlay_dir"])
|
|
|
|
def test_skills_optimized_dspy_kwargs(self, tmp_path: Path):
|
|
runner = self._make_runner(tmp_path)
|
|
kw = runner._backend_kwargs_for_condition("skills_optimized_dspy")
|
|
assert kw["skills_enabled"] is True
|
|
assert kw["overlay_dir"] == tmp_path / "dspy"
|
|
|
|
def test_skills_optimized_gepa_kwargs(self, tmp_path: Path):
|
|
runner = self._make_runner(tmp_path)
|
|
kw = runner._backend_kwargs_for_condition("skills_optimized_gepa")
|
|
assert kw["skills_enabled"] is True
|
|
assert kw["overlay_dir"] == tmp_path / "gepa"
|
|
|
|
|
|
class TestRunCondition:
|
|
def _make_runner(self, tmp_path: Path):
|
|
from openjarvis.evals.skill_benchmark import (
|
|
SkillBenchmarkConfig,
|
|
SkillBenchmarkRunner,
|
|
)
|
|
|
|
cfg = SkillBenchmarkConfig(
|
|
seeds=[42, 43],
|
|
max_samples=2,
|
|
output_dir=tmp_path,
|
|
)
|
|
return SkillBenchmarkRunner(cfg)
|
|
|
|
def test_run_condition_aggregates_per_seed(
|
|
self, tmp_path: Path, monkeypatch: Any
|
|
) -> None:
|
|
"""run_condition runs the eval once per seed and aggregates the
|
|
results into a ConditionResult."""
|
|
from openjarvis.evals.skill_benchmark import (
|
|
SkillBenchmarkRunner,
|
|
)
|
|
|
|
runner = self._make_runner(tmp_path)
|
|
|
|
# Stub _run_single_seed to return synthetic per-seed data
|
|
seed_results = {
|
|
42: {
|
|
"pass_rate": 0.5,
|
|
"per_task": {"task_001": True, "task_002": False},
|
|
"skill_invocations": {"research-skill": 1},
|
|
"total_tokens": 100,
|
|
"total_runtime_seconds": 10.0,
|
|
},
|
|
43: {
|
|
"pass_rate": 1.0,
|
|
"per_task": {"task_001": True, "task_002": True},
|
|
"skill_invocations": {"research-skill": 2},
|
|
"total_tokens": 200,
|
|
"total_runtime_seconds": 20.0,
|
|
},
|
|
}
|
|
|
|
def fake_run_single_seed(self_unused, condition, seed):
|
|
return seed_results[seed]
|
|
|
|
monkeypatch.setattr(
|
|
SkillBenchmarkRunner,
|
|
"_run_single_seed",
|
|
fake_run_single_seed,
|
|
)
|
|
|
|
result = runner.run_condition("no_skills")
|
|
|
|
assert result.condition == "no_skills"
|
|
assert result.seeds == [42, 43]
|
|
assert result.per_seed_pass_rate == {42: 0.5, 43: 1.0}
|
|
assert result.mean_pass_rate == 0.75
|
|
assert result.stddev_pass_rate > 0.0
|
|
# Per-task aggregation: each task gets a list of [seed1_pass, seed2_pass]
|
|
assert result.per_task_results["task_001"] == [True, True]
|
|
assert result.per_task_results["task_002"] == [False, True]
|
|
# Skill invocation counts summed across seeds
|
|
assert result.skill_invocation_counts["research-skill"] == 3
|
|
assert result.total_tokens == 300
|
|
assert result.total_runtime_seconds == 30.0
|
|
|
|
def test_run_condition_single_seed_zero_stddev(
|
|
self, tmp_path: Path, monkeypatch: Any
|
|
) -> None:
|
|
from openjarvis.evals.skill_benchmark import (
|
|
SkillBenchmarkConfig,
|
|
SkillBenchmarkRunner,
|
|
)
|
|
|
|
cfg = SkillBenchmarkConfig(
|
|
seeds=[42],
|
|
max_samples=1,
|
|
output_dir=tmp_path,
|
|
)
|
|
runner = SkillBenchmarkRunner(cfg)
|
|
|
|
def fake_run_single_seed(self_unused, condition, seed):
|
|
return {
|
|
"pass_rate": 0.42,
|
|
"per_task": {"task_001": False},
|
|
"skill_invocations": {},
|
|
"total_tokens": 50,
|
|
"total_runtime_seconds": 5.0,
|
|
}
|
|
|
|
monkeypatch.setattr(
|
|
SkillBenchmarkRunner,
|
|
"_run_single_seed",
|
|
fake_run_single_seed,
|
|
)
|
|
|
|
result = runner.run_condition("no_skills")
|
|
assert result.mean_pass_rate == 0.42
|
|
# Single seed → stddev is 0
|
|
assert result.stddev_pass_rate == 0.0
|
|
|
|
|
|
class TestRunAllConditions:
|
|
def test_run_all_conditions_invokes_all_four(
|
|
self, tmp_path: Path, monkeypatch: Any
|
|
) -> None:
|
|
from openjarvis.evals.skill_benchmark import (
|
|
ConditionResult,
|
|
SkillBenchmarkConfig,
|
|
SkillBenchmarkRunner,
|
|
)
|
|
|
|
cfg = SkillBenchmarkConfig(
|
|
seeds=[42],
|
|
max_samples=1,
|
|
output_dir=tmp_path,
|
|
)
|
|
runner = SkillBenchmarkRunner(cfg)
|
|
|
|
invoked: list = []
|
|
|
|
def fake_run_condition(self_unused, condition):
|
|
invoked.append(condition)
|
|
return ConditionResult(
|
|
condition=condition,
|
|
seeds=[42],
|
|
per_seed_pass_rate={42: 0.5},
|
|
mean_pass_rate=0.5,
|
|
stddev_pass_rate=0.0,
|
|
per_task_results={},
|
|
skill_invocation_counts={},
|
|
total_tokens=10,
|
|
total_runtime_seconds=1.0,
|
|
)
|
|
|
|
monkeypatch.setattr(
|
|
SkillBenchmarkRunner,
|
|
"run_condition",
|
|
fake_run_condition,
|
|
)
|
|
|
|
comparison = runner.run_all_conditions()
|
|
assert set(invoked) == {
|
|
"no_skills",
|
|
"skills_on",
|
|
"skills_optimized_dspy",
|
|
"skills_optimized_gepa",
|
|
}
|
|
assert len(comparison.results) == 4
|
|
|
|
def test_run_all_conditions_computes_deltas(
|
|
self, tmp_path: Path, monkeypatch: Any
|
|
) -> None:
|
|
from openjarvis.evals.skill_benchmark import (
|
|
ConditionResult,
|
|
SkillBenchmarkConfig,
|
|
SkillBenchmarkRunner,
|
|
)
|
|
|
|
cfg = SkillBenchmarkConfig(
|
|
seeds=[42],
|
|
max_samples=1,
|
|
output_dir=tmp_path,
|
|
)
|
|
runner = SkillBenchmarkRunner(cfg)
|
|
|
|
rates = {
|
|
"no_skills": 0.30,
|
|
"skills_on": 0.35,
|
|
"skills_optimized_dspy": 0.40,
|
|
"skills_optimized_gepa": 0.38,
|
|
}
|
|
|
|
def fake_run_condition(self_unused, condition):
|
|
return ConditionResult(
|
|
condition=condition,
|
|
seeds=[42],
|
|
per_seed_pass_rate={42: rates[condition]},
|
|
mean_pass_rate=rates[condition],
|
|
stddev_pass_rate=0.0,
|
|
per_task_results={},
|
|
skill_invocation_counts={},
|
|
total_tokens=0,
|
|
total_runtime_seconds=0.0,
|
|
)
|
|
|
|
monkeypatch.setattr(
|
|
SkillBenchmarkRunner,
|
|
"run_condition",
|
|
fake_run_condition,
|
|
)
|
|
|
|
comparison = runner.run_all_conditions()
|
|
# Deltas use the same names as the conditions
|
|
assert comparison.deltas["skills_on - no_skills"] == pytest.approx(0.05)
|
|
assert comparison.deltas["skills_optimized_dspy - skills_on"] == pytest.approx(
|
|
0.05
|
|
)
|
|
assert comparison.deltas["skills_optimized_gepa - skills_on"] == pytest.approx(
|
|
0.03
|
|
)
|
|
|
|
|
|
class TestWriteReport:
|
|
def test_write_report_creates_dated_markdown(self, tmp_path: Path) -> None:
|
|
from openjarvis.evals.skill_benchmark import (
|
|
ConditionComparison,
|
|
ConditionResult,
|
|
SkillBenchmarkConfig,
|
|
SkillBenchmarkRunner,
|
|
)
|
|
|
|
cfg = SkillBenchmarkConfig(output_dir=tmp_path)
|
|
runner = SkillBenchmarkRunner(cfg)
|
|
|
|
result = ConditionResult(
|
|
condition="no_skills",
|
|
seeds=[42, 43],
|
|
per_seed_pass_rate={42: 0.30, 43: 0.32},
|
|
mean_pass_rate=0.31,
|
|
stddev_pass_rate=0.014,
|
|
per_task_results={"task_001": [True, False]},
|
|
skill_invocation_counts={},
|
|
total_tokens=100,
|
|
total_runtime_seconds=12.5,
|
|
)
|
|
cmp = ConditionComparison(
|
|
config=cfg,
|
|
started_at="2026-04-08T00:00:00Z",
|
|
ended_at="2026-04-08T00:01:00Z",
|
|
results={"no_skills": result},
|
|
deltas={},
|
|
)
|
|
|
|
path = runner.write_report(cmp)
|
|
assert path.exists()
|
|
assert path.suffix == ".md"
|
|
content = path.read_text()
|
|
assert "no_skills" in content
|
|
assert "0.31" in content or "31.0" in content
|
|
assert "task_001" in content
|