diff --git a/tests/evals/test_deepplanning.py b/tests/evals/test_deepplanning.py new file mode 100644 index 00000000..d4db7903 --- /dev/null +++ b/tests/evals/test_deepplanning.py @@ -0,0 +1,87 @@ +"""Tests for DeepPlanning benchmark.""" + +from unittest.mock import MagicMock + +from openjarvis.evals.core.types import EvalRecord +from openjarvis.evals.datasets.deepplanning import DeepPlanningDataset +from openjarvis.evals.scorers.deepplanning_scorer import DeepPlanningScorer + + +def _mock_backend() -> MagicMock: + backend = MagicMock() + backend.generate.return_value = "CORRECT" + return backend + + +class TestDeepPlanningDataset: + def test_instantiation(self) -> None: + ds = DeepPlanningDataset() + assert ds.dataset_id == "deepplanning" + assert ds.dataset_name == "DeepPlanning" + + def test_has_required_methods(self) -> None: + ds = DeepPlanningDataset() + assert hasattr(ds, "load") + assert hasattr(ds, "iter_records") + assert hasattr(ds, "size") + + +class TestDeepPlanningScorer: + def test_instantiation(self) -> None: + s = DeepPlanningScorer(_mock_backend(), "test-model") + assert s.scorer_id == "deepplanning" + + def test_correct_plan(self) -> None: + s = DeepPlanningScorer(_mock_backend(), "test-model") + record = EvalRecord( + record_id="dp-1", + problem="## Task (travel)\nPlan a trip to Paris", + reference="Day 1: Flight to Paris...", + category="agentic", + subject="travel", + metadata={"task_type": "travel"}, + ) + is_correct, meta = s.score(record, "Day 1: Fly to Paris, visit Eiffel Tower") + assert is_correct is True + assert meta["match_type"] == "llm_judge" + + def test_incorrect_plan(self) -> None: + backend = MagicMock() + backend.generate.return_value = "INCORRECT - Missing budget constraint" + s = DeepPlanningScorer(backend, "test-model") + record = EvalRecord( + record_id="dp-2", + problem="## Task (shopping)\nBuild a cart", + reference="Cart: item A, item B, total $50", + category="agentic", + subject="shopping", + metadata={"task_type": "shopping"}, + ) + is_correct, meta = s.score(record, "Cart: item C, total $100") + assert is_correct is False + + def test_empty_response(self) -> None: + s = DeepPlanningScorer(_mock_backend(), "test-model") + record = EvalRecord( + record_id="dp-3", problem="task", + reference="answer", category="agentic", + ) + is_correct, meta = s.score(record, "") + assert is_correct is False + assert meta["reason"] == "empty_response" + + +class TestDeepPlanningCLI: + def test_in_benchmarks(self) -> None: + from openjarvis.evals.cli import BENCHMARKS + assert "deepplanning" in BENCHMARKS + + def test_build_dataset(self) -> None: + from openjarvis.evals.cli import _build_dataset + ds = _build_dataset("deepplanning") + assert ds.dataset_id == "deepplanning" + + def test_build_scorer(self) -> None: + from openjarvis.evals.cli import _build_scorer + s = _build_scorer("deepplanning", _mock_backend(), "test-model") + assert s.scorer_id == "deepplanning" diff --git a/tests/evals/test_paperarena.py b/tests/evals/test_paperarena.py new file mode 100644 index 00000000..022aeb9c --- /dev/null +++ b/tests/evals/test_paperarena.py @@ -0,0 +1,115 @@ +"""Tests for PaperArena benchmark.""" + +from unittest.mock import MagicMock + +from openjarvis.evals.core.types import EvalRecord +from openjarvis.evals.datasets.paperarena import PaperArenaDataset +from openjarvis.evals.scorers.paperarena_judge import PaperArenaScorer + + +def _mock_backend() -> MagicMock: + backend = MagicMock() + backend.generate.return_value = "CORRECT" + return backend + + +class TestPaperArenaDataset: + def test_instantiation(self) -> None: + ds = PaperArenaDataset() + assert ds.dataset_id == "paperarena" + assert ds.dataset_name == "PaperArena" + + def test_has_required_methods(self) -> None: + ds = PaperArenaDataset() + assert hasattr(ds, "load") + assert hasattr(ds, "iter_records") + assert hasattr(ds, "size") + + +class TestPaperArenaScorer: + def test_instantiation(self) -> None: + s = PaperArenaScorer(_mock_backend(), "test-model") + assert s.scorer_id == "paperarena" + + def test_mc_correct(self) -> None: + s = PaperArenaScorer(_mock_backend(), "test-model") + record = EvalRecord( + record_id="pa-1", + problem="## Question\nWhat is X?\nOptions:\n A) foo\n B) bar", + reference="A", + category="agentic", + subject="easy_mc", + metadata={"question_type": "MC"}, + ) + is_correct, meta = s.score(record, "The answer is A") + assert is_correct is True + assert meta["match_type"] == "exact_letter" + + def test_mc_wrong(self) -> None: + s = PaperArenaScorer(_mock_backend(), "test-model") + record = EvalRecord( + record_id="pa-2", + problem="Question", + reference="B", + category="agentic", + subject="medium_mc", + metadata={"question_type": "MC"}, + ) + is_correct, meta = s.score(record, "The answer is C") + assert is_correct is False + assert meta["candidate_letter"] == "C" + + def test_open_answer_judge(self) -> None: + s = PaperArenaScorer(_mock_backend(), "test-model") + record = EvalRecord( + record_id="pa-3", + problem="## Question\nExplain X", + reference="X is a method for...", + category="agentic", + subject="hard_oa", + metadata={"question_type": "OA"}, + ) + is_correct, meta = s.score(record, "X is a technique used to...") + assert is_correct is True + assert meta["match_type"] == "llm_judge" + + def test_closed_answer_judge(self) -> None: + backend = MagicMock() + backend.generate.return_value = "INCORRECT" + s = PaperArenaScorer(backend, "test-model") + record = EvalRecord( + record_id="pa-4", + problem="## Question\nWhat is the value?", + reference="42.5", + category="agentic", + subject="medium_ca", + metadata={"question_type": "CA"}, + ) + is_correct, meta = s.score(record, "The value is 100") + assert is_correct is False + + def test_empty_response(self) -> None: + s = PaperArenaScorer(_mock_backend(), "test-model") + record = EvalRecord( + record_id="pa-5", problem="q", + reference="a", category="agentic", + ) + is_correct, meta = s.score(record, "") + assert is_correct is False + assert meta["reason"] == "empty_response" + + +class TestPaperArenaCLI: + def test_in_benchmarks(self) -> None: + from openjarvis.evals.cli import BENCHMARKS + assert "paperarena" in BENCHMARKS + + def test_build_dataset(self) -> None: + from openjarvis.evals.cli import _build_dataset + ds = _build_dataset("paperarena") + assert ds.dataset_id == "paperarena" + + def test_build_scorer(self) -> None: + from openjarvis.evals.cli import _build_scorer + s = _build_scorer("paperarena", _mock_backend(), "test-model") + assert s.scorer_id == "paperarena" diff --git a/tests/test_feedback_collector.py b/tests/learning/test_feedback_collector.py similarity index 100% rename from tests/test_feedback_collector.py rename to tests/learning/test_feedback_collector.py diff --git a/tests/test_llm_optimizer.py b/tests/learning/test_llm_optimizer.py similarity index 100% rename from tests/test_llm_optimizer.py rename to tests/learning/test_llm_optimizer.py diff --git a/tests/test_multi_bench_runner.py b/tests/learning/test_multi_bench_runner.py similarity index 100% rename from tests/test_multi_bench_runner.py rename to tests/learning/test_multi_bench_runner.py diff --git a/tests/test_optimize_cmd.py b/tests/learning/test_optimize_cmd.py similarity index 100% rename from tests/test_optimize_cmd.py rename to tests/learning/test_optimize_cmd.py diff --git a/tests/test_optimize_store.py b/tests/learning/test_optimize_store.py similarity index 100% rename from tests/test_optimize_store.py rename to tests/learning/test_optimize_store.py diff --git a/tests/test_optimize_types.py b/tests/learning/test_optimize_types.py similarity index 100% rename from tests/test_optimize_types.py rename to tests/learning/test_optimize_types.py diff --git a/tests/test_optimizer_engine.py b/tests/learning/test_optimizer_engine.py similarity index 100% rename from tests/test_optimizer_engine.py rename to tests/learning/test_optimizer_engine.py diff --git a/tests/test_pareto.py b/tests/learning/test_pareto.py similarity index 100% rename from tests/test_pareto.py rename to tests/learning/test_pareto.py diff --git a/tests/test_personal_synthesizer.py b/tests/learning/test_personal_synthesizer.py similarity index 100% rename from tests/test_personal_synthesizer.py rename to tests/learning/test_personal_synthesizer.py diff --git a/tests/test_search_space.py b/tests/learning/test_search_space.py similarity index 100% rename from tests/test_search_space.py rename to tests/learning/test_search_space.py diff --git a/tests/test_trace_judge.py b/tests/learning/test_trace_judge.py similarity index 100% rename from tests/test_trace_judge.py rename to tests/learning/test_trace_judge.py diff --git a/tests/test_trial_runner.py b/tests/learning/test_trial_runner.py similarity index 100% rename from tests/test_trial_runner.py rename to tests/learning/test_trial_runner.py