From a9518e157425518c208ca6354b3a4112e350faab Mon Sep 17 00:00:00 2001 From: Jon Saad-Falcon Date: Mon, 2 Mar 2026 21:06:08 +0000 Subject: [PATCH] refactor: move evals/ into src/openjarvis/evals/ as proper subpackage Move the standalone evals framework from the project root into the openjarvis package. Rewrite all ~50+ import statements from 'from evals.' to 'from openjarvis.evals.' across the package, CLI, and tests. Remove the evals-specific pyproject.toml (no longer a standalone package). Update ruff per-file-ignores paths and fix line-length violations introduced by the longer import paths. Co-Authored-By: Claude Opus 4.6 --- evals/__main__.py | 6 -- evals/pyproject.toml | 24 ------ pyproject.toml | 4 +- src/openjarvis/cli/eval_cmd.py | 8 +- {evals => src/openjarvis/evals}/__init__.py | 0 src/openjarvis/evals/__main__.py | 6 ++ .../openjarvis/evals}/backends/__init__.py | 0 .../evals}/backends/jarvis_agent.py | 2 +- .../evals}/backends/jarvis_direct.py | 2 +- {evals => src/openjarvis/evals}/cli.py | 78 +++++++++-------- .../openjarvis/evals}/configs/full-suite.toml | 0 .../glm-4.7-flash-openhands-remaining.toml | 0 .../configs/glm-4.7-flash-openhands.toml | 0 .../configs/glm-4.7-fp8-openhands-gaia.toml | 0 .../openjarvis/evals}/configs/minimal.toml | 0 .../openjarvis/evals}/configs/single-run.toml | 0 .../openjarvis/evals}/core/__init__.py | 0 .../openjarvis/evals}/core/backend.py | 0 .../openjarvis/evals}/core/config.py | 2 +- .../openjarvis/evals}/core/dataset.py | 2 +- .../openjarvis/evals}/core/display.py | 2 +- .../openjarvis/evals}/core/runner.py | 8 +- .../openjarvis/evals}/core/scorer.py | 4 +- {evals => src/openjarvis/evals}/core/types.py | 0 .../openjarvis/evals}/datasets/__init__.py | 0 .../openjarvis/evals}/datasets/frames.py | 4 +- .../openjarvis/evals}/datasets/gaia.py | 4 +- .../openjarvis/evals}/datasets/gpqa.py | 4 +- .../openjarvis/evals}/datasets/hle.py | 4 +- .../openjarvis/evals}/datasets/ipw_mixed.py | 4 +- .../openjarvis/evals}/datasets/math500.py | 4 +- .../openjarvis/evals}/datasets/mmlu_pro.py | 4 +- .../evals}/datasets/natural_reasoning.py | 4 +- .../openjarvis/evals}/datasets/simpleqa.py | 4 +- .../openjarvis/evals}/datasets/supergpqa.py | 4 +- .../openjarvis/evals}/datasets/swebench.py | 4 +- .../evals}/datasets/swefficiency.py | 4 +- .../evals}/datasets/terminalbench.py | 4 +- .../evals}/datasets/terminalbench_native.py | 4 +- .../openjarvis/evals}/datasets/wildchat.py | 4 +- .../openjarvis/evals}/scorers/__init__.py | 0 .../openjarvis/evals}/scorers/frames_judge.py | 4 +- .../openjarvis/evals}/scorers/gaia_exact.py | 4 +- .../openjarvis/evals}/scorers/gpqa_mcq.py | 4 +- .../openjarvis/evals}/scorers/hle_judge.py | 6 +- .../openjarvis/evals}/scorers/ipw_mixed.py | 4 +- .../openjarvis/evals}/scorers/mmlu_pro_mcq.py | 4 +- .../evals}/scorers/reasoning_judge.py | 4 +- .../evals}/scorers/simpleqa_judge.py | 4 +- .../evals}/scorers/supergpqa_mcq.py | 4 +- .../evals}/scorers/swebench_structural.py | 4 +- .../evals}/scorers/swefficiency_structural.py | 4 +- .../evals}/scorers/terminalbench_judge.py | 4 +- .../terminalbench_native_structural.py | 4 +- .../evals}/scorers/wildchat_judge.py | 4 +- .../openjarvis/evals}/tests/__init__.py | 0 .../openjarvis/evals}/tests/conftest.py | 8 +- .../openjarvis/evals}/tests/test_backends.py | 12 +-- .../openjarvis/evals}/tests/test_cli_flags.py | 2 +- .../openjarvis/evals}/tests/test_config.py | 12 +-- .../openjarvis/evals}/tests/test_display.py | 4 +- .../openjarvis/evals}/tests/test_frames.py | 6 +- .../openjarvis/evals}/tests/test_gaia.py | 6 +- .../openjarvis/evals}/tests/test_runner.py | 10 ++- .../openjarvis/evals}/tests/test_supergpqa.py | 16 ++-- .../openjarvis/evals}/tests/test_types.py | 2 +- .../openjarvis/evals}/tests/test_wildchat.py | 6 +- tests/evals/test_benchmark_datasets.py | 86 +++++++++---------- tests/evals/test_display.py | 6 +- 69 files changed, 212 insertions(+), 230 deletions(-) delete mode 100644 evals/__main__.py delete mode 100644 evals/pyproject.toml rename {evals => src/openjarvis/evals}/__init__.py (100%) create mode 100644 src/openjarvis/evals/__main__.py rename {evals => src/openjarvis/evals}/backends/__init__.py (100%) rename {evals => src/openjarvis/evals}/backends/jarvis_agent.py (98%) rename {evals => src/openjarvis/evals}/backends/jarvis_direct.py (97%) rename {evals => src/openjarvis/evals}/cli.py (88%) rename {evals => src/openjarvis/evals}/configs/full-suite.toml (100%) rename {evals => src/openjarvis/evals}/configs/glm-4.7-flash-openhands-remaining.toml (100%) rename {evals => src/openjarvis/evals}/configs/glm-4.7-flash-openhands.toml (100%) rename {evals => src/openjarvis/evals}/configs/glm-4.7-fp8-openhands-gaia.toml (100%) rename {evals => src/openjarvis/evals}/configs/minimal.toml (100%) rename {evals => src/openjarvis/evals}/configs/single-run.toml (100%) rename {evals => src/openjarvis/evals}/core/__init__.py (100%) rename {evals => src/openjarvis/evals}/core/backend.py (100%) rename {evals => src/openjarvis/evals}/core/config.py (99%) rename {evals => src/openjarvis/evals}/core/dataset.py (94%) rename {evals => src/openjarvis/evals}/core/display.py (99%) rename {evals => src/openjarvis/evals}/core/runner.py (98%) rename {evals => src/openjarvis/evals}/core/scorer.py (92%) rename {evals => src/openjarvis/evals}/core/types.py (100%) rename {evals => src/openjarvis/evals}/datasets/__init__.py (100%) rename {evals => src/openjarvis/evals}/datasets/frames.py (97%) rename {evals => src/openjarvis/evals}/datasets/gaia.py (97%) rename {evals => src/openjarvis/evals}/datasets/gpqa.py (97%) rename {evals => src/openjarvis/evals}/datasets/hle.py (97%) rename {evals => src/openjarvis/evals}/datasets/ipw_mixed.py (98%) rename {evals => src/openjarvis/evals}/datasets/math500.py (96%) rename {evals => src/openjarvis/evals}/datasets/mmlu_pro.py (96%) rename {evals => src/openjarvis/evals}/datasets/natural_reasoning.py (96%) rename {evals => src/openjarvis/evals}/datasets/simpleqa.py (97%) rename {evals => src/openjarvis/evals}/datasets/supergpqa.py (96%) rename {evals => src/openjarvis/evals}/datasets/swebench.py (97%) rename {evals => src/openjarvis/evals}/datasets/swefficiency.py (97%) rename {evals => src/openjarvis/evals}/datasets/terminalbench.py (96%) rename {evals => src/openjarvis/evals}/datasets/terminalbench_native.py (97%) rename {evals => src/openjarvis/evals}/datasets/wildchat.py (96%) rename {evals => src/openjarvis/evals}/scorers/__init__.py (100%) rename {evals => src/openjarvis/evals}/scorers/frames_judge.py (96%) rename {evals => src/openjarvis/evals}/scorers/gaia_exact.py (98%) rename {evals => src/openjarvis/evals}/scorers/gpqa_mcq.py (96%) rename {evals => src/openjarvis/evals}/scorers/hle_judge.py (94%) rename {evals => src/openjarvis/evals}/scorers/ipw_mixed.py (97%) rename {evals => src/openjarvis/evals}/scorers/mmlu_pro_mcq.py (96%) rename {evals => src/openjarvis/evals}/scorers/reasoning_judge.py (97%) rename {evals => src/openjarvis/evals}/scorers/simpleqa_judge.py (97%) rename {evals => src/openjarvis/evals}/scorers/supergpqa_mcq.py (96%) rename {evals => src/openjarvis/evals}/scorers/swebench_structural.py (95%) rename {evals => src/openjarvis/evals}/scorers/swefficiency_structural.py (95%) rename {evals => src/openjarvis/evals}/scorers/terminalbench_judge.py (97%) rename {evals => src/openjarvis/evals}/scorers/terminalbench_native_structural.py (95%) rename {evals => src/openjarvis/evals}/scorers/wildchat_judge.py (97%) rename {evals => src/openjarvis/evals}/tests/__init__.py (100%) rename {evals => src/openjarvis/evals}/tests/conftest.py (94%) rename {evals => src/openjarvis/evals}/tests/test_backends.py (91%) rename {evals => src/openjarvis/evals}/tests/test_cli_flags.py (92%) rename {evals => src/openjarvis/evals}/tests/test_config.py (98%) rename {evals => src/openjarvis/evals}/tests/test_display.py (97%) rename {evals => src/openjarvis/evals}/tests/test_frames.py (94%) rename {evals => src/openjarvis/evals}/tests/test_gaia.py (96%) rename {evals => src/openjarvis/evals}/tests/test_runner.py (98%) rename {evals => src/openjarvis/evals}/tests/test_supergpqa.py (84%) rename {evals => src/openjarvis/evals}/tests/test_types.py (99%) rename {evals => src/openjarvis/evals}/tests/test_wildchat.py (96%) diff --git a/evals/__main__.py b/evals/__main__.py deleted file mode 100644 index 9b783560..00000000 --- a/evals/__main__.py +++ /dev/null @@ -1,6 +0,0 @@ -"""Allow running as ``python -m evals``.""" - -from evals.cli import main - -if __name__ == "__main__": - main() diff --git a/evals/pyproject.toml b/evals/pyproject.toml deleted file mode 100644 index 21cfe023..00000000 --- a/evals/pyproject.toml +++ /dev/null @@ -1,24 +0,0 @@ -[project] -name = "openjarvis-evals" -version = "0.1.0" -description = "Evaluation framework for OpenJarvis" -requires-python = ">=3.10" -dependencies = [ - "openjarvis>=1.0.0", - "click>=8", - "datasets>=2.14", - "huggingface-hub>=0.20", - "tqdm>=4.65", - "rich>=13", - "tomli>=2.0; python_version < '3.11'", -] - -[project.optional-dependencies] -dev = ["pytest>=8", "pytest-cov>=5"] - -[project.scripts] -openjarvis-eval = "evals.cli:main" - -[build-system] -requires = ["hatchling"] -build-backend = "hatchling.build" diff --git a/pyproject.toml b/pyproject.toml index 4b3b4efb..c0459b32 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -132,5 +132,5 @@ src = ["src", "tests"] select = ["E", "F", "I", "W"] [tool.ruff.lint.per-file-ignores] -"evals/datasets/*.py" = ["E501"] -"evals/scorers/*.py" = ["E501"] +"src/openjarvis/evals/datasets/*.py" = ["E501"] +"src/openjarvis/evals/scorers/*.py" = ["E501"] diff --git a/src/openjarvis/cli/eval_cmd.py b/src/openjarvis/cli/eval_cmd.py index eee684d9..ce5d116b 100644 --- a/src/openjarvis/cli/eval_cmd.py +++ b/src/openjarvis/cli/eval_cmd.py @@ -135,7 +135,7 @@ def eval_run( # Config-driven mode: load TOML suite, expand, run all if config_path is not None: try: - from evals.core.config import expand_suite, load_eval_config + from openjarvis.evals.core.config import expand_suite, load_eval_config except ImportError: console.print( "[red]Eval framework not available. " @@ -159,7 +159,7 @@ def eval_run( ) try: - from evals.cli import _run_single + from openjarvis.evals.cli import _run_single except ImportError: console.print( "[red]Eval CLI module not available.[/red]" @@ -195,7 +195,7 @@ def eval_run( ) try: - from evals.core.types import RunConfig + from openjarvis.evals.core.types import RunConfig except ImportError: console.print( "[red]Eval framework not available. " @@ -219,7 +219,7 @@ def eval_run( ) try: - from evals.cli import _run_single + from openjarvis.evals.cli import _run_single console.print( f"[cyan]Benchmark:[/cyan] {benchmark}\n" diff --git a/evals/__init__.py b/src/openjarvis/evals/__init__.py similarity index 100% rename from evals/__init__.py rename to src/openjarvis/evals/__init__.py diff --git a/src/openjarvis/evals/__main__.py b/src/openjarvis/evals/__main__.py new file mode 100644 index 00000000..c2a7457d --- /dev/null +++ b/src/openjarvis/evals/__main__.py @@ -0,0 +1,6 @@ +"""Allow running as ``python -m openjarvis.evals``.""" + +from openjarvis.evals.cli import main + +if __name__ == "__main__": + main() diff --git a/evals/backends/__init__.py b/src/openjarvis/evals/backends/__init__.py similarity index 100% rename from evals/backends/__init__.py rename to src/openjarvis/evals/backends/__init__.py diff --git a/evals/backends/jarvis_agent.py b/src/openjarvis/evals/backends/jarvis_agent.py similarity index 98% rename from evals/backends/jarvis_agent.py rename to src/openjarvis/evals/backends/jarvis_agent.py index 06cd9a60..6f9fca1c 100644 --- a/evals/backends/jarvis_agent.py +++ b/src/openjarvis/evals/backends/jarvis_agent.py @@ -5,7 +5,7 @@ from __future__ import annotations import time from typing import Any, Dict, List, Optional -from evals.core.backend import InferenceBackend +from openjarvis.evals.core.backend import InferenceBackend class JarvisAgentBackend(InferenceBackend): diff --git a/evals/backends/jarvis_direct.py b/src/openjarvis/evals/backends/jarvis_direct.py similarity index 97% rename from evals/backends/jarvis_direct.py rename to src/openjarvis/evals/backends/jarvis_direct.py index d215f0fa..107e4153 100644 --- a/evals/backends/jarvis_direct.py +++ b/src/openjarvis/evals/backends/jarvis_direct.py @@ -5,7 +5,7 @@ from __future__ import annotations import time from typing import Any, Dict, Optional -from evals.core.backend import InferenceBackend +from openjarvis.evals.core.backend import InferenceBackend class JarvisDirectBackend(InferenceBackend): diff --git a/evals/cli.py b/src/openjarvis/evals/cli.py similarity index 88% rename from evals/cli.py rename to src/openjarvis/evals/cli.py index aabba3d7..1318bc04 100644 --- a/evals/cli.py +++ b/src/openjarvis/evals/cli.py @@ -17,7 +17,7 @@ from rich.progress import ( TimeRemainingColumn, ) -from evals.core.display import ( +from openjarvis.evals.core.display import ( print_banner, print_completion, print_full_results, @@ -72,7 +72,7 @@ def _build_backend(backend_name: str, engine_key: Optional[str], telemetry: bool = False, gpu_metrics: bool = False): """Construct the appropriate backend.""" if backend_name == "jarvis-agent": - from evals.backends.jarvis_agent import JarvisAgentBackend + from openjarvis.evals.backends.jarvis_agent import JarvisAgentBackend return JarvisAgentBackend( engine_key=engine_key, agent_name=agent_name, @@ -81,7 +81,7 @@ def _build_backend(backend_name: str, engine_key: Optional[str], gpu_metrics=gpu_metrics, ) else: - from evals.backends.jarvis_direct import JarvisDirectBackend + from openjarvis.evals.backends.jarvis_direct import JarvisDirectBackend return JarvisDirectBackend( engine_key=engine_key, telemetry=telemetry, @@ -92,49 +92,51 @@ def _build_backend(backend_name: str, engine_key: Optional[str], def _build_dataset(benchmark: str): """Construct the dataset provider for a benchmark.""" if benchmark == "supergpqa": - from evals.datasets.supergpqa import SuperGPQADataset + from openjarvis.evals.datasets.supergpqa import SuperGPQADataset return SuperGPQADataset() elif benchmark == "gpqa": - from evals.datasets.gpqa import GPQADataset + from openjarvis.evals.datasets.gpqa import GPQADataset return GPQADataset() elif benchmark == "mmlu-pro": - from evals.datasets.mmlu_pro import MMLUProDataset + from openjarvis.evals.datasets.mmlu_pro import MMLUProDataset return MMLUProDataset() elif benchmark == "math500": - from evals.datasets.math500 import MATH500Dataset + from openjarvis.evals.datasets.math500 import MATH500Dataset return MATH500Dataset() elif benchmark == "natural-reasoning": - from evals.datasets.natural_reasoning import NaturalReasoningDataset + from openjarvis.evals.datasets.natural_reasoning import NaturalReasoningDataset return NaturalReasoningDataset() elif benchmark == "hle": - from evals.datasets.hle import HLEDataset + from openjarvis.evals.datasets.hle import HLEDataset return HLEDataset() elif benchmark == "simpleqa": - from evals.datasets.simpleqa import SimpleQADataset + from openjarvis.evals.datasets.simpleqa import SimpleQADataset return SimpleQADataset() elif benchmark == "wildchat": - from evals.datasets.wildchat import WildChatDataset + from openjarvis.evals.datasets.wildchat import WildChatDataset return WildChatDataset() elif benchmark == "ipw": - from evals.datasets.ipw_mixed import IPWDataset + from openjarvis.evals.datasets.ipw_mixed import IPWDataset return IPWDataset() elif benchmark == "gaia": - from evals.datasets.gaia import GAIADataset + from openjarvis.evals.datasets.gaia import GAIADataset return GAIADataset() elif benchmark == "frames": - from evals.datasets.frames import FRAMESDataset + from openjarvis.evals.datasets.frames import FRAMESDataset return FRAMESDataset() elif benchmark == "swebench": - from evals.datasets.swebench import SWEBenchDataset + from openjarvis.evals.datasets.swebench import SWEBenchDataset return SWEBenchDataset() elif benchmark == "swefficiency": - from evals.datasets.swefficiency import SWEfficiencyDataset + from openjarvis.evals.datasets.swefficiency import SWEfficiencyDataset return SWEfficiencyDataset() elif benchmark == "terminalbench": - from evals.datasets.terminalbench import TerminalBenchDataset + from openjarvis.evals.datasets.terminalbench import TerminalBenchDataset return TerminalBenchDataset() elif benchmark == "terminalbench-native": - from evals.datasets.terminalbench_native import TerminalBenchNativeDataset + from openjarvis.evals.datasets.terminalbench_native import ( + TerminalBenchNativeDataset, + ) return TerminalBenchNativeDataset() else: raise click.ClickException(f"Unknown benchmark: {benchmark}") @@ -143,46 +145,46 @@ def _build_dataset(benchmark: str): def _build_scorer(benchmark: str, judge_backend, judge_model: str): """Construct the scorer for a benchmark.""" if benchmark == "supergpqa": - from evals.scorers.supergpqa_mcq import SuperGPQAScorer + from openjarvis.evals.scorers.supergpqa_mcq import SuperGPQAScorer return SuperGPQAScorer(judge_backend, judge_model) elif benchmark == "gpqa": - from evals.scorers.gpqa_mcq import GPQAScorer + from openjarvis.evals.scorers.gpqa_mcq import GPQAScorer return GPQAScorer(judge_backend, judge_model) elif benchmark == "mmlu-pro": - from evals.scorers.mmlu_pro_mcq import MMLUProScorer + from openjarvis.evals.scorers.mmlu_pro_mcq import MMLUProScorer return MMLUProScorer(judge_backend, judge_model) elif benchmark == "math500" or benchmark == "natural-reasoning": - from evals.scorers.reasoning_judge import ReasoningJudgeScorer + from openjarvis.evals.scorers.reasoning_judge import ReasoningJudgeScorer return ReasoningJudgeScorer(judge_backend, judge_model) elif benchmark == "hle": - from evals.scorers.hle_judge import HLEScorer + from openjarvis.evals.scorers.hle_judge import HLEScorer return HLEScorer(judge_backend, judge_model) elif benchmark == "simpleqa": - from evals.scorers.simpleqa_judge import SimpleQAScorer + from openjarvis.evals.scorers.simpleqa_judge import SimpleQAScorer return SimpleQAScorer(judge_backend, judge_model) elif benchmark == "wildchat": - from evals.scorers.wildchat_judge import WildChatScorer + from openjarvis.evals.scorers.wildchat_judge import WildChatScorer return WildChatScorer(judge_backend, judge_model) elif benchmark == "ipw": - from evals.scorers.ipw_mixed import IPWMixedScorer + from openjarvis.evals.scorers.ipw_mixed import IPWMixedScorer return IPWMixedScorer(judge_backend, judge_model) elif benchmark == "gaia": - from evals.scorers.gaia_exact import GAIAScorer + from openjarvis.evals.scorers.gaia_exact import GAIAScorer return GAIAScorer(judge_backend, judge_model) elif benchmark == "frames": - from evals.scorers.frames_judge import FRAMESScorer + from openjarvis.evals.scorers.frames_judge import FRAMESScorer return FRAMESScorer(judge_backend, judge_model) elif benchmark == "swebench": - from evals.scorers.swebench_structural import SWEBenchScorer + from openjarvis.evals.scorers.swebench_structural import SWEBenchScorer return SWEBenchScorer(judge_backend, judge_model) elif benchmark == "swefficiency": - from evals.scorers.swefficiency_structural import SWEfficiencyScorer + from openjarvis.evals.scorers.swefficiency_structural import SWEfficiencyScorer return SWEfficiencyScorer(judge_backend, judge_model) elif benchmark == "terminalbench": - from evals.scorers.terminalbench_judge import TerminalBenchScorer + from openjarvis.evals.scorers.terminalbench_judge import TerminalBenchScorer return TerminalBenchScorer(judge_backend, judge_model) elif benchmark == "terminalbench-native": - from evals.scorers.terminalbench_native_structural import ( + from openjarvis.evals.scorers.terminalbench_native_structural import ( TerminalBenchNativeScorer, ) return TerminalBenchNativeScorer(judge_backend, judge_model) @@ -192,7 +194,7 @@ def _build_scorer(benchmark: str, judge_backend, judge_model: str): def _build_judge_backend(judge_model: str): """Build the judge backend (always cloud for LLM-as-judge).""" - from evals.backends.jarvis_direct import JarvisDirectBackend + from openjarvis.evals.backends.jarvis_direct import JarvisDirectBackend return JarvisDirectBackend(engine_key="cloud") @@ -217,7 +219,7 @@ def _print_summary( def _run_single(config, console: Optional[Console] = None) -> object: """Run a single eval from a RunConfig and return the summary.""" - from evals.core.runner import EvalRunner + from openjarvis.evals.core.runner import EvalRunner if console is None: console = Console() @@ -264,7 +266,7 @@ def _run_single(config, console: Optional[Console] = None) -> object: def _run_from_config(config_path: str, verbose: bool) -> None: """Load a TOML config and run the full models x benchmarks matrix.""" - from evals.core.config import expand_suite, load_eval_config + from openjarvis.evals.core.config import expand_suite, load_eval_config console = Console() @@ -381,7 +383,7 @@ def run(ctx, config_path, benchmark, backend, model, engine_key, agent_name, "(required when --config is not provided)" ) - from evals.core.types import RunConfig + from openjarvis.evals.core.types import RunConfig tool_list = [t.strip() for t in tools.split(",") if t.strip()] if tools else [] @@ -451,8 +453,8 @@ def run_all(model, engine_key, max_samples, max_workers, judge_model, """Run all benchmarks.""" _setup_logging(verbose) - from evals.core.runner import EvalRunner - from evals.core.types import RunConfig + from openjarvis.evals.core.runner import EvalRunner + from openjarvis.evals.core.types import RunConfig console = Console() diff --git a/evals/configs/full-suite.toml b/src/openjarvis/evals/configs/full-suite.toml similarity index 100% rename from evals/configs/full-suite.toml rename to src/openjarvis/evals/configs/full-suite.toml diff --git a/evals/configs/glm-4.7-flash-openhands-remaining.toml b/src/openjarvis/evals/configs/glm-4.7-flash-openhands-remaining.toml similarity index 100% rename from evals/configs/glm-4.7-flash-openhands-remaining.toml rename to src/openjarvis/evals/configs/glm-4.7-flash-openhands-remaining.toml diff --git a/evals/configs/glm-4.7-flash-openhands.toml b/src/openjarvis/evals/configs/glm-4.7-flash-openhands.toml similarity index 100% rename from evals/configs/glm-4.7-flash-openhands.toml rename to src/openjarvis/evals/configs/glm-4.7-flash-openhands.toml diff --git a/evals/configs/glm-4.7-fp8-openhands-gaia.toml b/src/openjarvis/evals/configs/glm-4.7-fp8-openhands-gaia.toml similarity index 100% rename from evals/configs/glm-4.7-fp8-openhands-gaia.toml rename to src/openjarvis/evals/configs/glm-4.7-fp8-openhands-gaia.toml diff --git a/evals/configs/minimal.toml b/src/openjarvis/evals/configs/minimal.toml similarity index 100% rename from evals/configs/minimal.toml rename to src/openjarvis/evals/configs/minimal.toml diff --git a/evals/configs/single-run.toml b/src/openjarvis/evals/configs/single-run.toml similarity index 100% rename from evals/configs/single-run.toml rename to src/openjarvis/evals/configs/single-run.toml diff --git a/evals/core/__init__.py b/src/openjarvis/evals/core/__init__.py similarity index 100% rename from evals/core/__init__.py rename to src/openjarvis/evals/core/__init__.py diff --git a/evals/core/backend.py b/src/openjarvis/evals/core/backend.py similarity index 100% rename from evals/core/backend.py rename to src/openjarvis/evals/core/backend.py diff --git a/evals/core/config.py b/src/openjarvis/evals/core/config.py similarity index 99% rename from evals/core/config.py rename to src/openjarvis/evals/core/config.py index fb6f7cdb..966d4fdc 100644 --- a/evals/core/config.py +++ b/src/openjarvis/evals/core/config.py @@ -7,7 +7,7 @@ import sys from pathlib import Path from typing import List -from evals.core.types import ( +from openjarvis.evals.core.types import ( BenchmarkConfig, DefaultsConfig, EvalSuiteConfig, diff --git a/evals/core/dataset.py b/src/openjarvis/evals/core/dataset.py similarity index 94% rename from evals/core/dataset.py rename to src/openjarvis/evals/core/dataset.py index 18271713..c3a31887 100644 --- a/evals/core/dataset.py +++ b/src/openjarvis/evals/core/dataset.py @@ -5,7 +5,7 @@ from __future__ import annotations from abc import ABC, abstractmethod from typing import Iterable, Optional -from evals.core.types import EvalRecord +from openjarvis.evals.core.types import EvalRecord class DatasetProvider(ABC): diff --git a/evals/core/display.py b/src/openjarvis/evals/core/display.py similarity index 99% rename from evals/core/display.py rename to src/openjarvis/evals/core/display.py index f427d775..e2efc9d9 100644 --- a/evals/core/display.py +++ b/src/openjarvis/evals/core/display.py @@ -12,7 +12,7 @@ from rich.table import Table if TYPE_CHECKING: from pathlib import Path - from evals.core.types import MetricStats, RunSummary + from openjarvis.evals.core.types import MetricStats, RunSummary OPENJARVIS_BANNER = r""" ___ _ _ diff --git a/evals/core/runner.py b/src/openjarvis/evals/core/runner.py similarity index 98% rename from evals/core/runner.py rename to src/openjarvis/evals/core/runner.py index d23fa882..07a9a0eb 100644 --- a/evals/core/runner.py +++ b/src/openjarvis/evals/core/runner.py @@ -11,10 +11,10 @@ from concurrent.futures import ThreadPoolExecutor, as_completed from pathlib import Path from typing import Any, Callable, Dict, List, Optional -from evals.core.backend import InferenceBackend -from evals.core.dataset import DatasetProvider -from evals.core.scorer import Scorer -from evals.core.types import EvalRecord, EvalResult, MetricStats, RunConfig, RunSummary +from openjarvis.evals.core.backend import InferenceBackend +from openjarvis.evals.core.dataset import DatasetProvider +from openjarvis.evals.core.scorer import Scorer +from openjarvis.evals.core.types import EvalRecord, EvalResult, MetricStats, RunConfig, RunSummary try: from openjarvis.telemetry.efficiency import compute_efficiency diff --git a/evals/core/scorer.py b/src/openjarvis/evals/core/scorer.py similarity index 92% rename from evals/core/scorer.py rename to src/openjarvis/evals/core/scorer.py index f466cde7..c67ae628 100644 --- a/evals/core/scorer.py +++ b/src/openjarvis/evals/core/scorer.py @@ -5,8 +5,8 @@ from __future__ import annotations from abc import ABC, abstractmethod from typing import Any, Dict, Optional, Tuple -from evals.core.backend import InferenceBackend -from evals.core.types import EvalRecord +from openjarvis.evals.core.backend import InferenceBackend +from openjarvis.evals.core.types import EvalRecord class Scorer(ABC): diff --git a/evals/core/types.py b/src/openjarvis/evals/core/types.py similarity index 100% rename from evals/core/types.py rename to src/openjarvis/evals/core/types.py diff --git a/evals/datasets/__init__.py b/src/openjarvis/evals/datasets/__init__.py similarity index 100% rename from evals/datasets/__init__.py rename to src/openjarvis/evals/datasets/__init__.py diff --git a/evals/datasets/frames.py b/src/openjarvis/evals/datasets/frames.py similarity index 97% rename from evals/datasets/frames.py rename to src/openjarvis/evals/datasets/frames.py index bd5a3a69..8c4df51d 100644 --- a/evals/datasets/frames.py +++ b/src/openjarvis/evals/datasets/frames.py @@ -8,8 +8,8 @@ from __future__ import annotations import random from typing import Iterable, List, MutableMapping, Optional, Sequence -from evals.core.dataset import DatasetProvider -from evals.core.types import EvalRecord +from openjarvis.evals.core.dataset import DatasetProvider +from openjarvis.evals.core.types import EvalRecord _DEFAULT_INPUT_PROMPT = """Please answer the question below. You should: diff --git a/evals/datasets/gaia.py b/src/openjarvis/evals/datasets/gaia.py similarity index 97% rename from evals/datasets/gaia.py rename to src/openjarvis/evals/datasets/gaia.py index 418b8746..d958ae1f 100644 --- a/evals/datasets/gaia.py +++ b/src/openjarvis/evals/datasets/gaia.py @@ -11,8 +11,8 @@ import shutil from pathlib import Path from typing import Iterable, List, MutableMapping, Optional, Sequence -from evals.core.dataset import DatasetProvider -from evals.core.types import EvalRecord +from openjarvis.evals.core.dataset import DatasetProvider +from openjarvis.evals.core.types import EvalRecord _DEFAULT_CACHE_DIR = Path.home() / ".cache" / "gaia_benchmark" diff --git a/evals/datasets/gpqa.py b/src/openjarvis/evals/datasets/gpqa.py similarity index 97% rename from evals/datasets/gpqa.py rename to src/openjarvis/evals/datasets/gpqa.py index 687090c4..9102f985 100644 --- a/evals/datasets/gpqa.py +++ b/src/openjarvis/evals/datasets/gpqa.py @@ -8,8 +8,8 @@ from __future__ import annotations import random from typing import Iterable, List, MutableMapping, Optional, Sequence -from evals.core.dataset import DatasetProvider -from evals.core.types import EvalRecord +from openjarvis.evals.core.dataset import DatasetProvider +from openjarvis.evals.core.types import EvalRecord def _format_options(options: Iterable[str]) -> str: diff --git a/evals/datasets/hle.py b/src/openjarvis/evals/datasets/hle.py similarity index 97% rename from evals/datasets/hle.py rename to src/openjarvis/evals/datasets/hle.py index 8b0e68b7..a44afd51 100644 --- a/evals/datasets/hle.py +++ b/src/openjarvis/evals/datasets/hle.py @@ -8,8 +8,8 @@ from __future__ import annotations import random from typing import Iterable, List, MutableMapping, Optional, Sequence -from evals.core.dataset import DatasetProvider -from evals.core.types import EvalRecord +from openjarvis.evals.core.dataset import DatasetProvider +from openjarvis.evals.core.types import EvalRecord # Fields whose presence signals a multimodal row. _MULTIMODAL_FIELDS = frozenset( diff --git a/evals/datasets/ipw_mixed.py b/src/openjarvis/evals/datasets/ipw_mixed.py similarity index 98% rename from evals/datasets/ipw_mixed.py rename to src/openjarvis/evals/datasets/ipw_mixed.py index e8fcef00..1cc85d8f 100644 --- a/evals/datasets/ipw_mixed.py +++ b/src/openjarvis/evals/datasets/ipw_mixed.py @@ -13,8 +13,8 @@ import random from pathlib import Path from typing import Any, Dict, Iterable, List, MutableMapping, Optional, Sequence -from evals.core.dataset import DatasetProvider -from evals.core.types import EvalRecord +from openjarvis.evals.core.dataset import DatasetProvider +from openjarvis.evals.core.types import EvalRecord LOGGER = logging.getLogger(__name__) diff --git a/evals/datasets/math500.py b/src/openjarvis/evals/datasets/math500.py similarity index 96% rename from evals/datasets/math500.py rename to src/openjarvis/evals/datasets/math500.py index 2aca3850..9178e74f 100644 --- a/evals/datasets/math500.py +++ b/src/openjarvis/evals/datasets/math500.py @@ -8,8 +8,8 @@ from __future__ import annotations import random from typing import Iterable, List, MutableMapping, Optional, Sequence -from evals.core.dataset import DatasetProvider -from evals.core.types import EvalRecord +from openjarvis.evals.core.dataset import DatasetProvider +from openjarvis.evals.core.types import EvalRecord _PROMPT_TEMPLATE = ( "Solve the following math problem step by step. " diff --git a/evals/datasets/mmlu_pro.py b/src/openjarvis/evals/datasets/mmlu_pro.py similarity index 96% rename from evals/datasets/mmlu_pro.py rename to src/openjarvis/evals/datasets/mmlu_pro.py index c04b6f8c..761d6669 100644 --- a/evals/datasets/mmlu_pro.py +++ b/src/openjarvis/evals/datasets/mmlu_pro.py @@ -8,8 +8,8 @@ from __future__ import annotations import random from typing import Iterable, List, MutableMapping, Optional, Sequence -from evals.core.dataset import DatasetProvider -from evals.core.types import EvalRecord +from openjarvis.evals.core.dataset import DatasetProvider +from openjarvis.evals.core.types import EvalRecord def _format_options(options: Iterable[str]) -> str: diff --git a/evals/datasets/natural_reasoning.py b/src/openjarvis/evals/datasets/natural_reasoning.py similarity index 96% rename from evals/datasets/natural_reasoning.py rename to src/openjarvis/evals/datasets/natural_reasoning.py index ca80756b..5ce98075 100644 --- a/evals/datasets/natural_reasoning.py +++ b/src/openjarvis/evals/datasets/natural_reasoning.py @@ -8,8 +8,8 @@ from __future__ import annotations import random from typing import Iterable, List, MutableMapping, Optional, Sequence -from evals.core.dataset import DatasetProvider -from evals.core.types import EvalRecord +from openjarvis.evals.core.dataset import DatasetProvider +from openjarvis.evals.core.types import EvalRecord _PROMPT_TEMPLATE = ( "Please solve the following reasoning problem. " diff --git a/evals/datasets/simpleqa.py b/src/openjarvis/evals/datasets/simpleqa.py similarity index 97% rename from evals/datasets/simpleqa.py rename to src/openjarvis/evals/datasets/simpleqa.py index 06dddc23..c309d84a 100644 --- a/evals/datasets/simpleqa.py +++ b/src/openjarvis/evals/datasets/simpleqa.py @@ -9,8 +9,8 @@ import ast import random from typing import Any, Dict, Iterable, List, MutableMapping, Optional, Sequence -from evals.core.dataset import DatasetProvider -from evals.core.types import EvalRecord +from openjarvis.evals.core.dataset import DatasetProvider +from openjarvis.evals.core.types import EvalRecord _PROMPT_TEMPLATE = """Please answer the following question with a short, factual response. Your answer should be a word, phrase, name, number, or date. diff --git a/evals/datasets/supergpqa.py b/src/openjarvis/evals/datasets/supergpqa.py similarity index 96% rename from evals/datasets/supergpqa.py rename to src/openjarvis/evals/datasets/supergpqa.py index 7291d827..f698ad65 100644 --- a/evals/datasets/supergpqa.py +++ b/src/openjarvis/evals/datasets/supergpqa.py @@ -8,8 +8,8 @@ from __future__ import annotations import random from typing import Iterable, List, MutableMapping, Optional, Sequence -from evals.core.dataset import DatasetProvider -from evals.core.types import EvalRecord +from openjarvis.evals.core.dataset import DatasetProvider +from openjarvis.evals.core.types import EvalRecord def _format_options(options: Iterable[str]) -> str: diff --git a/evals/datasets/swebench.py b/src/openjarvis/evals/datasets/swebench.py similarity index 97% rename from evals/datasets/swebench.py rename to src/openjarvis/evals/datasets/swebench.py index 50152e70..220693b5 100644 --- a/evals/datasets/swebench.py +++ b/src/openjarvis/evals/datasets/swebench.py @@ -9,8 +9,8 @@ import json import random from typing import Any, Iterable, List, MutableMapping, Optional, Sequence -from evals.core.dataset import DatasetProvider -from evals.core.types import EvalRecord +from openjarvis.evals.core.dataset import DatasetProvider +from openjarvis.evals.core.types import EvalRecord _HF_PATHS = { "verified": "princeton-nlp/SWE-bench_Verified", diff --git a/evals/datasets/swefficiency.py b/src/openjarvis/evals/datasets/swefficiency.py similarity index 97% rename from evals/datasets/swefficiency.py rename to src/openjarvis/evals/datasets/swefficiency.py index d553e59d..17ccc19a 100644 --- a/evals/datasets/swefficiency.py +++ b/src/openjarvis/evals/datasets/swefficiency.py @@ -9,8 +9,8 @@ import json import random from typing import Any, Iterable, List, MutableMapping, Optional, Sequence -from evals.core.dataset import DatasetProvider -from evals.core.types import EvalRecord +from openjarvis.evals.core.dataset import DatasetProvider +from openjarvis.evals.core.types import EvalRecord _HF_PATH = "swefficiency/swefficiency" diff --git a/evals/datasets/terminalbench.py b/src/openjarvis/evals/datasets/terminalbench.py similarity index 96% rename from evals/datasets/terminalbench.py rename to src/openjarvis/evals/datasets/terminalbench.py index f016afee..2c662315 100644 --- a/evals/datasets/terminalbench.py +++ b/src/openjarvis/evals/datasets/terminalbench.py @@ -8,8 +8,8 @@ from __future__ import annotations import random from typing import Iterable, List, MutableMapping, Optional, Sequence -from evals.core.dataset import DatasetProvider -from evals.core.types import EvalRecord +from openjarvis.evals.core.dataset import DatasetProvider +from openjarvis.evals.core.types import EvalRecord try: from datasets import load_dataset as _load_dataset # noqa: F401 diff --git a/evals/datasets/terminalbench_native.py b/src/openjarvis/evals/datasets/terminalbench_native.py similarity index 97% rename from evals/datasets/terminalbench_native.py rename to src/openjarvis/evals/datasets/terminalbench_native.py index 8d723b4e..97fc709e 100644 --- a/evals/datasets/terminalbench_native.py +++ b/src/openjarvis/evals/datasets/terminalbench_native.py @@ -10,8 +10,8 @@ import random from pathlib import Path from typing import Any, Dict, Iterable, List, Optional -from evals.core.dataset import DatasetProvider -from evals.core.types import EvalRecord +from openjarvis.evals.core.dataset import DatasetProvider +from openjarvis.evals.core.types import EvalRecord try: from terminal_bench import Task, TaskPaths diff --git a/evals/datasets/wildchat.py b/src/openjarvis/evals/datasets/wildchat.py similarity index 96% rename from evals/datasets/wildchat.py rename to src/openjarvis/evals/datasets/wildchat.py index 3d69d0a5..6b8c3a42 100644 --- a/evals/datasets/wildchat.py +++ b/src/openjarvis/evals/datasets/wildchat.py @@ -8,8 +8,8 @@ from __future__ import annotations import random from typing import Iterable, List, MutableMapping, Optional, Sequence -from evals.core.dataset import DatasetProvider -from evals.core.types import EvalRecord +from openjarvis.evals.core.dataset import DatasetProvider +from openjarvis.evals.core.types import EvalRecord class WildChatDataset(DatasetProvider): diff --git a/evals/scorers/__init__.py b/src/openjarvis/evals/scorers/__init__.py similarity index 100% rename from evals/scorers/__init__.py rename to src/openjarvis/evals/scorers/__init__.py diff --git a/evals/scorers/frames_judge.py b/src/openjarvis/evals/scorers/frames_judge.py similarity index 96% rename from evals/scorers/frames_judge.py rename to src/openjarvis/evals/scorers/frames_judge.py index 96076f24..8f5ba501 100644 --- a/evals/scorers/frames_judge.py +++ b/src/openjarvis/evals/scorers/frames_judge.py @@ -9,8 +9,8 @@ import logging import re from typing import Any, Dict, Optional, Tuple -from evals.core.scorer import LLMJudgeScorer -from evals.core.types import EvalRecord +from openjarvis.evals.core.scorer import LLMJudgeScorer +from openjarvis.evals.core.types import EvalRecord LOGGER = logging.getLogger(__name__) diff --git a/evals/scorers/gaia_exact.py b/src/openjarvis/evals/scorers/gaia_exact.py similarity index 98% rename from evals/scorers/gaia_exact.py rename to src/openjarvis/evals/scorers/gaia_exact.py index a52e1cc1..6050037b 100644 --- a/evals/scorers/gaia_exact.py +++ b/src/openjarvis/evals/scorers/gaia_exact.py @@ -10,8 +10,8 @@ import re import string from typing import Any, Dict, Optional, Tuple -from evals.core.scorer import LLMJudgeScorer -from evals.core.types import EvalRecord +from openjarvis.evals.core.scorer import LLMJudgeScorer +from openjarvis.evals.core.types import EvalRecord LOGGER = logging.getLogger(__name__) diff --git a/evals/scorers/gpqa_mcq.py b/src/openjarvis/evals/scorers/gpqa_mcq.py similarity index 96% rename from evals/scorers/gpqa_mcq.py rename to src/openjarvis/evals/scorers/gpqa_mcq.py index 7cc41ba7..5c211130 100644 --- a/evals/scorers/gpqa_mcq.py +++ b/src/openjarvis/evals/scorers/gpqa_mcq.py @@ -9,8 +9,8 @@ import logging import re from typing import Any, Dict, Optional, Tuple -from evals.core.scorer import LLMJudgeScorer -from evals.core.types import EvalRecord +from openjarvis.evals.core.scorer import LLMJudgeScorer +from openjarvis.evals.core.types import EvalRecord LOGGER = logging.getLogger(__name__) diff --git a/evals/scorers/hle_judge.py b/src/openjarvis/evals/scorers/hle_judge.py similarity index 94% rename from evals/scorers/hle_judge.py rename to src/openjarvis/evals/scorers/hle_judge.py index 783ee283..0fccd3e0 100644 --- a/evals/scorers/hle_judge.py +++ b/src/openjarvis/evals/scorers/hle_judge.py @@ -11,9 +11,9 @@ import logging import re from typing import Any, Dict, Optional, Tuple -from evals.core.scorer import LLMJudgeScorer -from evals.core.types import EvalRecord -from evals.scorers.reasoning_judge import reasoning_exact_match +from openjarvis.evals.core.scorer import LLMJudgeScorer +from openjarvis.evals.core.types import EvalRecord +from openjarvis.evals.scorers.reasoning_judge import reasoning_exact_match LOGGER = logging.getLogger(__name__) diff --git a/evals/scorers/ipw_mixed.py b/src/openjarvis/evals/scorers/ipw_mixed.py similarity index 97% rename from evals/scorers/ipw_mixed.py rename to src/openjarvis/evals/scorers/ipw_mixed.py index 67f5b44a..e4d8d132 100644 --- a/evals/scorers/ipw_mixed.py +++ b/src/openjarvis/evals/scorers/ipw_mixed.py @@ -11,8 +11,8 @@ import logging import re from typing import Any, Dict, Optional, Tuple -from evals.core.scorer import LLMJudgeScorer -from evals.core.types import EvalRecord +from openjarvis.evals.core.scorer import LLMJudgeScorer +from openjarvis.evals.core.types import EvalRecord LOGGER = logging.getLogger(__name__) diff --git a/evals/scorers/mmlu_pro_mcq.py b/src/openjarvis/evals/scorers/mmlu_pro_mcq.py similarity index 96% rename from evals/scorers/mmlu_pro_mcq.py rename to src/openjarvis/evals/scorers/mmlu_pro_mcq.py index 8442ee6e..951d4dac 100644 --- a/evals/scorers/mmlu_pro_mcq.py +++ b/src/openjarvis/evals/scorers/mmlu_pro_mcq.py @@ -9,8 +9,8 @@ import logging import re from typing import Any, Dict, Optional, Tuple -from evals.core.scorer import LLMJudgeScorer -from evals.core.types import EvalRecord +from openjarvis.evals.core.scorer import LLMJudgeScorer +from openjarvis.evals.core.types import EvalRecord LOGGER = logging.getLogger(__name__) diff --git a/evals/scorers/reasoning_judge.py b/src/openjarvis/evals/scorers/reasoning_judge.py similarity index 97% rename from evals/scorers/reasoning_judge.py rename to src/openjarvis/evals/scorers/reasoning_judge.py index 94311042..70d5ac9c 100644 --- a/evals/scorers/reasoning_judge.py +++ b/src/openjarvis/evals/scorers/reasoning_judge.py @@ -11,8 +11,8 @@ import re import string from typing import Any, Dict, Optional, Tuple -from evals.core.scorer import LLMJudgeScorer -from evals.core.types import EvalRecord +from openjarvis.evals.core.scorer import LLMJudgeScorer +from openjarvis.evals.core.types import EvalRecord LOGGER = logging.getLogger(__name__) diff --git a/evals/scorers/simpleqa_judge.py b/src/openjarvis/evals/scorers/simpleqa_judge.py similarity index 97% rename from evals/scorers/simpleqa_judge.py rename to src/openjarvis/evals/scorers/simpleqa_judge.py index 011e0a2e..121e5893 100644 --- a/evals/scorers/simpleqa_judge.py +++ b/src/openjarvis/evals/scorers/simpleqa_judge.py @@ -11,8 +11,8 @@ import re import string from typing import Any, Dict, Optional, Tuple -from evals.core.scorer import LLMJudgeScorer -from evals.core.types import EvalRecord +from openjarvis.evals.core.scorer import LLMJudgeScorer +from openjarvis.evals.core.types import EvalRecord LOGGER = logging.getLogger(__name__) diff --git a/evals/scorers/supergpqa_mcq.py b/src/openjarvis/evals/scorers/supergpqa_mcq.py similarity index 96% rename from evals/scorers/supergpqa_mcq.py rename to src/openjarvis/evals/scorers/supergpqa_mcq.py index ce6f9648..d2dc15d9 100644 --- a/evals/scorers/supergpqa_mcq.py +++ b/src/openjarvis/evals/scorers/supergpqa_mcq.py @@ -9,8 +9,8 @@ import logging import re from typing import Any, Dict, Optional, Tuple -from evals.core.scorer import LLMJudgeScorer -from evals.core.types import EvalRecord +from openjarvis.evals.core.scorer import LLMJudgeScorer +from openjarvis.evals.core.types import EvalRecord LOGGER = logging.getLogger(__name__) diff --git a/evals/scorers/swebench_structural.py b/src/openjarvis/evals/scorers/swebench_structural.py similarity index 95% rename from evals/scorers/swebench_structural.py rename to src/openjarvis/evals/scorers/swebench_structural.py index 8a789899..540c7946 100644 --- a/evals/scorers/swebench_structural.py +++ b/src/openjarvis/evals/scorers/swebench_structural.py @@ -11,8 +11,8 @@ from __future__ import annotations import re from typing import Any, Dict, Optional, Tuple -from evals.core.scorer import Scorer -from evals.core.types import EvalRecord +from openjarvis.evals.core.scorer import Scorer +from openjarvis.evals.core.types import EvalRecord _DIFF_MARKERS = [ r"^---\s", diff --git a/evals/scorers/swefficiency_structural.py b/src/openjarvis/evals/scorers/swefficiency_structural.py similarity index 95% rename from evals/scorers/swefficiency_structural.py rename to src/openjarvis/evals/scorers/swefficiency_structural.py index cef076c2..98efea26 100644 --- a/evals/scorers/swefficiency_structural.py +++ b/src/openjarvis/evals/scorers/swefficiency_structural.py @@ -12,8 +12,8 @@ from __future__ import annotations import re from typing import Any, Dict, Optional, Tuple -from evals.core.scorer import Scorer -from evals.core.types import EvalRecord +from openjarvis.evals.core.scorer import Scorer +from openjarvis.evals.core.types import EvalRecord _DIFF_MARKERS = [ r"^---\s", diff --git a/evals/scorers/terminalbench_judge.py b/src/openjarvis/evals/scorers/terminalbench_judge.py similarity index 97% rename from evals/scorers/terminalbench_judge.py rename to src/openjarvis/evals/scorers/terminalbench_judge.py index 6587227f..daf2a616 100644 --- a/evals/scorers/terminalbench_judge.py +++ b/src/openjarvis/evals/scorers/terminalbench_judge.py @@ -10,8 +10,8 @@ import logging import re from typing import Any, Dict, Optional, Tuple -from evals.core.scorer import LLMJudgeScorer -from evals.core.types import EvalRecord +from openjarvis.evals.core.scorer import LLMJudgeScorer +from openjarvis.evals.core.types import EvalRecord LOGGER = logging.getLogger(__name__) diff --git a/evals/scorers/terminalbench_native_structural.py b/src/openjarvis/evals/scorers/terminalbench_native_structural.py similarity index 95% rename from evals/scorers/terminalbench_native_structural.py rename to src/openjarvis/evals/scorers/terminalbench_native_structural.py index a7ca139b..4fe97f2a 100644 --- a/evals/scorers/terminalbench_native_structural.py +++ b/src/openjarvis/evals/scorers/terminalbench_native_structural.py @@ -9,8 +9,8 @@ from __future__ import annotations from typing import Any, Dict, Optional, Tuple -from evals.core.scorer import Scorer -from evals.core.types import EvalRecord +from openjarvis.evals.core.scorer import Scorer +from openjarvis.evals.core.types import EvalRecord class TerminalBenchNativeScorer(Scorer): diff --git a/evals/scorers/wildchat_judge.py b/src/openjarvis/evals/scorers/wildchat_judge.py similarity index 97% rename from evals/scorers/wildchat_judge.py rename to src/openjarvis/evals/scorers/wildchat_judge.py index 22923922..d4f01e11 100644 --- a/evals/scorers/wildchat_judge.py +++ b/src/openjarvis/evals/scorers/wildchat_judge.py @@ -9,8 +9,8 @@ import logging import re from typing import Any, Dict, Optional, Tuple -from evals.core.scorer import LLMJudgeScorer -from evals.core.types import EvalRecord +from openjarvis.evals.core.scorer import LLMJudgeScorer +from openjarvis.evals.core.types import EvalRecord LOGGER = logging.getLogger(__name__) diff --git a/evals/tests/__init__.py b/src/openjarvis/evals/tests/__init__.py similarity index 100% rename from evals/tests/__init__.py rename to src/openjarvis/evals/tests/__init__.py diff --git a/evals/tests/conftest.py b/src/openjarvis/evals/tests/conftest.py similarity index 94% rename from evals/tests/conftest.py rename to src/openjarvis/evals/tests/conftest.py index 902dd5b5..5d1af309 100644 --- a/evals/tests/conftest.py +++ b/src/openjarvis/evals/tests/conftest.py @@ -11,10 +11,10 @@ import pytest # Ensure evals package is importable from the repo root sys.path.insert(0, str(Path(__file__).resolve().parent.parent.parent)) -from evals.core.backend import InferenceBackend -from evals.core.dataset import DatasetProvider -from evals.core.scorer import Scorer -from evals.core.types import EvalRecord +from openjarvis.evals.core.backend import InferenceBackend +from openjarvis.evals.core.dataset import DatasetProvider +from openjarvis.evals.core.scorer import Scorer +from openjarvis.evals.core.types import EvalRecord # --------------------------------------------------------------------------- # Mock backend diff --git a/evals/tests/test_backends.py b/src/openjarvis/evals/tests/test_backends.py similarity index 91% rename from evals/tests/test_backends.py rename to src/openjarvis/evals/tests/test_backends.py index 5a431cc8..7196ee70 100644 --- a/evals/tests/test_backends.py +++ b/src/openjarvis/evals/tests/test_backends.py @@ -16,7 +16,7 @@ class TestJarvisDirectBackend: mock_builder.build.return_value = mock_system mock_builder_cls.return_value = mock_builder - from evals.backends.jarvis_direct import JarvisDirectBackend + from openjarvis.evals.backends.jarvis_direct import JarvisDirectBackend backend = JarvisDirectBackend() assert backend.backend_id == "jarvis-direct" @@ -33,7 +33,7 @@ class TestJarvisDirectBackend: mock_builder.build.return_value = MagicMock() mock_builder_cls.return_value = mock_builder - from evals.backends.jarvis_direct import JarvisDirectBackend + from openjarvis.evals.backends.jarvis_direct import JarvisDirectBackend JarvisDirectBackend(engine_key="cloud") mock_builder.engine.assert_called_with("cloud") @@ -54,7 +54,7 @@ class TestJarvisDirectBackend: mock_builder.build.return_value = mock_system mock_builder_cls.return_value = mock_builder - from evals.backends.jarvis_direct import JarvisDirectBackend + from openjarvis.evals.backends.jarvis_direct import JarvisDirectBackend backend = JarvisDirectBackend() result = backend.generate_full("What is 2+2?", model="test-model") @@ -77,7 +77,7 @@ class TestJarvisDirectBackend: mock_builder.build.return_value = mock_system mock_builder_cls.return_value = mock_builder - from evals.backends.jarvis_direct import JarvisDirectBackend + from openjarvis.evals.backends.jarvis_direct import JarvisDirectBackend backend = JarvisDirectBackend() text = backend.generate("Capital of France?", model="m") @@ -96,7 +96,7 @@ class TestJarvisAgentBackend: mock_builder.build.return_value = MagicMock() mock_builder_cls.return_value = mock_builder - from evals.backends.jarvis_agent import JarvisAgentBackend + from openjarvis.evals.backends.jarvis_agent import JarvisAgentBackend backend = JarvisAgentBackend( engine_key="cloud", agent_name="orchestrator", @@ -128,7 +128,7 @@ class TestJarvisAgentBackend: mock_builder.build.return_value = mock_system mock_builder_cls.return_value = mock_builder - from evals.backends.jarvis_agent import JarvisAgentBackend + from openjarvis.evals.backends.jarvis_agent import JarvisAgentBackend backend = JarvisAgentBackend(agent_name="orchestrator") result = backend.generate_full("What is 2+2?", model="gpt-4o") diff --git a/evals/tests/test_cli_flags.py b/src/openjarvis/evals/tests/test_cli_flags.py similarity index 92% rename from evals/tests/test_cli_flags.py rename to src/openjarvis/evals/tests/test_cli_flags.py index 24b54960..f79b65ba 100644 --- a/evals/tests/test_cli_flags.py +++ b/src/openjarvis/evals/tests/test_cli_flags.py @@ -4,7 +4,7 @@ from __future__ import annotations from click.testing import CliRunner -from evals.cli import main +from openjarvis.evals.cli import main class TestCompactFlag: diff --git a/evals/tests/test_config.py b/src/openjarvis/evals/tests/test_config.py similarity index 98% rename from evals/tests/test_config.py rename to src/openjarvis/evals/tests/test_config.py index 5cc4095c..a41f0161 100644 --- a/evals/tests/test_config.py +++ b/src/openjarvis/evals/tests/test_config.py @@ -7,8 +7,8 @@ from pathlib import Path import pytest -from evals.core.config import EvalConfigError, expand_suite, load_eval_config -from evals.core.types import ( +from openjarvis.evals.core.config import EvalConfigError, expand_suite, load_eval_config +from openjarvis.evals.core.types import ( BenchmarkConfig, DefaultsConfig, EvalSuiteConfig, @@ -597,7 +597,7 @@ class TestExpandSuite: class TestCLIConfig: def test_run_missing_benchmark_and_config(self): from click.testing import CliRunner - from evals.cli import main + from openjarvis.evals.cli import main runner = CliRunner() result = runner.invoke(main, ["run", "-m", "qwen3:8b"]) @@ -606,7 +606,7 @@ class TestCLIConfig: def test_run_missing_model_and_config(self): from click.testing import CliRunner - from evals.cli import main + from openjarvis.evals.cli import main runner = CliRunner() result = runner.invoke(main, ["run", "-b", "supergpqa"]) @@ -615,7 +615,7 @@ class TestCLIConfig: def test_run_config_file_not_found(self): from click.testing import CliRunner - from evals.cli import main + from openjarvis.evals.cli import main runner = CliRunner() result = runner.invoke(main, ["run", "--config", "/nonexistent.toml"]) @@ -630,7 +630,7 @@ class TestCLIConfig: from unittest.mock import patch from click.testing import CliRunner - from evals.cli import main + from openjarvis.evals.cli import main p = _write_toml(tmp_path, """\ [meta] diff --git a/evals/tests/test_display.py b/src/openjarvis/evals/tests/test_display.py similarity index 97% rename from evals/tests/test_display.py rename to src/openjarvis/evals/tests/test_display.py index 336aede2..49b24e97 100644 --- a/evals/tests/test_display.py +++ b/src/openjarvis/evals/tests/test_display.py @@ -6,7 +6,7 @@ from io import StringIO from rich.console import Console -from evals.core.display import ( +from openjarvis.evals.core.display import ( print_accuracy_panel, print_energy_table, print_latency_table, @@ -14,7 +14,7 @@ from evals.core.display import ( print_compact_table, print_full_results, ) -from evals.core.types import MetricStats, RunSummary +from openjarvis.evals.core.types import MetricStats, RunSummary def _make_summary(**overrides) -> RunSummary: diff --git a/evals/tests/test_frames.py b/src/openjarvis/evals/tests/test_frames.py similarity index 94% rename from evals/tests/test_frames.py rename to src/openjarvis/evals/tests/test_frames.py index c9047f37..aeea88fd 100644 --- a/evals/tests/test_frames.py +++ b/src/openjarvis/evals/tests/test_frames.py @@ -2,9 +2,9 @@ from __future__ import annotations -from evals.core.types import EvalRecord -from evals.scorers.frames_judge import _GRADER_TEMPLATE, FRAMESScorer -from evals.tests.conftest import MockBackend +from openjarvis.evals.core.types import EvalRecord +from openjarvis.evals.scorers.frames_judge import _GRADER_TEMPLATE, FRAMESScorer +from openjarvis.evals.tests.conftest import MockBackend class TestGraderTemplate: diff --git a/evals/tests/test_gaia.py b/src/openjarvis/evals/tests/test_gaia.py similarity index 96% rename from evals/tests/test_gaia.py rename to src/openjarvis/evals/tests/test_gaia.py index 9cdff9e7..4cce0043 100644 --- a/evals/tests/test_gaia.py +++ b/src/openjarvis/evals/tests/test_gaia.py @@ -2,8 +2,8 @@ from __future__ import annotations -from evals.core.types import EvalRecord -from evals.scorers.gaia_exact import ( +from openjarvis.evals.core.types import EvalRecord +from openjarvis.evals.scorers.gaia_exact import ( GAIAScorer, _is_float, _normalize_number_str, @@ -11,7 +11,7 @@ from evals.scorers.gaia_exact import ( _split_string, exact_match, ) -from evals.tests.conftest import MockBackend +from openjarvis.evals.tests.conftest import MockBackend class TestNormalization: diff --git a/evals/tests/test_runner.py b/src/openjarvis/evals/tests/test_runner.py similarity index 98% rename from evals/tests/test_runner.py rename to src/openjarvis/evals/tests/test_runner.py index f10ec043..80b65e17 100644 --- a/evals/tests/test_runner.py +++ b/src/openjarvis/evals/tests/test_runner.py @@ -6,9 +6,13 @@ import json import pytest -from evals.core.runner import EvalRunner, _metric_stats, _metric_stats_to_dict -from evals.core.types import EvalRecord, MetricStats, RunConfig -from evals.tests.conftest import MockBackend, MockDataset, MockScorer +from openjarvis.evals.core.runner import ( + EvalRunner, + _metric_stats, + _metric_stats_to_dict, +) +from openjarvis.evals.core.types import EvalRecord, MetricStats, RunConfig +from openjarvis.evals.tests.conftest import MockBackend, MockDataset, MockScorer class TestEvalRunner: diff --git a/evals/tests/test_supergpqa.py b/src/openjarvis/evals/tests/test_supergpqa.py similarity index 84% rename from evals/tests/test_supergpqa.py rename to src/openjarvis/evals/tests/test_supergpqa.py index 5dedb361..63fdf9ee 100644 --- a/evals/tests/test_supergpqa.py +++ b/src/openjarvis/evals/tests/test_supergpqa.py @@ -2,8 +2,8 @@ from __future__ import annotations -from evals.core.types import EvalRecord -from evals.tests.conftest import MockBackend +from openjarvis.evals.core.types import EvalRecord +from openjarvis.evals.tests.conftest import MockBackend class TestSuperGPQAScorer: @@ -25,7 +25,7 @@ class TestSuperGPQAScorer: ) def test_correct_extraction(self): - from evals.scorers.supergpqa_mcq import SuperGPQAScorer + from openjarvis.evals.scorers.supergpqa_mcq import SuperGPQAScorer backend = MockBackend(responses={}) backend._default_response = "B" @@ -39,7 +39,7 @@ class TestSuperGPQAScorer: assert meta["candidate_letter"] == "B" def test_incorrect_extraction(self): - from evals.scorers.supergpqa_mcq import SuperGPQAScorer + from openjarvis.evals.scorers.supergpqa_mcq import SuperGPQAScorer backend = MockBackend() backend._default_response = "A" @@ -52,7 +52,7 @@ class TestSuperGPQAScorer: assert meta["candidate_letter"] == "A" def test_missing_reference(self): - from evals.scorers.supergpqa_mcq import SuperGPQAScorer + from openjarvis.evals.scorers.supergpqa_mcq import SuperGPQAScorer backend = MockBackend() scorer = SuperGPQAScorer(backend, "gpt-4o") @@ -64,7 +64,7 @@ class TestSuperGPQAScorer: assert meta["reason"] == "missing_reference_letter" def test_no_extraction(self): - from evals.scorers.supergpqa_mcq import SuperGPQAScorer + from openjarvis.evals.scorers.supergpqa_mcq import SuperGPQAScorer backend = MockBackend() backend._default_response = "NONE" @@ -77,7 +77,7 @@ class TestSuperGPQAScorer: assert meta["reason"] == "no_choice_letter_extracted" def test_valid_letters_from_options(self): - from evals.scorers.supergpqa_mcq import SuperGPQAScorer + from openjarvis.evals.scorers.supergpqa_mcq import SuperGPQAScorer backend = MockBackend() scorer = SuperGPQAScorer(backend, "gpt-4o") @@ -92,7 +92,7 @@ class TestSuperGPQAScorer: assert letters == "ABCD" def test_extraction_with_verbose_response(self): - from evals.scorers.supergpqa_mcq import SuperGPQAScorer + from openjarvis.evals.scorers.supergpqa_mcq import SuperGPQAScorer backend = MockBackend() backend._default_response = "THE ANSWER IS: C" diff --git a/evals/tests/test_types.py b/src/openjarvis/evals/tests/test_types.py similarity index 99% rename from evals/tests/test_types.py rename to src/openjarvis/evals/tests/test_types.py index 31c437aa..0c0137a0 100644 --- a/evals/tests/test_types.py +++ b/src/openjarvis/evals/tests/test_types.py @@ -2,7 +2,7 @@ from __future__ import annotations -from evals.core.types import ( +from openjarvis.evals.core.types import ( BenchmarkConfig, DefaultsConfig, EvalRecord, diff --git a/evals/tests/test_wildchat.py b/src/openjarvis/evals/tests/test_wildchat.py similarity index 96% rename from evals/tests/test_wildchat.py rename to src/openjarvis/evals/tests/test_wildchat.py index fae2c9d3..351aa6ca 100644 --- a/evals/tests/test_wildchat.py +++ b/src/openjarvis/evals/tests/test_wildchat.py @@ -2,9 +2,9 @@ from __future__ import annotations -from evals.core.types import EvalRecord -from evals.scorers.wildchat_judge import WildChatScorer -from evals.tests.conftest import MockBackend +from openjarvis.evals.core.types import EvalRecord +from openjarvis.evals.scorers.wildchat_judge import WildChatScorer +from openjarvis.evals.tests.conftest import MockBackend class TestVerdictParsing: diff --git a/tests/evals/test_benchmark_datasets.py b/tests/evals/test_benchmark_datasets.py index 2fd6322f..4dd32311 100644 --- a/tests/evals/test_benchmark_datasets.py +++ b/tests/evals/test_benchmark_datasets.py @@ -23,91 +23,91 @@ class TestDatasetInstantiation: """Verify each dataset class can be instantiated with correct attributes.""" def test_supergpqa(self) -> None: - from evals.datasets.supergpqa import SuperGPQADataset + from openjarvis.evals.datasets.supergpqa import SuperGPQADataset ds = SuperGPQADataset() assert ds.dataset_id == "supergpqa" assert ds.dataset_name == "SuperGPQA" def test_gpqa(self) -> None: - from evals.datasets.gpqa import GPQADataset + from openjarvis.evals.datasets.gpqa import GPQADataset ds = GPQADataset() assert ds.dataset_id == "gpqa" assert ds.dataset_name == "GPQA" def test_mmlu_pro(self) -> None: - from evals.datasets.mmlu_pro import MMLUProDataset + from openjarvis.evals.datasets.mmlu_pro import MMLUProDataset ds = MMLUProDataset() assert ds.dataset_id == "mmlu-pro" assert ds.dataset_name == "MMLU-Pro" def test_math500(self) -> None: - from evals.datasets.math500 import MATH500Dataset + from openjarvis.evals.datasets.math500 import MATH500Dataset ds = MATH500Dataset() assert ds.dataset_id == "math500" assert ds.dataset_name == "MATH-500" def test_natural_reasoning(self) -> None: - from evals.datasets.natural_reasoning import NaturalReasoningDataset + from openjarvis.evals.datasets.natural_reasoning import NaturalReasoningDataset ds = NaturalReasoningDataset() assert ds.dataset_id == "natural-reasoning" assert ds.dataset_name == "Natural Reasoning" def test_hle(self) -> None: - from evals.datasets.hle import HLEDataset + from openjarvis.evals.datasets.hle import HLEDataset ds = HLEDataset() assert ds.dataset_id == "hle" assert ds.dataset_name == "HLE" def test_simpleqa(self) -> None: - from evals.datasets.simpleqa import SimpleQADataset + from openjarvis.evals.datasets.simpleqa import SimpleQADataset ds = SimpleQADataset() assert ds.dataset_id == "simpleqa" assert ds.dataset_name == "SimpleQA" def test_wildchat(self) -> None: - from evals.datasets.wildchat import WildChatDataset + from openjarvis.evals.datasets.wildchat import WildChatDataset ds = WildChatDataset() assert ds.dataset_id == "wildchat" assert ds.dataset_name == "WildChat" def test_ipw(self) -> None: - from evals.datasets.ipw_mixed import IPWDataset + from openjarvis.evals.datasets.ipw_mixed import IPWDataset ds = IPWDataset() assert ds.dataset_id == "ipw" assert ds.dataset_name == "IPW" def test_gaia(self) -> None: - from evals.datasets.gaia import GAIADataset + from openjarvis.evals.datasets.gaia import GAIADataset ds = GAIADataset() assert ds.dataset_id == "gaia" assert ds.dataset_name == "GAIA" def test_frames(self) -> None: - from evals.datasets.frames import FRAMESDataset + from openjarvis.evals.datasets.frames import FRAMESDataset ds = FRAMESDataset() assert ds.dataset_id == "frames" assert ds.dataset_name == "FRAMES" def test_swebench(self) -> None: - from evals.datasets.swebench import SWEBenchDataset + from openjarvis.evals.datasets.swebench import SWEBenchDataset ds = SWEBenchDataset() assert ds.dataset_id == "swebench" assert ds.dataset_name == "SWE-bench" def test_swefficiency(self) -> None: - from evals.datasets.swefficiency import SWEfficiencyDataset + from openjarvis.evals.datasets.swefficiency import SWEfficiencyDataset ds = SWEfficiencyDataset() assert ds.dataset_id == "swefficiency" assert ds.dataset_name == "SWEfficiency" def test_terminalbench(self) -> None: - from evals.datasets.terminalbench import TerminalBenchDataset + from openjarvis.evals.datasets.terminalbench import TerminalBenchDataset ds = TerminalBenchDataset() assert ds.dataset_id == "terminalbench" assert ds.dataset_name == "TerminalBench" def test_terminalbench_native(self) -> None: - from evals.datasets.terminalbench_native import ( + from openjarvis.evals.datasets.terminalbench_native import ( TerminalBenchNativeDataset, ) ds = TerminalBenchNativeDataset() @@ -131,74 +131,74 @@ class TestScorerInstantiation: """Verify each scorer class can be constructed.""" def test_supergpqa_scorer(self) -> None: - from evals.scorers.supergpqa_mcq import SuperGPQAScorer + from openjarvis.evals.scorers.supergpqa_mcq import SuperGPQAScorer s = SuperGPQAScorer(_mock_backend(), "test-model") assert s.scorer_id == "supergpqa" def test_gpqa_scorer(self) -> None: - from evals.scorers.gpqa_mcq import GPQAScorer + from openjarvis.evals.scorers.gpqa_mcq import GPQAScorer s = GPQAScorer(_mock_backend(), "test-model") assert s.scorer_id == "gpqa" def test_mmlu_pro_scorer(self) -> None: - from evals.scorers.mmlu_pro_mcq import MMLUProScorer + from openjarvis.evals.scorers.mmlu_pro_mcq import MMLUProScorer s = MMLUProScorer(_mock_backend(), "test-model") assert s.scorer_id == "mmlu-pro" def test_reasoning_judge_scorer(self) -> None: - from evals.scorers.reasoning_judge import ReasoningJudgeScorer + from openjarvis.evals.scorers.reasoning_judge import ReasoningJudgeScorer s = ReasoningJudgeScorer(_mock_backend(), "test-model") assert s.scorer_id == "reasoning_judge" def test_hle_scorer(self) -> None: - from evals.scorers.hle_judge import HLEScorer + from openjarvis.evals.scorers.hle_judge import HLEScorer s = HLEScorer(_mock_backend(), "test-model") assert s.scorer_id == "hle" def test_simpleqa_scorer(self) -> None: - from evals.scorers.simpleqa_judge import SimpleQAScorer + from openjarvis.evals.scorers.simpleqa_judge import SimpleQAScorer s = SimpleQAScorer(_mock_backend(), "test-model") assert s.scorer_id == "simpleqa" def test_wildchat_scorer(self) -> None: - from evals.scorers.wildchat_judge import WildChatScorer + from openjarvis.evals.scorers.wildchat_judge import WildChatScorer s = WildChatScorer(_mock_backend(), "test-model") assert s.scorer_id == "wildchat" def test_ipw_mixed_scorer(self) -> None: - from evals.scorers.ipw_mixed import IPWMixedScorer + from openjarvis.evals.scorers.ipw_mixed import IPWMixedScorer s = IPWMixedScorer(_mock_backend(), "test-model") assert s.scorer_id == "ipw" def test_gaia_scorer(self) -> None: - from evals.scorers.gaia_exact import GAIAScorer + from openjarvis.evals.scorers.gaia_exact import GAIAScorer s = GAIAScorer(_mock_backend(), "test-model") assert s.scorer_id == "gaia" def test_frames_scorer(self) -> None: - from evals.scorers.frames_judge import FRAMESScorer + from openjarvis.evals.scorers.frames_judge import FRAMESScorer s = FRAMESScorer(_mock_backend(), "test-model") assert s.scorer_id == "frames" def test_swebench_scorer(self) -> None: - from evals.scorers.swebench_structural import SWEBenchScorer + from openjarvis.evals.scorers.swebench_structural import SWEBenchScorer s = SWEBenchScorer(_mock_backend(), "test-model") assert s.scorer_id == "swebench" def test_swefficiency_scorer(self) -> None: - from evals.scorers.swefficiency_structural import ( + from openjarvis.evals.scorers.swefficiency_structural import ( SWEfficiencyScorer, ) s = SWEfficiencyScorer(_mock_backend(), "test-model") assert s.scorer_id == "swefficiency" def test_terminalbench_scorer(self) -> None: - from evals.scorers.terminalbench_judge import TerminalBenchScorer + from openjarvis.evals.scorers.terminalbench_judge import TerminalBenchScorer s = TerminalBenchScorer(_mock_backend(), "test-model") assert s.scorer_id == "terminalbench" def test_terminalbench_native_scorer(self) -> None: - from evals.scorers.terminalbench_native_structural import ( + from openjarvis.evals.scorers.terminalbench_native_structural import ( TerminalBenchNativeScorer, ) s = TerminalBenchNativeScorer(_mock_backend(), "test-model") @@ -222,7 +222,7 @@ class TestCLIFactories: @pytest.mark.parametrize("benchmark", ALL_BENCHMARKS) def test_build_dataset(self, benchmark: str) -> None: - from evals.cli import _build_dataset + from openjarvis.evals.cli import _build_dataset ds = _build_dataset(benchmark) assert ds is not None assert hasattr(ds, "load") @@ -231,20 +231,20 @@ class TestCLIFactories: @pytest.mark.parametrize("benchmark", ALL_BENCHMARKS) def test_build_scorer(self, benchmark: str) -> None: - from evals.cli import _build_scorer + from openjarvis.evals.cli import _build_scorer scorer = _build_scorer(benchmark, _mock_backend(), "test-model") assert scorer is not None assert hasattr(scorer, "score") def test_build_dataset_unknown(self) -> None: import click - from evals.cli import _build_dataset + from openjarvis.evals.cli import _build_dataset with pytest.raises(click.ClickException, match="Unknown benchmark"): _build_dataset("nonexistent") def test_build_scorer_unknown(self) -> None: import click - from evals.cli import _build_scorer + from openjarvis.evals.cli import _build_scorer with pytest.raises(click.ClickException, match="Unknown benchmark"): _build_scorer("nonexistent", _mock_backend(), "test-model") @@ -258,12 +258,12 @@ class TestConfigBenchmarks: """Verify KNOWN_BENCHMARKS includes all 15 benchmarks.""" def test_all_benchmarks_known(self) -> None: - from evals.core.config import KNOWN_BENCHMARKS + from openjarvis.evals.core.config import KNOWN_BENCHMARKS for b in ALL_BENCHMARKS: assert b in KNOWN_BENCHMARKS, f"{b} missing from KNOWN_BENCHMARKS" def test_benchmarks_count(self) -> None: - from evals.core.config import KNOWN_BENCHMARKS + from openjarvis.evals.core.config import KNOWN_BENCHMARKS assert len(KNOWN_BENCHMARKS) == 15 @@ -276,8 +276,8 @@ class TestStructuralScorers: """Test structural scorers that don't need LLM calls.""" def test_swebench_empty_response(self) -> None: - from evals.core.types import EvalRecord - from evals.scorers.swebench_structural import SWEBenchScorer + from openjarvis.evals.core.types import EvalRecord + from openjarvis.evals.scorers.swebench_structural import SWEBenchScorer scorer = SWEBenchScorer(_mock_backend(), "test-model") record = EvalRecord( record_id="swe-1", problem="Fix bug", reference="patch", @@ -288,8 +288,8 @@ class TestStructuralScorers: assert meta["reason"] == "empty_response" def test_swebench_with_diff(self) -> None: - from evals.core.types import EvalRecord - from evals.scorers.swebench_structural import SWEBenchScorer + from openjarvis.evals.core.types import EvalRecord + from openjarvis.evals.scorers.swebench_structural import SWEBenchScorer scorer = SWEBenchScorer(_mock_backend(), "test-model") record = EvalRecord( record_id="swe-2", problem="Fix bug", reference="patch", @@ -302,8 +302,8 @@ class TestStructuralScorers: assert meta["has_diff_markers"] is True def test_terminalbench_native_no_results(self) -> None: - from evals.core.types import EvalRecord - from evals.scorers.terminalbench_native_structural import ( + from openjarvis.evals.core.types import EvalRecord + from openjarvis.evals.scorers.terminalbench_native_structural import ( TerminalBenchNativeScorer, ) scorer = TerminalBenchNativeScorer(_mock_backend(), "test-model") @@ -316,8 +316,8 @@ class TestStructuralScorers: assert meta["reason"] == "no_test_results" def test_terminalbench_native_resolved(self) -> None: - from evals.core.types import EvalRecord - from evals.scorers.terminalbench_native_structural import ( + from openjarvis.evals.core.types import EvalRecord + from openjarvis.evals.scorers.terminalbench_native_structural import ( TerminalBenchNativeScorer, ) scorer = TerminalBenchNativeScorer(_mock_backend(), "test-model") diff --git a/tests/evals/test_display.py b/tests/evals/test_display.py index d6d21ea9..119d9054 100644 --- a/tests/evals/test_display.py +++ b/tests/evals/test_display.py @@ -1,11 +1,11 @@ -"""Tests for the Rich display helpers in evals.core.display.""" +"""Tests for the Rich display helpers in openjarvis.evals.core.display.""" from __future__ import annotations from io import StringIO from pathlib import Path -from evals.core.display import ( +from openjarvis.evals.core.display import ( print_banner, print_completion, print_metrics_table, @@ -14,7 +14,7 @@ from evals.core.display import ( print_subject_table, print_suite_summary, ) -from evals.core.types import MetricStats, RunSummary +from openjarvis.evals.core.types import MetricStats, RunSummary from rich.console import Console