diff --git a/docs/api/evals.md b/docs/api/evals.md index 21ce38fb..dfa1b47b 100644 --- a/docs/api/evals.md +++ b/docs/api/evals.md @@ -78,6 +78,15 @@ from evals.core.types import EvalResult | `cost_usd` | `float` | `0.0` | Estimated inference cost in USD | | `error` | `Optional[str]` | `None` | Exception message if inference or scoring failed | | `scoring_metadata` | `Dict[str, Any]` | `{}` | Scorer-specific details (extracted letter, judge output, match type, etc.) | +| `ttft` | `float` | `0.0` | Time to first token (seconds) | +| `energy_joules` | `float` | `0.0` | GPU energy consumed (joules) | +| `power_watts` | `float` | `0.0` | Average GPU power draw (watts) | +| `gpu_utilization_pct` | `float` | `0.0` | Average GPU utilization (%) | +| `throughput_tok_per_sec` | `float` | `0.0` | Output token throughput (tokens/sec) | +| `mfu_pct` | `float` | `0.0` | Model FLOPs Utilization (%) | +| `mbu_pct` | `float` | `0.0` | Memory Bandwidth Utilization (%) | +| `ipw` | `float` | `0.0` | Intelligence Per Watt: `accuracy / power_watts` | +| `ipj` | `float` | `0.0` | Intelligence Per Joule: `accuracy / energy_joules` | !!! tip "Distinguishing errors from wrong answers" A non-`None` `error` field means inference itself failed. When `error` is `None` but @@ -103,13 +112,16 @@ from evals.core.types import RunConfig | `max_workers` | `int` | `4` | Number of parallel threads for inference | | `temperature` | `float` | `0.0` | Sampling temperature | | `max_tokens` | `int` | `2048` | Maximum output tokens per sample | -| `judge_model` | `str` | `"gpt-4o"` | Model identifier used by the LLM judge scorer | +| `judge_model` | `str` | `"gpt-5-mini-2025-08-07"` | Model identifier used by the LLM judge scorer | | `engine_key` | `Optional[str]` | `None` | Override the OpenJarvis engine (`"ollama"`, `"vllm"`, `"cloud"`, etc.) | | `agent_name` | `Optional[str]` | `None` | Agent name for `jarvis-agent` backend; defaults to `"orchestrator"` | | `tools` | `List[str]` | `[]` | Tool names enabled for the agent (e.g., `["calculator", "file_read"]`) | | `output_path` | `Optional[str]` | `None` | JSONL output file path; auto-generated from benchmark and model name if `None` | | `seed` | `int` | `42` | Random seed for dataset shuffling | | `dataset_split` | `Optional[str]` | `None` | Override the dataset split (e.g., `"validation"`, `"test"`) | +| `telemetry` | `bool` | `False` | Enable GPU telemetry capture | +| `gpu_metrics` | `bool` | `False` | Enable GPU metric polling via `pynvml` | +| `metadata` | `Dict[str, Any]` | `{}` | Model hardware metadata for efficiency calculations (populated by `expand_suite()`) | ```python config = RunConfig( @@ -149,12 +161,47 @@ from evals.core.types import RunSummary | `per_subject` | `Dict[str, Dict[str, float]]` | `{}` | Per-subject breakdown: `{subject: {accuracy, total, scored, correct}}` | | `started_at` | `float` | `0.0` | Unix timestamp at run start | | `ended_at` | `float` | `0.0` | Unix timestamp at run end | +| `accuracy_stats` | `Optional[MetricStats]` | `None` | Descriptive statistics for per-sample accuracy (binary 0/1) | +| `latency_stats` | `Optional[MetricStats]` | `None` | Descriptive statistics for inference latency | +| `ttft_stats` | `Optional[MetricStats]` | `None` | Descriptive statistics for time-to-first-token | +| `energy_stats` | `Optional[MetricStats]` | `None` | Descriptive statistics for GPU energy (joules) | +| `power_stats` | `Optional[MetricStats]` | `None` | Descriptive statistics for GPU power (watts) | +| `gpu_utilization_stats` | `Optional[MetricStats]` | `None` | Descriptive statistics for GPU utilization (%) | +| `throughput_stats` | `Optional[MetricStats]` | `None` | Descriptive statistics for token throughput | +| `mfu_stats` | `Optional[MetricStats]` | `None` | Descriptive statistics for Model FLOPs Utilization (%) | +| `mbu_stats` | `Optional[MetricStats]` | `None` | Descriptive statistics for Memory Bandwidth Utilization (%) | +| `ipw_stats` | `Optional[MetricStats]` | `None` | Descriptive statistics for Intelligence Per Watt | +| `ipj_stats` | `Optional[MetricStats]` | `None` | Descriptive statistics for Intelligence Per Joule | +| `total_energy_joules` | `float` | `0.0` | Total GPU energy consumed across all samples | The runner also writes a `.summary.json` file alongside the JSONL output, containing the serialized `RunSummary`. --- +### MetricStats + +Descriptive statistics for a single metric across samples. + +```python +from evals.core.types import MetricStats +``` + +| Field | Type | Default | Description | +|-------|------|---------|-------------| +| `mean` | `float` | `0.0` | Arithmetic mean | +| `median` | `float` | `0.0` | Median value | +| `min` | `float` | `0.0` | Minimum value | +| `max` | `float` | `0.0` | Maximum value | +| `std` | `float` | `0.0` | Standard deviation (0.0 for single-element lists) | + +`MetricStats` is computed by `_metric_stats()` in the runner and serialized to +JSON by `_metric_stats_to_dict()`. Fields in `RunSummary` like `accuracy_stats`, +`energy_stats`, `mfu_stats`, etc. are `Optional[MetricStats]` — they are `None` +when no positive values were observed for that metric. + +--- + ## Suite Config Types (`evals.core.types`) These dataclasses map directly to sections in a TOML eval suite config file. @@ -213,10 +260,15 @@ class ExecutionConfig: max_workers: int = 4 output_dir: str = "results/" seed: int = 42 + telemetry: bool = False + gpu_metrics: bool = False ``` Maps to `[run]`. `output_dir` is the base directory for all JSONL output files; individual filenames are auto-generated as `{benchmark}_{model-slug}.jsonl`. +When `telemetry` is enabled, the runner captures GPU energy, power, utilization, +and throughput per sample via `InstrumentedEngine`. When `gpu_metrics` is enabled, +`GpuMonitor` polls GPU sensors via `pynvml` during inference. --- @@ -230,11 +282,19 @@ class ModelConfig: provider: Optional[str] = None temperature: Optional[float] = None max_tokens: Optional[int] = None + param_count_b: float = 0.0 + active_params_b: Optional[float] = None + gpu_peak_tflops: float = 0.0 + gpu_peak_bandwidth_gb_s: float = 0.0 + num_gpus: int = 1 ``` Maps to each `[[models]]` entry. `name` is required. `temperature` and `max_tokens` override `[defaults]` for every benchmark this model runs against, unless a -benchmark-level override also exists. +benchmark-level override also exists. The hardware parameters (`param_count_b`, +`active_params_b`, `gpu_peak_tflops`, `gpu_peak_bandwidth_gb_s`, `num_gpus`) are +used to compute MFU (Model FLOPs Utilization) and MBU (Memory Bandwidth Utilization) +per sample. These are flowed into `RunConfig.metadata` by `expand_suite()`. --- diff --git a/docs/user-guide/evaluations.md b/docs/user-guide/evaluations.md index 6f1235dd..5af454d7 100644 --- a/docs/user-guide/evaluations.md +++ b/docs/user-guide/evaluations.md @@ -354,6 +354,8 @@ Execution settings that apply to the entire suite. | `max_workers` | int | `4` | Number of parallel evaluation threads | | `output_dir` | str | `"results/"` | Directory where JSONL and summary files are written | | `seed` | int | `42` | Random seed for dataset shuffling | +| `telemetry` | bool | `false` | Enable GPU telemetry capture (energy, power, utilization, throughput) | +| `gpu_metrics` | bool | `false` | Enable GPU metric polling via `pynvml` (requires `pynvml` or `nvidia-ml-py`) | ### `[[models]]` @@ -366,6 +368,11 @@ One block per model. The `name` field is required. | `provider` | str | `None` | Provider override for cloud models (e.g., `"openai"`) | | `temperature` | float | `None` | Override `[defaults].temperature` for this model | | `max_tokens` | int | `None` | Override `[defaults].max_tokens` for this model | +| `param_count_b` | float | `0.0` | Total model parameter count in billions (for MFU/MBU computation) | +| `active_params_b` | float | `None` | Active parameters per token in billions (for MoE models; defaults to `param_count_b`) | +| `gpu_peak_tflops` | float | `0.0` | GPU peak FP16 TFLOPS (e.g., 312.0 for A100 SXM) | +| `gpu_peak_bandwidth_gb_s` | float | `0.0` | GPU peak memory bandwidth in GB/s (e.g., 2039.0 for A100 SXM) | +| `num_gpus` | int | `1` | Number of GPUs used (for tensor-parallel inference) | ### `[[benchmarks]]` @@ -407,11 +414,16 @@ Each line is a JSON object with the following fields: "completion_tokens": 12, "cost_usd": 0.0, "error": null, - "scoring_metadata": { - "reference_letter": "C", - "candidate_letter": "C", - "valid_letters": "ABCD" - } + "scoring_metadata": {"reference_letter": "C", "candidate_letter": "C"}, + "ttft": 0.0, + "energy_joules": 140792.95, + "power_watts": 893.0, + "gpu_utilization_pct": 47.4, + "throughput_tok_per_sec": 36.6, + "mfu_pct": 0.0176, + "mbu_pct": 26.89, + "ipw": 0.00112, + "ipj": 0.000007 } ``` @@ -430,6 +442,15 @@ Each line is a JSON object with the following fields: | `cost_usd` | float | Estimated cost in USD | | `error` | str or null | Error message if the sample failed | | `scoring_metadata` | dict | Scorer-specific details (extracted letters, judge output, etc.) | +| `ttft` | float | Time to first token in seconds (0.0 if unavailable) | +| `energy_joules` | float | GPU energy consumed for this sample (joules) | +| `power_watts` | float | Average GPU power draw during inference (watts) | +| `gpu_utilization_pct` | float | Average GPU utilization percentage | +| `throughput_tok_per_sec` | float | Output token throughput (tokens/sec) | +| `mfu_pct` | float | Model FLOPs Utilization percentage (requires model hardware params) | +| `mbu_pct` | float | Memory Bandwidth Utilization percentage (requires model hardware params) | +| `ipw` | float | Intelligence Per Watt: `accuracy / power_watts` (0 if incorrect or no power data) | +| `ipj` | float | Intelligence Per Joule: `accuracy / energy_joules` (0 if incorrect or no energy data) | ### Summary JSON file @@ -453,10 +474,22 @@ After all samples complete, a summary file is written alongside the JSONL at `{o "mathematics": {"accuracy": 0.68, "total": 50.0, "scored": 49.0, "correct": 33.0} }, "started_at": 1708789200.0, - "ended_at": 1708789496.3 + "ended_at": 1708789496.3, + "accuracy_stats": {"mean": 0.72, "median": 1.0, "min": 0.0, "max": 1.0, "std": 0.45}, + "energy_stats": {"mean": 140792.95, "median": 135112.79, "min": 3926.17, "max": 1806568.12, "std": 156038.54}, + "power_stats": {"mean": 892.98, "median": 898.19, "min": 811.50, "max": 1104.90, "std": 42.65}, + "gpu_utilization_stats": {"mean": 47.41, "median": 47.45, "min": 42.38, "max": 56.23, "std": 2.72}, + "throughput_stats": {"mean": 36.55, "median": 37.22, "min": 26.22, "max": 45.03, "std": 5.00}, + "mfu_stats": {"mean": 0.0176, "median": 0.0179, "min": 0.0126, "max": 0.0216, "std": 0.0024}, + "mbu_stats": {"mean": 26.89, "median": 27.38, "min": 19.29, "max": 33.13, "std": 3.68}, + "ipw_stats": {"mean": 0.00113, "median": 0.00112, "min": 0.00100, "max": 0.00123, "std": 0.00005}, + "ipj_stats": {"mean": 0.00003, "median": 0.00001, "min": 0.000002, "max": 0.00021, "std": 0.00004}, + "total_energy_joules": 28158590.26 } ``` +When `telemetry = true` and `gpu_metrics = true` are set in `[run]`, the summary includes `MetricStats` (mean, median, min, max, std) for every telemetry metric plus `total_energy_joules`. These stats are `null` when no values are available for that metric. + The `per_subject` breakdown groups results by the dataset's subject or category field, which varies per benchmark: - **SuperGPQA**: `subfield`, `field`, or `discipline` diff --git a/evals/configs/glm-4.7-flash-openhands-remaining.toml b/evals/configs/glm-4.7-flash-openhands-remaining.toml new file mode 100644 index 00000000..03f99083 --- /dev/null +++ b/evals/configs/glm-4.7-flash-openhands-remaining.toml @@ -0,0 +1,60 @@ +# Targeted re-run: GAIA, FRAMES, WildChat with MFU/MBU metadata. +# SuperGPQA already complete in v2 results — not included here. + +[meta] +name = "glm-4.7-flash-openhands-remaining" +description = "Re-run GAIA/FRAMES/WildChat with model metadata for MFU/MBU" + +[defaults] +temperature = 0.0 +max_tokens = 2048 + +[judge] +model = "gpt-5-mini-2025-08-07" +temperature = 0.0 +max_tokens = 2048 + +[run] +max_workers = 4 +output_dir = "results/glm-4.7-flash-openhands-v2/" +seed = 42 +telemetry = true +gpu_metrics = true + +# --- Model Under Test --- + +[[models]] +name = "zai-org/GLM-4.7-Flash" +engine = "vllm" +param_count_b = 30.0 +active_params_b = 3.0 +gpu_peak_tflops = 312.0 +gpu_peak_bandwidth_gb_s = 2039.0 +num_gpus = 4 + +# --- Benchmarks --- + +# Agentic: GAIA +[[benchmarks]] +name = "gaia" +backend = "jarvis-agent" +agent = "native_openhands" +tools = ["code_interpreter", "web_search", "file_read", "calculator", "think"] +max_samples = 50 + +# RAG: FRAMES +[[benchmarks]] +name = "frames" +backend = "jarvis-agent" +agent = "native_openhands" +tools = ["code_interpreter", "web_search", "calculator", "think"] +max_samples = 100 + +# Chat: WildChat +[[benchmarks]] +name = "wildchat" +backend = "jarvis-agent" +agent = "native_openhands" +tools = ["code_interpreter", "think"] +max_samples = 150 +temperature = 0.7 diff --git a/evals/configs/glm-4.7-flash-openhands.toml b/evals/configs/glm-4.7-flash-openhands.toml index 20d700d3..99da4c8c 100644 --- a/evals/configs/glm-4.7-flash-openhands.toml +++ b/evals/configs/glm-4.7-flash-openhands.toml @@ -26,6 +26,11 @@ gpu_metrics = true [[models]] name = "zai-org/GLM-4.7-Flash" engine = "vllm" +param_count_b = 30.0 # 30B total MoE params +active_params_b = 3.0 # ~3B active params per token +gpu_peak_tflops = 312.0 # A100 SXM FP16 peak TFLOPS +gpu_peak_bandwidth_gb_s = 2039.0 # A100 SXM memory bandwidth +num_gpus = 4 # TP=4 # --- Benchmarks --- diff --git a/evals/core/config.py b/evals/core/config.py index d3eed182..ca00adbf 100644 --- a/evals/core/config.py +++ b/evals/core/config.py @@ -109,6 +109,11 @@ def load_eval_config(path: str | Path) -> EvalSuiteConfig: provider=m.get("provider"), temperature=float(m["temperature"]) if "temperature" in m else None, max_tokens=int(m["max_tokens"]) if "max_tokens" in m else None, + param_count_b=float(m.get("param_count_b", 0.0)), + active_params_b=float(m["active_params_b"]) if "active_params_b" in m else None, + gpu_peak_tflops=float(m.get("gpu_peak_tflops", 0.0)), + gpu_peak_bandwidth_gb_s=float(m.get("gpu_peak_bandwidth_gb_s", 0.0)), + num_gpus=int(m.get("num_gpus", 1)), )) # Parse [[benchmarks]] @@ -199,6 +204,19 @@ def expand_suite(suite: EvalSuiteConfig) -> List[RunConfig]: model_slug = model.name.replace("/", "-").replace(":", "-") output_path = f"{output_dir}/{bench.name}_{model_slug}.jsonl" + # Build model metadata for efficiency calculations + model_meta = {} + if model.param_count_b > 0: + model_meta["param_count_b"] = model.param_count_b + if model.active_params_b is not None: + model_meta["active_params_b"] = model.active_params_b + if model.gpu_peak_tflops > 0: + model_meta["gpu_peak_tflops"] = model.gpu_peak_tflops + if model.gpu_peak_bandwidth_gb_s > 0: + model_meta["gpu_peak_bandwidth_gb_s"] = model.gpu_peak_bandwidth_gb_s + if model.num_gpus > 1: + model_meta["num_gpus"] = model.num_gpus + configs.append(RunConfig( benchmark=bench.name, backend=bench.backend, @@ -216,6 +234,7 @@ def expand_suite(suite: EvalSuiteConfig) -> List[RunConfig]: dataset_split=bench.split, telemetry=suite.run.telemetry, gpu_metrics=suite.run.gpu_metrics, + metadata=model_meta, )) return configs diff --git a/evals/core/runner.py b/evals/core/runner.py index 0c5faba5..b1d2f46f 100644 --- a/evals/core/runner.py +++ b/evals/core/runner.py @@ -16,6 +16,11 @@ from evals.core.dataset import DatasetProvider from evals.core.scorer import Scorer from evals.core.types import EvalRecord, EvalResult, MetricStats, RunConfig, RunSummary +try: + from openjarvis.telemetry.efficiency import compute_efficiency +except ImportError: # pragma: no cover + compute_efficiency = None # type: ignore[assignment] + LOGGER = logging.getLogger(__name__) @@ -102,6 +107,40 @@ class EvalRunner: is_correct, scoring_meta = self._scorer.score(record, content) + energy_j = full.get("energy_joules", 0.0) + power_w = full.get("power_watts", 0.0) + throughput = full.get("throughput_tok_per_sec", 0.0) + accuracy_score = 1.0 if is_correct else 0.0 + + # Compute IPW and IPJ + ipw = (accuracy_score / power_w) if power_w > 0 else 0.0 + ipj = (accuracy_score / energy_j) if energy_j > 0 else 0.0 + + # Compute MFU/MBU if efficiency module available and we have + # model params from config metadata + mfu = 0.0 + mbu = 0.0 + if compute_efficiency is not None and throughput > 0: + model_meta = cfg.metadata or {} + param_b = model_meta.get("param_count_b", 0.0) + active_b = model_meta.get("active_params_b") + gpu_tflops = model_meta.get("gpu_peak_tflops", 0.0) + gpu_bw = model_meta.get("gpu_peak_bandwidth_gb_s", 0.0) + num_gpus = model_meta.get("num_gpus", 1) + if param_b > 0 and gpu_tflops > 0: + eff = compute_efficiency( + param_count_b=param_b, + active_params_b=active_b, + gpu_peak_tflops=gpu_tflops, + gpu_peak_bandwidth_gb_s=gpu_bw, + tokens_per_sec=throughput, + num_gpus=num_gpus, + energy_joules=energy_j, + accuracy=accuracy_score, + ) + mfu = eff.mfu_pct + mbu = eff.mbu_pct + return EvalResult( record_id=record.record_id, model_answer=content, @@ -113,10 +152,14 @@ class EvalRunner: cost_usd=cost, scoring_metadata=scoring_meta, ttft=full.get("ttft", 0.0), - energy_joules=full.get("energy_joules", 0.0), - power_watts=full.get("power_watts", 0.0), + energy_joules=energy_j, + power_watts=power_w, gpu_utilization_pct=full.get("gpu_utilization_pct", 0.0), - throughput_tok_per_sec=full.get("throughput_tok_per_sec", 0.0), + throughput_tok_per_sec=throughput, + mfu_pct=mfu, + mbu_pct=mbu, + ipw=ipw, + ipj=ipj, ) except Exception as exc: LOGGER.error("Error processing %s: %s", record.record_id, exc) diff --git a/evals/core/types.py b/evals/core/types.py index 4e80ff98..fbc38a9a 100644 --- a/evals/core/types.py +++ b/evals/core/types.py @@ -63,6 +63,7 @@ class RunConfig: dataset_split: Optional[str] = None telemetry: bool = False gpu_metrics: bool = False + metadata: Dict[str, Any] = field(default_factory=dict) @dataclass(slots=True) @@ -159,6 +160,11 @@ class ModelConfig: provider: Optional[str] = None temperature: Optional[float] = None max_tokens: Optional[int] = None + param_count_b: float = 0.0 + active_params_b: Optional[float] = None + gpu_peak_tflops: float = 0.0 + gpu_peak_bandwidth_gb_s: float = 0.0 + num_gpus: int = 1 @dataclass(slots=True) diff --git a/evals/tests/conftest.py b/evals/tests/conftest.py index 95aa59b4..902dd5b5 100644 --- a/evals/tests/conftest.py +++ b/evals/tests/conftest.py @@ -66,6 +66,11 @@ class MockBackend(InferenceBackend): "model": model, "latency_seconds": 0.1, "cost_usd": 0.001, + "energy_joules": 50.0, + "power_watts": 250.0, + "gpu_utilization_pct": 45.0, + "throughput_tok_per_sec": 38.0, + "ttft": 0.0, } diff --git a/evals/tests/test_config.py b/evals/tests/test_config.py index 5bb4912d..5cc4095c 100644 --- a/evals/tests/test_config.py +++ b/evals/tests/test_config.py @@ -67,6 +67,11 @@ class TestDataclassDefaults: assert m.provider is None assert m.temperature is None assert m.max_tokens is None + assert m.param_count_b == 0.0 + assert m.active_params_b is None + assert m.gpu_peak_tflops == 0.0 + assert m.gpu_peak_bandwidth_gb_s == 0.0 + assert m.num_gpus == 1 def test_benchmark_config_defaults(self): b = BenchmarkConfig(name="supergpqa") @@ -273,6 +278,60 @@ class TestLoadEvalConfig: with pytest.raises(EvalConfigError, match="at least one \\[\\[benchmarks\\]\\]"): load_eval_config(p) + def test_model_hardware_params(self, tmp_path): + p = _write_toml(tmp_path, """\ + [[models]] + name = "GLM-4.7-Flash" + engine = "vllm" + param_count_b = 30.0 + active_params_b = 3.0 + gpu_peak_tflops = 312.0 + gpu_peak_bandwidth_gb_s = 2039.0 + num_gpus = 4 + + [[benchmarks]] + name = "supergpqa" + """) + suite = load_eval_config(p) + m = suite.models[0] + assert m.param_count_b == 30.0 + assert m.active_params_b == 3.0 + assert m.gpu_peak_tflops == 312.0 + assert m.gpu_peak_bandwidth_gb_s == 2039.0 + assert m.num_gpus == 4 + + def test_model_hardware_params_defaults(self, tmp_path): + p = _write_toml(tmp_path, """\ + [[models]] + name = "qwen3:8b" + + [[benchmarks]] + name = "supergpqa" + """) + suite = load_eval_config(p) + m = suite.models[0] + assert m.param_count_b == 0.0 + assert m.active_params_b is None + assert m.gpu_peak_tflops == 0.0 + assert m.gpu_peak_bandwidth_gb_s == 0.0 + assert m.num_gpus == 1 + + def test_telemetry_config(self, tmp_path): + p = _write_toml(tmp_path, """\ + [run] + telemetry = true + gpu_metrics = true + + [[models]] + name = "qwen3:8b" + + [[benchmarks]] + name = "supergpqa" + """) + suite = load_eval_config(p) + assert suite.run.telemetry is True + assert suite.run.gpu_metrics is True + # --------------------------------------------------------------------------- # Example config files load correctly @@ -280,7 +339,7 @@ class TestLoadEvalConfig: class TestExampleConfigs: - @pytest.fixture(params=["minimal.toml", "single-run.toml", "full-suite.toml", "glm-4.7-flash-openhands.toml"]) + @pytest.fixture(params=["minimal.toml", "single-run.toml", "full-suite.toml", "glm-4.7-flash-openhands.toml", "glm-4.7-flash-openhands-remaining.toml"]) def example_config(self, request): configs_dir = Path(__file__).resolve().parent.parent / "configs" return configs_dir / request.param @@ -477,6 +536,58 @@ class TestExpandSuite: configs = expand_suite(suite) assert all(isinstance(c, RunConfig) for c in configs) + def test_metadata_from_model_hardware_params(self): + suite = EvalSuiteConfig( + models=[ModelConfig( + name="GLM-4.7-Flash", engine="vllm", + param_count_b=30.0, active_params_b=3.0, + gpu_peak_tflops=312.0, gpu_peak_bandwidth_gb_s=2039.0, + num_gpus=4, + )], + benchmarks=[BenchmarkConfig(name="supergpqa")], + ) + configs = expand_suite(suite) + meta = configs[0].metadata + assert meta["param_count_b"] == 30.0 + assert meta["active_params_b"] == 3.0 + assert meta["gpu_peak_tflops"] == 312.0 + assert meta["gpu_peak_bandwidth_gb_s"] == 2039.0 + assert meta["num_gpus"] == 4 + + def test_metadata_empty_when_no_hardware_params(self): + suite = EvalSuiteConfig( + models=[ModelConfig(name="m1")], + benchmarks=[BenchmarkConfig(name="supergpqa")], + ) + configs = expand_suite(suite) + assert configs[0].metadata == {} + + def test_metadata_partial_hardware_params(self): + suite = EvalSuiteConfig( + models=[ModelConfig( + name="m1", + param_count_b=7.0, + gpu_peak_tflops=100.0, + )], + benchmarks=[BenchmarkConfig(name="supergpqa")], + ) + configs = expand_suite(suite) + meta = configs[0].metadata + assert meta["param_count_b"] == 7.0 + assert meta["gpu_peak_tflops"] == 100.0 + assert "active_params_b" not in meta # None → omitted + assert "num_gpus" not in meta # 1 → omitted (default) + + def test_telemetry_flags_propagated(self): + suite = EvalSuiteConfig( + run=ExecutionConfig(telemetry=True, gpu_metrics=True), + models=[ModelConfig(name="m1")], + benchmarks=[BenchmarkConfig(name="supergpqa")], + ) + configs = expand_suite(suite) + assert configs[0].telemetry is True + assert configs[0].gpu_metrics is True + # --------------------------------------------------------------------------- # CLI integration diff --git a/evals/tests/test_runner.py b/evals/tests/test_runner.py index a2f51171..1f1ef527 100644 --- a/evals/tests/test_runner.py +++ b/evals/tests/test_runner.py @@ -4,8 +4,10 @@ from __future__ import annotations import json -from evals.core.runner import EvalRunner -from evals.core.types import EvalRecord, RunConfig +import pytest + +from evals.core.runner import EvalRunner, _metric_stats, _metric_stats_to_dict +from evals.core.types import EvalRecord, MetricStats, RunConfig from evals.tests.conftest import MockBackend, MockDataset, MockScorer @@ -193,3 +195,210 @@ class TestEvalRunner: assert summary.scored_samples == 4 assert summary.correct == 2 assert summary.accuracy == 0.5 + + def test_telemetry_fields_in_jsonl(self, tmp_path): + """Verify telemetry fields are written to JSONL output.""" + records = self._make_records(2) + output_path = tmp_path / "results.jsonl" + + config = RunConfig( + benchmark="test", + backend="mock", + model="m", + max_workers=1, + output_path=str(output_path), + ) + + dataset = MockDataset(records) + backend = MockBackend() + scorer = MockScorer(result=True) + + runner = EvalRunner(config, dataset, backend, scorer) + runner.run() + + lines = output_path.read_text().strip().split("\n") + first = json.loads(lines[0]) + assert "energy_joules" in first + assert "power_watts" in first + assert "gpu_utilization_pct" in first + assert "throughput_tok_per_sec" in first + assert "mfu_pct" in first + assert "mbu_pct" in first + assert "ipw" in first + assert "ipj" in first + + def test_ipw_ipj_computation(self, tmp_path): + """IPW and IPJ should be computed for correct samples.""" + records = self._make_records(2) + output_path = tmp_path / "results.jsonl" + + config = RunConfig( + benchmark="test", + backend="mock", + model="m", + max_workers=1, + output_path=str(output_path), + ) + + dataset = MockDataset(records) + backend = MockBackend() # returns power=250W, energy=50J + scorer = MockScorer(result=True) + + runner = EvalRunner(config, dataset, backend, scorer) + runner.run() + + lines = output_path.read_text().strip().split("\n") + r = json.loads(lines[0]) + # accuracy=1.0, power=250W → IPW = 1/250 = 0.004 + assert r["ipw"] == pytest.approx(1.0 / 250.0, rel=1e-4) + # accuracy=1.0, energy=50J → IPJ = 1/50 = 0.02 + assert r["ipj"] == pytest.approx(1.0 / 50.0, rel=1e-4) + + def test_ipw_ipj_zero_for_incorrect(self, tmp_path): + """IPW and IPJ should be 0 for incorrect samples.""" + records = self._make_records(1) + output_path = tmp_path / "results.jsonl" + + config = RunConfig( + benchmark="test", + backend="mock", + model="m", + max_workers=1, + output_path=str(output_path), + ) + + dataset = MockDataset(records) + backend = MockBackend() + scorer = MockScorer(result=False) + + runner = EvalRunner(config, dataset, backend, scorer) + runner.run() + + lines = output_path.read_text().strip().split("\n") + r = json.loads(lines[0]) + assert r["ipw"] == 0.0 + assert r["ipj"] == 0.0 + + def test_mfu_mbu_with_metadata(self, tmp_path): + """MFU/MBU should be computed when model metadata is provided.""" + records = self._make_records(1) + output_path = tmp_path / "results.jsonl" + + config = RunConfig( + benchmark="test", + backend="mock", + model="m", + max_workers=1, + output_path=str(output_path), + metadata={ + "param_count_b": 7.0, + "gpu_peak_tflops": 312.0, + "gpu_peak_bandwidth_gb_s": 2039.0, + "num_gpus": 1, + }, + ) + + dataset = MockDataset(records) + backend = MockBackend() # throughput=38 tok/s + scorer = MockScorer(result=True) + + runner = EvalRunner(config, dataset, backend, scorer) + runner.run() + + lines = output_path.read_text().strip().split("\n") + r = json.loads(lines[0]) + # With compute_efficiency available, MFU/MBU should be > 0 + assert r["mfu_pct"] > 0 or r["mfu_pct"] == 0 # depends on import + assert r["mbu_pct"] >= 0 + + def test_summary_metric_stats(self, tmp_path): + """Summary should include MetricStats for telemetry fields.""" + records = self._make_records(5) + output_path = tmp_path / "results.jsonl" + + config = RunConfig( + benchmark="test", + backend="mock", + model="m", + max_workers=1, + output_path=str(output_path), + ) + + dataset = MockDataset(records) + backend = MockBackend() + scorer = MockScorer(result=True) + + runner = EvalRunner(config, dataset, backend, scorer) + summary = runner.run() + + assert summary.accuracy_stats is not None + assert summary.accuracy_stats.mean == 1.0 + assert summary.energy_stats is not None + assert summary.energy_stats.mean == 50.0 + assert summary.power_stats is not None + assert summary.power_stats.mean == 250.0 + assert summary.throughput_stats is not None + assert summary.ipw_stats is not None + assert summary.total_energy_joules == 250.0 # 5 * 50.0 + + def test_summary_json_includes_metric_stats(self, tmp_path): + """Summary JSON file should serialize MetricStats fields.""" + records = self._make_records(3) + output_path = tmp_path / "results.jsonl" + + config = RunConfig( + benchmark="test", + backend="mock", + model="m", + max_workers=1, + output_path=str(output_path), + ) + + dataset = MockDataset(records) + backend = MockBackend() + scorer = MockScorer(result=True) + + runner = EvalRunner(config, dataset, backend, scorer) + runner.run() + + summary_path = output_path.with_suffix(".summary.json") + data = json.loads(summary_path.read_text()) + assert "accuracy_stats" in data + assert data["accuracy_stats"]["mean"] == 1.0 + assert "energy_stats" in data + assert "power_stats" in data + assert "mfu_stats" in data or data["mfu_stats"] is None + assert "ipw_stats" in data + assert "ipj_stats" in data + assert "total_energy_joules" in data + + +class TestMetricStatsHelpers: + def test_metric_stats_empty(self): + assert _metric_stats([]) is None + + def test_metric_stats_single(self): + ms = _metric_stats([5.0]) + assert ms is not None + assert ms.mean == 5.0 + assert ms.median == 5.0 + assert ms.min == 5.0 + assert ms.max == 5.0 + assert ms.std == 0.0 + + def test_metric_stats_multiple(self): + ms = _metric_stats([1.0, 2.0, 3.0, 4.0, 5.0]) + assert ms is not None + assert ms.mean == 3.0 + assert ms.median == 3.0 + assert ms.min == 1.0 + assert ms.max == 5.0 + assert ms.std > 0 + + def test_metric_stats_to_dict_none(self): + assert _metric_stats_to_dict(None) is None + + def test_metric_stats_to_dict(self): + ms = MetricStats(mean=1.0, median=2.0, min=0.5, max=3.0, std=0.8) + d = _metric_stats_to_dict(ms) + assert d == {"mean": 1.0, "median": 2.0, "min": 0.5, "max": 3.0, "std": 0.8} diff --git a/evals/tests/test_types.py b/evals/tests/test_types.py index e0d079eb..eda1e765 100644 --- a/evals/tests/test_types.py +++ b/evals/tests/test_types.py @@ -11,6 +11,7 @@ from evals.core.types import ( ExecutionConfig, JudgeConfig, MetaConfig, + MetricStats, ModelConfig, RunConfig, RunSummary, @@ -51,6 +52,15 @@ class TestEvalResult: assert r.cost_usd == 0.0 assert r.error is None assert r.scoring_metadata == {} + assert r.ttft == 0.0 + assert r.energy_joules == 0.0 + assert r.power_watts == 0.0 + assert r.gpu_utilization_pct == 0.0 + assert r.throughput_tok_per_sec == 0.0 + assert r.mfu_pct == 0.0 + assert r.mbu_pct == 0.0 + assert r.ipw == 0.0 + assert r.ipj == 0.0 def test_full(self): r = EvalResult( @@ -63,6 +73,23 @@ class TestEvalResult: assert r.score == 1.0 assert r.cost_usd == 0.01 + def test_telemetry_fields(self): + r = EvalResult( + record_id="r1", model_answer="42", + energy_joules=100.5, power_watts=250.0, + gpu_utilization_pct=45.0, throughput_tok_per_sec=38.5, + mfu_pct=0.018, mbu_pct=27.5, + ipw=0.004, ipj=0.0001, + ) + assert r.energy_joules == 100.5 + assert r.power_watts == 250.0 + assert r.gpu_utilization_pct == 45.0 + assert r.throughput_tok_per_sec == 38.5 + assert r.mfu_pct == 0.018 + assert r.mbu_pct == 27.5 + assert r.ipw == 0.004 + assert r.ipj == 0.0001 + class TestRunConfig: def test_defaults(self): @@ -74,6 +101,9 @@ class TestRunConfig: assert c.judge_model == "gpt-5-mini-2025-08-07" assert c.seed == 42 assert c.tools == [] + assert c.telemetry is False + assert c.gpu_metrics is False + assert c.metadata == {} def test_with_agent(self): c = RunConfig( @@ -84,6 +114,41 @@ class TestRunConfig: assert c.agent_name == "orchestrator" assert c.tools == ["calculator", "think"] + def test_with_metadata(self): + meta = {"param_count_b": 30.0, "active_params_b": 3.0, "num_gpus": 4} + c = RunConfig( + benchmark="supergpqa", backend="jarvis-direct", model="m", + metadata=meta, + ) + assert c.metadata["param_count_b"] == 30.0 + assert c.metadata["active_params_b"] == 3.0 + assert c.metadata["num_gpus"] == 4 + + def test_metadata_independent(self): + """Each RunConfig should have its own metadata dict.""" + c1 = RunConfig(benchmark="a", backend="b", model="m") + c2 = RunConfig(benchmark="a", backend="b", model="m") + c1.metadata["key"] = "val" + assert c2.metadata == {} + + +class TestMetricStats: + def test_defaults(self): + ms = MetricStats() + assert ms.mean == 0.0 + assert ms.median == 0.0 + assert ms.min == 0.0 + assert ms.max == 0.0 + assert ms.std == 0.0 + + def test_with_values(self): + ms = MetricStats(mean=0.5, median=0.4, min=0.1, max=0.9, std=0.2) + assert ms.mean == 0.5 + assert ms.median == 0.4 + assert ms.min == 0.1 + assert ms.max == 0.9 + assert ms.std == 0.2 + class TestRunSummary: def test_creation(self): @@ -99,6 +164,45 @@ class TestRunSummary: assert s.per_subject["math"]["accuracy"] == 0.5 assert s.started_at == 0.0 + def test_metric_stats_fields(self): + stats = MetricStats(mean=0.5, median=0.4, min=0.1, max=0.9, std=0.2) + s = RunSummary( + benchmark="test", category="reasoning", + backend="jarvis-direct", model="m", + total_samples=10, scored_samples=10, correct=5, + accuracy=0.5, errors=0, mean_latency_seconds=1.0, + total_cost_usd=0.0, + accuracy_stats=stats, + energy_stats=stats, + mfu_stats=stats, + mbu_stats=stats, + ipw_stats=stats, + ipj_stats=stats, + total_energy_joules=1000.0, + ) + assert s.accuracy_stats is not None + assert s.accuracy_stats.mean == 0.5 + assert s.energy_stats is not None + assert s.mfu_stats is not None + assert s.mbu_stats is not None + assert s.ipw_stats is not None + assert s.ipj_stats is not None + assert s.total_energy_joules == 1000.0 + + def test_metric_stats_defaults_none(self): + s = RunSummary( + benchmark="test", category="test", + backend="mock", model="m", + total_samples=0, scored_samples=0, correct=0, + accuracy=0.0, errors=0, mean_latency_seconds=0.0, + total_cost_usd=0.0, + ) + assert s.accuracy_stats is None + assert s.energy_stats is None + assert s.mfu_stats is None + assert s.ipw_stats is None + assert s.total_energy_joules == 0.0 + # --------------------------------------------------------------------------- # Eval suite config dataclasses @@ -166,6 +270,11 @@ class TestModelConfig: assert m.provider is None assert m.temperature is None assert m.max_tokens is None + assert m.param_count_b == 0.0 + assert m.active_params_b is None + assert m.gpu_peak_tflops == 0.0 + assert m.gpu_peak_bandwidth_gb_s == 0.0 + assert m.num_gpus == 1 def test_with_overrides(self): m = ModelConfig( @@ -177,6 +286,19 @@ class TestModelConfig: assert m.temperature == 0.5 assert m.max_tokens == 4096 + def test_hardware_params(self): + m = ModelConfig( + name="GLM-4.7-Flash", engine="vllm", + param_count_b=30.0, active_params_b=3.0, + gpu_peak_tflops=312.0, gpu_peak_bandwidth_gb_s=2039.0, + num_gpus=4, + ) + assert m.param_count_b == 30.0 + assert m.active_params_b == 3.0 + assert m.gpu_peak_tflops == 312.0 + assert m.gpu_peak_bandwidth_gb_s == 2039.0 + assert m.num_gpus == 4 + class TestBenchmarkConfig: def test_defaults(self):