OpenJarvis/tests/evals/comparison/test_subprocess_runner.py

"""Tests for openjarvis.evals.backends.external._subprocess_runner."""

from __future__ import annotations

import json
from pathlib import Path
from unittest.mock import MagicMock, patch

import pytest

from openjarvis.evals.backends.external._subprocess_runner import (
    EnergySample,
    SubprocessResult,
    run_one_shot,
)


class TestSubprocessResult:
    def test_dataclass_fields(self) -> None:
        result = SubprocessResult(
            stdout="hello",
            stderr="",
            exit_code=0,
            latency_seconds=1.5,
            energy_joules=100.0,
            peak_power_w=20.0,
            sampler_method="nvml",
            parsed_json={"content": "x"},
        )
        assert result.stdout == "hello"
        assert result.exit_code == 0
        assert result.energy_joules == 100.0
        assert result.parsed_json == {"content": "x"}


class TestEnergySample:
    def test_dataclass_fields(self) -> None:
        s = EnergySample(timestamp=1.0, watts=15.5)
        assert s.timestamp == 1.0
        assert s.watts == 15.5


class TestRunOneShot:
    def test_successful_run_returns_parsed_json(self, tmp_path: Path) -> None:
        output_json = tmp_path / "out.json"
        output_json.write_text(
            json.dumps(
                {
                    "content": "hello",
                    "usage": {
                        "prompt_tokens": 10,
                        "completion_tokens": 5,
                        "total_tokens": 15,
                    },
                    "trajectory": [],
                    "tool_calls": 0,
                    "turn_count": 1,
                    "error": None,
                }
            )
        )
        with patch("subprocess.Popen") as MockPopen:
            mock_proc = MagicMock()
            mock_proc.communicate.return_value = ("done", "")
            mock_proc.returncode = 0
            MockPopen.return_value = mock_proc

            result = run_one_shot(
                cmd=["echo", "ok"],
                env={},
                timeout=10.0,
                output_json_path=output_json,
            )
        assert result.exit_code == 0
        assert result.parsed_json["content"] == "hello"
        assert result.error is None

    def test_nonzero_exit_records_crash(self, tmp_path: Path) -> None:
        output_json = tmp_path / "out.json"
        with patch("subprocess.Popen") as MockPopen:
            mock_proc = MagicMock()
            mock_proc.communicate.return_value = ("", "boom")
            mock_proc.returncode = 139
            MockPopen.return_value = mock_proc

            result = run_one_shot(
                cmd=["false"],
                env={},
                timeout=10.0,
                output_json_path=output_json,
            )
        assert result.exit_code == 139
        assert result.error == "subprocess_crash"
        assert "boom" in result.stderr

    def test_timeout_kills_process(self, tmp_path: Path) -> None:
        import subprocess as sp

        output_json = tmp_path / "out.json"
        with patch("subprocess.Popen") as MockPopen:
            mock_proc = MagicMock()
            mock_proc.communicate.side_effect = [
                sp.TimeoutExpired(cmd="x", timeout=10),
                ("", ""),
            ]
            mock_proc.returncode = -9
            MockPopen.return_value = mock_proc

            result = run_one_shot(
                cmd=["sleep", "100"],
                env={},
                timeout=10.0,
                output_json_path=output_json,
            )
        assert result.error == "timeout"
        mock_proc.terminate.assert_called_once()

    def test_malformed_json_recorded(self, tmp_path: Path) -> None:
        output_json = tmp_path / "out.json"
        output_json.write_text("not json {{{")
        with patch("subprocess.Popen") as MockPopen:
            mock_proc = MagicMock()
            mock_proc.communicate.return_value = ("", "")
            mock_proc.returncode = 0
            MockPopen.return_value = mock_proc

            result = run_one_shot(
                cmd=["echo"],
                env={},
                timeout=10.0,
                output_json_path=output_json,
            )
        assert result.error == "malformed_runner_output"

    def test_missing_output_json_recorded(self, tmp_path: Path) -> None:
        output_json = tmp_path / "never_created.json"
        with patch("subprocess.Popen") as MockPopen:
            mock_proc = MagicMock()
            mock_proc.communicate.return_value = ("", "")
            mock_proc.returncode = 0
            MockPopen.return_value = mock_proc

            result = run_one_shot(
                cmd=["echo"],
                env={},
                timeout=10.0,
                output_json_path=output_json,
            )
        assert result.error == "invalid_runner_output"


class TestEnergySampler:
    def test_fallback_chain_reaches_unavailable(
        self, monkeypatch: pytest.MonkeyPatch
    ) -> None:
        """When all samplers fail to initialize, return the null sampler."""
        from openjarvis.evals.backends.external import _subprocess_runner as m

        # Force every probe to fail
        monkeypatch.setattr(m, "_try_start_nvml", lambda: None)
        monkeypatch.setattr(m, "_try_start_powermetrics", lambda: None)
        monkeypatch.setattr(m, "_try_start_rocm_smi", lambda: None)
        monkeypatch.setattr(m, "_try_start_rapl", lambda: None)

        sampler = m._start_sampler()
        assert sampler.method == "unavailable"
        assert sampler.stop() == []