mirror of
https://github.com/open-jarvis/OpenJarvis.git
synced 2026-07-28 05:12:26 +00:00
211 lines
6.8 KiB
Python
211 lines
6.8 KiB
Python
"""Tests for openjarvis.optimize.feedback.judge module."""
|
|
|
|
from __future__ import annotations
|
|
|
|
from unittest.mock import MagicMock
|
|
|
|
from openjarvis.core.types import StepType, Trace, TraceStep
|
|
from openjarvis.learning.optimize.feedback.judge import TraceJudge, _parse_score
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# _parse_score unit tests
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
class TestParseScore:
|
|
"""Tests for the internal _parse_score helper."""
|
|
|
|
def test_plain_decimal(self) -> None:
|
|
assert _parse_score("0.85") == 0.85
|
|
|
|
def test_score_prefix(self) -> None:
|
|
assert _parse_score("Score: 0.90") == 0.90
|
|
|
|
def test_rating_prefix(self) -> None:
|
|
assert _parse_score("Rating: 0.75\nGood work.") == 0.75
|
|
|
|
def test_fraction_format(self) -> None:
|
|
score = _parse_score("Rating: 7/10\nSolid answer")
|
|
assert abs(score - 0.7) < 1e-6
|
|
|
|
def test_fraction_eight_over_ten(self) -> None:
|
|
score = _parse_score("Score: 8/10")
|
|
assert abs(score - 0.8) < 1e-6
|
|
|
|
def test_whole_number_over_one_treated_as_ten_scale(self) -> None:
|
|
score = _parse_score("Quality: 9\nExcellent")
|
|
assert abs(score - 0.9) < 1e-6
|
|
|
|
def test_zero_score(self) -> None:
|
|
assert _parse_score("Score: 0.0\nTerrible") == 0.0
|
|
|
|
def test_perfect_score(self) -> None:
|
|
assert _parse_score("1.0\nPerfect") == 1.0
|
|
|
|
def test_no_score_defaults_to_half(self) -> None:
|
|
assert _parse_score("This is just a comment.") == 0.5
|
|
|
|
def test_negative_clamped_to_zero(self) -> None:
|
|
# Edge case: if somehow negative appeared
|
|
score = _parse_score("Score: 0.0")
|
|
assert score == 0.0
|
|
|
|
def test_over_one_on_ten_scale_capped(self) -> None:
|
|
score = _parse_score("Score: 11")
|
|
assert score <= 1.0
|
|
|
|
def test_quality_prefix(self) -> None:
|
|
assert _parse_score("Quality: 0.65") == 0.65
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# TraceJudge
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def _make_trace(**overrides) -> Trace:
|
|
"""Create a minimal Trace for testing."""
|
|
defaults = dict(
|
|
trace_id="trace-001",
|
|
query="What is 2+2?",
|
|
agent="orchestrator",
|
|
model="qwen3:8b",
|
|
result="The answer is 4.",
|
|
)
|
|
defaults.update(overrides)
|
|
return Trace(**defaults)
|
|
|
|
|
|
class TestTraceJudgeInit:
|
|
"""TraceJudge constructor stores backend and model."""
|
|
|
|
def test_stores_backend_and_model(self) -> None:
|
|
backend = MagicMock()
|
|
judge = TraceJudge(backend=backend, model="judge-model")
|
|
assert judge._backend is backend
|
|
assert judge._model == "judge-model"
|
|
|
|
|
|
class TestScoreTrace:
|
|
"""TraceJudge.score_trace calls backend and parses result."""
|
|
|
|
def test_returns_score_and_feedback(self) -> None:
|
|
backend = MagicMock()
|
|
backend.generate.return_value = (
|
|
"Score: 0.85\nGood reasoning and correct answer."
|
|
)
|
|
judge = TraceJudge(backend=backend, model="judge-model")
|
|
|
|
trace = _make_trace()
|
|
score, feedback = judge.score_trace(trace)
|
|
|
|
assert score == 0.85
|
|
assert "Good reasoning" in feedback
|
|
backend.generate.assert_called_once()
|
|
|
|
def test_prompt_includes_query(self) -> None:
|
|
backend = MagicMock()
|
|
backend.generate.return_value = "0.9\nGreat"
|
|
judge = TraceJudge(backend=backend, model="m")
|
|
|
|
trace = _make_trace(query="Explain gravity")
|
|
judge.score_trace(trace)
|
|
|
|
call_args = backend.generate.call_args
|
|
prompt = call_args[0][0]
|
|
assert "Explain gravity" in prompt
|
|
|
|
def test_prompt_includes_result(self) -> None:
|
|
backend = MagicMock()
|
|
backend.generate.return_value = "0.7\nOk"
|
|
judge = TraceJudge(backend=backend, model="m")
|
|
|
|
trace = _make_trace(result="Gravity is a fundamental force.")
|
|
judge.score_trace(trace)
|
|
|
|
call_args = backend.generate.call_args
|
|
prompt = call_args[0][0]
|
|
assert "Gravity is a fundamental force." in prompt
|
|
|
|
def test_prompt_includes_steps(self) -> None:
|
|
backend = MagicMock()
|
|
backend.generate.return_value = "0.8\nGood"
|
|
judge = TraceJudge(backend=backend, model="m")
|
|
|
|
step = TraceStep(
|
|
step_type=StepType.GENERATE,
|
|
timestamp=1.0,
|
|
duration_seconds=0.5,
|
|
input={"content": "some input"},
|
|
output={"content": "some output"},
|
|
)
|
|
trace = _make_trace(steps=[step])
|
|
judge.score_trace(trace)
|
|
|
|
call_args = backend.generate.call_args
|
|
prompt = call_args[0][0]
|
|
assert "generate" in prompt
|
|
assert "some input" in prompt
|
|
|
|
def test_uses_system_prompt(self) -> None:
|
|
backend = MagicMock()
|
|
backend.generate.return_value = "0.5"
|
|
judge = TraceJudge(backend=backend, model="m")
|
|
|
|
judge.score_trace(_make_trace())
|
|
|
|
call_args = backend.generate.call_args
|
|
assert call_args[1]["system"] != ""
|
|
|
|
def test_fraction_score_parsing(self) -> None:
|
|
backend = MagicMock()
|
|
backend.generate.return_value = "Rating: 7/10\nDecent answer"
|
|
judge = TraceJudge(backend=backend, model="m")
|
|
|
|
score, _ = judge.score_trace(_make_trace())
|
|
assert abs(score - 0.7) < 1e-6
|
|
|
|
|
|
class TestBatchEvaluate:
|
|
"""TraceJudge.batch_evaluate processes multiple traces."""
|
|
|
|
def test_returns_one_result_per_trace(self) -> None:
|
|
backend = MagicMock()
|
|
backend.generate.return_value = "0.8\nGood"
|
|
judge = TraceJudge(backend=backend, model="m")
|
|
|
|
traces = [_make_trace(trace_id=f"t{i}") for i in range(3)]
|
|
results = judge.batch_evaluate(traces)
|
|
|
|
assert len(results) == 3
|
|
assert all(isinstance(r, tuple) and len(r) == 2 for r in results)
|
|
|
|
def test_calls_score_trace_for_each(self) -> None:
|
|
backend = MagicMock()
|
|
backend.generate.return_value = "0.6\nAverage"
|
|
judge = TraceJudge(backend=backend, model="m")
|
|
|
|
traces = [_make_trace(trace_id=f"t{i}") for i in range(5)]
|
|
judge.batch_evaluate(traces)
|
|
|
|
assert backend.generate.call_count == 5
|
|
|
|
def test_empty_list_returns_empty(self) -> None:
|
|
backend = MagicMock()
|
|
judge = TraceJudge(backend=backend, model="m")
|
|
assert judge.batch_evaluate([]) == []
|
|
|
|
def test_different_scores_per_trace(self) -> None:
|
|
backend = MagicMock()
|
|
backend.generate.side_effect = [
|
|
"Score: 0.9\nExcellent",
|
|
"Score: 0.3\nPoor",
|
|
]
|
|
judge = TraceJudge(backend=backend, model="m")
|
|
|
|
traces = [_make_trace(trace_id="a"), _make_trace(trace_id="b")]
|
|
results = judge.batch_evaluate(traces)
|
|
|
|
assert results[0][0] == 0.9
|
|
assert results[1][0] == 0.3
|