diff --git a/src/openjarvis/optimize/personal/__init__.py b/src/openjarvis/optimize/personal/__init__.py new file mode 100644 index 00000000..f22ff8e5 --- /dev/null +++ b/src/openjarvis/optimize/personal/__init__.py @@ -0,0 +1,17 @@ +"""Personal benchmark system -- synthesize benchmarks from interaction traces.""" + +from openjarvis.optimize.personal.dataset import PersonalBenchmarkDataset +from openjarvis.optimize.personal.scorer import PersonalBenchmarkScorer +from openjarvis.optimize.personal.synthesizer import ( + PersonalBenchmark, + PersonalBenchmarkSample, + PersonalBenchmarkSynthesizer, +) + +__all__ = [ + "PersonalBenchmark", + "PersonalBenchmarkSample", + "PersonalBenchmarkSynthesizer", + "PersonalBenchmarkDataset", + "PersonalBenchmarkScorer", +] diff --git a/src/openjarvis/optimize/personal/dataset.py b/src/openjarvis/optimize/personal/dataset.py new file mode 100644 index 00000000..efe6e540 --- /dev/null +++ b/src/openjarvis/optimize/personal/dataset.py @@ -0,0 +1,54 @@ +"""DatasetProvider adapter for personal benchmarks.""" + +from __future__ import annotations + +from typing import Iterable, List, Optional + +from openjarvis.evals.core.dataset import DatasetProvider +from openjarvis.evals.core.types import EvalRecord +from openjarvis.optimize.personal.synthesizer import PersonalBenchmark + + +class PersonalBenchmarkDataset(DatasetProvider): + """Wraps a PersonalBenchmark as a DatasetProvider for EvalRunner.""" + + dataset_id: str = "personal" + dataset_name: str = "Personal Benchmark" + + def __init__(self, benchmark: PersonalBenchmark) -> None: + self._benchmark = benchmark + self._records: List[EvalRecord] = [] + + def load( + self, + *, + max_samples: Optional[int] = None, + split: Optional[str] = None, + seed: Optional[int] = None, + ) -> None: + """Convert :class:`PersonalBenchmarkSample` instances to :class:`EvalRecord`.""" + samples = self._benchmark.samples + if max_samples is not None: + samples = samples[:max_samples] + self._records = [ + EvalRecord( + record_id=s.trace_id, + problem=s.query, + reference=s.reference_answer, + category=s.category, + subject=s.agent or "general", + metadata=s.metadata, + ) + for s in samples + ] + + def iter_records(self) -> Iterable[EvalRecord]: + """Iterate over loaded records.""" + return iter(self._records) + + def size(self) -> int: + """Return the number of loaded records.""" + return len(self._records) + + +__all__ = ["PersonalBenchmarkDataset"] diff --git a/src/openjarvis/optimize/personal/scorer.py b/src/openjarvis/optimize/personal/scorer.py new file mode 100644 index 00000000..8ed0bec1 --- /dev/null +++ b/src/openjarvis/optimize/personal/scorer.py @@ -0,0 +1,47 @@ +"""LLM-judge scorer for personal benchmarks.""" + +from __future__ import annotations + +from typing import Any, Dict, Optional, Tuple + +from openjarvis.evals.core.backend import InferenceBackend +from openjarvis.evals.core.scorer import LLMJudgeScorer +from openjarvis.evals.core.types import EvalRecord + + +class PersonalBenchmarkScorer(LLMJudgeScorer): + """Judges a candidate response against the best-known response from traces.""" + + scorer_id: str = "personal_judge" + + def __init__(self, judge_backend: InferenceBackend, judge_model: str) -> None: + super().__init__(judge_backend, judge_model) + + def score( + self, record: EvalRecord, model_answer: str, + ) -> Tuple[Optional[bool], Dict[str, Any]]: + """Compare *model_answer* against *record.reference* using the judge LLM. + + Returns ``(is_correct, metadata)`` where *is_correct* indicates whether + the candidate answer is at least as good as the reference. + """ + prompt = ( + "Compare these two answers to the query.\n\n" + f"Query: {record.problem}\n\n" + "Reference answer (known good):\n" + f"{record.reference}\n\n" + "Candidate answer:\n" + f"{model_answer}\n\n" + "Is the candidate answer at least as good as the reference? " + 'Respond with exactly "YES" or "NO" on the first line, ' + "then explain your reasoning." + ) + response = self._ask_judge( + prompt, system="You are an impartial quality judge.", + ) + first_line = response.strip().split("\n")[0].strip().upper() + is_correct = first_line.startswith("YES") + return is_correct, {"judge_response": response} + + +__all__ = ["PersonalBenchmarkScorer"] diff --git a/src/openjarvis/optimize/personal/synthesizer.py b/src/openjarvis/optimize/personal/synthesizer.py new file mode 100644 index 00000000..ecd0f289 --- /dev/null +++ b/src/openjarvis/optimize/personal/synthesizer.py @@ -0,0 +1,119 @@ +"""Synthesize personal benchmarks from interaction traces.""" + +from __future__ import annotations + +import time +from collections import defaultdict +from dataclasses import dataclass, field +from typing import Any, Dict, List + +from openjarvis.traces.store import TraceStore + + +@dataclass(slots=True) +class PersonalBenchmarkSample: + """A single sample in a personal benchmark.""" + + trace_id: str + query: str + reference_answer: str # best known answer from traces + agent: str = "" + category: str = "chat" + feedback_score: float = 0.0 + metadata: Dict[str, Any] = field(default_factory=dict) + + +@dataclass(slots=True) +class PersonalBenchmark: + """A synthesized benchmark from user interaction traces.""" + + workflow_id: str + samples: List[PersonalBenchmarkSample] = field(default_factory=list) + created_at: float = 0.0 + + +def _query_class_key(agent: str, query: str) -> str: + """Compute a grouping key from agent name and query prefix.""" + prefix = query[:50].strip().lower() + return f"{agent}::{prefix}" + + +def _infer_category(agent: str) -> str: + """Heuristic to map an agent name to an eval category.""" + agent_lower = agent.lower() + if any(tok in agent_lower for tok in ("react", "openhands", "orchestrator")): + return "agentic" + if any(tok in agent_lower for tok in ("rag", "memory", "retriev")): + return "rag" + if any(tok in agent_lower for tok in ("reason", "math", "code")): + return "reasoning" + return "chat" + + +class PersonalBenchmarkSynthesizer: + """Mines interaction traces into a reusable personal benchmark.""" + + def __init__(self, trace_store: TraceStore) -> None: + self._store = trace_store + + def synthesize( + self, + workflow_id: str = "default", + min_feedback: float = 0.7, + max_samples: int = 100, + ) -> PersonalBenchmark: + """Build a personal benchmark from high-quality traces. + + 1. Query traces that have feedback >= *min_feedback*. + 2. Group by query class (agent + first 50 chars of query). + 3. For each class, pick the trace with the highest feedback as reference. + 4. Return a :class:`PersonalBenchmark` capped at *max_samples*. + """ + # Fetch a large pool of traces (limit high enough to cover most stores) + all_traces = self._store.list_traces(limit=10_000) + + # Filter to traces with sufficient feedback + qualified = [ + t + for t in all_traces + if t.feedback is not None and t.feedback >= min_feedback + ] + + # Group by query class + groups: Dict[str, list] = defaultdict(list) + for trace in qualified: + key = _query_class_key(trace.agent, trace.query) + groups[key].append(trace) + + # Pick best trace per class + samples: List[PersonalBenchmarkSample] = [] + for _key, traces in groups.items(): + best = max(traces, key=lambda t: t.feedback or 0.0) + samples.append( + PersonalBenchmarkSample( + trace_id=best.trace_id, + query=best.query, + reference_answer=best.result, + agent=best.agent, + category=_infer_category(best.agent), + feedback_score=best.feedback or 0.0, + metadata=best.metadata, + ), + ) + + # Sort deterministically (highest feedback first) and cap + samples.sort(key=lambda s: (-s.feedback_score, s.trace_id)) + samples = samples[:max_samples] + + return PersonalBenchmark( + workflow_id=workflow_id, + samples=samples, + created_at=time.time(), + ) + + +__all__ = [ + "PersonalBenchmark", + "PersonalBenchmarkSample", + "PersonalBenchmarkSynthesizer", +] diff --git a/tests/test_personal_synthesizer.py b/tests/test_personal_synthesizer.py new file mode 100644 index 00000000..ba6dd1c5 --- /dev/null +++ b/tests/test_personal_synthesizer.py @@ -0,0 +1,357 @@ +"""Tests for the personal benchmark system.""" + +from __future__ import annotations + +import tempfile +from pathlib import Path +from typing import Any, Dict +from unittest.mock import MagicMock + +import pytest + +from openjarvis.core.types import Trace +from openjarvis.evals.core.types import EvalRecord +from openjarvis.optimize.personal.dataset import PersonalBenchmarkDataset +from openjarvis.optimize.personal.scorer import PersonalBenchmarkScorer +from openjarvis.optimize.personal.synthesizer import ( + PersonalBenchmark, + PersonalBenchmarkSample, + PersonalBenchmarkSynthesizer, +) +from openjarvis.traces.store import TraceStore + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +def _make_trace( + trace_id: str = "t1", + query: str = "What is 2+2?", + agent: str = "simple", + result: str = "4", + feedback: float | None = 0.9, + model: str = "test-model", + engine: str = "test-engine", + metadata: Dict[str, Any] | None = None, +) -> Trace: + return Trace( + trace_id=trace_id, + query=query, + agent=agent, + result=result, + feedback=feedback, + model=model, + engine=engine, + metadata=metadata or {}, + ) + + +@pytest.fixture() +def trace_store(tmp_path: Path) -> TraceStore: + """Provide a fresh TraceStore backed by a temporary SQLite database.""" + return TraceStore(tmp_path / "traces.db") + + +# --------------------------------------------------------------------------- +# PersonalBenchmarkSample defaults +# --------------------------------------------------------------------------- + + +class TestPersonalBenchmarkSampleDefaults: + def test_default_category(self) -> None: + sample = PersonalBenchmarkSample( + trace_id="t1", query="hello", reference_answer="world", + ) + assert sample.category == "chat" + + def test_default_agent_empty(self) -> None: + sample = PersonalBenchmarkSample( + trace_id="t1", query="q", reference_answer="a", + ) + assert sample.agent == "" + + def test_default_feedback_score_zero(self) -> None: + sample = PersonalBenchmarkSample( + trace_id="t1", query="q", reference_answer="a", + ) + assert sample.feedback_score == 0.0 + + def test_default_metadata_empty(self) -> None: + sample = PersonalBenchmarkSample( + trace_id="t1", query="q", reference_answer="a", + ) + assert sample.metadata == {} + + +# --------------------------------------------------------------------------- +# PersonalBenchmarkSynthesizer +# --------------------------------------------------------------------------- + + +class TestPersonalBenchmarkSynthesizer: + def test_synthesize_creates_benchmark_from_traces( + self, trace_store: TraceStore, + ) -> None: + trace_store.save(_make_trace(trace_id="t1", feedback=0.9)) + trace_store.save(_make_trace(trace_id="t2", query="What is 3+3?", feedback=0.8)) + synth = PersonalBenchmarkSynthesizer(trace_store) + bm = synth.synthesize(workflow_id="wf1") + assert bm.workflow_id == "wf1" + assert len(bm.samples) == 2 + assert bm.created_at > 0 + + def test_filter_by_min_feedback(self, trace_store: TraceStore) -> None: + trace_store.save(_make_trace(trace_id="t1", feedback=0.9)) + trace_store.save(_make_trace(trace_id="t2", feedback=0.5)) + trace_store.save(_make_trace(trace_id="t3", feedback=0.3)) + synth = PersonalBenchmarkSynthesizer(trace_store) + bm = synth.synthesize(min_feedback=0.7) + assert len(bm.samples) == 1 + assert bm.samples[0].trace_id == "t1" + + def test_none_feedback_excluded(self, trace_store: TraceStore) -> None: + trace_store.save(_make_trace(trace_id="t1", feedback=None)) + trace_store.save(_make_trace(trace_id="t2", feedback=0.8)) + synth = PersonalBenchmarkSynthesizer(trace_store) + bm = synth.synthesize(min_feedback=0.5) + assert len(bm.samples) == 1 + assert bm.samples[0].trace_id == "t2" + + def test_grouping_by_query_class(self, trace_store: TraceStore) -> None: + """Same agent + same query prefix -> same group, so only one sample.""" + trace_store.save( + _make_trace(trace_id="t1", query="What is 2+2?", agent="simple", feedback=0.8), + ) + trace_store.save( + _make_trace(trace_id="t2", query="What is 2+2?", agent="simple", feedback=0.95), + ) + synth = PersonalBenchmarkSynthesizer(trace_store) + bm = synth.synthesize() + # Should collapse into one sample (same group) + assert len(bm.samples) == 1 + + def test_picks_highest_feedback_per_group(self, trace_store: TraceStore) -> None: + trace_store.save( + _make_trace(trace_id="t1", query="Tell me a joke", agent="simple", feedback=0.7, result="bad joke"), + ) + trace_store.save( + _make_trace(trace_id="t2", query="Tell me a joke", agent="simple", feedback=0.99, result="great joke"), + ) + synth = PersonalBenchmarkSynthesizer(trace_store) + bm = synth.synthesize() + assert len(bm.samples) == 1 + assert bm.samples[0].trace_id == "t2" + assert bm.samples[0].reference_answer == "great joke" + assert bm.samples[0].feedback_score == 0.99 + + def test_different_agents_separate_groups(self, trace_store: TraceStore) -> None: + trace_store.save( + _make_trace(trace_id="t1", query="Hello", agent="simple", feedback=0.9), + ) + trace_store.save( + _make_trace(trace_id="t2", query="Hello", agent="orchestrator", feedback=0.8), + ) + synth = PersonalBenchmarkSynthesizer(trace_store) + bm = synth.synthesize() + assert len(bm.samples) == 2 + + def test_max_samples_limit(self, trace_store: TraceStore) -> None: + for i in range(10): + trace_store.save( + _make_trace( + trace_id=f"t{i}", + query=f"Unique question number {i}", + feedback=0.8 + i * 0.01, + ), + ) + synth = PersonalBenchmarkSynthesizer(trace_store) + bm = synth.synthesize(max_samples=3) + assert len(bm.samples) == 3 + + def test_empty_traces_returns_empty_benchmark( + self, trace_store: TraceStore, + ) -> None: + synth = PersonalBenchmarkSynthesizer(trace_store) + bm = synth.synthesize() + assert bm.samples == [] + assert bm.workflow_id == "default" + + def test_category_inferred_from_agent(self, trace_store: TraceStore) -> None: + trace_store.save( + _make_trace(trace_id="t1", agent="orchestrator", feedback=0.9), + ) + trace_store.save( + _make_trace( + trace_id="t2", + query="Different query", + agent="simple", + feedback=0.9, + ), + ) + synth = PersonalBenchmarkSynthesizer(trace_store) + bm = synth.synthesize() + categories = {s.agent: s.category for s in bm.samples} + assert categories["orchestrator"] == "agentic" + assert categories["simple"] == "chat" + + def test_samples_sorted_by_feedback_desc(self, trace_store: TraceStore) -> None: + trace_store.save( + _make_trace(trace_id="t1", query="Q1", feedback=0.75), + ) + trace_store.save( + _make_trace(trace_id="t2", query="Q2", feedback=0.95), + ) + trace_store.save( + _make_trace(trace_id="t3", query="Q3", feedback=0.85), + ) + synth = PersonalBenchmarkSynthesizer(trace_store) + bm = synth.synthesize(min_feedback=0.7) + scores = [s.feedback_score for s in bm.samples] + assert scores == sorted(scores, reverse=True) + + +# --------------------------------------------------------------------------- +# PersonalBenchmarkDataset +# --------------------------------------------------------------------------- + + +class TestPersonalBenchmarkDataset: + def _make_benchmark(self) -> PersonalBenchmark: + return PersonalBenchmark( + workflow_id="test", + samples=[ + PersonalBenchmarkSample( + trace_id=f"t{i}", + query=f"Query {i}", + reference_answer=f"Answer {i}", + agent="simple", + category="chat", + feedback_score=0.9, + ) + for i in range(5) + ], + ) + + def test_load_creates_records(self) -> None: + ds = PersonalBenchmarkDataset(self._make_benchmark()) + ds.load() + assert ds.size() == 5 + + def test_iter_records(self) -> None: + ds = PersonalBenchmarkDataset(self._make_benchmark()) + ds.load() + records = list(ds.iter_records()) + assert len(records) == 5 + assert all(isinstance(r, EvalRecord) for r in records) + + def test_record_fields_mapped(self) -> None: + ds = PersonalBenchmarkDataset(self._make_benchmark()) + ds.load() + rec = list(ds.iter_records())[0] + assert rec.record_id == "t0" + assert rec.problem == "Query 0" + assert rec.reference == "Answer 0" + assert rec.category == "chat" + assert rec.subject == "simple" + + def test_max_samples(self) -> None: + ds = PersonalBenchmarkDataset(self._make_benchmark()) + ds.load(max_samples=2) + assert ds.size() == 2 + + def test_size_before_load(self) -> None: + ds = PersonalBenchmarkDataset(self._make_benchmark()) + assert ds.size() == 0 + + def test_dataset_id_and_name(self) -> None: + ds = PersonalBenchmarkDataset(self._make_benchmark()) + assert ds.dataset_id == "personal" + assert ds.dataset_name == "Personal Benchmark" + + def test_empty_benchmark(self) -> None: + bm = PersonalBenchmark(workflow_id="empty") + ds = PersonalBenchmarkDataset(bm) + ds.load() + assert ds.size() == 0 + assert list(ds.iter_records()) == [] + + def test_subject_defaults_to_general(self) -> None: + bm = PersonalBenchmark( + workflow_id="test", + samples=[ + PersonalBenchmarkSample( + trace_id="t1", + query="q", + reference_answer="a", + agent="", # empty agent + ), + ], + ) + ds = PersonalBenchmarkDataset(bm) + ds.load() + rec = list(ds.iter_records())[0] + assert rec.subject == "general" + + +# --------------------------------------------------------------------------- +# PersonalBenchmarkScorer +# --------------------------------------------------------------------------- + + +class TestPersonalBenchmarkScorer: + def _make_scorer(self, judge_response: str) -> PersonalBenchmarkScorer: + backend = MagicMock() + backend.generate.return_value = judge_response + return PersonalBenchmarkScorer(backend, "judge-model") + + def _make_record(self) -> EvalRecord: + return EvalRecord( + record_id="r1", + problem="What is 2+2?", + reference="4", + category="chat", + ) + + def test_score_yes(self) -> None: + scorer = self._make_scorer("YES\nThe answer is correct.") + is_correct, meta = scorer.score(self._make_record(), "4") + assert is_correct is True + assert "judge_response" in meta + + def test_score_no(self) -> None: + scorer = self._make_scorer("NO\nThe answer is incorrect.") + is_correct, meta = scorer.score(self._make_record(), "5") + assert is_correct is False + assert "judge_response" in meta + + def test_score_yes_case_insensitive(self) -> None: + scorer = self._make_scorer("yes, the answer is good") + is_correct, _ = scorer.score(self._make_record(), "4") + assert is_correct is True + + def test_score_no_multiline(self) -> None: + scorer = self._make_scorer("NO\nLine2\nLine3") + is_correct, _ = scorer.score(self._make_record(), "wrong") + assert is_correct is False + + def test_judge_receives_prompt_with_query_and_reference(self) -> None: + backend = MagicMock() + backend.generate.return_value = "YES" + scorer = PersonalBenchmarkScorer(backend, "judge-model") + record = self._make_record() + scorer.score(record, "4") + call_args = backend.generate.call_args + prompt = call_args[0][0] + assert "What is 2+2?" in prompt + assert "4" in prompt # reference + + def test_scorer_id(self) -> None: + scorer = self._make_scorer("YES") + assert scorer.scorer_id == "personal_judge" + + def test_score_empty_response_treated_as_no(self) -> None: + scorer = self._make_scorer("") + is_correct, _ = scorer.score(self._make_record(), "4") + assert is_correct is False