mirror of
https://github.com/open-jarvis/OpenJarvis.git
synced 2026-07-28 13:26:48 +00:00
142 lines
5.0 KiB
Python
142 lines
5.0 KiB
Python
"""Tests for SkillOptimizer (Plan 2A) — buckets traces by skill, runs DSPy/GEPA."""
|
|
|
|
from __future__ import annotations
|
|
|
|
from pathlib import Path
|
|
from typing import Any, List
|
|
|
|
from openjarvis.core.events import EventBus
|
|
from openjarvis.core.types import StepType, Trace, TraceStep
|
|
from openjarvis.skills.manager import SkillManager
|
|
|
|
|
|
class _FakeTraceStore:
|
|
def __init__(self, traces: List[Trace]) -> None:
|
|
self._traces = traces
|
|
|
|
def list_traces(self, *, limit: int = 100, **_kwargs: Any) -> List[Trace]:
|
|
return list(self._traces[:limit])
|
|
|
|
|
|
def _make_skill_trace(skill_name: str, feedback: float = 1.0) -> Trace:
|
|
return Trace(
|
|
query=f"query for {skill_name}",
|
|
agent="native_react",
|
|
model="qwen3.5:9b",
|
|
engine="ollama",
|
|
steps=[
|
|
TraceStep(
|
|
step_type=StepType.TOOL_CALL,
|
|
timestamp=0.0,
|
|
input={"tool": f"skill_{skill_name}", "arguments": {}},
|
|
output={"success": True, "result": "ok"},
|
|
metadata={"skill": skill_name, "skill_kind": "instructional"},
|
|
),
|
|
],
|
|
outcome="success",
|
|
feedback=feedback,
|
|
result="result text",
|
|
)
|
|
|
|
|
|
def _make_manager_with_skill(name: str, tmp_path: Path) -> SkillManager:
|
|
skill_dir = tmp_path / "skills" / name
|
|
skill_dir.mkdir(parents=True)
|
|
(skill_dir / "SKILL.md").write_text(
|
|
f"---\nname: {name}\ndescription: Original description\n---\nBody"
|
|
)
|
|
mgr = SkillManager(bus=EventBus())
|
|
mgr.discover(paths=[tmp_path / "skills"])
|
|
return mgr
|
|
|
|
|
|
class TestSkillOptimizerBucketing:
|
|
def test_buckets_traces_by_skill_name(self, tmp_path: Path):
|
|
from openjarvis.learning.agents.skill_optimizer import SkillOptimizer
|
|
|
|
traces = [
|
|
_make_skill_trace("research-skill"),
|
|
_make_skill_trace("research-skill"),
|
|
_make_skill_trace("code-skill"),
|
|
]
|
|
optimizer = SkillOptimizer(min_traces_per_skill=1)
|
|
buckets = optimizer._bucket_traces_by_skill(traces)
|
|
|
|
assert "research-skill" in buckets
|
|
assert "code-skill" in buckets
|
|
assert len(buckets["research-skill"]) == 2
|
|
assert len(buckets["code-skill"]) == 1
|
|
|
|
def test_skips_traces_without_skill_metadata(self, tmp_path: Path):
|
|
from openjarvis.learning.agents.skill_optimizer import SkillOptimizer
|
|
|
|
# A trace with no skill metadata in the tool call
|
|
plain_trace = Trace(
|
|
query="plain",
|
|
steps=[
|
|
TraceStep(
|
|
step_type=StepType.TOOL_CALL,
|
|
timestamp=0.0,
|
|
input={"tool": "calculator", "arguments": {}},
|
|
output={"success": True, "result": "42"},
|
|
metadata={},
|
|
),
|
|
],
|
|
)
|
|
optimizer = SkillOptimizer(min_traces_per_skill=1)
|
|
buckets = optimizer._bucket_traces_by_skill([plain_trace])
|
|
assert buckets == {}
|
|
|
|
|
|
class TestSkillOptimizerOptimize:
|
|
def test_skips_skills_below_min_traces(self, tmp_path: Path):
|
|
from openjarvis.learning.agents.skill_optimizer import SkillOptimizer
|
|
|
|
traces = [_make_skill_trace("research-skill") for _ in range(3)]
|
|
store = _FakeTraceStore(traces)
|
|
mgr = _make_manager_with_skill("research-skill", tmp_path)
|
|
|
|
optimizer = SkillOptimizer(min_traces_per_skill=20)
|
|
results = optimizer.optimize(store, mgr, overlay_dir=tmp_path / "overlays")
|
|
|
|
assert "research-skill" in results
|
|
assert results["research-skill"].status == "skipped"
|
|
assert results["research-skill"].trace_count == 3
|
|
|
|
def test_optimizes_skill_with_enough_traces(
|
|
self, tmp_path: Path, monkeypatch: Any
|
|
) -> None:
|
|
from openjarvis.learning.agents.skill_optimizer import (
|
|
SkillOptimizer,
|
|
_OptimizerOutput,
|
|
)
|
|
|
|
traces = [_make_skill_trace("research-skill") for _ in range(25)]
|
|
store = _FakeTraceStore(traces)
|
|
mgr = _make_manager_with_skill("research-skill", tmp_path)
|
|
|
|
# Mock the underlying DSPy call
|
|
def fake_run(self_unused, skill_name, skill_traces):
|
|
return _OptimizerOutput(
|
|
description="An optimized description",
|
|
few_shot=[
|
|
{"input": "hello", "output": "world"},
|
|
],
|
|
)
|
|
|
|
monkeypatch.setattr(SkillOptimizer, "_run_dspy", fake_run)
|
|
|
|
optimizer = SkillOptimizer(min_traces_per_skill=10)
|
|
results = optimizer.optimize(store, mgr, overlay_dir=tmp_path / "overlays")
|
|
|
|
assert results["research-skill"].status == "optimized"
|
|
assert results["research-skill"].trace_count == 25
|
|
# Overlay file should exist
|
|
overlay_path = tmp_path / "overlays" / "research-skill" / "optimized.toml"
|
|
assert overlay_path.exists()
|
|
# And contain the optimized description
|
|
content = overlay_path.read_text()
|
|
assert "An optimized description" in content
|
|
assert "hello" in content
|
|
assert "world" in content
|