feat: add FakeEngine and scenario_harness fixture for agent lifecycle testing

This commit is contained in:
Jon Saad-Falcon
2026-03-16 20:41:24 -07:00
parent f85ee81d53
commit a3f3f047da
5 changed files with 216 additions and 0 deletions
+47
View File
@@ -0,0 +1,47 @@
"""Shared fixtures for agent tests."""
from __future__ import annotations
import pytest
from openjarvis.agents.executor import AgentExecutor
from openjarvis.agents.manager import AgentManager
from openjarvis.agents.scheduler import AgentScheduler
from openjarvis.core.events import EventBus
from tests.agents.fake_engine import FakeEngine
from tests.agents.scenario_harness import FakeSystem, ScenarioHarness
@pytest.fixture
def scenario_harness(tmp_path):
"""Wire up real components for agent lifecycle testing."""
from openjarvis.agents.monitor_operative import MonitorOperativeAgent
from openjarvis.core.registry import AgentRegistry
# Re-register agent types (conftest auto-clears registries)
if not AgentRegistry.contains("monitor_operative"):
AgentRegistry.register("monitor_operative")(MonitorOperativeAgent)
db_path = str(tmp_path / "agents.db")
bus = EventBus(record_history=True)
manager = AgentManager(db_path=db_path)
engine = FakeEngine([{"content": "Default response."}])
system = FakeSystem(engine=engine)
executor = AgentExecutor(manager=manager, event_bus=bus)
executor.set_system(system)
scheduler = AgentScheduler(
manager=manager, executor=executor, event_bus=bus,
)
return ScenarioHarness(
manager=manager,
executor=executor,
scheduler=scheduler,
bus=bus,
engine=engine,
system=system,
db_path=db_path,
)
+70
View File
@@ -0,0 +1,70 @@
"""Scripted inference engine for deterministic agent testing."""
from __future__ import annotations
from typing import Any, AsyncIterator, Dict, List
from openjarvis.engine._stubs import InferenceEngine
class FakeEngine(InferenceEngine):
"""Returns pre-defined responses in order. Captures prompts for assertions."""
engine_id = "fake"
def __init__(self, responses: list[dict]) -> None:
self._responses = list(responses)
self._call_count = 0
self._last_messages: list | None = None
@property
def call_count(self) -> int:
return self._call_count
@property
def last_messages(self) -> list | None:
return self._last_messages
def generate(
self,
messages: list,
*,
model: str,
temperature: float = 0.7,
max_tokens: int = 1024,
**kw: Any,
) -> Dict[str, Any]:
self._last_messages = messages
idx = min(self._call_count, len(self._responses) - 1)
resp = self._responses[idx]
self._call_count += 1
# Support raising exceptions for error testing
if "raise" in resp:
raise resp["raise"]
result: Dict[str, Any] = {
"content": resp.get("content", ""),
"usage": resp.get("usage", {
"prompt_tokens": 10,
"completion_tokens": 5,
"total_tokens": 15,
}),
"model": model,
"finish_reason": "tool_calls" if resp.get("tool_calls") else "stop",
}
if resp.get("tool_calls"):
result["tool_calls"] = resp["tool_calls"]
return result
async def stream(
self, messages: list, *, model: str, **kw: Any,
) -> AsyncIterator[str]:
result = self.generate(messages, model=model, **kw)
yield result["content"]
def list_models(self) -> List[str]:
return ["fake-model"]
def health(self) -> bool:
return True
+38
View File
@@ -0,0 +1,38 @@
"""Shared harness dataclasses for agent lifecycle scenario tests."""
from __future__ import annotations
from dataclasses import dataclass, field
from typing import Any
from openjarvis.agents.executor import AgentExecutor
from openjarvis.agents.manager import AgentManager
from openjarvis.agents.scheduler import AgentScheduler
from openjarvis.core.events import EventBus
from tests.agents.fake_engine import FakeEngine
@dataclass(slots=True)
class FakeSystem:
"""Lightweight stand-in for JarvisSystem — just engine + model."""
engine: FakeEngine
model: str = "fake-model"
memory_backend: Any = None
channel_backend: Any = None
tools: list = field(default_factory=list)
config: Any = None
session_store: Any = None
@dataclass(slots=True)
class ScenarioHarness:
"""All components needed for an agent lifecycle scenario test."""
manager: AgentManager
executor: AgentExecutor
scheduler: AgentScheduler
bus: EventBus
engine: FakeEngine
system: FakeSystem
db_path: str
+42
View File
@@ -0,0 +1,42 @@
"""Tests for FakeEngine."""
from __future__ import annotations
import pytest
from tests.agents.fake_engine import FakeEngine
def test_fake_engine_returns_responses_in_order():
engine = FakeEngine([
{"content": "first"},
{"content": "second"},
])
r1 = engine.generate([], model="m")
r2 = engine.generate([], model="m")
assert r1["content"] == "first"
assert r2["content"] == "second"
assert engine.call_count == 2
def test_fake_engine_repeats_last_response():
engine = FakeEngine([{"content": "only"}])
engine.generate([], model="m")
r = engine.generate([], model="m")
assert r["content"] == "only"
def test_fake_engine_raises_on_request():
engine = FakeEngine([{"raise": ValueError("boom")}])
with pytest.raises(ValueError, match="boom"):
engine.generate([], model="m")
def test_fake_engine_tool_calls():
engine = FakeEngine([{
"content": "",
"tool_calls": [{"id": "1", "function": {"name": "think", "arguments": "{}"}}],
}])
r = engine.generate([], model="m")
assert r["finish_reason"] == "tool_calls"
assert len(r["tool_calls"]) == 1
@@ -0,0 +1,19 @@
"""Smoke test for the scenario harness fixture."""
from __future__ import annotations
from tests.agents.scenario_harness import ScenarioHarness
def test_harness_creates_and_runs_agent(scenario_harness: ScenarioHarness):
"""Verify the harness wires up correctly — create agent, run tick."""
h = scenario_harness
agent = h.manager.create_agent("Smoke Test", config={
"schedule_type": "manual",
"instruction": "Say hello.",
})
h.executor.execute_tick(agent["id"])
updated = h.manager.get_agent(agent["id"])
assert updated["status"] == "idle"
assert updated["total_runs"] == 1
assert updated["summary_memory"] != ""