From 04205d8f87b262ac795c1787cf505a2f53faa493 Mon Sep 17 00:00:00 2001 From: Jon Saad-Falcon <41205309+jonsaadfalcon@users.noreply.github.com> Date: Mon, 16 Mar 2026 20:36:15 -0700 Subject: [PATCH 01/44] fix: normalize TOML arrays to comma-separated strings for str-typed config fields --- src/openjarvis/core/config.py | 8 +++++++- src/openjarvis/scheduler/scheduler.py | 7 ++++++- tests/core/test_config.py | 11 +++++++++++ 3 files changed, 24 insertions(+), 2 deletions(-) diff --git a/src/openjarvis/core/config.py b/src/openjarvis/core/config.py index 1067c1a4..55c36164 100644 --- a/src/openjarvis/core/config.py +++ b/src/openjarvis/core/config.py @@ -1100,7 +1100,8 @@ def _apply_toml_section(target: Any, section: Dict[str, Any]) -> None: """Overlay TOML key/value pairs onto a dataclass instance. Recursively handles nested dicts when the target attribute is itself - a dataclass. + a dataclass. Normalises TOML arrays to comma-separated strings when + the target field is annotated as ``str``. """ for key, value in section.items(): if hasattr(target, key): @@ -1111,6 +1112,11 @@ def _apply_toml_section(target: Any, section: Dict[str, Any]) -> None: else: setattr(target, key, value) else: + # Normalise TOML arrays → comma-separated string when field is str + if isinstance(value, list) and hasattr(target, "__dataclass_fields__"): + field_obj = target.__dataclass_fields__.get(key) + if field_obj is not None and field_obj.type in ("str", str): + value = ",".join(str(v) for v in value) setattr(target, key, value) diff --git a/src/openjarvis/scheduler/scheduler.py b/src/openjarvis/scheduler/scheduler.py index 3e04b5ef..8702ee21 100644 --- a/src/openjarvis/scheduler/scheduler.py +++ b/src/openjarvis/scheduler/scheduler.py @@ -218,8 +218,13 @@ class TaskScheduler: try: if self._system is not None: + raw_tools = ( + task.tools + if isinstance(task.tools, list) + else task.tools.split(",") + ) tools_list = ( - [t.strip() for t in task.tools.split(",") if t.strip()] + [t.strip() for t in raw_tools if t.strip()] if task.tools else [] ) diff --git a/tests/core/test_config.py b/tests/core/test_config.py index 28407d26..7034cb46 100644 --- a/tests/core/test_config.py +++ b/tests/core/test_config.py @@ -449,6 +449,17 @@ class TestSchedulerConfig: # --------------------------------------------------------------------------- +class TestApplyTomlSectionListNormalization: + def test_apply_toml_section_list_to_str_field(self) -> None: + """TOML arrays assigned to str-typed fields should be joined with ','.""" + from openjarvis.core.config import ToolsConfig, _apply_toml_section + + target = ToolsConfig() + _apply_toml_section(target, {"enabled": ["code_interpreter", "web_search", "file_read"]}) + assert isinstance(target.enabled, str) + assert target.enabled == "code_interpreter,web_search,file_read" + + class TestWhatsAppBaileysChannelConfig: def test_defaults(self) -> None: wc = WhatsAppBaileysChannelConfig() From f85ee81d532e3a38985fe124302b923555e32139 Mon Sep 17 00:00:00 2001 From: Jon Saad-Falcon <41205309+jonsaadfalcon@users.noreply.github.com> Date: Mon, 16 Mar 2026 20:38:10 -0700 Subject: [PATCH 02/44] fix: prevent race on rapid Run Now clicks, auto-recover error-state agents, fix immediate messages - Acquire tick BEFORE spawning thread to prevent race condition on concurrent Run Now clicks - Auto-recover agents in error/needs_attention state when Run Now is clicked - Auto-recover error-state agents when receiving immediate messages - End tick on system build failure to avoid stuck running state - Add test verifying concurrent start_tick raises ValueError --- src/openjarvis/server/agent_manager_routes.py | 23 +++++++++++++++++-- tests/server/test_agent_manager_routes.py | 18 +++++++++++++++ 2 files changed, 39 insertions(+), 2 deletions(-) diff --git a/src/openjarvis/server/agent_manager_routes.py b/src/openjarvis/server/agent_manager_routes.py index 45a93f1a..544ee707 100644 --- a/src/openjarvis/server/agent_manager_routes.py +++ b/src/openjarvis/server/agent_manager_routes.py @@ -287,8 +287,18 @@ def create_agent_manager_router( raise HTTPException(status_code=404, detail="Agent not found") if agent["status"] == "archived": raise HTTPException(status_code=400, detail="Agent is archived") - if agent["status"] == "running": - raise HTTPException(status_code=409, detail="Agent is already running") + + # Auto-recover from error/needs_attention state + if agent["status"] in ("error", "needs_attention"): + manager.update_agent(agent_id, status="idle") + + # Acquire tick BEFORE spawning thread — prevents race + try: + manager.start_tick(agent_id) + except ValueError: + raise HTTPException( + status_code=409, detail="Agent is already running" + ) def _run_tick(): try: @@ -303,6 +313,7 @@ def create_agent_manager_router( system = SystemBuilder().build() executor.set_system(system) except Exception as build_err: + manager.end_tick(agent_id) manager.update_agent(agent_id, status="error") manager.update_summary_memory( agent_id, @@ -400,6 +411,14 @@ def create_agent_manager_router( @agents_router.post("/{agent_id}/messages") def send_message(agent_id: str, req: SendMessageRequest): + agent = manager.get_agent(agent_id) + if not agent: + raise HTTPException(status_code=404, detail="Agent not found") + + # Auto-recover error-state agents on immediate messages + if req.mode == "immediate" and agent["status"] in ("error", "needs_attention"): + manager.update_agent(agent_id, status="idle") + msg = manager.send_message(agent_id, req.content, mode=req.mode) return msg diff --git a/tests/server/test_agent_manager_routes.py b/tests/server/test_agent_manager_routes.py index b694edd5..80e78b6f 100644 --- a/tests/server/test_agent_manager_routes.py +++ b/tests/server/test_agent_manager_routes.py @@ -191,3 +191,21 @@ class TestAgentManagerRoutes: assert "channels" in state assert "messages" in state assert "checkpoint" in state + + +def test_run_agent_concurrent_returns_409(tmp_path): + """Rapid Run Now clicks should not spawn multiple ticks.""" + from openjarvis.agents.manager import AgentManager + + mgr = AgentManager(db_path=str(tmp_path / "test.db")) + agent = mgr.create_agent("Test", config={"schedule_type": "manual"}) + aid = agent["id"] + + # Simulate first click acquiring the tick + mgr.start_tick(aid) + + # Second click should fail + with pytest.raises(ValueError, match="already executing a tick"): + mgr.start_tick(aid) + + mgr.end_tick(aid) From a3f3f047dac871acb91be1701a78efc603b5cc8c Mon Sep 17 00:00:00 2001 From: Jon Saad-Falcon <41205309+jonsaadfalcon@users.noreply.github.com> Date: Mon, 16 Mar 2026 20:41:24 -0700 Subject: [PATCH 03/44] feat: add FakeEngine and scenario_harness fixture for agent lifecycle testing --- tests/agents/conftest.py | 47 ++++++++++++++ tests/agents/fake_engine.py | 70 +++++++++++++++++++++ tests/agents/scenario_harness.py | 38 +++++++++++ tests/agents/test_fake_engine.py | 42 +++++++++++++ tests/agents/test_scenario_harness_smoke.py | 19 ++++++ 5 files changed, 216 insertions(+) create mode 100644 tests/agents/conftest.py create mode 100644 tests/agents/fake_engine.py create mode 100644 tests/agents/scenario_harness.py create mode 100644 tests/agents/test_fake_engine.py create mode 100644 tests/agents/test_scenario_harness_smoke.py diff --git a/tests/agents/conftest.py b/tests/agents/conftest.py new file mode 100644 index 00000000..41277334 --- /dev/null +++ b/tests/agents/conftest.py @@ -0,0 +1,47 @@ +"""Shared fixtures for agent tests.""" + +from __future__ import annotations + +import pytest + +from openjarvis.agents.executor import AgentExecutor +from openjarvis.agents.manager import AgentManager +from openjarvis.agents.scheduler import AgentScheduler +from openjarvis.core.events import EventBus +from tests.agents.fake_engine import FakeEngine +from tests.agents.scenario_harness import FakeSystem, ScenarioHarness + + +@pytest.fixture +def scenario_harness(tmp_path): + """Wire up real components for agent lifecycle testing.""" + from openjarvis.agents.monitor_operative import MonitorOperativeAgent + from openjarvis.core.registry import AgentRegistry + + # Re-register agent types (conftest auto-clears registries) + if not AgentRegistry.contains("monitor_operative"): + AgentRegistry.register("monitor_operative")(MonitorOperativeAgent) + + db_path = str(tmp_path / "agents.db") + bus = EventBus(record_history=True) + manager = AgentManager(db_path=db_path) + + engine = FakeEngine([{"content": "Default response."}]) + system = FakeSystem(engine=engine) + + executor = AgentExecutor(manager=manager, event_bus=bus) + executor.set_system(system) + + scheduler = AgentScheduler( + manager=manager, executor=executor, event_bus=bus, + ) + + return ScenarioHarness( + manager=manager, + executor=executor, + scheduler=scheduler, + bus=bus, + engine=engine, + system=system, + db_path=db_path, + ) diff --git a/tests/agents/fake_engine.py b/tests/agents/fake_engine.py new file mode 100644 index 00000000..805b05a4 --- /dev/null +++ b/tests/agents/fake_engine.py @@ -0,0 +1,70 @@ +"""Scripted inference engine for deterministic agent testing.""" + +from __future__ import annotations + +from typing import Any, AsyncIterator, Dict, List + +from openjarvis.engine._stubs import InferenceEngine + + +class FakeEngine(InferenceEngine): + """Returns pre-defined responses in order. Captures prompts for assertions.""" + + engine_id = "fake" + + def __init__(self, responses: list[dict]) -> None: + self._responses = list(responses) + self._call_count = 0 + self._last_messages: list | None = None + + @property + def call_count(self) -> int: + return self._call_count + + @property + def last_messages(self) -> list | None: + return self._last_messages + + def generate( + self, + messages: list, + *, + model: str, + temperature: float = 0.7, + max_tokens: int = 1024, + **kw: Any, + ) -> Dict[str, Any]: + self._last_messages = messages + idx = min(self._call_count, len(self._responses) - 1) + resp = self._responses[idx] + self._call_count += 1 + + # Support raising exceptions for error testing + if "raise" in resp: + raise resp["raise"] + + result: Dict[str, Any] = { + "content": resp.get("content", ""), + "usage": resp.get("usage", { + "prompt_tokens": 10, + "completion_tokens": 5, + "total_tokens": 15, + }), + "model": model, + "finish_reason": "tool_calls" if resp.get("tool_calls") else "stop", + } + if resp.get("tool_calls"): + result["tool_calls"] = resp["tool_calls"] + return result + + async def stream( + self, messages: list, *, model: str, **kw: Any, + ) -> AsyncIterator[str]: + result = self.generate(messages, model=model, **kw) + yield result["content"] + + def list_models(self) -> List[str]: + return ["fake-model"] + + def health(self) -> bool: + return True diff --git a/tests/agents/scenario_harness.py b/tests/agents/scenario_harness.py new file mode 100644 index 00000000..0e387175 --- /dev/null +++ b/tests/agents/scenario_harness.py @@ -0,0 +1,38 @@ +"""Shared harness dataclasses for agent lifecycle scenario tests.""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Any + +from openjarvis.agents.executor import AgentExecutor +from openjarvis.agents.manager import AgentManager +from openjarvis.agents.scheduler import AgentScheduler +from openjarvis.core.events import EventBus +from tests.agents.fake_engine import FakeEngine + + +@dataclass(slots=True) +class FakeSystem: + """Lightweight stand-in for JarvisSystem — just engine + model.""" + + engine: FakeEngine + model: str = "fake-model" + memory_backend: Any = None + channel_backend: Any = None + tools: list = field(default_factory=list) + config: Any = None + session_store: Any = None + + +@dataclass(slots=True) +class ScenarioHarness: + """All components needed for an agent lifecycle scenario test.""" + + manager: AgentManager + executor: AgentExecutor + scheduler: AgentScheduler + bus: EventBus + engine: FakeEngine + system: FakeSystem + db_path: str diff --git a/tests/agents/test_fake_engine.py b/tests/agents/test_fake_engine.py new file mode 100644 index 00000000..e0815612 --- /dev/null +++ b/tests/agents/test_fake_engine.py @@ -0,0 +1,42 @@ +"""Tests for FakeEngine.""" + +from __future__ import annotations + +import pytest + +from tests.agents.fake_engine import FakeEngine + + +def test_fake_engine_returns_responses_in_order(): + engine = FakeEngine([ + {"content": "first"}, + {"content": "second"}, + ]) + r1 = engine.generate([], model="m") + r2 = engine.generate([], model="m") + assert r1["content"] == "first" + assert r2["content"] == "second" + assert engine.call_count == 2 + + +def test_fake_engine_repeats_last_response(): + engine = FakeEngine([{"content": "only"}]) + engine.generate([], model="m") + r = engine.generate([], model="m") + assert r["content"] == "only" + + +def test_fake_engine_raises_on_request(): + engine = FakeEngine([{"raise": ValueError("boom")}]) + with pytest.raises(ValueError, match="boom"): + engine.generate([], model="m") + + +def test_fake_engine_tool_calls(): + engine = FakeEngine([{ + "content": "", + "tool_calls": [{"id": "1", "function": {"name": "think", "arguments": "{}"}}], + }]) + r = engine.generate([], model="m") + assert r["finish_reason"] == "tool_calls" + assert len(r["tool_calls"]) == 1 diff --git a/tests/agents/test_scenario_harness_smoke.py b/tests/agents/test_scenario_harness_smoke.py new file mode 100644 index 00000000..32776740 --- /dev/null +++ b/tests/agents/test_scenario_harness_smoke.py @@ -0,0 +1,19 @@ +"""Smoke test for the scenario harness fixture.""" + +from __future__ import annotations + +from tests.agents.scenario_harness import ScenarioHarness + + +def test_harness_creates_and_runs_agent(scenario_harness: ScenarioHarness): + """Verify the harness wires up correctly — create agent, run tick.""" + h = scenario_harness + agent = h.manager.create_agent("Smoke Test", config={ + "schedule_type": "manual", + "instruction": "Say hello.", + }) + h.executor.execute_tick(agent["id"]) + updated = h.manager.get_agent(agent["id"]) + assert updated["status"] == "idle" + assert updated["total_runs"] == 1 + assert updated["summary_memory"] != "" From 941016f25523b0178a9236f3cfa4068ddd0766cd Mon Sep 17 00:00:00 2001 From: Jon Saad-Falcon <41205309+jonsaadfalcon@users.noreply.github.com> Date: Mon, 16 Mar 2026 20:46:18 -0700 Subject: [PATCH 04/44] feat: add 12 agent lifecycle scenario tests with real SQLite state --- .../agents/test_agent_lifecycle_scenarios.py | 538 ++++++++++++++++++ 1 file changed, 538 insertions(+) create mode 100644 tests/agents/test_agent_lifecycle_scenarios.py diff --git a/tests/agents/test_agent_lifecycle_scenarios.py b/tests/agents/test_agent_lifecycle_scenarios.py new file mode 100644 index 00000000..221bb8bc --- /dev/null +++ b/tests/agents/test_agent_lifecycle_scenarios.py @@ -0,0 +1,538 @@ +"""Agent lifecycle scenario tests. + +Twelve+ scenarios exercising the full managed-agent stack (manager, executor, +scheduler) with real SQLite state and a scripted FakeEngine. +""" + +from __future__ import annotations + +from unittest.mock import patch + +import pytest + +from openjarvis.agents.errors import RetryableError +from openjarvis.core.events import EventType +from tests.agents.scenario_harness import ScenarioHarness + +# --------------------------------------------------------------------------- +# Scenario 1: Manual agent full lifecycle +# --------------------------------------------------------------------------- + + +def test_manual_agent_full_lifecycle(scenario_harness: ScenarioHarness) -> None: + """Create a manual agent, run a tick, verify status/runs/memory.""" + h = scenario_harness + h.engine._responses = [{"content": "Tick 1 completed."}] + h.engine._call_count = 0 + + agent = h.manager.create_agent( + name="manual-agent", + config={"schedule_type": "manual", "instruction": "Do something."}, + ) + aid = agent["id"] + + h.executor.execute_tick(aid) + + agent = h.manager.get_agent(aid) + assert agent is not None + assert agent["status"] == "idle" + assert agent["total_runs"] == 1 + assert "Tick 1 completed." in agent["summary_memory"] + + +# --------------------------------------------------------------------------- +# Scenario 2: Interval-scheduled agent +# --------------------------------------------------------------------------- + + +def test_interval_scheduled_agent(scenario_harness: ScenarioHarness) -> None: + """Interval agent does not fire until time advances past interval.""" + h = scenario_harness + h.engine._responses = [{"content": "Interval response."}] + h.engine._call_count = 0 + + base_time = 1_000_000.0 + + agent = h.manager.create_agent( + name="interval-agent", + config={ + "schedule_type": "interval", + "schedule_value": 60, + "instruction": "Check status.", + }, + ) + aid = agent["id"] + + # Register at base_time — next_fire = base_time + 60 + with patch("openjarvis.agents.scheduler.time") as mock_time: + mock_time.time.return_value = base_time + h.scheduler.register_agent(aid) + + info = h.scheduler._agents[aid] + assert info["next_fire"] == pytest.approx(base_time + 60, abs=1) + + # At base_time + 30 the agent should NOT fire + with patch("openjarvis.agents.scheduler.time") as mock_time: + mock_time.time.return_value = base_time + 30 + h.scheduler._check_due_agents() + + agent = h.manager.get_agent(aid) + assert agent is not None + assert agent["total_runs"] == 0 + + # At base_time + 61 the agent SHOULD fire + with patch("openjarvis.agents.scheduler.time") as mock_time: + mock_time.time.return_value = base_time + 61 + h.scheduler._check_due_agents() + + agent = h.manager.get_agent(aid) + assert agent is not None + assert agent["total_runs"] == 1 + assert "Interval response." in agent["summary_memory"] + + +# --------------------------------------------------------------------------- +# Scenario 3: Cron-scheduled agent +# --------------------------------------------------------------------------- + + +def test_cron_scheduled_agent(scenario_harness: ScenarioHarness) -> None: + """Cron agent gets next_fire set correctly.""" + h = scenario_harness + + agent = h.manager.create_agent( + name="cron-agent", + config={ + "schedule_type": "cron", + "schedule_value": "0 9 * * *", + "instruction": "Daily check.", + }, + ) + aid = agent["id"] + + h.scheduler.register_agent(aid) + + info = h.scheduler._agents[aid] + assert info["schedule_type"] == "cron" + # next_fire should be in the future + import time + + assert info["next_fire"] > time.time() + + +# --------------------------------------------------------------------------- +# Scenario 4: Queued message delivery +# --------------------------------------------------------------------------- + + +def test_queued_message_delivery(scenario_harness: ScenarioHarness) -> None: + """Queue 3 messages, run tick, verify all delivered and in prompt.""" + h = scenario_harness + h.engine._responses = [{"content": "Processed all messages."}] + h.engine._call_count = 0 + + agent = h.manager.create_agent( + name="msg-agent", + config={"schedule_type": "manual", "instruction": "Handle messages."}, + ) + aid = agent["id"] + + h.manager.send_message(aid, "Message one", mode="queued") + h.manager.send_message(aid, "Message two", mode="queued") + h.manager.send_message(aid, "Message three", mode="queued") + + # Verify 3 pending before tick + pending = h.manager.get_pending_messages(aid) + assert len(pending) == 3 + + h.executor.execute_tick(aid) + + # All should be delivered + pending = h.manager.get_pending_messages(aid) + assert len(pending) == 0 + + # Engine should have been called with messages in the prompt + assert h.engine.last_messages is not None + prompt_text = " ".join( + str(getattr(m, "content", m)) for m in h.engine.last_messages + ) + assert "Message one" in prompt_text + assert "Message two" in prompt_text + assert "Message three" in prompt_text + + # Response stored + agent = h.manager.get_agent(aid) + assert agent is not None + assert "Processed all messages." in agent["summary_memory"] + + +# --------------------------------------------------------------------------- +# Scenario 5: Immediate message +# --------------------------------------------------------------------------- + + +def test_immediate_message(scenario_harness: ScenarioHarness) -> None: + """Send an immediate message, run tick, assert response stored.""" + h = scenario_harness + h.engine._responses = [{"content": "Immediate reply."}] + h.engine._call_count = 0 + + agent = h.manager.create_agent( + name="imm-agent", + config={"schedule_type": "manual", "instruction": "Reply to user."}, + ) + aid = agent["id"] + + h.manager.send_message(aid, "Urgent question", mode="immediate") + + h.executor.execute_tick(aid) + + # The response should be stored + messages = h.manager.list_messages(aid) + responses = [m for m in messages if m["direction"] == "agent_to_user"] + assert len(responses) >= 1 + assert any("Immediate reply." in r["content"] for r in responses) + + +# --------------------------------------------------------------------------- +# Scenario 6: Budget enforcement +# --------------------------------------------------------------------------- + + +def test_budget_enforcement(scenario_harness: ScenarioHarness) -> None: + """Agent with max_cost runs tick, total_runs incremented, stays idle + because cost is below threshold.""" + h = scenario_harness + # The default usage in FakeEngine reports 0 cost (no cost key in metadata), + # so total_cost stays at 0 which is below max_cost=0.01. + h.engine._responses = [{"content": "Budget check."}] + h.engine._call_count = 0 + + agent = h.manager.create_agent( + name="budget-agent", + config={ + "schedule_type": "manual", + "max_cost": 0.01, + "instruction": "Cheap task.", + }, + ) + aid = agent["id"] + + h.executor.execute_tick(aid) + + agent = h.manager.get_agent(aid) + assert agent is not None + assert agent["total_runs"] == 1 + # Cost is 0 (FakeEngine doesn't report cost), so stays idle + assert agent["status"] == "idle" + + +# --------------------------------------------------------------------------- +# Scenario 7: Error + retry (success on 3rd attempt) +# --------------------------------------------------------------------------- + + +def test_error_retry_success(scenario_harness: ScenarioHarness) -> None: + """FakeEngine raises RetryableError first 2 times, succeeds on 3rd.""" + h = scenario_harness + h.engine._responses = [ + {"raise": RetryableError("transient-1")}, + {"raise": RetryableError("transient-2")}, + {"content": "Success after retries."}, + ] + h.engine._call_count = 0 + + agent = h.manager.create_agent( + name="retry-agent", + config={"schedule_type": "manual", "instruction": "Retry test."}, + ) + aid = agent["id"] + + # Patch retry_delay to avoid real sleeps + with patch("openjarvis.agents.executor.time.sleep"): + h.executor.execute_tick(aid) + + assert h.engine.call_count == 3 + agent = h.manager.get_agent(aid) + assert agent is not None + assert agent["status"] == "idle" + assert "Success after retries." in agent["summary_memory"] + + +# --------------------------------------------------------------------------- +# Scenario 7b: Error exhaustion +# --------------------------------------------------------------------------- + + +def test_error_exhaustion(scenario_harness: ScenarioHarness) -> None: + """FakeEngine always raises RetryableError. Agent ends in error state.""" + h = scenario_harness + h.engine._responses = [ + {"raise": RetryableError("fail-1")}, + {"raise": RetryableError("fail-2")}, + {"raise": RetryableError("fail-3")}, + ] + h.engine._call_count = 0 + + agent = h.manager.create_agent( + name="exhaust-agent", + config={"schedule_type": "manual", "instruction": "Always fail."}, + ) + aid = agent["id"] + + with patch("openjarvis.agents.executor.time.sleep"): + h.executor.execute_tick(aid) + + agent = h.manager.get_agent(aid) + assert agent is not None + assert agent["status"] == "error" + + +# --------------------------------------------------------------------------- +# Scenario 8: Stall detection + recovery +# --------------------------------------------------------------------------- + + +def test_stall_detection_and_recovery(scenario_harness: ScenarioHarness) -> None: + """Set agent to running with stale last_activity, reconcile detects stall, + then recover resets to idle.""" + h = scenario_harness + import time as real_time + + agent = h.manager.create_agent( + name="stall-agent", + config={ + "schedule_type": "manual", + "timeout_seconds": 300, + "max_stall_retries": 5, + "instruction": "Long task.", + }, + ) + aid = agent["id"] + + # Simulate: agent is running with stale activity + h.manager.start_tick(aid) + ten_minutes_ago = real_time.time() - 600 + h.manager.update_agent(aid, last_activity_at=ten_minutes_ago) + + # Reconcile should detect the stall + h.scheduler._reconcile() + + agent = h.manager.get_agent(aid) + assert agent is not None + # Stall detection should have released the concurrency guard (end_tick) + # and incremented stall_retries + assert agent["status"] == "idle" + assert agent["stall_retries"] == 1 + + # Verify stall event was published + stall_events = [ + e for e in h.bus.history if e.event_type == EventType.AGENT_STALL_DETECTED + ] + assert len(stall_events) >= 1 + assert stall_events[0].data["agent_id"] == aid + + # Now recover the agent explicitly + h.manager.recover_agent(aid) + agent = h.manager.get_agent(aid) + assert agent is not None + assert agent["status"] == "idle" + + +# --------------------------------------------------------------------------- +# Scenario 9: Pause / resume +# --------------------------------------------------------------------------- + + +def test_pause_resume(scenario_harness: ScenarioHarness) -> None: + """Paused agent does not fire; resumed agent fires normally.""" + h = scenario_harness + h.engine._responses = [{"content": "Resumed response."}] + h.engine._call_count = 0 + + base_time = 2_000_000.0 + + agent = h.manager.create_agent( + name="pause-agent", + config={ + "schedule_type": "interval", + "schedule_value": 60, + "instruction": "Pause test.", + }, + ) + aid = agent["id"] + + # Register at base_time + with patch("openjarvis.agents.scheduler.time") as mock_time: + mock_time.time.return_value = base_time + h.scheduler.register_agent(aid) + + # Pause the agent + h.manager.pause_agent(aid) + + # Advance past interval — should NOT fire because paused + with patch("openjarvis.agents.scheduler.time") as mock_time: + mock_time.time.return_value = base_time + 61 + h.scheduler._check_due_agents() + + agent = h.manager.get_agent(aid) + assert agent is not None + assert agent["total_runs"] == 0 + + # Resume + h.manager.resume_agent(aid) + + # Now advance again — should fire + with patch("openjarvis.agents.scheduler.time") as mock_time: + mock_time.time.return_value = base_time + 61 + h.scheduler._check_due_agents() + + agent = h.manager.get_agent(aid) + assert agent is not None + assert agent["total_runs"] == 1 + + +# --------------------------------------------------------------------------- +# Scenario 10: Multi-agent concurrent scheduling +# --------------------------------------------------------------------------- + + +def test_multi_agent_scheduling(scenario_harness: ScenarioHarness) -> None: + """Three agents with different intervals fire at the right times.""" + h = scenario_harness + h.engine._responses = [{"content": "Agent response."}] + h.engine._call_count = 0 + + base_time = 3_000_000.0 + + agents = [] + for i, interval in enumerate([30, 60, 120]): + a = h.manager.create_agent( + name=f"multi-{i}", + config={ + "schedule_type": "interval", + "schedule_value": interval, + "instruction": f"Task {i}.", + }, + ) + agents.append(a) + with patch("openjarvis.agents.scheduler.time") as mock_time: + mock_time.time.return_value = base_time + h.scheduler.register_agent(a["id"]) + + # At base_time + 35, only agent 0 (30s interval) should fire + h.engine._call_count = 0 + with patch("openjarvis.agents.scheduler.time") as mock_time: + mock_time.time.return_value = base_time + 35 + h.scheduler._check_due_agents() + + a0 = h.manager.get_agent(agents[0]["id"]) + a1 = h.manager.get_agent(agents[1]["id"]) + a2 = h.manager.get_agent(agents[2]["id"]) + assert a0 is not None and a0["total_runs"] == 1 + assert a1 is not None and a1["total_runs"] == 0 + assert a2 is not None and a2["total_runs"] == 0 + + # At base_time + 65, agents 0 and 1 should have fired (but agent 0 + # next_fire was reset to base_time + 35 + 30 = base_time + 65, so it + # fires again at exactly 65) + h.engine._responses = [{"content": "Agent response again."}] + h.engine._call_count = 0 + with patch("openjarvis.agents.scheduler.time") as mock_time: + mock_time.time.return_value = base_time + 65 + h.scheduler._check_due_agents() + + a0 = h.manager.get_agent(agents[0]["id"]) + a1 = h.manager.get_agent(agents[1]["id"]) + a2 = h.manager.get_agent(agents[2]["id"]) + assert a0 is not None and a0["total_runs"] == 2 + assert a1 is not None and a1["total_runs"] == 1 + assert a2 is not None and a2["total_runs"] == 0 + + # At base_time + 125, all three should have fired + h.engine._responses = [{"content": "All fired."}] + h.engine._call_count = 0 + with patch("openjarvis.agents.scheduler.time") as mock_time: + mock_time.time.return_value = base_time + 125 + h.scheduler._check_due_agents() + + a0 = h.manager.get_agent(agents[0]["id"]) + a1 = h.manager.get_agent(agents[1]["id"]) + a2 = h.manager.get_agent(agents[2]["id"]) + assert a0 is not None and a0["total_runs"] >= 3 + assert a1 is not None and a1["total_runs"] >= 2 + assert a2 is not None and a2["total_runs"] == 1 + + +# --------------------------------------------------------------------------- +# Scenario 11: Template instantiation +# --------------------------------------------------------------------------- + + +def test_template_instantiation(scenario_harness: ScenarioHarness) -> None: + """Each built-in template creates an agent with correct config fields.""" + h = scenario_harness + + templates = h.manager.list_templates() + + for tpl in templates: + tpl_id = tpl.get("id") + if not tpl_id: + continue + + agent = h.manager.create_from_template(tpl_id, name=f"from-{tpl_id}") + config = agent["config"] + + # Every template has a schedule_type and schedule_value + assert "schedule_type" in config + assert "schedule_value" in config + # Every template has a system_prompt + assert "system_prompt" in config + # Agent type should be set + assert agent["agent_type"] == tpl.get("agent_type", "monitor_operative") + + +# --------------------------------------------------------------------------- +# Scenario 12: Memory persistence across ticks +# --------------------------------------------------------------------------- + + +def test_memory_persistence_across_ticks(scenario_harness: ScenarioHarness) -> None: + """Tick 1 summary becomes part of tick 2 engine prompt (Previous context).""" + h = scenario_harness + h.engine._responses = [{"content": "Findings from tick one."}] + h.engine._call_count = 0 + + agent = h.manager.create_agent( + name="memory-agent", + config={ + "schedule_type": "manual", + "instruction": "Investigate topic X.", + }, + ) + aid = agent["id"] + + # --- Tick 1 --- + h.executor.execute_tick(aid) + + agent = h.manager.get_agent(aid) + assert agent is not None + assert "Findings from tick one." in agent["summary_memory"] + + # --- Tick 2 --- + h.engine._responses = [{"content": "Tick two builds on prior context."}] + h.engine._call_count = 0 + + h.executor.execute_tick(aid) + + # The engine should have received the tick-1 summary in its prompt + assert h.engine.last_messages is not None + prompt_text = " ".join( + str(getattr(m, "content", m)) for m in h.engine.last_messages + ) + assert "Findings from tick one." in prompt_text + + # Tick 2 summary should now be stored + agent = h.manager.get_agent(aid) + assert agent is not None + assert "Tick two builds on prior context." in agent["summary_memory"] From cf362897cd95f936235e7b9c474c163a6d1fb52b Mon Sep 17 00:00:00 2001 From: Jon Saad-Falcon <41205309+jonsaadfalcon@users.noreply.github.com> Date: Mon, 16 Mar 2026 21:05:37 -0700 Subject: [PATCH 05/44] feat: add Gmail/Twitter channel adapters, curated templates, channel contract tests, and QA runbook - Add GmailChannel (OAuth2 + polling) and TwitterChannel (tweepy v2 API) - Update research_monitor, inbox_triager, code_reviewer templates with curated tool sets - Add channel_send/channel_list tools to all templates - Add parametrized contract tests for all 28 channel adapters (195 tests) - Add Gmail mocked tests (17 tests) and Twitter mocked tests (9 tests) - Add agent-channel E2E tests with WebChatChannel - Add live_channel pytest marker - Add manual QA runbook (docs/testing/agent-qa-runbook.md) --- docs/testing/agent-qa-runbook.md | 75 +++++ pyproject.toml | 7 + .../agents/templates/code_reviewer.toml | 4 +- .../agents/templates/inbox_triager.toml | 4 +- .../agents/templates/research_monitor.toml | 4 +- src/openjarvis/channels/__init__.py | 2 + src/openjarvis/channels/gmail.py | 298 ++++++++++++++++++ src/openjarvis/channels/twitter.py | 242 ++++++++++++++ tests/channels/conftest.py | 65 ++++ tests/channels/test_agent_channel_e2e.py | 64 ++++ tests/channels/test_channel_contract.py | 85 +++++ tests/channels/test_tier1_gmail.py | 209 ++++++++++++ tests/channels/test_tier2_twitter.py | 136 ++++++++ tests/core/test_config.py | 3 +- 14 files changed, 1191 insertions(+), 7 deletions(-) create mode 100644 docs/testing/agent-qa-runbook.md create mode 100644 src/openjarvis/channels/gmail.py create mode 100644 src/openjarvis/channels/twitter.py create mode 100644 tests/channels/conftest.py create mode 100644 tests/channels/test_agent_channel_e2e.py create mode 100644 tests/channels/test_channel_contract.py create mode 100644 tests/channels/test_tier1_gmail.py create mode 100644 tests/channels/test_tier2_twitter.py diff --git a/docs/testing/agent-qa-runbook.md b/docs/testing/agent-qa-runbook.md new file mode 100644 index 00000000..df79fcee --- /dev/null +++ b/docs/testing/agent-qa-runbook.md @@ -0,0 +1,75 @@ +# Agent QA Runbook + +Manual testing scenarios for persistent agents in the CLI and desktop app. + +## Environment Setup + +| Prerequisite | Command / Check | +|---|---| +| Ollama running with model | `ollama list` shows `qwen3:8b` | +| OpenJarvis initialized | `uv run jarvis doctor` all green | +| Rust extension built | `uv run maturin develop -m rust/crates/openjarvis-python/Cargo.toml` | +| Desktop app running | `uv run jarvis serve` + `cd frontend && npm run dev` | +| Slack credentials | `SLACK_BOT_TOKEN`, `SLACK_APP_TOKEN` set, bot invited to test channel | +| Gmail credentials | OAuth credentials.json downloaded, token generated | +| Twitter credentials | All 5 env vars set (bearer + OAuth 1.0a) | +| Discord credentials | Bot token set, bot invited to test server | +| Telegram credentials | Bot token from @BotFather, test chat ID known | +| Email credentials | SMTP/IMAP host + credentials for test account | + +## CLI Agent Scenarios + +| # | Scenario | Steps | Expected Result | Pass | +|---|----------|-------|-----------------|------| +| 1 | Template launch | `jarvis agents launch`, pick research_monitor | Agent created, config printed with curated tools | [ ] | +| 2 | Manual run | `jarvis agents run ` | Output shows reasoning + tool calls, status -> idle | [ ] | +| 3 | Immediate ask | `jarvis agents ask "summarize recent AI news"` | Synchronous response in terminal | [ ] | +| 4 | Queued instruct | `jarvis agents instruct "focus on diffusion"`, then `jarvis agents run ` | Queued -> delivered, response in `jarvis agents messages ` | [ ] | +| 5 | Status check | `jarvis agents status` after 3+ runs | total_runs, total_cost, last_run_at populated | [ ] | +| 6 | Pause/resume | `jarvis agents pause `, verify skipped, `jarvis agents resume `, verify fires | Status toggles correctly | [ ] | +| 7 | Daemon scheduling | `jarvis agents daemon` with interval agent (60s) | 3+ ticks fire on schedule, memory accumulates | [ ] | +| 8 | Budget exhaustion | Set max_cost=0.001, run until exceeded | Status becomes budget_exceeded | [ ] | +| 9 | Error recovery | Kill Ollama mid-tick, then `jarvis agents recover ` | Error -> recover -> idle with checkpoint | [ ] | +| 10 | Channel binding | `jarvis agents bind --slack #test`, run tick | Agent sends to Slack | [ ] | +| 11 | Multi-agent | Launch 3 agents, different intervals, run daemon | All fire independently | [ ] | +| 12 | Template tools | Create from each template, `jarvis agents info ` | All curated default tools listed | [ ] | + +## Desktop App Scenarios + +| # | Scenario | Steps | Expected Result | Pass | +|---|----------|-------|-----------------|------| +| 1 | Template wizard | New Agent -> pick each template -> complete wizard | Agent appears in grid with correct config | [ ] | +| 2 | Custom agent | New Agent -> Custom -> manual schedule, pick tools, set credentials | Tools + creds saved, agent created | [ ] | +| 3 | Run Now | Click Run Now on agent card | Status dot: green -> blue -> green, stats increment | [ ] | +| 4 | Immediate chat | Interact tab -> type message -> send (immediate mode) | Response appears in chat UI | [ ] | +| 5 | Queued chat | Interact tab -> send (queued mode) -> click Run Now | Message delivered on tick, response appears | [ ] | +| 6 | Task management | Tasks tab -> create task -> run agent | Task status updates, findings populated | [ ] | +| 7 | Memory inspection | Run 3+ ticks -> Memory tab | Summary memory reflects agent's accumulated knowledge | [ ] | +| 8 | Trace inspection | Run tick -> Logs tab | Trace steps visible with tool calls and results | [ ] | +| 9 | Learning | Enable trace-driven learning -> Learning tab -> trigger | Learning log entries appear | [ ] | +| 10 | Error + recovery | Stop Ollama -> run agent -> verify error badge -> click Recover | Error state shown, recovery resets to idle | [ ] | + +## Channel-Specific QA Matrix + +| Channel | Send Test | Receive Test | Thread/Reply Test | Agent Template | Pass | +|---------|-----------|-------------|-------------------|----------------|------| +| Slack | Post to #test-channel | Socket Mode incoming msg | Reply in thread (thread_ts) | inbox_triager | [ ] | +| Gmail | Send email to test recipient | Poll unread -> handler fires | Reply in thread (threadId) | inbox_triager | [ ] | +| Email (SMTP/IMAP) | Send via SMTP | IMAP poll UNSEEN | In-Reply-To header | inbox_triager | [ ] | +| iMessage (BlueBubbles) | Send to phone number | N/A (send-only) | N/A | research_monitor | [ ] | +| Twitter/X | Post tweet + send DM | Poll mentions | Reply (in_reply_to_tweet_id) | research_monitor | [ ] | +| Discord | Post to #test-channel | Gateway message event | N/A | code_reviewer | [ ] | +| Telegram | Send to test chat | Long-poll update | reply_to_message_id | research_monitor | [ ] | +| WhatsApp (Baileys) | Send to test number | Baileys incoming msg | N/A | inbox_triager | [ ] | + +## Stress & Edge Cases + +| # | Scenario | How to Test | Pass Criteria | Pass | +|---|----------|-------------|---------------|------| +| 1 | Message flood | Queue 50 messages via CLI, run tick | All 50 delivered, response generated | [ ] | +| 2 | Long-running daemon | Run daemon for 1 hour, 60s interval agent | No memory leak, no stall, ~60 ticks | [ ] | +| 3 | Rapid pause/resume | Script: pause -> resume -> pause -> resume during tick | Clean state, no corruption | [ ] | +| 4 | Credential revocation | Revoke Slack token mid-tick | Agent gets tool error, tick completes, status not corrupted | [ ] | +| 5 | Multi-agent load | 10 agents on daemon, mix of intervals | All fire on schedule, no interference | [ ] | +| 6 | Large response handling | Agent produces 10k+ char response | summary_memory truncated to 2000 chars, full response in messages | [ ] | +| 7 | Checkpoint integrity | Kill process mid-tick, restart, recover | Checkpoint restored, agent resumes cleanly | [ ] | diff --git a/pyproject.toml b/pyproject.toml index 6a50980e..987e76a2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -77,6 +77,12 @@ channel-rocketchat = ["rocketchat-API>=1.30"] channel-zulip = ["zulip>=0.9"] channel-twitch = ["twitchio>=2.6"] channel-nostr = ["pynostr>=0.6"] +channel-gmail = [ + "google-api-python-client>=2.0", + "google-auth-oauthlib>=1.0", + "google-auth-httplib2>=0.2", +] +channel-twitter = ["tweepy>=4.14"] browser = ["playwright>=1.40"] media = ["openai>=1.30"] pdf = ["pdfplumber>=0.10"] @@ -117,6 +123,7 @@ markers = [ "apple: requires Apple Silicon", "macos15: requires macOS 15+ (Sequoia) for Apple FM", "slow: long-running test", + "live_channel: requires real channel credentials (env vars)", ] [tool.ruff] diff --git a/src/openjarvis/agents/templates/code_reviewer.toml b/src/openjarvis/agents/templates/code_reviewer.toml index f31152a9..4a93f40a 100644 --- a/src/openjarvis/agents/templates/code_reviewer.toml +++ b/src/openjarvis/agents/templates/code_reviewer.toml @@ -5,11 +5,11 @@ description = "Monitors a repository for changes, reviews code, and stores feedb agent_type = "monitor_operative" schedule_type = "interval" schedule_value = "3600" -tools = ["file_read", "shell_exec", "memory_store", "think"] +tools = ["file_read", "file_write", "shell_exec", "git_status", "git_diff", "git_commit", "git_log", "apply_patch", "code_interpreter", "memory_store", "memory_retrieve", "think", "channel_send", "channel_list"] max_turns = 15 temperature = 0.2 memory_extraction = "scratchpad" observation_compression = "summarize" retrieval_strategy = "keyword" task_decomposition = "monolithic" -system_prompt = "You are a code review agent. Monitor the repository for recent changes, review code quality, identify potential bugs, and store your findings." +system_prompt = "You are a code review agent. Monitor the repository for recent changes, review code quality, identify potential bugs, and store your findings. After completing your review, send a summary of findings and any suggested patches to bound channels using channel_send." diff --git a/src/openjarvis/agents/templates/inbox_triager.toml b/src/openjarvis/agents/templates/inbox_triager.toml index 1087b2b1..9999c271 100644 --- a/src/openjarvis/agents/templates/inbox_triager.toml +++ b/src/openjarvis/agents/templates/inbox_triager.toml @@ -5,11 +5,11 @@ description = "Monitors email and messaging channels, categorizes and summarizes agent_type = "monitor_operative" schedule_type = "interval" schedule_value = "1800" -tools = ["web_search", "memory_store", "memory_retrieve", "think"] +tools = ["channel_send", "channel_list", "memory_store", "memory_retrieve", "think", "web_search", "file_write"] max_turns = 20 temperature = 0.3 memory_extraction = "structured_json" observation_compression = "summarize" retrieval_strategy = "semantic" task_decomposition = "phased" -system_prompt = "You are an inbox triage agent. Monitor incoming messages, categorize them by priority and topic, summarize key information, and flag items that need attention." +system_prompt = "You are an inbox triage agent. Monitor incoming messages, categorize them by priority and topic, summarize key information, and flag items that need attention. When triaging messages, use channel_send to forward urgent items or send status updates to bound channels." diff --git a/src/openjarvis/agents/templates/research_monitor.toml b/src/openjarvis/agents/templates/research_monitor.toml index ef1aad1e..ff087f9b 100644 --- a/src/openjarvis/agents/templates/research_monitor.toml +++ b/src/openjarvis/agents/templates/research_monitor.toml @@ -5,11 +5,11 @@ description = "Periodically searches for papers/news on a topic, stores findings agent_type = "monitor_operative" schedule_type = "cron" schedule_value = "0 9 * * *" -tools = ["web_search", "memory_store", "memory_retrieve", "think"] +tools = ["web_search", "http_request", "file_read", "file_write", "memory_store", "memory_retrieve", "think", "channel_send", "channel_list"] max_turns = 25 temperature = 0.3 memory_extraction = "structured_json" observation_compression = "summarize" retrieval_strategy = "hybrid_with_self_eval" task_decomposition = "phased" -system_prompt = "You are a research monitor agent. Your job is to search for new papers, articles, and developments on your assigned topics. Store important findings in memory. Be thorough but concise in your summaries." +system_prompt = "You are a research monitor agent. Your job is to search for new papers, articles, and developments on your assigned topics. Store important findings in memory. Be thorough but concise in your summaries. After completing your research, send a summary of findings to any bound channels using the channel_send tool." diff --git a/src/openjarvis/channels/__init__.py b/src/openjarvis/channels/__init__.py index df908166..e98ce6e8 100644 --- a/src/openjarvis/channels/__init__.py +++ b/src/openjarvis/channels/__init__.py @@ -38,6 +38,8 @@ _CHANNEL_MODULES = [ "zulip_channel", "twitch_channel", "nostr_channel", + "twitter", + "gmail", ] for _mod in _CHANNEL_MODULES: diff --git a/src/openjarvis/channels/gmail.py b/src/openjarvis/channels/gmail.py new file mode 100644 index 00000000..95f8c729 --- /dev/null +++ b/src/openjarvis/channels/gmail.py @@ -0,0 +1,298 @@ +"""GmailChannel — native Gmail API adapter using OAuth2.""" + +from __future__ import annotations + +import base64 +import logging +import os +import threading +from email.mime.text import MIMEText +from typing import Any, Dict, List, Optional + +from openjarvis.channels._stubs import ( + BaseChannel, + ChannelHandler, + ChannelMessage, + ChannelStatus, +) +from openjarvis.core.events import EventBus, EventType +from openjarvis.core.registry import ChannelRegistry + +logger = logging.getLogger(__name__) + + +@ChannelRegistry.register("gmail") +class GmailChannel(BaseChannel): + """Native Gmail channel adapter using the Gmail API with OAuth2. + + Parameters + ---------- + credentials_path: + Path to the OAuth2 ``credentials.json`` file. + Falls back to ``GMAIL_CREDENTIALS_PATH`` env var. + token_path: + Path to the stored OAuth2 token file. + Falls back to ``GMAIL_TOKEN_PATH`` env var. + user_id: + Gmail user ID (default ``"me"`` for the authenticated user). + poll_interval: + Seconds between inbox polls (default 30). + bus: + Optional event bus for publishing channel events. + """ + + channel_id = "gmail" + + def __init__( + self, + credentials_path: str = "", + *, + token_path: str = "", + user_id: str = "me", + poll_interval: int = 30, + bus: Optional[EventBus] = None, + ) -> None: + self._credentials_path = credentials_path or os.environ.get( + "GMAIL_CREDENTIALS_PATH", "", + ) + self._token_path = token_path or os.environ.get( + "GMAIL_TOKEN_PATH", "", + ) + self._user_id = user_id + self._poll_interval = poll_interval + self._bus = bus + self._handlers: List[ChannelHandler] = [] + self._status = ChannelStatus.DISCONNECTED + self._service: Any = None + self._listener_thread: Optional[threading.Thread] = None + self._stop_event = threading.Event() + + # -- connection lifecycle --------------------------------------------------- + + def connect(self) -> None: + """Load OAuth2 credentials and build the Gmail API service.""" + if not self._credentials_path and not self._token_path: + logger.warning("No Gmail credentials configured") + self._status = ChannelStatus.ERROR + return + + self._stop_event.clear() + self._status = ChannelStatus.CONNECTING + + try: + from google.oauth2.credentials import Credentials + from google_auth_oauthlib.flow import InstalledAppFlow + from googleapiclient.discovery import build + + SCOPES = ["https://www.googleapis.com/auth/gmail.modify"] + + creds: Any = None + if self._token_path and os.path.exists(self._token_path): + creds = Credentials.from_authorized_user_file( + self._token_path, SCOPES, + ) + + if not creds or not creds.valid: + if creds and creds.expired and creds.refresh_token: + from google.auth.transport.requests import Request + + creds.refresh(Request()) + elif self._credentials_path and os.path.exists( + self._credentials_path, + ): + flow = InstalledAppFlow.from_client_secrets_file( + self._credentials_path, SCOPES, + ) + creds = flow.run_local_server(port=0) + else: + logger.warning("Gmail credentials not found or invalid") + self._status = ChannelStatus.ERROR + return + + # Save token for future use + if self._token_path and creds: + with open(self._token_path, "w") as token_file: + token_file.write(creds.to_json()) + + self._service = build("gmail", "v1", credentials=creds) + self._status = ChannelStatus.CONNECTED + logger.info("Gmail channel connected") + + # Start polling thread + self._listener_thread = threading.Thread( + target=self._poll_loop, daemon=True, + ) + self._listener_thread.start() + except ImportError: + logger.warning( + "Google API libraries not installed; " + "install with: pip install openjarvis[channel-gmail]", + ) + self._status = ChannelStatus.ERROR + except Exception: + logger.debug("Gmail connect failed", exc_info=True) + self._status = ChannelStatus.ERROR + + def disconnect(self) -> None: + """Stop the polling thread and clear the service.""" + self._stop_event.set() + if self._listener_thread is not None: + self._listener_thread.join(timeout=5.0) + self._listener_thread = None + self._service = None + self._status = ChannelStatus.DISCONNECTED + + # -- send / receive -------------------------------------------------------- + + def send( + self, + channel: str, + content: str, + *, + conversation_id: str = "", + metadata: Dict[str, Any] | None = None, + ) -> bool: + """Send an email via the Gmail API. + + ``channel`` is the recipient email address. + """ + if self._service is None: + logger.warning("Cannot send: Gmail service not connected") + return False + + try: + msg = MIMEText(content) + msg["To"] = channel + msg["From"] = self._user_id + msg["Subject"] = (metadata or {}).get( + "subject", "Message from OpenJarvis", + ) + + raw = base64.urlsafe_b64encode(msg.as_bytes()).decode("utf-8") + body: Dict[str, Any] = {"raw": raw} + if conversation_id: + body["threadId"] = conversation_id + + self._service.users().messages().send( + userId=self._user_id, body=body, + ).execute() + + self._publish_sent(channel, content, conversation_id) + return True + except Exception: + logger.debug("Gmail send failed", exc_info=True) + return False + + def status(self) -> ChannelStatus: + """Return the current connection status.""" + if self._status == ChannelStatus.CONNECTED and self._service is None: + return ChannelStatus.ERROR + return self._status + + def list_channels(self) -> List[str]: + """Return available channel identifiers.""" + return ["inbox"] + + def on_message(self, handler: ChannelHandler) -> None: + """Register a callback for incoming Gmail messages.""" + self._handlers.append(handler) + + # -- internal helpers ------------------------------------------------------- + + def _poll_loop(self) -> None: + """Poll Gmail inbox for unread messages.""" + while not self._stop_event.is_set(): + try: + if self._service is None: + break + + results = ( + self._service.users() + .messages() + .list(userId=self._user_id, q="is:unread") + .execute() + ) + messages = results.get("messages", []) + + for msg_ref in messages: + msg_id = msg_ref["id"] + msg = ( + self._service.users() + .messages() + .get(userId=self._user_id, id=msg_id, format="full") + .execute() + ) + + headers = { + h["name"]: h["value"] + for h in msg.get("payload", {}).get("headers", []) + } + + # Extract body text + body = "" + payload = msg.get("payload", {}) + if payload.get("body", {}).get("data"): + body = base64.urlsafe_b64decode( + payload["body"]["data"], + ).decode("utf-8", errors="replace") + elif payload.get("parts"): + for part in payload["parts"]: + if part.get("mimeType") == "text/plain": + data = part.get("body", {}).get("data", "") + if data: + body = base64.urlsafe_b64decode( + data, + ).decode("utf-8", errors="replace") + break + + cm = ChannelMessage( + channel="gmail", + sender=headers.get("From", ""), + content=body, + message_id=msg_id, + conversation_id=msg.get("threadId", ""), + ) + + for handler in self._handlers: + try: + handler(cm) + except Exception: + logger.exception("Gmail handler error") + + if self._bus is not None: + self._bus.publish( + EventType.CHANNEL_MESSAGE_RECEIVED, + { + "channel": cm.channel, + "sender": cm.sender, + "content": cm.content, + "message_id": cm.message_id, + }, + ) + + # Mark message as read + self._service.users().messages().modify( + userId=self._user_id, + id=msg_id, + body={"removeLabelIds": ["UNREAD"]}, + ).execute() + + except Exception: + logger.debug("Gmail poll error", exc_info=True) + + self._stop_event.wait(self._poll_interval) + + def _publish_sent(self, channel: str, content: str, conversation_id: str) -> None: + """Publish a CHANNEL_MESSAGE_SENT event on the bus.""" + if self._bus is not None: + self._bus.publish( + EventType.CHANNEL_MESSAGE_SENT, + { + "channel": channel, + "content": content, + "conversation_id": conversation_id, + }, + ) + + +__all__ = ["GmailChannel"] diff --git a/src/openjarvis/channels/twitter.py b/src/openjarvis/channels/twitter.py new file mode 100644 index 00000000..5a14dfd2 --- /dev/null +++ b/src/openjarvis/channels/twitter.py @@ -0,0 +1,242 @@ +"""TwitterChannel — native Twitter/X API adapter using tweepy.""" + +from __future__ import annotations + +import logging +import os +import threading +from typing import Any, Dict, List, Optional + +from openjarvis.channels._stubs import ( + BaseChannel, + ChannelHandler, + ChannelMessage, + ChannelStatus, +) +from openjarvis.core.events import EventBus, EventType +from openjarvis.core.registry import ChannelRegistry + +logger = logging.getLogger(__name__) + + +@ChannelRegistry.register("twitter") +class TwitterChannel(BaseChannel): + """Native Twitter/X channel adapter using tweepy. + + Parameters + ---------- + bearer_token: + Twitter API v2 Bearer Token. Falls back to ``TWITTER_BEARER_TOKEN``. + api_key: + Twitter API Key (consumer key). Falls back to ``TWITTER_API_KEY``. + api_secret: + Twitter API Secret (consumer secret). Falls back to ``TWITTER_API_SECRET``. + access_token: + Twitter Access Token. Falls back to ``TWITTER_ACCESS_TOKEN``. + access_secret: + Twitter Access Token Secret. Falls back to ``TWITTER_ACCESS_SECRET``. + poll_interval: + Seconds between mention polls (default 60). + bus: + Optional event bus for publishing channel events. + """ + + channel_id = "twitter" + + def __init__( + self, + bearer_token: str = "", + *, + api_key: str = "", + api_secret: str = "", + access_token: str = "", + access_secret: str = "", + poll_interval: int = 60, + bus: Optional[EventBus] = None, + ) -> None: + self._bearer_token = bearer_token or os.environ.get( + "TWITTER_BEARER_TOKEN", "", + ) + self._api_key = api_key or os.environ.get("TWITTER_API_KEY", "") + self._api_secret = api_secret or os.environ.get("TWITTER_API_SECRET", "") + self._access_token = access_token or os.environ.get( + "TWITTER_ACCESS_TOKEN", "", + ) + self._access_secret = access_secret or os.environ.get( + "TWITTER_ACCESS_SECRET", "", + ) + self._poll_interval = poll_interval + self._bus = bus + self._handlers: List[ChannelHandler] = [] + self._status = ChannelStatus.DISCONNECTED + self._client: Any = None + self._poll_thread: Optional[threading.Thread] = None + self._stop_event = threading.Event() + + # -- connection lifecycle --------------------------------------------------- + + def connect(self) -> None: + """Build a tweepy Client and optionally start polling for mentions.""" + if not self._bearer_token: + logger.warning("No Twitter bearer token configured") + self._status = ChannelStatus.ERROR + return + + self._stop_event.clear() + + try: + import tweepy # noqa: F401 + + self._client = tweepy.Client( + bearer_token=self._bearer_token, + consumer_key=self._api_key or None, + consumer_secret=self._api_secret or None, + access_token=self._access_token or None, + access_token_secret=self._access_secret or None, + ) + self._status = ChannelStatus.CONNECTED + logger.info("Twitter channel connected") + + if self._access_token: + self._poll_thread = threading.Thread( + target=self._poll_loop, daemon=True, + ) + self._poll_thread.start() + except ImportError: + logger.info("tweepy not installed; Twitter channel unavailable") + self._status = ChannelStatus.ERROR + except Exception: + logger.debug("Twitter connect failed", exc_info=True) + self._status = ChannelStatus.ERROR + + def disconnect(self) -> None: + """Stop the polling thread and disconnect.""" + self._stop_event.set() + if self._poll_thread is not None: + self._poll_thread.join(timeout=5.0) + self._poll_thread = None + self._client = None + self._status = ChannelStatus.DISCONNECTED + + # -- send / receive -------------------------------------------------------- + + def send( + self, + channel: str, + content: str, + *, + conversation_id: str = "", + metadata: Dict[str, Any] | None = None, + ) -> bool: + """Send a tweet or direct message. + + If *channel* is numeric, sends a DM via ``create_direct_message()``. + Otherwise sends a tweet via ``create_tweet()``. If *conversation_id* + is provided, it is used as ``in_reply_to_tweet_id``. + """ + if self._client is None: + logger.warning("Cannot send: Twitter client not connected") + return False + + try: + if channel.isdigit(): + # Direct message to a user ID + self._client.create_direct_message( + participant_id=int(channel), + text=content, + ) + else: + kwargs: Dict[str, Any] = {"text": content} + if conversation_id: + kwargs["in_reply_to_tweet_id"] = int(conversation_id) + self._client.create_tweet(**kwargs) + + self._publish_sent(channel, content, conversation_id) + return True + except Exception: + logger.debug("Twitter send failed", exc_info=True) + return False + + def status(self) -> ChannelStatus: + """Return the current connection status.""" + return self._status + + def list_channels(self) -> List[str]: + """Return available channel identifiers.""" + return ["timeline", "dm"] + + def on_message(self, handler: ChannelHandler) -> None: + """Register a callback for incoming messages.""" + self._handlers.append(handler) + + # -- internal helpers ------------------------------------------------------- + + def _poll_loop(self) -> None: + """Poll for new mentions in a background thread.""" + since_id: Optional[str] = None + + while not self._stop_event.is_set(): + try: + kwargs: Dict[str, Any] = {} + if since_id: + kwargs["since_id"] = since_id + + me = self._client.get_me() + if me and me.data: + user_id = me.data.id + else: + self._stop_event.wait(self._poll_interval) + continue + + response = self._client.get_users_mentions( + id=user_id, **kwargs, + ) + + if response and response.data: + for tweet in response.data: + since_id = str(tweet.id) + cm = ChannelMessage( + channel="twitter", + sender=str(getattr(tweet, "author_id", "")), + content=tweet.text, + message_id=str(tweet.id), + conversation_id=str( + getattr(tweet, "conversation_id", ""), + ), + ) + for handler in self._handlers: + try: + handler(cm) + except Exception: + logger.exception("Twitter handler error") + if self._bus is not None: + self._bus.publish( + EventType.CHANNEL_MESSAGE_RECEIVED, + { + "channel": cm.channel, + "sender": cm.sender, + "content": cm.content, + "message_id": cm.message_id, + }, + ) + except Exception: + logger.debug("Twitter poll error", exc_info=True) + + self._stop_event.wait(self._poll_interval) + + def _publish_sent( + self, channel: str, content: str, conversation_id: str, + ) -> None: + """Publish a CHANNEL_MESSAGE_SENT event on the bus.""" + if self._bus is not None: + self._bus.publish( + EventType.CHANNEL_MESSAGE_SENT, + { + "channel": channel, + "content": content, + "conversation_id": conversation_id, + }, + ) + + +__all__ = ["TwitterChannel"] diff --git a/tests/channels/conftest.py b/tests/channels/conftest.py new file mode 100644 index 00000000..b1d377ea --- /dev/null +++ b/tests/channels/conftest.py @@ -0,0 +1,65 @@ +"""Shared fixtures for channel integration tests.""" + +from __future__ import annotations + +import os +from contextlib import contextmanager + +import pytest + +from openjarvis.core.events import EventBus + +from tests.agents.fake_engine import FakeEngine +from tests.agents.scenario_harness import FakeSystem, ScenarioHarness + + +@pytest.fixture +def scenario_harness(tmp_path): + """Wire up real components for agent lifecycle testing (channels copy).""" + from openjarvis.agents.executor import AgentExecutor + from openjarvis.agents.manager import AgentManager + from openjarvis.agents.monitor_operative import MonitorOperativeAgent + from openjarvis.agents.scheduler import AgentScheduler + from openjarvis.core.registry import AgentRegistry + + if not AgentRegistry.contains("monitor_operative"): + AgentRegistry.register("monitor_operative")(MonitorOperativeAgent) + + db_path = str(tmp_path / "agents.db") + bus = EventBus(record_history=True) + manager = AgentManager(db_path=db_path) + engine = FakeEngine([{"content": "Default response."}]) + system = FakeSystem(engine=engine) + executor = AgentExecutor(manager=manager, event_bus=bus) + executor.set_system(system) + scheduler = AgentScheduler(manager=manager, executor=executor, event_bus=bus) + + return ScenarioHarness( + manager=manager, executor=executor, scheduler=scheduler, + bus=bus, engine=engine, system=system, db_path=db_path, + ) + + +@pytest.fixture +def fake_event_bus() -> EventBus: + """EventBus with history recording for asserting event emissions.""" + return EventBus(record_history=True) + + +@contextmanager +def credential_env(channel_name: str, **creds: str): + """Context manager that sets env vars for a channel, cleans up after.""" + prefix = channel_name.upper() + old_values = {} + for key, value in creds.items(): + env_key = f"{prefix}_{key.upper()}" + old_values[env_key] = os.environ.get(env_key) + os.environ[env_key] = value + try: + yield + finally: + for env_key, old in old_values.items(): + if old is None: + os.environ.pop(env_key, None) + else: + os.environ[env_key] = old diff --git a/tests/channels/test_agent_channel_e2e.py b/tests/channels/test_agent_channel_e2e.py new file mode 100644 index 00000000..1eaefa97 --- /dev/null +++ b/tests/channels/test_agent_channel_e2e.py @@ -0,0 +1,64 @@ +"""Agent-channel E2E tests — using WebChatChannel (in-memory, no external deps).""" + +from __future__ import annotations + +from tests.agents.scenario_harness import ScenarioHarness + + +def test_agent_sends_to_webchat(scenario_harness: ScenarioHarness): + """Agent runs and completes tick with WebChatChannel available.""" + from openjarvis.channels.webchat import WebChatChannel + + h = scenario_harness + webchat = WebChatChannel() + webchat.connect() + + agent = h.manager.create_agent("Channel Agent", config={ + "schedule_type": "manual", + "instruction": "Send a report to the general channel.", + }) + + h.executor.execute_tick(agent["id"]) + data = h.manager.get_agent(agent["id"]) + assert data["status"] == "idle" + assert data["total_runs"] == 1 + webchat.disconnect() + + +def test_channel_failure_does_not_crash_agent(scenario_harness: ScenarioHarness): + """Agent continues if channel send fails.""" + h = scenario_harness + agent = h.manager.create_agent("Resilient Agent", config={ + "schedule_type": "manual", + "instruction": "Try to send a message.", + }) + + h.executor.execute_tick(agent["id"]) + data = h.manager.get_agent(agent["id"]) + assert data["status"] == "idle" + assert data["total_runs"] == 1 + + +def test_agent_with_channel_binding(scenario_harness: ScenarioHarness): + """Agent with a channel binding completes tick without error.""" + from openjarvis.channels.webchat import WebChatChannel + + h = scenario_harness + webchat = WebChatChannel() + webchat.connect() + + agent = h.manager.create_agent("Bound Agent", config={ + "schedule_type": "manual", + "instruction": "Monitor and report.", + }) + aid = agent["id"] + + # Bind a channel + h.manager.bind_channel(aid, "webchat", config={"channel": "general"}) + bindings = h.manager.list_channel_bindings(aid) + assert len(bindings) == 1 + + h.executor.execute_tick(aid) + data = h.manager.get_agent(aid) + assert data["status"] == "idle" + webchat.disconnect() diff --git a/tests/channels/test_channel_contract.py b/tests/channels/test_channel_contract.py new file mode 100644 index 00000000..44f04252 --- /dev/null +++ b/tests/channels/test_channel_contract.py @@ -0,0 +1,85 @@ +"""Parametrized contract tests — verify every channel implements BaseChannel correctly. + +These tests run without any credentials and verify that all channel adapters +handle graceful degradation (no crashes on missing auth, idempotent disconnect, +valid enum returns). +""" + +from __future__ import annotations + +import importlib + +import pytest + +import openjarvis.channels # noqa: F401 — trigger registration +from openjarvis.channels._stubs import ChannelStatus +from openjarvis.core.registry import ChannelRegistry + +# Collect channel classes at import time (before registry gets cleared). +# We store the actual class objects, not registry keys, so they survive +# the autouse _clean_registries fixture. +importlib.reload(openjarvis.channels) +_ALL_CHANNELS = [ + (key, ChannelRegistry.get(key)) for key in ChannelRegistry.keys() +] + + +@pytest.fixture(params=_ALL_CHANNELS, ids=lambda x: x[0]) +def channel_entry(request): + """Parametrized fixture yielding (channel_key, channel_cls) tuples. + + Uses the class reference captured at import time — does not depend + on the registry being populated at test time. + """ + return request.param + + +def test_has_channel_id(channel_entry): + key, channel_cls = channel_entry + assert hasattr(channel_cls, "channel_id") + assert isinstance(channel_cls.channel_id, str) + assert len(channel_cls.channel_id) > 0 + + +def test_connect_no_credentials_no_crash(channel_entry): + key, channel_cls = channel_entry + ch = channel_cls() + ch.connect() + # Should not raise — just set status to ERROR or DISCONNECTED + + +def test_disconnect_idempotent(channel_entry): + key, channel_cls = channel_entry + ch = channel_cls() + ch.disconnect() + ch.disconnect() # Second call should not raise + + +def test_status_returns_valid_enum(channel_entry): + key, channel_cls = channel_entry + ch = channel_cls() + s = ch.status() + assert isinstance(s, ChannelStatus) + + +def test_list_channels_returns_list(channel_entry): + key, channel_cls = channel_entry + ch = channel_cls() + result = ch.list_channels() + assert isinstance(result, list) + + +def test_send_no_credentials_returns_false(channel_entry): + key, channel_cls = channel_entry + if key == "webchat": + pytest.skip("WebChatChannel is in-memory and always succeeds") + ch = channel_cls() + result = ch.send("test", "hello") + assert result is False + + +def test_on_message_accepts_handler(channel_entry): + key, channel_cls = channel_entry + ch = channel_cls() + ch.on_message(lambda msg: None) + # Should not raise diff --git a/tests/channels/test_tier1_gmail.py b/tests/channels/test_tier1_gmail.py new file mode 100644 index 00000000..75292689 --- /dev/null +++ b/tests/channels/test_tier1_gmail.py @@ -0,0 +1,209 @@ +"""Tests for the GmailChannel adapter.""" + +from __future__ import annotations + +import base64 +import os +from unittest.mock import MagicMock, patch + +import pytest + +from openjarvis.channels._stubs import ChannelStatus +from openjarvis.channels.gmail import GmailChannel +from openjarvis.core.events import EventBus, EventType +from openjarvis.core.registry import ChannelRegistry + + +@pytest.fixture(autouse=True) +def _register_gmail(): + """Re-register after any registry clear.""" + if not ChannelRegistry.contains("gmail"): + ChannelRegistry.register_value("gmail", GmailChannel) + + +class TestRegistration: + def test_gmail_channel_registered(self): + assert ChannelRegistry.contains("gmail") + + def test_channel_id(self): + ch = GmailChannel() + assert ch.channel_id == "gmail" + + +class TestNoCredentials: + def test_gmail_no_credentials_status(self): + ch = GmailChannel() + ch.connect() + assert ch.status() in (ChannelStatus.ERROR, ChannelStatus.DISCONNECTED) + + def test_gmail_send_no_credentials_returns_false(self): + ch = GmailChannel() + result = ch.send("recipient@example.com", "Hello!") + assert result is False + + +class TestSend: + def test_gmail_send_constructs_mime(self): + ch = GmailChannel() + mock_service = MagicMock() + ch._service = mock_service + ch._status = ChannelStatus.CONNECTED + + result = ch.send("recipient@example.com", "Hello Gmail!") + assert result is True + + # Verify send was called + mock_service.users().messages().send.assert_called_once() + call_kwargs = mock_service.users().messages().send.call_args[1] + assert call_kwargs["userId"] == "me" + + # Verify the body contains base64-encoded MIME message + body = call_kwargs["body"] + assert "raw" in body + decoded = base64.urlsafe_b64decode(body["raw"]).decode("utf-8") + assert "recipient@example.com" in decoded + assert "Hello Gmail!" in decoded + + def test_gmail_send_with_thread_id(self): + ch = GmailChannel() + mock_service = MagicMock() + ch._service = mock_service + ch._status = ChannelStatus.CONNECTED + + result = ch.send( + "recipient@example.com", + "Thread reply", + conversation_id="thread-abc123", + ) + assert result is True + + call_kwargs = mock_service.users().messages().send.call_args[1] + body = call_kwargs["body"] + assert body["threadId"] == "thread-abc123" + + def test_gmail_send_with_subject_metadata(self): + ch = GmailChannel() + mock_service = MagicMock() + ch._service = mock_service + ch._status = ChannelStatus.CONNECTED + + result = ch.send( + "recipient@example.com", + "Body text", + metadata={"subject": "Custom Subject"}, + ) + assert result is True + + call_kwargs = mock_service.users().messages().send.call_args[1] + decoded = base64.urlsafe_b64decode( + call_kwargs["body"]["raw"], + ).decode("utf-8") + assert "Custom Subject" in decoded + + def test_gmail_send_exception_returns_false(self): + ch = GmailChannel() + mock_service = MagicMock() + mock_service.users().messages().send().execute.side_effect = ( + RuntimeError("API error") + ) + ch._service = mock_service + ch._status = ChannelStatus.CONNECTED + + result = ch.send("recipient@example.com", "Hello!") + assert result is False + + +class TestListChannels: + def test_gmail_list_channels(self): + ch = GmailChannel() + assert ch.list_channels() == ["inbox"] + + +class TestOnMessage: + def test_gmail_on_message_registers_handler(self): + ch = GmailChannel() + handler = MagicMock() + ch.on_message(handler) + assert handler in ch._handlers + + +class TestEventBus: + def test_gmail_event_bus_integration(self): + bus = EventBus(record_history=True) + ch = GmailChannel(bus=bus) + mock_service = MagicMock() + ch._service = mock_service + ch._status = ChannelStatus.CONNECTED + + ch.send("recipient@example.com", "Hello!") + + event_types = [e.event_type for e in bus.history] + assert EventType.CHANNEL_MESSAGE_SENT in event_types + + +class TestStatus: + def test_disconnected_initially(self): + ch = GmailChannel() + assert ch.status() == ChannelStatus.DISCONNECTED + + def test_status_error_when_connected_but_no_service(self): + ch = GmailChannel() + ch._status = ChannelStatus.CONNECTED + ch._service = None + assert ch.status() == ChannelStatus.ERROR + + +class TestDisconnect: + def test_disconnect(self): + ch = GmailChannel() + ch._service = MagicMock() + ch._status = ChannelStatus.CONNECTED + ch.disconnect() + assert ch.status() == ChannelStatus.DISCONNECTED + assert ch._service is None + + +class TestEnvVarFallback: + def test_credentials_path_env_var(self): + with patch.dict(os.environ, {"GMAIL_CREDENTIALS_PATH": "/tmp/creds.json"}): + ch = GmailChannel() + assert ch._credentials_path == "/tmp/creds.json" + + def test_token_path_env_var(self): + with patch.dict(os.environ, {"GMAIL_TOKEN_PATH": "/tmp/token.json"}): + ch = GmailChannel() + assert ch._token_path == "/tmp/token.json" + + def test_constructor_overrides_env(self): + with patch.dict(os.environ, {"GMAIL_CREDENTIALS_PATH": "/env/creds.json"}): + ch = GmailChannel(credentials_path="/explicit/creds.json") + assert ch._credentials_path == "/explicit/creds.json" + + +@pytest.mark.live_channel +class TestLive: + def test_gmail_send_live(self): + creds_path = os.environ.get("GMAIL_CREDENTIALS_PATH", "") + token_path = os.environ.get("GMAIL_TOKEN_PATH", "") + recipient = os.environ.get("GMAIL_TEST_RECIPIENT", "") + + if not creds_path and not token_path: + pytest.skip("No Gmail credentials configured") + if not recipient: + pytest.skip("No GMAIL_TEST_RECIPIENT configured") + + ch = GmailChannel( + credentials_path=creds_path, + token_path=token_path, + ) + ch.connect() + if ch.status() != ChannelStatus.CONNECTED: + pytest.skip("Gmail channel failed to connect") + + result = ch.send( + recipient, + "OpenJarvis Gmail channel test message", + metadata={"subject": "OpenJarvis Test"}, + ) + assert result is True + ch.disconnect() diff --git a/tests/channels/test_tier2_twitter.py b/tests/channels/test_tier2_twitter.py new file mode 100644 index 00000000..fcc15924 --- /dev/null +++ b/tests/channels/test_tier2_twitter.py @@ -0,0 +1,136 @@ +"""Tests for the TwitterChannel adapter.""" + +from __future__ import annotations + +import os +from unittest.mock import MagicMock + +import pytest + +from openjarvis.channels._stubs import ChannelStatus +from openjarvis.channels.twitter import TwitterChannel +from openjarvis.core.events import EventBus, EventType +from openjarvis.core.registry import ChannelRegistry + + +@pytest.fixture(autouse=True) +def _register_twitter(): + """Re-register after any registry clear.""" + if not ChannelRegistry.contains("twitter"): + ChannelRegistry.register_value("twitter", TwitterChannel) + + +def test_twitter_channel_registered(): + """Twitter channel should be discoverable via the registry.""" + assert ChannelRegistry.contains("twitter") + cls = ChannelRegistry.get("twitter") + assert cls is TwitterChannel + + +def test_twitter_no_credentials_status(): + """Without credentials, connect() sets status to ERROR.""" + ch = TwitterChannel() + ch.connect() + assert ch.status() == ChannelStatus.ERROR + + +def test_twitter_send_no_credentials_returns_false(): + """send() returns False when client is not connected.""" + ch = TwitterChannel() + result = ch.send("timeline", "Hello Twitter!") + assert result is False + + +def test_twitter_send_tweet(): + """send() with a non-numeric channel calls create_tweet.""" + ch = TwitterChannel(bearer_token="test-bearer") + mock_client = MagicMock() + ch._client = mock_client + ch._status = ChannelStatus.CONNECTED + + result = ch.send("timeline", "Hello from Jarvis!") + assert result is True + mock_client.create_tweet.assert_called_once_with(text="Hello from Jarvis!") + + +def test_twitter_send_dm_when_channel_is_numeric(): + """send() with a numeric channel calls create_direct_message.""" + ch = TwitterChannel(bearer_token="test-bearer") + mock_client = MagicMock() + ch._client = mock_client + ch._status = ChannelStatus.CONNECTED + + result = ch.send("12345678", "Hello via DM!") + assert result is True + mock_client.create_direct_message.assert_called_once_with( + participant_id=12345678, + text="Hello via DM!", + ) + + +def test_twitter_send_reply(): + """send() with conversation_id sets in_reply_to_tweet_id.""" + ch = TwitterChannel(bearer_token="test-bearer") + mock_client = MagicMock() + ch._client = mock_client + ch._status = ChannelStatus.CONNECTED + + result = ch.send( + "timeline", "This is a reply", conversation_id="999888777", + ) + assert result is True + mock_client.create_tweet.assert_called_once_with( + text="This is a reply", + in_reply_to_tweet_id=999888777, + ) + + +def test_twitter_list_channels(): + """list_channels() returns the expected identifiers.""" + ch = TwitterChannel() + assert ch.list_channels() == ["timeline", "dm"] + + +def test_twitter_event_bus_integration(): + """Successful send publishes CHANNEL_MESSAGE_SENT on the event bus.""" + bus = EventBus(record_history=True) + ch = TwitterChannel(bearer_token="test-bearer", bus=bus) + mock_client = MagicMock() + ch._client = mock_client + ch._status = ChannelStatus.CONNECTED + + ch.send("timeline", "Event test!") + + event_types = [e.event_type for e in bus.history] + assert EventType.CHANNEL_MESSAGE_SENT in event_types + + sent_event = next( + e for e in bus.history + if e.event_type == EventType.CHANNEL_MESSAGE_SENT + ) + assert sent_event.data["content"] == "Event test!" + assert sent_event.data["channel"] == "timeline" + + +@pytest.mark.live_channel +def test_twitter_connect_live(): + """Live test: connect with real credentials from env vars. + + Skipped unless all required env vars are set. + """ + required_vars = [ + "TWITTER_BEARER_TOKEN", + "TWITTER_API_KEY", + "TWITTER_API_SECRET", + "TWITTER_ACCESS_TOKEN", + "TWITTER_ACCESS_SECRET", + ] + for var in required_vars: + if not os.environ.get(var): + pytest.skip(f"Missing env var {var}") + + ch = TwitterChannel() + ch.connect() + assert ch.status() == ChannelStatus.CONNECTED + ch.disconnect() + assert ch.status() == ChannelStatus.DISCONNECTED diff --git a/tests/core/test_config.py b/tests/core/test_config.py index 7034cb46..e8e8090f 100644 --- a/tests/core/test_config.py +++ b/tests/core/test_config.py @@ -455,7 +455,8 @@ class TestApplyTomlSectionListNormalization: from openjarvis.core.config import ToolsConfig, _apply_toml_section target = ToolsConfig() - _apply_toml_section(target, {"enabled": ["code_interpreter", "web_search", "file_read"]}) + tools = ["code_interpreter", "web_search", "file_read"] + _apply_toml_section(target, {"enabled": tools}) assert isinstance(target.enabled, str) assert target.enabled == "code_interpreter,web_search,file_read" From 593132f6c194e69348874d4b47bdd284fd1b286a Mon Sep 17 00:00:00 2001 From: Jon Saad-Falcon <41205309+jonsaadfalcon@users.noreply.github.com> Date: Mon, 16 Mar 2026 21:06:13 -0700 Subject: [PATCH 06/44] fix: sort imports in channels conftest --- tests/channels/conftest.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/channels/conftest.py b/tests/channels/conftest.py index b1d377ea..1ed8dcd8 100644 --- a/tests/channels/conftest.py +++ b/tests/channels/conftest.py @@ -8,7 +8,6 @@ from contextlib import contextmanager import pytest from openjarvis.core.events import EventBus - from tests.agents.fake_engine import FakeEngine from tests.agents.scenario_harness import FakeSystem, ScenarioHarness From 43b1240a440c6342de85bce1434afd562fcaa3c9 Mon Sep 17 00:00:00 2001 From: Jon Saad-Falcon <41205309+jonsaadfalcon@users.noreply.github.com> Date: Wed, 25 Mar 2026 20:51:12 -0700 Subject: [PATCH 07/44] feat: remote engine host configuration via CLI (#104) Add jarvis config set command, --host flag to jarvis init, and improved error messages for configuring remote LLM engine endpoints. Closes #104. --- pyproject.toml | 1 + src/openjarvis/cli/ask.py | 124 ++++++++++++------ src/openjarvis/cli/config_cmd.py | 98 +++++++++++++++ src/openjarvis/cli/hints.py | 6 +- src/openjarvis/cli/init_cmd.py | 63 +++++++--- src/openjarvis/core/config.py | 102 ++++++++++++++- tests/cli/test_config_set.py | 154 +++++++++++++++++++++++ tests/cli/test_hints.py | 10 ++ tests/cli/test_init_host.py | 117 +++++++++++++++++ tests/core/test_config_key_validation.py | 54 ++++++++ uv.lock | 111 +++++++++++++++- 11 files changed, 776 insertions(+), 64 deletions(-) create mode 100644 tests/cli/test_config_set.py create mode 100644 tests/cli/test_init_host.py create mode 100644 tests/core/test_config_key_validation.py diff --git a/pyproject.toml b/pyproject.toml index 67e820eb..4af236ee 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -17,6 +17,7 @@ dependencies = [ "python-telegram-bot>=22.6", "rich>=13", "tomli>=2.0; python_version < '3.11'", + "tomlkit>=0.12", ] [project.optional-dependencies] diff --git a/src/openjarvis/cli/ask.py b/src/openjarvis/cli/ask.py index 43014909..594338e4 100644 --- a/src/openjarvis/cli/ask.py +++ b/src/openjarvis/cli/ask.py @@ -44,7 +44,8 @@ def _get_memory_backend(config): if key == "sqlite": backend = MemoryRegistry.create( - key, db_path=config.memory.db_path, + key, + db_path=config.memory.db_path, ) else: backend = MemoryRegistry.create(key) @@ -106,8 +107,7 @@ def _run_agent( if not AgentRegistry.contains(agent_name): raise click.ClickException( - f"Unknown agent: {agent_name}. " - f"Available: {', '.join(AgentRegistry.keys())}" + f"Unknown agent: {agent_name}. Available: {', '.join(AgentRegistry.keys())}" ) agent_cls = AgentRegistry.get(agent_name) @@ -117,6 +117,7 @@ def _run_agent( if tool_names: # Trigger tool registration import openjarvis.tools # noqa: F401 + tools = _build_tools(tool_names, config, engine, model_name) # Build agent with appropriate kwargs @@ -149,7 +150,10 @@ def _run_agent( max_context_tokens=config.memory.context_max_tokens, ) context_messages = inject_context( - query_text, [], backend, config=ctx_cfg, + query_text, + [], + backend, + config=ctx_cfg, ) for msg in context_messages: ctx.conversation.add(msg) @@ -169,9 +173,7 @@ def _print_profile( ) -> None: """Print an inference telemetry profile table from EventBus history.""" # Collect all INFERENCE_END events (agents may fire multiple) - inf_events = [ - e for e in bus.history if e.event_type == EventType.INFERENCE_END - ] + inf_events = [e for e in bus.history if e.event_type == EventType.INFERENCE_END] if not inf_events: console.print("[dim]No inference telemetry recorded.[/dim]") return @@ -186,13 +188,15 @@ def _print_profile( for e in inf_events ) total_prompt = sum( - e.data.get("usage", {}).get("prompt_tokens", 0) - for e in inf_events + e.data.get("usage", {}).get("prompt_tokens", 0) for e in inf_events ) total_energy = sum(e.data.get("energy_joules", 0.0) for e in inf_events) avg_power = 0.0 - power_vals = [e.data.get("power_watts", 0.0) for e in inf_events - if e.data.get("power_watts", 0.0) > 0] + power_vals = [ + e.data.get("power_watts", 0.0) + for e in inf_events + if e.data.get("power_watts", 0.0) > 0 + ] if power_vals: avg_power = sum(power_vals) / len(power_vals) @@ -284,29 +288,42 @@ def _print_profile( @click.option("-m", "--model", "model_name", default=None, help="Model to use.") @click.option("-e", "--engine", "engine_key", default=None, help="Engine backend.") @click.option( - "-t", "--temperature", default=None, type=float, + "-t", + "--temperature", + default=None, + type=float, help="Sampling temperature (default: from config).", ) @click.option( - "--max-tokens", default=None, type=int, + "--max-tokens", + default=None, + type=int, help="Max tokens to generate (default: from config).", ) @click.option("--json", "output_json", is_flag=True, help="Output raw JSON result.") @click.option("--no-stream", is_flag=True, help="Disable streaming (sync mode).") @click.option( - "--no-context", is_flag=True, + "--no-context", + is_flag=True, help="Disable memory context injection.", ) @click.option( - "-a", "--agent", "agent_name", default=None, + "-a", + "--agent", + "agent_name", + default=None, help="Agent to use (simple, orchestrator).", ) @click.option( - "--tools", "tool_names", default=None, + "--tools", + "tool_names", + default=None, help="Comma-separated tool names to enable (e.g. calculator,think).", ) @click.option( - "--profile", "enable_profile", is_flag=True, + "--profile", + "enable_profile", + is_flag=True, help="Print inference telemetry profile (latency, tokens, energy, IPW).", ) def ask( @@ -377,7 +394,10 @@ def ask( " [cyan]ollama serve[/cyan] — start Ollama\n" " [cyan]vllm serve [/cyan] — start vLLM\n" " [cyan]llama-server -m [/cyan] — start llama.cpp\n\n" - "Or set OPENAI_API_KEY / ANTHROPIC_API_KEY for cloud inference." + "Or set OPENAI_API_KEY / ANTHROPIC_API_KEY for cloud inference.\n\n" + "[dim]To use a remote engine:[/dim]\n" + " [cyan]jarvis config set engine.ollama.host http://:11434[/cyan]\n" + " [dim]or[/dim] [cyan]export OLLAMA_HOST=http://:11434[/cyan]" ) sys.exit(1) @@ -385,6 +405,7 @@ def ask( # Apply security guardrails from openjarvis.security import setup_security + sec = setup_security(config, engine, bus) engine = sec.engine @@ -427,12 +448,14 @@ def ask( # the user would have gotten without the analyzer. if not user_set_max_tokens: suggested = adjust_tokens_for_model( - complexity_result.suggested_max_tokens, model_name, + complexity_result.suggested_max_tokens, + model_name, ) max_tokens = max(suggested, config.intelligence.max_tokens) logger.debug( "Using complexity-suggested max_tokens=%d (model=%s)", - max_tokens, model_name, + max_tokens, + model_name, ) # Agent mode @@ -444,8 +467,15 @@ def ask( ) try: result = _run_agent( - agent_name, query_text, engine, model_name, - parsed_tools, config, bus, temperature, max_tokens, + agent_name, + query_text, + engine, + model_name, + parsed_tools, + config, + bus, + temperature, + max_tokens, capability_policy=sec.capability_policy, ) except EngineConnectionError as exc: @@ -454,25 +484,33 @@ def ask( sys.exit(1) if output_json: - click.echo(json_mod.dumps({ - "content": result.content, - "turns": result.turns, - "tool_results": [ + click.echo( + json_mod.dumps( { - "tool_name": tr.tool_name, - "content": tr.content, - "success": tr.success, - } - for tr in result.tool_results - ], - }, indent=2)) + "content": result.content, + "turns": result.turns, + "tool_results": [ + { + "tool_name": tr.tool_name, + "content": tr.content, + "success": tr.success, + } + for tr in result.tool_results + ], + }, + indent=2, + ) + ) else: click.echo(result.content) if enable_profile: _print_profile( - bus, time.monotonic() - wall_start, - engine_name, model_name, console, + bus, + time.monotonic() - wall_start, + engine_name, + model_name, + console, complexity_result=complexity_result, ) @@ -493,17 +531,18 @@ def ask( ContextConfig, inject_context, ) + backend = _get_memory_backend(config) if backend is not None: ctx_cfg = ContextConfig( top_k=config.memory.context_top_k, min_score=config.memory.context_min_score, - max_context_tokens=( - config.memory.context_max_tokens - ), + max_context_tokens=(config.memory.context_max_tokens), ) messages = inject_context( - query_text, messages, backend, + query_text, + messages, + backend, config=ctx_cfg, ) except Exception as exc: @@ -531,8 +570,11 @@ def ask( if enable_profile: _print_profile( - bus, time.monotonic() - wall_start, - engine_name, model_name, console, + bus, + time.monotonic() - wall_start, + engine_name, + model_name, + console, complexity_result=complexity_result, ) diff --git a/src/openjarvis/cli/config_cmd.py b/src/openjarvis/cli/config_cmd.py index 324a348a..43e5be3f 100644 --- a/src/openjarvis/cli/config_cmd.py +++ b/src/openjarvis/cli/config_cmd.py @@ -4,9 +4,11 @@ from __future__ import annotations import json import os +import re from pathlib import Path import click +import httpx from rich.console import Console from rich.panel import Panel from rich.syntax import Syntax @@ -274,4 +276,100 @@ def hardware() -> None: config.add_command(show_group, "show") +def _probe_engine_host(url: str, console: Console) -> None: + """Probe an engine host URL and print reachability status.""" + try: + resp = httpx.get(url.rstrip("/") + "/", timeout=2.0) + if resp.status_code < 500: + console.print(f" [green]Reachable[/green] ({url})") + else: + console.print( + f" [yellow]Warning:[/yellow] Host returned status " + f"{resp.status_code} — config saved anyway." + ) + except Exception: + console.print( + f" [yellow]Warning:[/yellow] Host unreachable ({url}) " + f"— config saved anyway." + ) + + +def _coerce_value(value: str, target_type: type) -> object: + """Coerce a CLI string value to the target Python type.""" + if target_type is bool: + low = value.lower() + if low in ("true", "1", "yes"): + return True + if low in ("false", "0", "no"): + return False + raise ValueError( + f"Invalid boolean value: {value!r} (expected: true/false, yes/no, 1/0)" + ) + if target_type is int: + return int(value) + if target_type is float: + return float(value) + return value + + +@click.command("set") +@click.argument("key") +@click.argument("value") +def set_config(key: str, value: str) -> None: + """Set a configuration value (e.g. jarvis config set engine.ollama.host URL).""" + import tomlkit + + from openjarvis.core.config import DEFAULT_CONFIG_DIR, validate_config_key + + console = Console(stderr=True) + + # Validate key + try: + target_type = validate_config_key(key) + except ValueError as exc: + console.print(f"[red]Error:[/red] {exc}") + raise SystemExit(1) + + # Coerce value + try: + typed_value = _coerce_value(value, target_type) + except (ValueError, TypeError) as exc: + console.print( + f"[red]Error:[/red] Cannot convert {value!r} to " + f"{target_type.__name__}: {exc}" + ) + raise SystemExit(1) + + # Load or create TOML document + config_path = Path( + os.environ.get("OPENJARVIS_CONFIG", DEFAULT_CONFIG_DIR / "config.toml") + ) + if config_path.exists(): + doc = tomlkit.parse(config_path.read_text()) + else: + doc = tomlkit.document() + config_path.parent.mkdir(parents=True, exist_ok=True) + + # Set nested key + parts = key.split(".") + current = doc + for part in parts[:-1]: + if part not in current: + current.add(part, tomlkit.table()) + current = current[part] + current[parts[-1]] = typed_value + + # Write back + config_path.write_text(tomlkit.dumps(doc)) + + console.print(f"[green]Set[/green] {key} = {value!r}") + + # Probe engine host if applicable + if re.match(r"^engine\.\w+\.host$", key): + _probe_engine_host(value, console) + + +config.add_command(set_config, "set") + + __all__ = ["config"] diff --git a/src/openjarvis/cli/hints.py b/src/openjarvis/cli/hints.py index 9c8617cf..c2b1e3cd 100644 --- a/src/openjarvis/cli/hints.py +++ b/src/openjarvis/cli/hints.py @@ -22,7 +22,11 @@ def hint_no_engine(engine_name: Optional[str] = None) -> str: f"[yellow]Hint:[/yellow] Engine '{name}' is not reachable.\n" f" Make sure the {name} server is running.\n" " Run [bold]jarvis doctor[/bold] to check all engines.\n" - " Run [bold]jarvis quickstart[/bold] for guided setup." + " Run [bold]jarvis quickstart[/bold] for guided setup.\n" + "\n" + " [dim]To use a remote engine:[/dim]\n" + f" [cyan]jarvis config set engine.{name}.host http://:[/cyan]\n" + f" [dim]or[/dim] [cyan]export OLLAMA_HOST=http://:11434[/cyan]" ) diff --git a/src/openjarvis/cli/init_cmd.py b/src/openjarvis/cli/init_cmd.py index 8d729b7e..a60f0217 100644 --- a/src/openjarvis/cli/init_cmd.py +++ b/src/openjarvis/cli/init_cmd.py @@ -6,6 +6,7 @@ from pathlib import Path from typing import Optional import click +import httpx from rich.console import Console from rich.markup import escape from rich.panel import Panel @@ -25,8 +26,14 @@ from openjarvis.core.config import ( # Engines supported by ``jarvis init --engine``. _SUPPORTED_ENGINES = [ - "ollama", "vllm", "sglang", "llamacpp", "mlx", "lmstudio", - "exo", "nexa", + "ollama", + "vllm", + "sglang", + "llamacpp", + "mlx", + "lmstudio", + "exo", + "nexa", ] @@ -70,7 +77,7 @@ def _next_steps_text(engine: str, model: str = "") -> str: f" ollama pull {pull_model}\n" "\n" " 3. Try it out:\n" - " jarvis ask \"Hello\"\n" + ' jarvis ask "Hello"\n' "\n" " Run `jarvis doctor` to verify your setup." ), @@ -82,7 +89,7 @@ def _next_steps_text(engine: str, model: str = "") -> str: " vllm serve Qwen/Qwen3-4B\n" "\n" " 2. Try it out:\n" - " jarvis ask \"Hello\"\n" + ' jarvis ask "Hello"\n' "\n" " Run `jarvis doctor` to verify your setup." ), @@ -94,7 +101,7 @@ def _next_steps_text(engine: str, model: str = "") -> str: " llama-server -m path/to/model.gguf\n" "\n" " 2. Try it out:\n" - " jarvis ask \"Hello\"\n" + ' jarvis ask "Hello"\n' "\n" " Run `jarvis doctor` to verify your setup." ), @@ -106,7 +113,7 @@ def _next_steps_text(engine: str, model: str = "") -> str: " python -m sglang.launch_server --model-path Qwen/Qwen3-8B\n" "\n" " 2. Try it out:\n" - " jarvis ask \"Hello\"\n" + ' jarvis ask "Hello"\n' "\n" " Run `jarvis doctor` to verify your setup." ), @@ -118,7 +125,7 @@ def _next_steps_text(engine: str, model: str = "") -> str: " mlx_lm.server --model mlx-community/Qwen2.5-7B-4bit\n" "\n" " 2. Try it out:\n" - " jarvis ask \"Hello\"\n" + ' jarvis ask "Hello"\n' "\n" " Run `jarvis doctor` to verify your setup." ), @@ -131,7 +138,7 @@ def _next_steps_text(engine: str, model: str = "") -> str: " 2. Load a model and start the local server (port 1234)\n" "\n" " 3. Try it out:\n" - " jarvis ask \"Hello\"\n" + ' jarvis ask "Hello"\n' "\n" " Run `jarvis doctor` to verify your setup." ), @@ -141,7 +148,7 @@ def _next_steps_text(engine: str, model: str = "") -> str: " pip install exo\n" " exo\n\n" " 2. Try it out:\n" - " jarvis ask \"Hello\"\n\n" + ' jarvis ask "Hello"\n\n' " Run `jarvis doctor` to verify your setup." ), "nexa": ( @@ -150,7 +157,7 @@ def _next_steps_text(engine: str, model: str = "") -> str: " pip install nexaai\n" " nexa server\n\n" " 2. Try it out:\n" - " jarvis ask \"Hello\"\n\n" + ' jarvis ask "Hello"\n\n' " Run `jarvis doctor` to verify your setup." ), } @@ -177,6 +184,7 @@ def _quick_privacy_check(console: Console) -> None: def _do_download(engine: str, model: str, spec, console: Console) -> None: """Dispatch model download based on engine type.""" import os + if engine == "ollama": host = os.environ.get("OLLAMA_HOST", "http://localhost:11434").rstrip("/") ollama_pull(host, model, console) @@ -228,12 +236,18 @@ def _do_download(engine: str, model: str, spec, console: Console) -> None: @click.option( "--no-download", is_flag=True, default=False, help="Skip the model download prompt." ) +@click.option( + "--host", + default=None, + help="Remote engine host URL (e.g. http://192.168.1.50:11434).", +) def init( force: bool, config: Optional[Path], full_config: bool = False, engine: Optional[str] = None, no_download: bool = False, + host: Optional[str] = None, ) -> None: """Detect hardware and generate ~/.openjarvis/config.toml.""" console = Console() @@ -267,9 +281,7 @@ def init( console.print("[bold]Detecting running inference engines...[/bold]") running = _detect_running_engines() if running: - console.print( - f" Found running: [green]{', '.join(running)}[/green]" - ) + console.print(f" Found running: [green]{', '.join(running)}[/green]") else: console.print(" No running engines detected.") @@ -313,13 +325,31 @@ def init( default=default, ) + # Probe remote host if specified + if host: + console.print("\n[bold]Checking remote host...[/bold]") + try: + resp = httpx.get(host.rstrip("/") + "/", timeout=2.0) + if resp.status_code < 500: + console.print(f" [green]Reachable[/green] ({host})") + else: + console.print( + f" [yellow]Warning:[/yellow] Host returned status " + f"{resp.status_code} — writing config anyway." + ) + except Exception: + console.print( + f" [yellow]Warning:[/yellow] Host unreachable ({host}) " + f"— writing config anyway." + ) + if config: toml_content = config.read_text() else: if full_config: - toml_content = generate_default_toml(hw, engine=engine) + toml_content = generate_default_toml(hw, engine=engine, host=host) else: - toml_content = generate_minimal_toml(hw, engine=engine) + toml_content = generate_minimal_toml(hw, engine=engine, host=host) DEFAULT_CONFIG_DIR.mkdir(parents=True, exist_ok=True) if config: @@ -341,8 +371,7 @@ def init( soul_path = DEFAULT_CONFIG_DIR / "SOUL.md" if not soul_path.exists(): soul_path.write_text( - "# Agent Persona\n\n" - "You are Jarvis, a helpful personal AI assistant.\n" + "# Agent Persona\n\nYou are Jarvis, a helpful personal AI assistant.\n" ) memory_path = DEFAULT_CONFIG_DIR / "MEMORY.md" diff --git a/src/openjarvis/core/config.py b/src/openjarvis/core/config.py index 9a829fd4..d8f1f10f 100644 --- a/src/openjarvis/core/config.py +++ b/src/openjarvis/core/config.py @@ -1157,6 +1157,80 @@ class JarvisConfig: self.tools.storage = value +# --------------------------------------------------------------------------- +# Config key validation +# --------------------------------------------------------------------------- + +# Sections that users may set via ``jarvis config set``. +# ``hardware`` is auto-detected and not user-settable. +_SETTABLE_SECTIONS = frozenset(JarvisConfig.__dataclass_fields__.keys()) - {"hardware"} + + +def validate_config_key(dotted_key: str) -> type: + """Validate a dotted config key and return the leaf field's Python type. + + Raises :class:`ValueError` when the key does not map to a known field. + The function walks the ``JarvisConfig`` dataclass hierarchy using + ``dataclasses.fields()``. + + Examples:: + + validate_config_key("engine.ollama.host") # -> str + validate_config_key("intelligence.temperature") # -> float + """ + from dataclasses import fields as dc_fields + + parts = dotted_key.split(".") + if len(parts) < 2: + raise ValueError( + f"Config key must have at least two segments (e.g. engine.default), " + f"got: {dotted_key!r}" + ) + + if parts[0] not in _SETTABLE_SECTIONS: + raise ValueError( + f"Unknown config key: {dotted_key!r} " + f"(valid top-level sections: {sorted(_SETTABLE_SECTIONS)})" + ) + + # Walk the dataclass tree + current_cls = JarvisConfig + for i, part in enumerate(parts): + field_map = {f.name: f for f in dc_fields(current_cls)} + if part not in field_map: + path_so_far = ".".join(parts[: i + 1]) + raise ValueError( + f"Unknown config key: {dotted_key!r} " + f"(no field {part!r} at {path_so_far}; " + f"valid fields: {sorted(field_map.keys())})" + ) + fld = field_map[part] + # Resolve the type — unwrap Optional, etc. + fld_type = fld.type + if isinstance(fld_type, str): + # Evaluate forward references in the config module namespace + import openjarvis.core.config as _cfg_mod + + fld_type = eval(fld_type, vars(_cfg_mod)) # noqa: S307 + + if i == len(parts) - 1: + # Leaf — return the primitive type + return fld_type + else: + # Must be a nested dataclass + if not hasattr(fld_type, "__dataclass_fields__"): + path_so_far = ".".join(parts[: i + 1]) + raise ValueError( + f"Unknown config key: {dotted_key!r} " + f"({path_so_far} is a leaf of type {fld_type.__name__}, " + f"not a section)" + ) + current_cls = fld_type + + # Should not reach here, but satisfy type checker + raise ValueError(f"Unknown config key: {dotted_key!r}") + + # --------------------------------------------------------------------------- # TOML loading # --------------------------------------------------------------------------- @@ -1283,7 +1357,9 @@ def load_config(path: Optional[Path] = None) -> JarvisConfig: # --------------------------------------------------------------------------- -def generate_minimal_toml(hw: HardwareInfo, engine: str | None = None) -> str: +def generate_minimal_toml( + hw: HardwareInfo, engine: str | None = None, *, host: str | None = None +) -> str: """Render a minimal TOML config with only essential settings.""" engine = engine or recommend_engine(hw) model = recommend_model(hw, engine) @@ -1291,6 +1367,14 @@ def generate_minimal_toml(hw: HardwareInfo, engine: str | None = None) -> str: if hw.gpu: mem_label = "unified memory" if hw.gpu.vendor == "apple" else "VRAM" gpu_comment = f"\n# GPU: {hw.gpu.name} ({hw.gpu.vram_gb} GB {mem_label})" + if host: + engine_host_section = f'\n[engine.{engine}]\nhost = "{host}"\n' + else: + engine_host_section = ( + f"\n[engine.{engine}]\n" + f'# host = "http://localhost:11434" ' + f"# set to remote URL if engine runs elsewhere\n" + ) return f"""\ # OpenJarvis configuration # Hardware: {hw.cpu_brand} ({hw.cpu_count} cores, {hw.ram_gb} GB RAM){gpu_comment} @@ -1298,7 +1382,7 @@ def generate_minimal_toml(hw: HardwareInfo, engine: str | None = None) -> str: [engine] default = "{engine}" - +{engine_host_section} [intelligence] default_model = "{model}" @@ -1310,7 +1394,9 @@ enabled = ["code_interpreter", "web_search", "file_read", "shell_exec"] """ -def generate_default_toml(hw: HardwareInfo, engine: str | None = None) -> str: +def generate_default_toml( + hw: HardwareInfo, engine: str | None = None, *, host: str | None = None +) -> str: """Render a commented TOML string suitable for ``~/.openjarvis/config.toml``.""" engine = engine or recommend_engine(hw) model = recommend_model(hw, engine) @@ -1322,7 +1408,7 @@ def generate_default_toml(hw: HardwareInfo, engine: str | None = None) -> str: if model: model_comment = " # recommended for your hardware" - return f"""\ + result = f"""\ # OpenJarvis configuration # Generated by `jarvis init` # @@ -1517,6 +1603,13 @@ ssrf_protection = true # assistant_name = "Jarvis" # assistant_has_own_number = false """ + if host: + import re as _re + + pattern = _re.escape(f"[engine.{engine}]") + r"\nhost = \"[^\"]*\"" + replacement = f'[engine.{engine}]\\nhost = "{host}"' + result = _re.sub(pattern, replacement, result) + return result __all__ = [ @@ -1581,4 +1674,5 @@ __all__ = [ "load_config", "recommend_engine", "recommend_model", + "validate_config_key", ] diff --git a/tests/cli/test_config_set.py b/tests/cli/test_config_set.py new file mode 100644 index 00000000..54162256 --- /dev/null +++ b/tests/cli/test_config_set.py @@ -0,0 +1,154 @@ +"""Tests for ``jarvis config set`` command.""" + +from __future__ import annotations + +import os +from pathlib import Path +from unittest import mock + +from click.testing import CliRunner + +from openjarvis.cli import cli + + +class TestConfigSet: + def test_set_creates_config_file(self, tmp_path: Path) -> None: + """config set creates config.toml if it doesn't exist.""" + config_file = tmp_path / "config.toml" + env = {"OPENJARVIS_CONFIG": str(config_file)} + with mock.patch.dict(os.environ, env): + result = CliRunner().invoke( + cli, ["config", "set", "engine.default", "vllm"] + ) + assert result.exit_code == 0 + assert config_file.exists() + content = config_file.read_text() + assert "vllm" in content + + def test_set_engine_ollama_host(self, tmp_path: Path) -> None: + """config set writes engine.ollama.host correctly.""" + config_file = tmp_path / "config.toml" + config_file.write_text('[engine]\ndefault = "ollama"\n') + env = {"OPENJARVIS_CONFIG": str(config_file)} + with ( + mock.patch.dict(os.environ, env), + mock.patch("openjarvis.cli.config_cmd.httpx"), + ): + result = CliRunner().invoke( + cli, + ["config", "set", "engine.ollama.host", "http://192.168.1.50:11434"], + ) + assert result.exit_code == 0 + content = config_file.read_text() + assert "http://192.168.1.50:11434" in content + + def test_set_preserves_existing_keys(self, tmp_path: Path) -> None: + """config set preserves other keys in the file.""" + config_file = tmp_path / "config.toml" + config_file.write_text( + '[engine]\ndefault = "ollama"\n\n' + "[intelligence]\n" + 'default_model = "qwen2.5:3b"\n' + ) + env = {"OPENJARVIS_CONFIG": str(config_file)} + with mock.patch.dict(os.environ, env): + result = CliRunner().invoke( + cli, ["config", "set", "engine.default", "vllm"] + ) + assert result.exit_code == 0 + content = config_file.read_text() + assert "vllm" in content + assert "qwen2.5:3b" in content + + def test_set_invalid_key_rejected(self, tmp_path: Path) -> None: + """config set rejects unknown keys.""" + config_file = tmp_path / "config.toml" + config_file.write_text("") + env = {"OPENJARVIS_CONFIG": str(config_file)} + with mock.patch.dict(os.environ, env): + result = CliRunner().invoke( + cli, ["config", "set", "engine.olllama.host", "http://x:1234"] + ) + assert result.exit_code != 0 + assert "Unknown config key" in result.output + + def test_set_engine_host_probes_reachable(self, tmp_path: Path) -> None: + """config set probes engine host and reports reachability.""" + config_file = tmp_path / "config.toml" + config_file.write_text("") + env = {"OPENJARVIS_CONFIG": str(config_file)} + with ( + mock.patch.dict(os.environ, env), + mock.patch("openjarvis.cli.config_cmd.httpx") as mock_httpx, + ): + mock_httpx.get.return_value = mock.Mock(status_code=200) + result = CliRunner().invoke( + cli, + ["config", "set", "engine.ollama.host", "http://myserver:11434"], + ) + assert result.exit_code == 0 + assert "Reachable" in result.output + + def test_set_engine_host_probes_unreachable(self, tmp_path: Path) -> None: + """config set warns when engine host is unreachable, but still writes.""" + config_file = tmp_path / "config.toml" + config_file.write_text("") + env = {"OPENJARVIS_CONFIG": str(config_file)} + with ( + mock.patch.dict(os.environ, env), + mock.patch("openjarvis.cli.config_cmd.httpx") as mock_httpx, + ): + mock_httpx.get.side_effect = Exception("Connection refused") + result = CliRunner().invoke( + cli, + ["config", "set", "engine.ollama.host", "http://myserver:11434"], + ) + assert result.exit_code == 0 + output_lower = result.output.lower() + assert "unreachable" in output_lower or "warning" in output_lower + content = config_file.read_text() + assert "http://myserver:11434" in content + + def test_set_integer_value(self, tmp_path: Path) -> None: + """config set coerces integer values.""" + config_file = tmp_path / "config.toml" + config_file.write_text("") + env = {"OPENJARVIS_CONFIG": str(config_file)} + with mock.patch.dict(os.environ, env): + result = CliRunner().invoke( + cli, ["config", "set", "intelligence.max_tokens", "2048"] + ) + assert result.exit_code == 0 + content = config_file.read_text() + assert "2048" in content + + def test_set_float_value(self, tmp_path: Path) -> None: + """config set coerces float values.""" + config_file = tmp_path / "config.toml" + config_file.write_text("") + env = {"OPENJARVIS_CONFIG": str(config_file)} + with mock.patch.dict(os.environ, env): + result = CliRunner().invoke( + cli, ["config", "set", "intelligence.temperature", "0.9"] + ) + assert result.exit_code == 0 + content = config_file.read_text() + assert "0.9" in content + + def test_set_missing_args(self) -> None: + """config set with missing args shows usage error.""" + result = CliRunner().invoke(cli, ["config", "set"]) + assert result.exit_code != 0 + + def test_set_shows_confirmation(self, tmp_path: Path) -> None: + """config set prints a confirmation message.""" + config_file = tmp_path / "config.toml" + config_file.write_text("") + env = {"OPENJARVIS_CONFIG": str(config_file)} + with mock.patch.dict(os.environ, env): + result = CliRunner().invoke( + cli, ["config", "set", "engine.default", "vllm"] + ) + assert result.exit_code == 0 + assert "Set" in result.output + assert "engine.default" in result.output diff --git a/tests/cli/test_hints.py b/tests/cli/test_hints.py index 0ba4797a..67953d16 100644 --- a/tests/cli/test_hints.py +++ b/tests/cli/test_hints.py @@ -35,3 +35,13 @@ class TestHintFunctions: def test_hint_no_model_with_name(self): msg = hint_no_model("qwen3:8b") assert "qwen3:8b" in msg + + def test_hint_no_engine_includes_remote_tip(self): + msg = hint_no_engine() + assert "config set" in msg + assert "OLLAMA_HOST" in msg + + def test_hint_no_engine_with_name_includes_remote_tip(self): + msg = hint_no_engine("vllm") + assert "config set" in msg + assert "engine.vllm.host" in msg diff --git a/tests/cli/test_init_host.py b/tests/cli/test_init_host.py new file mode 100644 index 00000000..aed663dc --- /dev/null +++ b/tests/cli/test_init_host.py @@ -0,0 +1,117 @@ +"""Tests for ``jarvis init --host`` option.""" + +from __future__ import annotations + +from pathlib import Path +from unittest import mock + +from click.testing import CliRunner + +from openjarvis.cli import cli +from openjarvis.core.config import generate_default_toml, generate_minimal_toml + +_NO_DL = "--no-download" + + +class TestInitHost: + def test_init_host_writes_to_config(self, tmp_path: Path) -> None: + """jarvis init --host writes the host into config.toml.""" + config_dir = tmp_path / ".openjarvis" + config_path = config_dir / "config.toml" + with ( + mock.patch("openjarvis.cli.init_cmd.DEFAULT_CONFIG_DIR", config_dir), + mock.patch("openjarvis.cli.init_cmd.DEFAULT_CONFIG_PATH", config_path), + mock.patch("openjarvis.cli.init_cmd.PrivacyScanner"), + ): + result = CliRunner().invoke( + cli, + [ + "init", + "--engine", + "ollama", + "--host", + "http://192.168.1.50:11434", + _NO_DL, + ], + ) + assert result.exit_code == 0 + content = config_path.read_text() + assert "http://192.168.1.50:11434" in content + + def test_init_host_with_vllm(self, tmp_path: Path) -> None: + """jarvis init --host applies to the selected engine.""" + config_dir = tmp_path / ".openjarvis" + config_path = config_dir / "config.toml" + with ( + mock.patch("openjarvis.cli.init_cmd.DEFAULT_CONFIG_DIR", config_dir), + mock.patch("openjarvis.cli.init_cmd.DEFAULT_CONFIG_PATH", config_path), + mock.patch("openjarvis.cli.init_cmd.PrivacyScanner"), + ): + result = CliRunner().invoke( + cli, + ["init", "--engine", "vllm", "--host", "http://10.0.0.5:8000", _NO_DL], + ) + assert result.exit_code == 0 + content = config_path.read_text() + assert "http://10.0.0.5:8000" in content + + def test_init_host_probes_and_reports(self, tmp_path: Path) -> None: + """jarvis init --host shows reachability status.""" + config_dir = tmp_path / ".openjarvis" + config_path = config_dir / "config.toml" + with ( + mock.patch("openjarvis.cli.init_cmd.DEFAULT_CONFIG_DIR", config_dir), + mock.patch("openjarvis.cli.init_cmd.DEFAULT_CONFIG_PATH", config_path), + mock.patch("openjarvis.cli.init_cmd.PrivacyScanner"), + mock.patch("openjarvis.cli.init_cmd.httpx") as mock_httpx, + ): + mock_httpx.get.side_effect = Exception("Connection refused") + result = CliRunner().invoke( + cli, + ["init", "--engine", "ollama", "--host", "http://bad:11434", _NO_DL], + ) + assert result.exit_code == 0 + output_lower = result.output.lower() + assert "unreachable" in output_lower or "warning" in output_lower + + def test_init_without_host_still_works(self, tmp_path: Path) -> None: + """jarvis init without --host still produces valid config.""" + config_dir = tmp_path / ".openjarvis" + config_path = config_dir / "config.toml" + with ( + mock.patch("openjarvis.cli.init_cmd.DEFAULT_CONFIG_DIR", config_dir), + mock.patch("openjarvis.cli.init_cmd.DEFAULT_CONFIG_PATH", config_path), + mock.patch("openjarvis.cli.init_cmd.PrivacyScanner"), + ): + result = CliRunner().invoke(cli, ["init", "--engine", "ollama", _NO_DL]) + assert result.exit_code == 0 + content = config_path.read_text() + assert "[engine]" in content + + +class TestGenerateTomlHost: + def test_minimal_toml_with_host(self) -> None: + from openjarvis.core.config import HardwareInfo + + hw = HardwareInfo() + toml_str = generate_minimal_toml( + hw, engine="ollama", host="http://remote:11434" + ) + assert "http://remote:11434" in toml_str + assert "[engine.ollama]" in toml_str + + def test_minimal_toml_without_host_has_comment(self) -> None: + from openjarvis.core.config import HardwareInfo + + hw = HardwareInfo() + toml_str = generate_minimal_toml(hw, engine="ollama") + assert "# host" in toml_str + + def test_default_toml_with_host(self) -> None: + from openjarvis.core.config import HardwareInfo + + hw = HardwareInfo() + toml_str = generate_default_toml( + hw, engine="ollama", host="http://remote:11434" + ) + assert "http://remote:11434" in toml_str diff --git a/tests/core/test_config_key_validation.py b/tests/core/test_config_key_validation.py new file mode 100644 index 00000000..badc706e --- /dev/null +++ b/tests/core/test_config_key_validation.py @@ -0,0 +1,54 @@ +"""Tests for config key validation.""" + +from __future__ import annotations + +import pytest + +from openjarvis.core.config import validate_config_key + + +class TestValidateConfigKey: + def test_valid_engine_default(self): + field_type = validate_config_key("engine.default") + assert field_type is str + + def test_valid_engine_ollama_host(self): + field_type = validate_config_key("engine.ollama.host") + assert field_type is str + + def test_valid_intelligence_temperature(self): + field_type = validate_config_key("intelligence.temperature") + assert field_type is float + + def test_valid_intelligence_max_tokens(self): + field_type = validate_config_key("intelligence.max_tokens") + assert field_type is int + + def test_valid_agent_default_agent(self): + field_type = validate_config_key("agent.default_agent") + assert field_type is str + + def test_invalid_top_level_key(self): + with pytest.raises(ValueError, match="Unknown config key"): + validate_config_key("nonexistent.foo") + + def test_invalid_nested_key(self): + with pytest.raises(ValueError, match="Unknown config key"): + validate_config_key("engine.olllama.host") + + def test_invalid_leaf_key(self): + with pytest.raises(ValueError, match="Unknown config key"): + validate_config_key("engine.ollama.nonexistent") + + def test_empty_key(self): + with pytest.raises(ValueError): + validate_config_key("") + + def test_single_segment(self): + with pytest.raises(ValueError): + validate_config_key("engine") + + def test_hardware_key_rejected(self): + """Hardware is auto-detected, not user-settable.""" + with pytest.raises(ValueError, match="Unknown config key"): + validate_config_key("hardware.cpu_count") diff --git a/uv.lock b/uv.lock index da90bd67..e6821504 100644 --- a/uv.lock +++ b/uv.lock @@ -893,6 +893,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/ae/3a/dbeec9d1ee0844c679f6bb5d6ad4e9f198b1224f4e7a32825f47f6192b0c/cffi-2.0.0-cp314-cp314t-win_arm64.whl", hash = "sha256:0a1527a803f0a659de1af2e1fd700213caba79377e27e4693648c2923da066f9", size = 184195, upload-time = "2025-09-08T23:23:43.004Z" }, ] +[[package]] +name = "cfgv" +version = "3.5.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/4e/b5/721b8799b04bf9afe054a3899c6cf4e880fcf8563cc71c15610242490a0c/cfgv-3.5.0.tar.gz", hash = "sha256:d5b1034354820651caa73ede66a6294d6e95c1b00acc5e9b098e917404669132", size = 7334, upload-time = "2025-11-19T20:55:51.612Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/db/3c/33bac158f8ab7f89b2e59426d5fe2e4f63f7ed25df84c036890172b412b5/cfgv-3.5.0-py2.py3-none-any.whl", hash = "sha256:a8dc6b26ad22ff227d2634a65cb388215ce6cc96bbcc5cfde7641ae87e8dacc0", size = 7445, upload-time = "2025-11-19T20:55:50.744Z" }, +] + [[package]] name = "charset-normalizer" version = "3.4.4" @@ -1632,6 +1641,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/3f/27/4570e78fc0bf5ea0ca45eb1de3818a23787af9b390c0b0a0033a1b8236f9/diskcache-5.6.3-py3-none-any.whl", hash = "sha256:5e31b2d5fbad117cc363ebaf6b689474db18a1f6438bc82358b024abd4c2ca19", size = 45550, upload-time = "2023-08-31T06:11:58.822Z" }, ] +[[package]] +name = "distlib" +version = "0.4.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/96/8e/709914eb2b5749865801041647dc7f4e6d00b549cfe88b65ca192995f07c/distlib-0.4.0.tar.gz", hash = "sha256:feec40075be03a04501a973d81f633735b4b69f98b05450592310c0f401a4e0d", size = 614605, upload-time = "2025-07-17T16:52:00.465Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/33/6b/e0547afaf41bf2c42e52430072fa5658766e3d65bd4b03a563d1b6336f57/distlib-0.4.0-py2.py3-none-any.whl", hash = "sha256:9659f7d87e46584a30b5780e43ac7a2143098441670ff0a49d5f9034c54a6c16", size = 469047, upload-time = "2025-07-17T16:51:58.613Z" }, +] + [[package]] name = "distro" version = "1.9.0" @@ -2467,6 +2485,7 @@ wheels = [ name = "griffelib" version = "2.0.0" source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/ad/06/eccbd311c9e2b3ca45dbc063b93134c57a1ccc7607c5e545264ad092c4a9/griffelib-2.0.0.tar.gz", hash = "sha256:e504d637a089f5cab9b5daf18f7645970509bf4f53eda8d79ed71cce8bd97934", size = 166312, upload-time = "2026-03-23T21:06:55.954Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/4d/51/c936033e16d12b627ea334aaaaf42229c37620d0f15593456ab69ab48161/griffelib-2.0.0-py3-none-any.whl", hash = "sha256:01284878c966508b6d6f1dbff9b6fa607bc062d8261c5c7253cb285b06422a7f", size = 142004, upload-time = "2026-02-09T19:09:40.561Z" }, ] @@ -2695,6 +2714,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/a8/af/48ac8483240de756d2438c380746e7130d1c6f75802ef22f3c6d49982787/huggingface_hub-0.36.2-py3-none-any.whl", hash = "sha256:48f0c8eac16145dfce371e9d2d7772854a4f591bcb56c9cf548accf531d54270", size = 566395, upload-time = "2026-02-06T09:24:11.133Z" }, ] +[[package]] +name = "identify" +version = "2.6.18" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/46/c4/7fb4db12296cdb11893d61c92048fe617ee853f8523b9b296ac03b43757e/identify-2.6.18.tar.gz", hash = "sha256:873ac56a5e3fd63e7438a7ecbc4d91aca692eb3fefa4534db2b7913f3fc352fd", size = 99580, upload-time = "2026-03-15T18:39:50.319Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/46/33/92ef41c6fad0233e41d3d84ba8e8ad18d1780f1e5d99b3c683e6d7f98b63/identify-2.6.18-py2.py3-none-any.whl", hash = "sha256:8db9d3c8ea9079db92cafb0ebf97abdc09d52e97f4dcf773a2e694048b7cd737", size = 99394, upload-time = "2026-03-15T18:39:48.915Z" }, +] + [[package]] name = "idna" version = "3.11" @@ -4237,6 +4265,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/df/93/a7b983643d1253bb223234b5b226e69de6cda02b76cdca7770f684b795f5/ninja-1.13.0-py3-none-win_arm64.whl", hash = "sha256:3c0b40b1f0bba764644385319028650087b4c1b18cdfa6f45cb39a3669b81aa9", size = 290806, upload-time = "2025-08-11T15:10:18.018Z" }, ] +[[package]] +name = "nodeenv" +version = "1.10.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/24/bf/d1bda4f6168e0b2e9e5958945e01910052158313224ada5ce1fb2e1113b8/nodeenv-1.10.0.tar.gz", hash = "sha256:996c191ad80897d076bdfba80a41994c2b47c68e224c542b48feba42ba00f8bb", size = 55611, upload-time = "2025-12-20T14:08:54.006Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/88/b2/d0896bdcdc8d28a7fc5717c305f1a861c26e18c05047949fb371034d98bd/nodeenv-1.10.0-py2.py3-none-any.whl", hash = "sha256:5bb13e3eed2923615535339b3c620e76779af4cb4c6a90deccc9e36b274d3827", size = 23438, upload-time = "2025-12-20T14:08:52.782Z" }, +] + [[package]] name = "numba" version = "0.61.2" @@ -4682,6 +4719,7 @@ dependencies = [ { name = "python-telegram-bot" }, { name = "rich" }, { name = "tomli", marker = "python_full_version < '3.11'" }, + { name = "tomlkit" }, ] [package.optional-dependencies] @@ -4734,6 +4772,7 @@ dashboard = [ ] dev = [ { name = "maturin" }, + { name = "pre-commit" }, { name = "pytest" }, { name = "pytest-asyncio" }, { name = "pytest-cov" }, @@ -4772,6 +4811,9 @@ inference-cloud = [ { name = "anthropic" }, { name = "openai" }, ] +inference-gemma = [ + { name = "pygemma" }, +] inference-google = [ { name = "google-genai" }, ] @@ -4895,7 +4937,9 @@ requires-dist = [ { name = "pdfplumber", marker = "extra == 'pdf'", specifier = ">=0.10" }, { name = "playwright", marker = "extra == 'browser'", specifier = ">=1.40" }, { name = "praw", marker = "extra == 'channel-reddit'", specifier = ">=7.0" }, + { name = "pre-commit", marker = "extra == 'dev'", specifier = ">=3.0" }, { name = "pydantic", marker = "extra == 'server'", specifier = ">=2.0" }, + { name = "pygemma", marker = "extra == 'inference-gemma'", specifier = ">=0.1.3" }, { name = "pymessenger", marker = "extra == 'channel-messenger'", specifier = ">=0.0.7" }, { name = "pynostr", marker = "extra == 'channel-nostr'", specifier = ">=0.6" }, { name = "pynvml", marker = "extra == 'energy-all'", specifier = ">=12.0" }, @@ -4917,6 +4961,7 @@ requires-dist = [ { name = "tavily-python", marker = "extra == 'tools-search'", specifier = ">=0.3" }, { name = "textual", marker = "extra == 'dashboard'", specifier = ">=0.80" }, { name = "tomli", marker = "python_full_version < '3.11'", specifier = ">=2.0" }, + { name = "tomlkit", specifier = ">=0.12" }, { name = "torch", marker = "extra == 'memory-colbert'", specifier = ">=2.0" }, { name = "torch", marker = "extra == 'orchestrator-training'", specifier = ">=2.0" }, { name = "transformers", marker = "extra == 'orchestrator-training'", specifier = ">=4.40" }, @@ -4930,7 +4975,7 @@ requires-dist = [ { name = "zeus-ml", extras = ["apple"], marker = "extra == 'energy-apple'" }, { name = "zulip", marker = "extra == 'channel-zulip'", specifier = ">=0.9" }, ] -provides-extras = ["dev", "inference-mlx", "inference-vllm", "inference-cloud", "inference-google", "inference-litellm", "tools-search", "memory-faiss", "memory-colbert", "memory-pdf", "memory-bm25", "server", "openhands", "gpu-metrics", "energy-amd", "energy-apple", "energy-all", "orchestrator-training", "learning-dspy", "learning-gepa", "channel-telegram", "channel-discord", "channel-slack", "channel-line", "channel-viber", "channel-messenger", "channel-reddit", "channel-mastodon", "channel-xmpp", "channel-rocketchat", "channel-zulip", "channel-twitch", "channel-nostr", "browser", "media", "pdf", "scheduler", "security-signing", "sandbox-wasm", "sandbox-docker", "dashboard", "speech", "speech-deepgram", "eval-wandb", "eval-sheets", "docs"] +provides-extras = ["dev", "inference-mlx", "inference-vllm", "inference-cloud", "inference-google", "inference-litellm", "inference-gemma", "tools-search", "memory-faiss", "memory-colbert", "memory-pdf", "memory-bm25", "server", "openhands", "gpu-metrics", "energy-amd", "energy-apple", "energy-all", "orchestrator-training", "learning-dspy", "learning-gepa", "channel-telegram", "channel-discord", "channel-slack", "channel-line", "channel-viber", "channel-messenger", "channel-reddit", "channel-mastodon", "channel-xmpp", "channel-rocketchat", "channel-zulip", "channel-twitch", "channel-nostr", "browser", "media", "pdf", "scheduler", "security-signing", "sandbox-wasm", "sandbox-docker", "dashboard", "speech", "speech-deepgram", "eval-wandb", "eval-sheets", "docs"] [package.metadata.requires-dev] dev = [{ name = "maturin", specifier = ">=1.12.6" }] @@ -5601,6 +5646,22 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/96/5c/8af904314e42d5401afcfaff69940dc448e974f80f7aa39b241a4fbf0cf1/prawcore-2.4.0-py3-none-any.whl", hash = "sha256:29af5da58d85704b439ad3c820873ad541f4535e00bb98c66f0fbcc8c603065a", size = 17203, upload-time = "2023-10-01T23:30:47.651Z" }, ] +[[package]] +name = "pre-commit" +version = "4.5.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "cfgv" }, + { name = "identify" }, + { name = "nodeenv" }, + { name = "pyyaml" }, + { name = "virtualenv" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/40/f1/6d86a29246dfd2e9b6237f0b5823717f60cad94d47ddc26afa916d21f525/pre_commit-4.5.1.tar.gz", hash = "sha256:eb545fcff725875197837263e977ea257a402056661f09dae08e4b149b030a61", size = 198232, upload-time = "2025-12-16T21:14:33.552Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/5d/19/fd3ef348460c80af7bb4669ea7926651d1f95c23ff2df18b9d24bab4f3fa/pre_commit-4.5.1-py2.py3-none-any.whl", hash = "sha256:3b3afd891e97337708c1674210f8eba659b52a38ea5f822ff142d10786221f77", size = 226437, upload-time = "2025-12-16T21:14:32.409Z" }, +] + [[package]] name = "primp" version = "1.1.3" @@ -6386,6 +6447,16 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/a0/c4/b4d4827c93ef43c01f599ef31453ccc1c132b353284fc6c87d535c233129/pyee-13.0.1-py3-none-any.whl", hash = "sha256:af2f8fede4171ef667dfded53f96e2ed0d6e6bd7ee3bb46437f77e3b57689228", size = 15659, upload-time = "2026-02-14T21:12:26.263Z" }, ] +[[package]] +name = "pygemma" +version = "0.1.3.post3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/ed/23/ec5446b040c35be02431dffcc04f808b2cb8007af03d811b76a4fd3d08a0/pygemma-0.1.3.post3.tar.gz", hash = "sha256:2456accacd5643514ef28b43931261f86ece5e61e1da4fe51e3e0ce7ec0ac109", size = 3997, upload-time = "2024-03-19T02:09:42.599Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/08/51/426d2e3b3b1253bacb281357d698815c432456ad0827d49cc09e456ac3e1/pygemma-0.1.3.post3-cp311-cp311-macosx_14_0_arm64.whl", hash = "sha256:d1a61added2ea6765130b55e12e2fc305afbddb77b2cb88e257fcec91c766464", size = 1650822, upload-time = "2024-03-19T02:09:40.274Z" }, + { url = "https://files.pythonhosted.org/packages/72/4a/4bd0aafd5e9073c9a4c3926f56bcc681290cc0811479638290fc7e189b7e/pygemma-0.1.3.post3-cp311-cp311-manylinux1_x86_64.whl", hash = "sha256:5a54175b7ecf76b3fcd6df061f7cc7d3eab90c1f0e00b6884ca9948852e3dd04", size = 1848325, upload-time = "2024-03-19T11:14:02.634Z" }, +] + [[package]] name = "pygments" version = "2.19.2" @@ -6559,6 +6630,19 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/ec/57/56b9bcc3c9c6a792fcbaf139543cee77261f3651ca9da0c93f5c1221264b/python_dateutil-2.9.0.post0-py2.py3-none-any.whl", hash = "sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427", size = 229892, upload-time = "2024-03-01T18:36:18.57Z" }, ] +[[package]] +name = "python-discovery" +version = "1.2.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "filelock" }, + { name = "platformdirs" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/9c/90/bcce6b46823c9bec1757c964dc37ed332579be512e17a30e9698095dcae4/python_discovery-1.2.0.tar.gz", hash = "sha256:7d33e350704818b09e3da2bd419d37e21e7c30db6e0977bb438916e06b41b5b1", size = 58055, upload-time = "2026-03-19T01:43:08.248Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c2/3c/2005227cb951df502412de2fa781f800663cccbef8d90ec6f1b371ac2c0d/python_discovery-1.2.0-py3-none-any.whl", hash = "sha256:1e108f1bbe2ed0ef089823d28805d5ad32be8e734b86a5f212bf89b71c266e4a", size = 31524, upload-time = "2026-03-19T01:43:07.045Z" }, +] + [[package]] name = "python-dotenv" version = "1.2.1" @@ -8373,6 +8457,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/23/d1/136eb2cb77520a31e1f64cbae9d33ec6df0d78bdf4160398e86eec8a8754/tomli-2.4.0-py3-none-any.whl", hash = "sha256:1f776e7d669ebceb01dee46484485f43a4048746235e683bcdffacdf1fb4785a", size = 14477, upload-time = "2026-01-11T11:22:37.446Z" }, ] +[[package]] +name = "tomlkit" +version = "0.14.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/c3/af/14b24e41977adb296d6bd1fb59402cf7d60ce364f90c890bd2ec65c43b5a/tomlkit-0.14.0.tar.gz", hash = "sha256:cf00efca415dbd57575befb1f6634c4f42d2d87dbba376128adb42c121b87064", size = 187167, upload-time = "2026-01-13T01:14:53.304Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b5/11/87d6d29fb5d237229d67973a6c9e06e048f01cf4994dee194ab0ea841814/tomlkit-0.14.0-py3-none-any.whl", hash = "sha256:592064ed85b40fa213469f81ac584f67a4f2992509a7c3ea2d632208623a3680", size = 39310, upload-time = "2026-01-13T01:14:51.965Z" }, +] + [[package]] name = "torch" version = "2.10.0" @@ -8932,6 +9025,22 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/5d/c1/f01846f10c8383b197a67b4cdfea6eea476aab678ed9f6f5d9ccc8c8dddf/viberbot-1.0.12-py3-none-any.whl", hash = "sha256:ca43fea2945d650c2ef2cbd777f3c546c795bf945278f6620ceda42f4101d801", size = 26164, upload-time = "2021-01-27T13:45:43.726Z" }, ] +[[package]] +name = "virtualenv" +version = "21.2.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "distlib" }, + { name = "filelock" }, + { name = "platformdirs" }, + { name = "python-discovery" }, + { name = "typing-extensions", marker = "python_full_version < '3.11'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/aa/92/58199fe10049f9703c2666e809c4f686c54ef0a68b0f6afccf518c0b1eb9/virtualenv-21.2.0.tar.gz", hash = "sha256:1720dc3a62ef5b443092e3f499228599045d7fea4c79199770499df8becf9098", size = 5840618, upload-time = "2026-03-09T17:24:38.013Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c6/59/7d02447a55b2e55755011a647479041bc92a82e143f96a8195cb33bd0a1c/virtualenv-21.2.0-py3-none-any.whl", hash = "sha256:1bd755b504931164a5a496d217c014d098426cddc79363ad66ac78125f9d908f", size = 5825084, upload-time = "2026-03-09T17:24:35.378Z" }, +] + [[package]] name = "vllm" version = "0.17.1" From 1d24c64f117b6fccc6e18ece9a423b7604e2dbfe Mon Sep 17 00:00:00 2001 From: Jon Saad-Falcon <41205309+jonsaadfalcon@users.noreply.github.com> Date: Wed, 25 Mar 2026 21:16:57 -0700 Subject: [PATCH 08/44] =?UTF-8?q?fix(evals):=20PinchBench=20harness=20fixe?= =?UTF-8?q?s=20=E2=80=94=20scores=20from=2026%=20to=2084%=20(#124)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(evals): PinchBench harness fixes — scores from 26% to 84% Multiple infrastructure bugs prevented PinchBench from producing accurate scores. This commit fixes the eval harness so model scores match the official leaderboard (Qwen3.5-397B: 26% → 84%, GPT-5.4: 5% → 58%). Key fixes: - EvalRunner: wrap generation in PinchBenchTaskEnv context so workspace files persist through grading (was deleting before scorer ran) - EvalRunner: detect task_env datasets and force sequential processing (CWD changes aren't thread-safe) - EvalRunner: fix episode_mode auto-detection to check for real iter_episodes() override instead of hasattr() (always True) - Scorer: use "params" field in transcripts to match PinchBench grade() functions (was "arguments") - Scorer: add None guard in _trace_to_transcript for tool_calls - Scorer: capture final assistant text response in transcript so text-only tasks (like sanity check) can be graded - Scorer: add _tool_results_to_transcript() helper for EvalRunner path - native_openhands: add native function-calling support (tools= parameter) with text-based fallback, matching monitor_operative - Tools: add Python fallbacks for file_read, file_write, shell_exec, calculator, think, http_request when openjarvis_rust unavailable - Security: add Python fallback for is_sensitive_file() - Config: add "pinchbench" to KNOWN_BENCHMARKS - Add PinchBench eval configs for Qwen3.5-397B and GPT-5.4 Co-Authored-By: Claude Opus 4.6 (1M context) * fix(evals): fix hybrid grading crash when grading_weights is None The 4 tasks with grading_type=hybrid (task_10, task_13, task_16_market, task_22) crashed because grading_weights was explicitly None in task metadata. dict.get("key", default) returns None (not the default) when the key exists with value None. Use `or` to coalesce None to defaults. Co-Authored-By: Claude Opus 4.6 (1M context) * fix(evals): handle MagicMock datasets in runner type checks The iter_episodes and create_task_env type identity checks fail with AttributeError when the dataset is a MagicMock (used in tracker tests). Wrap in try/except to default to False for non-DatasetProvider objects. Co-Authored-By: Claude Opus 4.6 (1M context) --------- Co-authored-by: Claude Opus 4.6 (1M context) --- src/openjarvis/agents/native_openhands.py | 47 +++++++++++- .../evals/configs/pinchbench-gpt54.toml | 34 +++++++++ .../evals/configs/pinchbench-qwen397b.toml | 35 +++++++++ src/openjarvis/evals/core/config.py | 1 + src/openjarvis/evals/core/runner.py | 74 ++++++++++++++++--- src/openjarvis/evals/scorers/pinchbench.py | 70 ++++++++++++++++-- src/openjarvis/security/file_policy.py | 22 +++++- src/openjarvis/tools/calculator.py | 13 +++- src/openjarvis/tools/file_read.py | 9 ++- src/openjarvis/tools/file_write.py | 22 +++--- src/openjarvis/tools/http_request.py | 10 ++- src/openjarvis/tools/shell_exec.py | 53 ++++++------- src/openjarvis/tools/think.py | 9 ++- tests/evals/test_benchmark_datasets.py | 2 +- tests/evals/test_pinchbench_grading.py | 4 +- 15 files changed, 331 insertions(+), 74 deletions(-) create mode 100644 src/openjarvis/evals/configs/pinchbench-gpt54.toml create mode 100644 src/openjarvis/evals/configs/pinchbench-qwen397b.toml diff --git a/src/openjarvis/agents/native_openhands.py b/src/openjarvis/agents/native_openhands.py index cf5579cd..9b518809 100644 --- a/src/openjarvis/agents/native_openhands.py +++ b/src/openjarvis/agents/native_openhands.py @@ -283,14 +283,23 @@ class NativeOpenHandsAgent(ToolUsingAgent): last_content = "" total_usage = {"prompt_tokens": 0, "completion_tokens": 0, "total_tokens": 0} + # Build OpenAI-format tool schemas for native function calling + openai_tools = ( + self._executor.get_openai_tools() if self._tools else [] + ) + for _turn in range(self._max_turns): turns += 1 # Truncate before every generate call -- tool results may have # expanded the context beyond what the model supports. messages = self._truncate_if_needed(messages) + gen_kwargs: dict[str, Any] = {} + if openai_tools: + gen_kwargs["tools"] = openai_tools + try: - result = self._generate(messages) + result = self._generate(messages, **gen_kwargs) except Exception as exc: error_str = str(exc) if "400" in error_str: @@ -318,6 +327,42 @@ class NativeOpenHandsAgent(ToolUsingAgent): content = self._strip_think_tags(content) last_content = content + # --- Native function-calling path (OpenAI, Anthropic, etc.) --- + raw_tool_calls = result.get("tool_calls", []) + if raw_tool_calls: + native_calls = [ + ToolCall( + id=tc.get("id", f"call_{turns}_{i}"), + name=tc.get("name", ""), + arguments=tc.get("arguments", "{}"), + ) + for i, tc in enumerate(raw_tool_calls) + ] + messages.append( + Message( + role=Role.ASSISTANT, + content=content, + tool_calls=native_calls, + ) + ) + for tc in native_calls: + tool_result = self._executor.execute(tc) + all_tool_results.append(tool_result) + obs_text = tool_result.content + if len(obs_text) > 4000: + obs_text = obs_text[:4000] + "\n\n[Output truncated]" + messages.append( + Message( + role=Role.TOOL, + content=obs_text, + tool_call_id=tc.id, + name=tc.name, + ) + ) + continue + + # --- Text-based fallback (CodeAct / Action-Input format) --- + # Try to extract code code = self._extract_code(content) if code: diff --git a/src/openjarvis/evals/configs/pinchbench-gpt54.toml b/src/openjarvis/evals/configs/pinchbench-gpt54.toml new file mode 100644 index 00000000..e1feee6a --- /dev/null +++ b/src/openjarvis/evals/configs/pinchbench-gpt54.toml @@ -0,0 +1,34 @@ +# PinchBench eval: gpt-5.4 (cloud) +# Agent: native_openhands — all PinchBench-required tools enabled + +[meta] +name = "pinchbench-gpt54" +description = "PinchBench on gpt-5.4-2026-03-05" + +[defaults] +temperature = 0.6 +max_tokens = 8192 + +[judge] +model = "claude-opus-4-5" +temperature = 0.0 +engine = "cloud" + +[run] +max_workers = 1 +output_dir = "results/pinchbench-gpt54/" +seed = 42 + +[[models]] +name = "gpt-5.4-2026-03-05" +engine = "cloud" + +[[benchmarks]] +name = "pinchbench" +backend = "jarvis-agent" +agent = "native_openhands" +tools = [ + "think", "file_read", "file_write", "web_search", "shell_exec", + "code_interpreter", "browser_navigate", "image_generate", + "calculator", "http_request", "pdf_extract", +] diff --git a/src/openjarvis/evals/configs/pinchbench-qwen397b.toml b/src/openjarvis/evals/configs/pinchbench-qwen397b.toml new file mode 100644 index 00000000..b6b1493b --- /dev/null +++ b/src/openjarvis/evals/configs/pinchbench-qwen397b.toml @@ -0,0 +1,35 @@ +# PinchBench eval: Qwen3.5-397B-A17B-FP8 (vLLM, TP=8) +# Agent: native_openhands — all PinchBench-required tools enabled + +[meta] +name = "pinchbench-qwen397b" +description = "PinchBench on Qwen/Qwen3.5-397B-A17B-FP8 (vLLM, TP=8)" + +[defaults] +temperature = 0.6 +max_tokens = 8192 + +[judge] +model = "claude-opus-4-5" +temperature = 0.0 +engine = "cloud" + +[run] +max_workers = 1 +output_dir = "results/pinchbench-qwen397b/" +seed = 42 + +[[models]] +name = "Qwen/Qwen3.5-397B-A17B-FP8" +engine = "vllm" +num_gpus = 8 + +[[benchmarks]] +name = "pinchbench" +backend = "jarvis-agent" +agent = "native_openhands" +tools = [ + "think", "file_read", "file_write", "web_search", "shell_exec", + "code_interpreter", "browser_navigate", "image_generate", + "calculator", "http_request", "pdf_extract", +] diff --git a/src/openjarvis/evals/core/config.py b/src/openjarvis/evals/core/config.py index ac62d191..ab864c71 100644 --- a/src/openjarvis/evals/core/config.py +++ b/src/openjarvis/evals/core/config.py @@ -43,6 +43,7 @@ KNOWN_BENCHMARKS = { "knowledge_base", "coding_task", "coding_assistant", "security_scanner", "daily_digest", "doc_qa", "browser_assistant", + "pinchbench", } diff --git a/src/openjarvis/evals/core/runner.py b/src/openjarvis/evals/core/runner.py index e3d8716e..9effbad2 100644 --- a/src/openjarvis/evals/core/runner.py +++ b/src/openjarvis/evals/core/runner.py @@ -96,11 +96,19 @@ class EvalRunner: seed=cfg.seed, ) - # Auto-enable episode_mode when the dataset has iter_episodes() - # (i.e. it is a lifelong/sequential benchmark like LifelongAgentBench). - # This is enforced at the runner level so it applies regardless of - # how the runner is invoked (CLI, SDK, tests, etc.). - if not cfg.episode_mode and hasattr(self._dataset, "iter_episodes"): + # Auto-enable episode_mode when the dataset *overrides* + # iter_episodes() (i.e. it is a lifelong/sequential benchmark like + # LifelongAgentBench). The base DatasetProvider always defines a + # default iter_episodes() that wraps each record in its own episode, + # so hasattr() is always True — we must check for a real override. + from openjarvis.evals.core.dataset import DatasetProvider as _DP + try: + _overrides_episodes = ( + type(self._dataset).iter_episodes is not _DP.iter_episodes + ) + except AttributeError: + _overrides_episodes = False + if not cfg.episode_mode and _overrides_episodes: LOGGER.info( "%s requires sequential episode processing — " "auto-enabling episode_mode.", @@ -109,6 +117,15 @@ class EvalRunner: cfg = dataclasses.replace(cfg, episode_mode=True) self._config = cfg + # Detect if dataset provides task environments (e.g. PinchBench) + try: + self._has_task_env = ( + type(self._dataset).create_task_env + is not _DP.create_task_env + ) + except AttributeError: + self._has_task_env = False + records = list(self._dataset.iter_records()) LOGGER.info( "Running %s: %d samples, backend=%s, model=%s, workers=%d, " @@ -145,6 +162,15 @@ class EvalRunner: try: if cfg.episode_mode: self._run_episode_mode(records, progress_callback, total) + elif self._has_task_env: + # Task environments (PinchBench etc.) change CWD — + # must process sequentially for thread safety. + for record in records: + result = self._process_one(record) + self._results.append(result) + self._flush_result(result) + if progress_callback is not None: + progress_callback(len(self._results), total) else: with ThreadPoolExecutor(max_workers=cfg.max_workers) as pool: futures = { @@ -224,17 +250,41 @@ class EvalRunner: ) if cfg.system_prompt: gen_kwargs["system"] = cfg.system_prompt - full = self._backend.generate_full( - record.problem, - **gen_kwargs, - ) - content = full.get("content", "") + + if getattr(self, "_has_task_env", False): + from contextlib import nullcontext + task_env = self._dataset.create_task_env(record) + ctx = task_env if task_env is not None else nullcontext() + with ctx: + full = self._backend.generate_full( + record.problem, + **gen_kwargs, + ) + full = full or {} + # Store tool results for the scorer to build transcripts + record.metadata["tool_results"] = full.get( + "tool_results", [], + ) + # Score INSIDE context so workspace files still exist + content = full.get("content", "") + is_correct, scoring_meta = self._scorer.score( + record, content, + ) + else: + full = self._backend.generate_full( + record.problem, + **gen_kwargs, + ) + full = full or {} + content = full.get("content", "") + is_correct, scoring_meta = self._scorer.score( + record, content, + ) + usage = full.get("usage", {}) latency = full.get("latency_seconds", 0.0) cost = full.get("cost_usd", 0.0) - is_correct, scoring_meta = self._scorer.score(record, content) - energy_j = full.get("energy_joules", 0.0) power_w = full.get("power_watts", 0.0) throughput = full.get("throughput_tok_per_sec", 0.0) diff --git a/src/openjarvis/evals/scorers/pinchbench.py b/src/openjarvis/evals/scorers/pinchbench.py index 68269ec0..693f3d12 100644 --- a/src/openjarvis/evals/scorers/pinchbench.py +++ b/src/openjarvis/evals/scorers/pinchbench.py @@ -52,12 +52,12 @@ def events_to_transcript(events: List[Any]) -> List[Dict[str, Any]]: if etype == EventType.TOOL_CALL_START or etype == EventType.TOOL_CALL_START.value: tool_name = event.metadata.get("tool", "unknown") mapped = _TOOL_NAME_MAP.get(tool_name, tool_name) - arguments = event.metadata.get("arguments", {}) + arguments = event.metadata.get("arguments") or {} transcript.append({ "type": "message", "message": { "role": "assistant", - "content": [{"type": "toolCall", "name": mapped, "arguments": arguments}], + "content": [{"type": "toolCall", "name": mapped, "params": arguments}], }, }) elif etype == EventType.TOOL_CALL_END or etype == EventType.TOOL_CALL_END.value: @@ -78,12 +78,14 @@ def _trace_to_transcript(trace: Any) -> List[Dict[str, Any]]: transcript: List[Dict[str, Any]] = [] for turn in trace.turns: for tc in getattr(turn, "tool_calls", []): + if tc is None: + continue mapped = _TOOL_NAME_MAP.get(tc["name"], tc["name"]) transcript.append({ "type": "message", "message": { "role": "assistant", - "content": [{"type": "toolCall", "name": mapped, "arguments": tc.get("arguments", {})}], + "content": [{"type": "toolCall", "name": mapped, "params": tc.get("arguments") or {}}], }, }) transcript.append({ @@ -93,6 +95,42 @@ def _trace_to_transcript(trace: Any) -> List[Dict[str, Any]]: "content": [{"text": tc.get("result", "")}], }, }) + + # Capture final assistant text response (for tasks graded on text output) + response_text = getattr(trace, "response_text", "") + if response_text: + transcript.append({ + "type": "message", + "message": { + "role": "assistant", + "content": [{"type": "text", "text": response_text}], + }, + }) + return transcript + + +def _tool_results_to_transcript( + tool_results: List[Dict[str, Any]], +) -> List[Dict[str, Any]]: + """Build transcript from JarvisAgentBackend tool_results list.""" + transcript: List[Dict[str, Any]] = [] + for tr in tool_results: + tool_name = tr.get("tool_name", "unknown") + mapped = _TOOL_NAME_MAP.get(tool_name, tool_name) + transcript.append({ + "type": "message", + "message": { + "role": "assistant", + "content": [{"type": "toolCall", "name": mapped, "params": {}}], + }, + }) + transcript.append({ + "type": "message", + "message": { + "role": "toolResult", + "content": [{"text": tr.get("content", "")}], + }, + }) return transcript @@ -118,8 +156,10 @@ def _summarize_transcript(transcript: List[Dict[str, Any]]) -> str: for item in msg.get("content", []): if item.get("type") == "toolCall": parts.append( - f"Tool: {item.get('name')}({json.dumps(item.get('arguments', {}))})" + f"Tool: {item.get('name')}({json.dumps(item.get('params', {}))})" ) + elif item.get("type") == "text": + parts.append(f"Assistant: {item.get('text', '')}") elif role == "toolResult": content = msg.get("content", []) if content: @@ -372,7 +412,7 @@ def _grade_hybrid( judge_model: str, ) -> Dict[str, Any]: """Run both automated and LLM judge grading, combine with weights.""" - weights = record.metadata.get("grading_weights", {"automated": 0.5, "llm_judge": 0.5}) + weights = record.metadata.get("grading_weights") or {"automated": 0.5, "llm_judge": 0.5} auto = _grade_automated(record, transcript, workspace_path) llm = _grade_llm_judge(record, transcript, workspace_path, judge_backend, judge_model) @@ -426,7 +466,24 @@ class PinchBenchScorer(LLMJudgeScorer): self, record: EvalRecord, model_answer: str, ) -> Tuple[Optional[bool], Dict[str, Any]]: trace = record.metadata.get("query_trace") - transcript = _trace_to_transcript(trace) if trace else [] + if trace: + transcript = _trace_to_transcript(trace) + else: + # No trace — build transcript from tool_results if available + tool_results = record.metadata.get("tool_results", []) + transcript = _tool_results_to_transcript(tool_results) + + # Always append final model answer as assistant text message + # so grading functions that check for text responses can find it + if model_answer: + transcript.append({ + "type": "message", + "message": { + "role": "assistant", + "content": [{"type": "text", "text": model_answer}], + }, + }) + result = grade_pinchbench_task( record=record, transcript=transcript, @@ -442,4 +499,5 @@ __all__ = [ "PinchBenchScorer", "events_to_transcript", "grade_pinchbench_task", + "_tool_results_to_transcript", ] diff --git a/src/openjarvis/security/file_policy.py b/src/openjarvis/security/file_policy.py index fc4006ae..36482440 100644 --- a/src/openjarvis/security/file_policy.py +++ b/src/openjarvis/security/file_policy.py @@ -30,11 +30,27 @@ def is_sensitive_file(path: Union[str, Path]) -> bool: Checks both the filename and the full name against ``DEFAULT_SENSITIVE_PATTERNS`` using :func:`fnmatch.fnmatch`. + Uses the Rust implementation when available, falls back to Python. """ - from openjarvis._rust_bridge import get_rust_module + try: + from openjarvis._rust_bridge import get_rust_module - _rust = get_rust_module() - return _rust.is_sensitive_file(str(path)) + _rust = get_rust_module() + return _rust.is_sensitive_file(str(path)) + except ImportError: + return _is_sensitive_file_py(str(path)) + + +def _is_sensitive_file_py(path_str: str) -> bool: + """Pure-Python fallback for sensitive file detection.""" + import fnmatch + + p = Path(path_str) + name = p.name + for pattern in DEFAULT_SENSITIVE_PATTERNS: + if fnmatch.fnmatch(name, pattern) or fnmatch.fnmatch(str(p), pattern): + return True + return False def filter_sensitive_paths(paths: Iterable[Union[str, Path]]) -> List[Path]: diff --git a/src/openjarvis/tools/calculator.py b/src/openjarvis/tools/calculator.py index 14ca4bc4..2ec101eb 100644 --- a/src/openjarvis/tools/calculator.py +++ b/src/openjarvis/tools/calculator.py @@ -89,10 +89,15 @@ def _safe_eval_node(node: ast.AST) -> Any: def safe_eval(expression: str) -> float: - """Evaluate a math expression safely — always via Rust backend.""" - from openjarvis._rust_bridge import get_rust_module - _rust = get_rust_module() - return float(_rust.CalculatorTool().execute(expression)) + """Evaluate a math expression safely — Rust backend with Python fallback.""" + try: + from openjarvis._rust_bridge import get_rust_module + _rust = get_rust_module() + return float(_rust.CalculatorTool().execute(expression)) + except ImportError: + import ast as _ast + tree = _ast.parse(expression, mode="eval") + return float(_safe_eval_node(tree.body)) @ToolRegistry.register("calculator") diff --git a/src/openjarvis/tools/file_read.py b/src/openjarvis/tools/file_read.py index 36fb258e..6e5b7e38 100644 --- a/src/openjarvis/tools/file_read.py +++ b/src/openjarvis/tools/file_read.py @@ -114,10 +114,15 @@ class FileReadTool(BaseTool): content=f"File too large: {size} bytes (max {_MAX_SIZE_BYTES}).", success=False, ) - from openjarvis._rust_bridge import get_rust_module - _rust = get_rust_module() try: + from openjarvis._rust_bridge import get_rust_module + _rust = get_rust_module() text = _rust.FileReadTool().execute(str(path)) + except ImportError: + try: + text = path.read_text(encoding="utf-8") + except UnicodeDecodeError: + text = path.read_text(encoding="utf-8", errors="replace") except Exception as exc: return ToolResult( tool_name="file_read", diff --git a/src/openjarvis/tools/file_write.py b/src/openjarvis/tools/file_write.py index d006280c..71be96df 100644 --- a/src/openjarvis/tools/file_write.py +++ b/src/openjarvis/tools/file_write.py @@ -155,26 +155,26 @@ class FileWriteTool(BaseTool): success=False, ) - from openjarvis._rust_bridge import get_rust_module - _rust = get_rust_module() if mode == "write": try: + from openjarvis._rust_bridge import get_rust_module + _rust = get_rust_module() _rust.FileWriteTool().execute(str(path), content) + except ImportError: + try: + path.write_text(content, encoding="utf-8") + except OSError as exc: + return ToolResult( + tool_name="file_write", + content=f"Write error: {exc}", + success=False, + ) except Exception as exc: return ToolResult( tool_name="file_write", content=f"Write error: {exc}", success=False, ) - elif False: # dead code — all write modes go through Rust - try: - path.write_text(content, encoding="utf-8") - except OSError as exc: - return ToolResult( - tool_name="file_write", - content=f"Write error: {exc}", - success=False, - ) else: # append mode — always Python try: diff --git a/src/openjarvis/tools/http_request.py b/src/openjarvis/tools/http_request.py index 4cde4d60..d519815a 100644 --- a/src/openjarvis/tools/http_request.py +++ b/src/openjarvis/tools/http_request.py @@ -103,9 +103,13 @@ class HttpRequestTool(BaseTool): body = params.get("body") timeout = params.get("timeout", 30) - from openjarvis._rust_bridge import get_rust_module - _rust = get_rust_module() - if not headers: + _rust = None + try: + from openjarvis._rust_bridge import get_rust_module + _rust = get_rust_module() + except ImportError: + pass + if _rust is not None and not headers: try: content = _rust.HttpRequestTool().execute(url, method, body) return ToolResult( diff --git a/src/openjarvis/tools/shell_exec.py b/src/openjarvis/tools/shell_exec.py index 3952d26d..6611df7a 100644 --- a/src/openjarvis/tools/shell_exec.py +++ b/src/openjarvis/tools/shell_exec.py @@ -125,32 +125,33 @@ class ShellExecTool(BaseTool): if val is not None: env[key] = val - from openjarvis._rust_bridge import get_rust_module - _rust = get_rust_module() - if True: - try: - output = _rust.ShellExecTool().execute(command, working_dir) - return ToolResult( - tool_name="shell_exec", - content=output or "(no output)", - success=True, - metadata={ - "returncode": 0, - "timeout_used": timeout, - "working_dir": working_dir, - }, - ) - except Exception as exc: - return ToolResult( - tool_name="shell_exec", - content=str(exc), - success=False, - metadata={ - "returncode": -1, - "timeout_used": timeout, - "working_dir": working_dir, - }, - ) + try: + from openjarvis._rust_bridge import get_rust_module + _rust = get_rust_module() + output = _rust.ShellExecTool().execute(command, working_dir) + return ToolResult( + tool_name="shell_exec", + content=output or "(no output)", + success=True, + metadata={ + "returncode": 0, + "timeout_used": timeout, + "working_dir": working_dir, + }, + ) + except ImportError: + pass # Fall through to subprocess below + except Exception as exc: + return ToolResult( + tool_name="shell_exec", + content=str(exc), + success=False, + metadata={ + "returncode": -1, + "timeout_used": timeout, + "working_dir": working_dir, + }, + ) try: result = subprocess.run( command, diff --git a/src/openjarvis/tools/think.py b/src/openjarvis/tools/think.py index 225c888e..699bbe03 100644 --- a/src/openjarvis/tools/think.py +++ b/src/openjarvis/tools/think.py @@ -40,9 +40,12 @@ class ThinkTool(BaseTool): def execute(self, **params: Any) -> ToolResult: thought = params.get("thought", "") - from openjarvis._rust_bridge import get_rust_module - _rust = get_rust_module() - content = _rust.ThinkTool().execute(thought) + try: + from openjarvis._rust_bridge import get_rust_module + _rust = get_rust_module() + content = _rust.ThinkTool().execute(thought) + except ImportError: + content = thought return ToolResult( tool_name="think", content=content, diff --git a/tests/evals/test_benchmark_datasets.py b/tests/evals/test_benchmark_datasets.py index ee5e73a3..e569fa48 100644 --- a/tests/evals/test_benchmark_datasets.py +++ b/tests/evals/test_benchmark_datasets.py @@ -313,7 +313,7 @@ class TestConfigBenchmarks: def test_benchmarks_count(self) -> None: from openjarvis.evals.core.config import KNOWN_BENCHMARKS - assert len(KNOWN_BENCHMARKS) == 25 + assert len(KNOWN_BENCHMARKS) == 26 # --------------------------------------------------------------------------- diff --git a/tests/evals/test_pinchbench_grading.py b/tests/evals/test_pinchbench_grading.py index aa107506..d2396bfc 100644 --- a/tests/evals/test_pinchbench_grading.py +++ b/tests/evals/test_pinchbench_grading.py @@ -74,7 +74,7 @@ def grade(transcript, workspace_path): { "type": "toolCall", "name": "read_file", - "arguments": {"path": "a.txt"}, + "params": {"path": "a.txt"}, } ], }, @@ -124,7 +124,7 @@ class TestSummarizeTranscript: { "type": "toolCall", "name": "read_file", - "arguments": {"path": "a.txt"}, + "params": {"path": "a.txt"}, } ], }, From 2259280f2310767e444746905d144b80b4256c39 Mon Sep 17 00:00:00 2001 From: Jon Saad-Falcon <41205309+jonsaadfalcon@users.noreply.github.com> Date: Wed, 25 Mar 2026 21:17:32 -0700 Subject: [PATCH 09/44] feat: auto-clone repo on desktop app first launch (#122) Auto-clone the OpenJarvis repo on first desktop app launch instead of showing an error. Uses git clone --depth 1 to ~/OpenJarvis. Closes #122. --- desktop/src-tauri/src/lib.rs | 91 ++++++++++++++++++++++++++++++++---- 1 file changed, 82 insertions(+), 9 deletions(-) diff --git a/desktop/src-tauri/src/lib.rs b/desktop/src-tauri/src/lib.rs index 14dcf64e..5f6163e3 100644 --- a/desktop/src-tauri/src/lib.rs +++ b/desktop/src-tauri/src/lib.rs @@ -460,17 +460,90 @@ async fn boot_backend(backend: SharedBackend, status: SharedStatus) { return; } - let project_root = find_project_root(); + let mut project_root = find_project_root(); if project_root.is_none() { - let mut s = status.lock().await; - s.error = Some( - "Could not find the OpenJarvis project directory. \ - Clone it with: git clone https://github.com/open-jarvis/OpenJarvis.git ~/OpenJarvis \ - then relaunch." - .into(), - ); - return; + // Auto-clone on first launch + let git_bin = resolve_bin("git"); + + // Check that git is installed + if !std::path::Path::new(&git_bin).exists() && git_bin == "git" { + let mut s = status.lock().await; + s.error = Some( + "Could not find 'git'. \ + Install it from https://git-scm.com then relaunch." + .into(), + ); + return; + } + + let target_path = std::path::PathBuf::from(home_dir()).join("OpenJarvis"); + let clone_target = target_path.display().to_string(); + + // If the directory exists but is not a valid project, don't overwrite + if target_path.exists() && !target_path.join("pyproject.toml").exists() { + let mut s = status.lock().await; + s.error = Some(format!( + "{} exists but is not a valid OpenJarvis project. \ + Remove it and relaunch, or set OPENJARVIS_ROOT to the correct path.", + clone_target, + )); + return; + } + + { + let mut s = status.lock().await; + s.detail = "Downloading OpenJarvis (first launch)...".into(); + } + + let clone_result = tokio::process::Command::new(&git_bin) + .args([ + "clone", + "--depth", + "1", + "https://github.com/open-jarvis/OpenJarvis.git", + &clone_target, + ]) + .stdout(std::process::Stdio::null()) + .stderr(std::process::Stdio::piped()) + .spawn(); + + match clone_result { + Ok(child) => match child.wait_with_output().await { + Ok(output) if output.status.success() => { + project_root = Some(target_path); + } + Ok(output) => { + let stderr = String::from_utf8_lossy(&output.stderr); + let mut s = status.lock().await; + s.error = Some(format!( + "Failed to download OpenJarvis: {}. \ + Clone manually: git clone https://github.com/open-jarvis/OpenJarvis.git {}", + stderr.trim(), + clone_target, + )); + return; + } + Err(e) => { + let mut s = status.lock().await; + s.error = Some(format!( + "Failed to download OpenJarvis: {}. \ + Clone manually: git clone https://github.com/open-jarvis/OpenJarvis.git {}", + e, clone_target, + )); + return; + } + }, + Err(e) => { + let mut s = status.lock().await; + s.error = Some(format!( + "Could not run git: {}. \ + Install git from https://git-scm.com then relaunch.", + e, + )); + return; + } + } } // Kill any leftover server on our port from a previous run From 1ff155d4458928dec1a13ae399888a0ae91960ca Mon Sep 17 00:00:00 2001 From: Jon Saad-Falcon <41205309+jonsaadfalcon@users.noreply.github.com> Date: Wed, 25 Mar 2026 21:31:32 -0700 Subject: [PATCH 10/44] fix(ci): skip live and cloud tests in CI The gemma_cpp live tests require local model weights and env vars (GEMMA_CPP_MODEL_PATH, etc.) that are not available in CI, causing 4 failures since the gemma-cpp-engine PR was merged. Add marker filters to the pytest invocation so live and cloud tests are skipped. Co-Authored-By: Claude Opus 4.6 (1M context) --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 05b90a58..d9b4f9cf 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -63,7 +63,7 @@ jobs: run: uv run maturin develop --manifest-path rust/crates/openjarvis-python/Cargo.toml - name: Run tests - run: uv run pytest tests/ -v --tb=short + run: uv run pytest tests/ -v --tb=short -m "not live and not cloud" rust: runs-on: ubuntu-latest From f7d2cce86b88f195b263e1fb7d553b22b8510f62 Mon Sep 17 00:00:00 2001 From: Jana Bergant Date: Thu, 26 Mar 2026 08:29:31 +0100 Subject: [PATCH 11/44] fix: correct Qwen3.5 model sizes and MLX repos in catalog The model catalog listed non-existent Qwen3.5 sizes (3B, 8B, 14B) and pointed to MLX community repos that don't exist, causing `jarvis init` to recommend models that cannot be downloaded on Apple Silicon. Replace with the actual Qwen3.5 model family sizes (0.8B, 2B, 9B, 27B) and verified mlx-community repo URLs from HuggingFace. Fixes #129 Co-Authored-By: Claude Opus 4.6 (1M context) --- src/openjarvis/cli/doctor_cmd.py | 45 ++++---------- src/openjarvis/cli/hints.py | 2 +- src/openjarvis/cli/init_cmd.py | 36 ++++++----- src/openjarvis/cli/quickstart_cmd.py | 8 +-- src/openjarvis/evals/configs/smoke_apple.toml | 4 +- src/openjarvis/intelligence/model_catalog.py | 61 +++++++++++-------- tests/cli/test_init_guidance.py | 42 ++++--------- tests/cli/test_model_pull.py | 15 ++--- tests/core/test_recommend_model.py | 20 +++--- .../test_model_catalog_extended.py | 30 ++++----- 10 files changed, 123 insertions(+), 140 deletions(-) diff --git a/src/openjarvis/cli/doctor_cmd.py b/src/openjarvis/cli/doctor_cmd.py index f79a536b..dbfe2280 100644 --- a/src/openjarvis/cli/doctor_cmd.py +++ b/src/openjarvis/cli/doctor_cmd.py @@ -35,17 +35,13 @@ def _check_python_version() -> CheckResult: version_str = f"{ver.major}.{ver.minor}.{ver.micro}" if (ver.major, ver.minor) >= (3, 10): return CheckResult("Python version", "ok", version_str) - return CheckResult( - "Python version", "fail", f"{version_str} (requires >= 3.10)" - ) + return CheckResult("Python version", "fail", f"{version_str} (requires >= 3.10)") def _check_config_exists() -> CheckResult: """Check that the config file exists.""" if DEFAULT_CONFIG_PATH.exists(): - return CheckResult( - "Config file", "ok", str(DEFAULT_CONFIG_PATH) - ) + return CheckResult("Config file", "ok", str(DEFAULT_CONFIG_PATH)) return CheckResult( "Config file", "warn", @@ -57,16 +53,12 @@ def _check_config_exists() -> CheckResult: def _check_config_parses() -> CheckResult: """Check that the config file parses successfully.""" if not DEFAULT_CONFIG_PATH.exists(): - return CheckResult( - "Config parsing", "warn", "Skipped (no config file)" - ) + return CheckResult("Config parsing", "warn", "Skipped (no config file)") try: load_config() return CheckResult("Config parsing", "ok", "Config loaded successfully") except Exception as exc: - return CheckResult( - "Config parsing", "fail", f"Parse error: {exc}" - ) + return CheckResult("Config parsing", "fail", f"Parse error: {exc}") def _ensure_engines_imported() -> None: @@ -102,22 +94,16 @@ def _check_engines() -> List[CheckResult]: try: engine = _discovery._make_engine(key, config) if engine.health(): - results.append( - CheckResult(f"Engine: {key}", "ok", "Reachable") - ) + results.append(CheckResult(f"Engine: {key}", "ok", "Reachable")) else: - results.append( - CheckResult(f"Engine: {key}", "warn", "Unreachable") - ) + results.append(CheckResult(f"Engine: {key}", "warn", "Unreachable")) except Exception as exc: results.append( CheckResult(f"Engine: {key}", "warn", f"Unreachable ({exc})") ) if not results: - results.append( - CheckResult("Engines", "warn", "No engines registered") - ) + results.append(CheckResult("Engines", "warn", "No engines registered")) return results @@ -154,7 +140,7 @@ def _check_models() -> List[CheckResult]: f"Models: {key}", "warn", "No models available", - details="Pull a model (e.g. `ollama pull qwen3.5:3b`).", + details="Pull a model (e.g. `ollama pull qwen3.5:2b`).", ) ) except Exception: @@ -168,9 +154,7 @@ def _check_default_model() -> CheckResult: try: config = load_config() except Exception: - return CheckResult( - "Default model", "warn", "Skipped (config unavailable)" - ) + return CheckResult("Default model", "warn", "Skipped (config unavailable)") default_model = config.intelligence.default_model if not default_model: @@ -221,9 +205,7 @@ def _check_optional_deps() -> List[CheckResult]: for pkg, install_hint, description in optional_packages: try: __import__(pkg) - results.append( - CheckResult(f"Optional: {description}", "ok", "Installed") - ) + results.append(CheckResult(f"Optional: {description}", "ok", "Installed")) except Exception: results.append( CheckResult( @@ -266,8 +248,7 @@ def _check_nodejs() -> CheckResult: "warn", f"{version_str} (requires >= v22)", details=( - "Upgrade Node.js for ClaudeCodeAgent and WhatsApp " - "Baileys support." + "Upgrade Node.js for ClaudeCodeAgent and WhatsApp Baileys support." ), ) except Exception as exc: @@ -335,7 +316,5 @@ def doctor(as_json: bool) -> None: warn_count = sum(1 for c in checks if c.status == "warn") fail_count = sum(1 for c in checks if c.status == "fail") console.print() - console.print( - f" {ok_count} passed, {warn_count} warnings, {fail_count} failures" - ) + console.print(f" {ok_count} passed, {warn_count} warnings, {fail_count} failures") console.print() diff --git a/src/openjarvis/cli/hints.py b/src/openjarvis/cli/hints.py index 9c8617cf..792bd0b1 100644 --- a/src/openjarvis/cli/hints.py +++ b/src/openjarvis/cli/hints.py @@ -36,6 +36,6 @@ def hint_no_model(model_name: Optional[str] = None) -> str: ) return ( "[yellow]Hint:[/yellow] No models available.\n" - " Pull a model first: [bold]ollama pull qwen3.5:3b[/bold]\n" + " Pull a model first: [bold]ollama pull qwen3.5:2b[/bold]\n" " Run [bold]jarvis model list[/bold] to see available models." ) diff --git a/src/openjarvis/cli/init_cmd.py b/src/openjarvis/cli/init_cmd.py index 8d729b7e..8eaa4bfe 100644 --- a/src/openjarvis/cli/init_cmd.py +++ b/src/openjarvis/cli/init_cmd.py @@ -25,8 +25,14 @@ from openjarvis.core.config import ( # Engines supported by ``jarvis init --engine``. _SUPPORTED_ENGINES = [ - "ollama", "vllm", "sglang", "llamacpp", "mlx", "lmstudio", - "exo", "nexa", + "ollama", + "vllm", + "sglang", + "llamacpp", + "mlx", + "lmstudio", + "exo", + "nexa", ] @@ -57,7 +63,7 @@ def _detect_running_engines() -> list[str]: def _next_steps_text(engine: str, model: str = "") -> str: """Return engine-specific next-steps guidance after init.""" - pull_model = model or "qwen3.5:3b" + pull_model = model or "qwen3.5:2b" steps: dict[str, str] = { "ollama": ( "Next steps:\n" @@ -70,7 +76,7 @@ def _next_steps_text(engine: str, model: str = "") -> str: f" ollama pull {pull_model}\n" "\n" " 3. Try it out:\n" - " jarvis ask \"Hello\"\n" + ' jarvis ask "Hello"\n' "\n" " Run `jarvis doctor` to verify your setup." ), @@ -82,7 +88,7 @@ def _next_steps_text(engine: str, model: str = "") -> str: " vllm serve Qwen/Qwen3-4B\n" "\n" " 2. Try it out:\n" - " jarvis ask \"Hello\"\n" + ' jarvis ask "Hello"\n' "\n" " Run `jarvis doctor` to verify your setup." ), @@ -94,7 +100,7 @@ def _next_steps_text(engine: str, model: str = "") -> str: " llama-server -m path/to/model.gguf\n" "\n" " 2. Try it out:\n" - " jarvis ask \"Hello\"\n" + ' jarvis ask "Hello"\n' "\n" " Run `jarvis doctor` to verify your setup." ), @@ -106,7 +112,7 @@ def _next_steps_text(engine: str, model: str = "") -> str: " python -m sglang.launch_server --model-path Qwen/Qwen3-8B\n" "\n" " 2. Try it out:\n" - " jarvis ask \"Hello\"\n" + ' jarvis ask "Hello"\n' "\n" " Run `jarvis doctor` to verify your setup." ), @@ -118,7 +124,7 @@ def _next_steps_text(engine: str, model: str = "") -> str: " mlx_lm.server --model mlx-community/Qwen2.5-7B-4bit\n" "\n" " 2. Try it out:\n" - " jarvis ask \"Hello\"\n" + ' jarvis ask "Hello"\n' "\n" " Run `jarvis doctor` to verify your setup." ), @@ -131,7 +137,7 @@ def _next_steps_text(engine: str, model: str = "") -> str: " 2. Load a model and start the local server (port 1234)\n" "\n" " 3. Try it out:\n" - " jarvis ask \"Hello\"\n" + ' jarvis ask "Hello"\n' "\n" " Run `jarvis doctor` to verify your setup." ), @@ -141,7 +147,7 @@ def _next_steps_text(engine: str, model: str = "") -> str: " pip install exo\n" " exo\n\n" " 2. Try it out:\n" - " jarvis ask \"Hello\"\n\n" + ' jarvis ask "Hello"\n\n' " Run `jarvis doctor` to verify your setup." ), "nexa": ( @@ -150,7 +156,7 @@ def _next_steps_text(engine: str, model: str = "") -> str: " pip install nexaai\n" " nexa server\n\n" " 2. Try it out:\n" - " jarvis ask \"Hello\"\n\n" + ' jarvis ask "Hello"\n\n' " Run `jarvis doctor` to verify your setup." ), } @@ -177,6 +183,7 @@ def _quick_privacy_check(console: Console) -> None: def _do_download(engine: str, model: str, spec, console: Console) -> None: """Dispatch model download based on engine type.""" import os + if engine == "ollama": host = os.environ.get("OLLAMA_HOST", "http://localhost:11434").rstrip("/") ollama_pull(host, model, console) @@ -267,9 +274,7 @@ def init( console.print("[bold]Detecting running inference engines...[/bold]") running = _detect_running_engines() if running: - console.print( - f" Found running: [green]{', '.join(running)}[/green]" - ) + console.print(f" Found running: [green]{', '.join(running)}[/green]") else: console.print(" No running engines detected.") @@ -341,8 +346,7 @@ def init( soul_path = DEFAULT_CONFIG_DIR / "SOUL.md" if not soul_path.exists(): soul_path.write_text( - "# Agent Persona\n\n" - "You are Jarvis, a helpful personal AI assistant.\n" + "# Agent Persona\n\nYou are Jarvis, a helpful personal AI assistant.\n" ) memory_path = DEFAULT_CONFIG_DIR / "MEMORY.md" diff --git a/src/openjarvis/cli/quickstart_cmd.py b/src/openjarvis/cli/quickstart_cmd.py index 67c8db1b..06d6f4aa 100644 --- a/src/openjarvis/cli/quickstart_cmd.py +++ b/src/openjarvis/cli/quickstart_cmd.py @@ -106,8 +106,7 @@ def quickstart(force: bool) -> None: console.print("[bold cyan][2/5][/bold cyan] Writing config...") if DEFAULT_CONFIG_PATH.exists() and not force: console.print( - f" [dim]Config already exists at" - f" {DEFAULT_CONFIG_PATH} (skip)[/dim]" + f" [dim]Config already exists at {DEFAULT_CONFIG_PATH} (skip)[/dim]" ) else: toml_content = generate_default_toml(hw) @@ -144,7 +143,7 @@ def quickstart(force: bool) -> None: if not _check_model_available(active_engine): console.print(" [yellow]No models found.[/yellow]") console.print( - " Pull a model first (e.g. [bold]ollama pull qwen3.5:3b[/bold])." + " Pull a model first (e.g. [bold]ollama pull qwen3.5:2b[/bold])." ) raise SystemExit(1) console.print(" [green]Models available.[/green]") @@ -157,6 +156,5 @@ def quickstart(force: bool) -> None: console.print() console.print( - '[bold green]Setup complete![/bold green]' - ' Try: [bold]jarvis ask "Hello"[/bold]' + '[bold green]Setup complete![/bold green] Try: [bold]jarvis ask "Hello"[/bold]' ) diff --git a/src/openjarvis/evals/configs/smoke_apple.toml b/src/openjarvis/evals/configs/smoke_apple.toml index 478fb014..a058577c 100644 --- a/src/openjarvis/evals/configs/smoke_apple.toml +++ b/src/openjarvis/evals/configs/smoke_apple.toml @@ -32,9 +32,9 @@ name = "mlx-community/Qwen2.5-7B-4bit" engine = "mlx" param_count_b = 7.0 -# Ollama GGUF — pull with: ollama pull qwen3.5:3b +# Ollama GGUF — pull with: ollama pull qwen3.5:2b [[models]] -name = "qwen3.5:3b" +name = "qwen3.5:2b" engine = "ollama" # ── Benchmarks ──────────────────────────────────────────────────────────────── diff --git a/src/openjarvis/intelligence/model_catalog.py b/src/openjarvis/intelligence/model_catalog.py index ee36976f..af561c6a 100644 --- a/src/openjarvis/intelligence/model_catalog.py +++ b/src/openjarvis/intelligence/model_catalog.py @@ -40,48 +40,62 @@ BUILTIN_MODELS: List[ModelSpec] = [ # Local models — Qwen3.5 (MoE) # ----------------------------------------------------------------------- ModelSpec( - model_id="qwen3.5:3b", - name="Qwen3.5 3B", - parameter_count_b=3.0, - active_parameter_count_b=0.6, + model_id="qwen3.5:0.8b", + name="Qwen3.5 0.8B", + parameter_count_b=0.8, + active_parameter_count_b=0.15, context_length=131072, supported_engines=("ollama", "vllm", "llamacpp", "sglang", "mlx"), provider="alibaba", metadata={ "architecture": "moe", - "hf_repo": "Qwen/Qwen3.5-3B", - "gguf_file": "qwen3.5-3b-q4_k_m.gguf", - "mlx_repo": "mlx-community/Qwen3.5-3B-4bit", + "hf_repo": "Qwen/Qwen3.5-0.8B", + "mlx_repo": "mlx-community/Qwen3.5-0.8B-OptiQ-4bit", }, ), ModelSpec( - model_id="qwen3.5:8b", - name="Qwen3.5 8B", - parameter_count_b=8.0, - active_parameter_count_b=1.0, + model_id="qwen3.5:2b", + name="Qwen3.5 2B", + parameter_count_b=2.0, + active_parameter_count_b=0.4, context_length=131072, supported_engines=("ollama", "vllm", "llamacpp", "sglang", "mlx"), provider="alibaba", metadata={ "architecture": "moe", - "hf_repo": "Qwen/Qwen3.5-8B", - "gguf_file": "qwen3.5-8b-q4_k_m.gguf", - "mlx_repo": "mlx-community/Qwen3.5-8B-4bit", + "hf_repo": "Qwen/Qwen3.5-2B", + "mlx_repo": "mlx-community/Qwen3.5-2B-OptiQ-4bit", }, ), ModelSpec( - model_id="qwen3.5:14b", - name="Qwen3.5 14B", - parameter_count_b=14.0, - active_parameter_count_b=2.0, + model_id="qwen3.5:9b", + name="Qwen3.5 9B", + parameter_count_b=9.0, + active_parameter_count_b=1.5, context_length=131072, supported_engines=("ollama", "vllm", "llamacpp", "sglang", "mlx"), provider="alibaba", metadata={ "architecture": "moe", - "hf_repo": "Qwen/Qwen3.5-14B", - "gguf_file": "qwen3.5-14b-q4_k_m.gguf", - "mlx_repo": "mlx-community/Qwen3.5-14B-4bit", + "hf_repo": "Qwen/Qwen3.5-9B", + "gguf_file": "qwen3.5-9b-q4_k_m.gguf", + "mlx_repo": "mlx-community/Qwen3.5-9B-MLX-4bit", + }, + ), + ModelSpec( + model_id="qwen3.5:27b", + name="Qwen3.5 27B", + parameter_count_b=27.0, + active_parameter_count_b=3.0, + context_length=131072, + min_vram_gb=16.0, + supported_engines=("ollama", "vllm", "llamacpp", "sglang", "mlx"), + provider="alibaba", + metadata={ + "architecture": "moe", + "hf_repo": "Qwen/Qwen3.5-27B", + "gguf_file": "qwen3.5-27b-q4_k_m.gguf", + "mlx_repo": "mlx-community/Qwen3.5-27B-4bit-DWQ", }, ), ModelSpec( @@ -235,7 +249,7 @@ BUILTIN_MODELS: List[ModelSpec] = [ "architecture": "moe", "hf_repo": "Qwen/Qwen3.5-4B", "gguf_file": "qwen3.5-4b-q4_k_m.gguf", - "mlx_repo": "mlx-community/Qwen3.5-4B-4bit", + "mlx_repo": "mlx-community/Qwen3.5-4B-OptiQ-4bit", }, ), ModelSpec( @@ -490,8 +504,7 @@ BUILTIN_MODELS: List[ModelSpec] = [ metadata={ "architecture": "moe", "hf_repo": ( - "TeichAI/GLM-4.7-Flash-Claude-" - "Opus-4.5-High-Reasoning-Distill-GGUF" + "TeichAI/GLM-4.7-Flash-Claude-Opus-4.5-High-Reasoning-Distill-GGUF" ), "teacher": "Claude Opus 4.5", "quantization": "GGUF Q4_K_M / Q8_0", diff --git a/tests/cli/test_init_guidance.py b/tests/cli/test_init_guidance.py index acca3c0c..dfde2c37 100644 --- a/tests/cli/test_init_guidance.py +++ b/tests/cli/test_init_guidance.py @@ -23,9 +23,7 @@ class TestInitShowsNextSteps: mock.patch("openjarvis.cli.init_cmd.DEFAULT_CONFIG_PATH", config_path), mock.patch("openjarvis.cli.init_cmd.PrivacyScanner"), ): - result = CliRunner().invoke( - cli, ["init", "--engine", "llamacpp", _NO_DL] - ) + result = CliRunner().invoke(cli, ["init", "--engine", "llamacpp", _NO_DL]) assert result.exit_code == 0 assert "Getting Started" in result.output assert "jarvis ask" in result.output @@ -40,9 +38,7 @@ class TestInitShowsNextSteps: mock.patch("openjarvis.cli.init_cmd.DEFAULT_CONFIG_PATH", config_path), mock.patch("openjarvis.cli.init_cmd.PrivacyScanner"), ): - result = CliRunner().invoke( - cli, ["init", "--engine", "llamacpp", _NO_DL] - ) + result = CliRunner().invoke(cli, ["init", "--engine", "llamacpp", _NO_DL]) assert result.exit_code == 0 assert "[engine]" in result.output assert "[intelligence]" in result.output @@ -57,12 +53,12 @@ class TestNextStepsOllama: assert "jarvis doctor" in text def test_next_steps_ollama_with_model(self) -> None: - text = _next_steps_text("ollama", "qwen3.5:14b") - assert "ollama pull qwen3.5:14b" in text + text = _next_steps_text("ollama", "qwen3.5:27b") + assert "ollama pull qwen3.5:27b" in text def test_next_steps_ollama_default_model(self) -> None: text = _next_steps_text("ollama") - assert "ollama pull qwen3.5:3b" in text + assert "ollama pull qwen3.5:2b" in text class TestNextStepsVllm: @@ -102,9 +98,7 @@ class TestMinimalConfig: mock.patch("openjarvis.cli.init_cmd.DEFAULT_CONFIG_PATH", config_path), mock.patch("openjarvis.cli.init_cmd.PrivacyScanner"), ): - result = CliRunner().invoke( - cli, ["init", "--engine", "ollama", _NO_DL] - ) + result = CliRunner().invoke(cli, ["init", "--engine", "ollama", _NO_DL]) assert result.exit_code == 0 content = config_path.read_text() # Minimal config should be short @@ -158,9 +152,7 @@ class TestInitDownloadPrompt: mock.patch("openjarvis.cli.init_cmd.DEFAULT_CONFIG_PATH", config_path), mock.patch("openjarvis.cli.init_cmd.PrivacyScanner"), ): - result = CliRunner().invoke( - cli, ["init", "--engine", "ollama", _NO_DL] - ) + result = CliRunner().invoke(cli, ["init", "--engine", "ollama", _NO_DL]) assert result.exit_code == 0 assert "Download" not in result.output @@ -175,13 +167,10 @@ class TestInitEmptyModelFallback: mock.patch("openjarvis.cli.init_cmd.recommend_model", return_value=""), mock.patch("openjarvis.cli.init_cmd.PrivacyScanner"), ): - result = CliRunner().invoke( - cli, ["init", "--engine", "llamacpp"] - ) + result = CliRunner().invoke(cli, ["init", "--engine", "llamacpp"]) assert result.exit_code == 0 assert ( - "Not enough memory" in result.output - or "not enough memory" in result.output + "Not enough memory" in result.output or "not enough memory" in result.output ) @@ -200,9 +189,7 @@ class TestNextStepsExoNexa: class TestInitDownloadDispatch: - def test_init_ollama_download_calls_ollama_pull( - self, tmp_path: Path - ) -> None: + def test_init_ollama_download_calls_ollama_pull(self, tmp_path: Path) -> None: config_dir = tmp_path / ".openjarvis" config_path = config_dir / "config.toml" with ( @@ -228,9 +215,7 @@ class TestInitDownloadDispatch: mock.patch("openjarvis.cli.init_cmd.DEFAULT_CONFIG_PATH", config_path), mock.patch("openjarvis.cli.init_cmd.PrivacyScanner"), ): - result = CliRunner().invoke( - cli, ["init", "--engine", "vllm"], input="y\n" - ) + result = CliRunner().invoke(cli, ["init", "--engine", "vllm"], input="y\n") assert result.exit_code == 0 assert "automatically" in result.output @@ -245,12 +230,11 @@ class TestInitPrivacyHook: mock.patch("openjarvis.cli.init_cmd.PrivacyScanner") as MockScanner, ): from openjarvis.cli.scan_cmd import ScanResult + instance = MockScanner.return_value instance.run_quick.return_value = [ ScanResult("FileVault", "ok", "FileVault enabled", "darwin"), ] - result = CliRunner().invoke( - cli, ["init", "--engine", "llamacpp", _NO_DL] - ) + result = CliRunner().invoke(cli, ["init", "--engine", "llamacpp", _NO_DL]) assert result.exit_code == 0 assert "jarvis scan" in result.output diff --git a/tests/cli/test_model_pull.py b/tests/cli/test_model_pull.py index 3a724950..13f74067 100644 --- a/tests/cli/test_model_pull.py +++ b/tests/cli/test_model_pull.py @@ -15,6 +15,7 @@ class TestOllamaPull: def test_ollama_pull_success(self) -> None: import io + console = Console(file=io.StringIO()) mock_lines = [ '{"status": "pulling manifest"}', @@ -28,7 +29,7 @@ class TestOllamaPull: mock_resp.__exit__ = mock.MagicMock(return_value=False) with mock.patch("httpx.stream", return_value=mock_resp): - result = ollama_pull("http://localhost:11434", "qwen3.5:3b", console) + result = ollama_pull("http://localhost:11434", "qwen3.5:2b", console) assert result is True def test_ollama_pull_connect_error(self) -> None: @@ -38,7 +39,7 @@ class TestOllamaPull: console = Console(file=io.StringIO()) with mock.patch("httpx.stream", side_effect=httpx.ConnectError("refused")): - result = ollama_pull("http://localhost:11434", "qwen3.5:3b", console) + result = ollama_pull("http://localhost:11434", "qwen3.5:2b", console) assert result is False @@ -58,14 +59,14 @@ class TestPullCliMultiEngine: mock_run.return_value = mock.MagicMock(returncode=0) result = runner.invoke( - cli, ["model", "pull", "qwen3.5:8b", "--engine", "llamacpp"] + cli, ["model", "pull", "qwen3.5:9b", "--engine", "llamacpp"] ) assert result.exit_code == 0 mock_run.assert_called_once() call_args = mock_run.call_args[0][0] assert "huggingface-cli" in call_args - assert "qwen3.5-8b-q4_k_m.gguf" in call_args + assert "qwen3.5-9b-q4_k_m.gguf" in call_args def test_pull_mlx_uses_huggingface_cli(self) -> None: from openjarvis.cli import cli @@ -80,14 +81,14 @@ class TestPullCliMultiEngine: mock_run.return_value = mock.MagicMock(returncode=0) result = runner.invoke( - cli, ["model", "pull", "qwen3.5:8b", "--engine", "mlx"] + cli, ["model", "pull", "qwen3.5:9b", "--engine", "mlx"] ) assert result.exit_code == 0 mock_run.assert_called_once() call_args = mock_run.call_args[0][0] assert "huggingface-cli" in call_args - assert "mlx-community/Qwen3.5-8B-4bit" in call_args + assert "mlx-community/Qwen3.5-9B-MLX-4bit" in call_args def test_pull_llamacpp_huggingface_cli_not_found(self) -> None: from openjarvis.cli import cli @@ -101,7 +102,7 @@ class TestPullCliMultiEngine: mock_cfg.return_value.engine.ollama_host = None result = runner.invoke( - cli, ["model", "pull", "qwen3.5:8b", "--engine", "llamacpp"] + cli, ["model", "pull", "qwen3.5:9b", "--engine", "llamacpp"] ) assert result.exit_code != 0 diff --git a/tests/core/test_recommend_model.py b/tests/core/test_recommend_model.py index 61600b69..1a5cc5aa 100644 --- a/tests/core/test_recommend_model.py +++ b/tests/core/test_recommend_model.py @@ -27,7 +27,7 @@ class TestRecommendModelGpu: result = recommend_model(hw, "ollama") # 14B * 0.5 * 1.1 = 7.7 GB; available = 8 * 0.9 = 7.2 → too big # 8B * 0.5 * 1.1 = 4.4 GB; available = 7.2 → fits - assert result == "qwen3.5:8b" + assert result == "qwen3.5:9b" def test_4gb_gpu_picks_qwen35_4b(self) -> None: hw = HardwareInfo( @@ -47,7 +47,7 @@ class TestRecommendModelGpu: ) result = recommend_model(hw, "ollama") # 3B * 0.5 * 1.1 = 1.65 GB; available = 2 * 0.9 = 1.8 → fits - assert result == "qwen3.5:3b" + assert result == "qwen3.5:2b" def test_multi_gpu_picks_larger_model(self) -> None: hw = HardwareInfo( @@ -80,8 +80,9 @@ class TestRecommendModelCpuOnly: hw = HardwareInfo(platform="linux", ram_gb=16.0, gpu=None) result = recommend_model(hw, "llamacpp") # available = (16 - 4) * 0.8 = 9.6 GB - # 14B * 0.5 * 1.1 = 7.7 → fits - assert result == "qwen3.5:14b" + # 27B * 0.5 * 1.1 = 14.85 → too big + # 9B * 0.5 * 1.1 = 4.95 → fits + assert result == "qwen3.5:9b" def test_cpu_only_8gb_ram(self) -> None: hw = HardwareInfo(platform="linux", ram_gb=8.0, gpu=None) @@ -127,7 +128,7 @@ class TestRecommendModelMlx: gpu=GpuInfo(vendor="apple", name="Apple M1", vram_gb=8.0, count=1), ) result = recommend_model(hw, "mlx") - assert result == "qwen3.5:8b" + assert result == "qwen3.5:9b" def test_apple_silicon_16gb_mlx(self) -> None: hw = HardwareInfo( @@ -136,7 +137,10 @@ class TestRecommendModelMlx: gpu=GpuInfo(vendor="apple", name="Apple M2", vram_gb=16.0, count=1), ) result = recommend_model(hw, "mlx") - assert result == "qwen3.5:14b" + # available = 16 * 0.9 = 14.4 GB + # 27B * 0.5 * 1.1 = 14.85 → too big + # 9B * 0.5 * 1.1 = 4.95 → fits + assert result == "qwen3.5:9b" def test_apple_silicon_32gb_mlx(self) -> None: hw = HardwareInfo( @@ -145,7 +149,7 @@ class TestRecommendModelMlx: gpu=GpuInfo(vendor="apple", name="Apple M2 Pro", vram_gb=32.0, count=1), ) result = recommend_model(hw, "mlx") - assert result == "qwen3.5:14b" + assert result == "qwen3.5:27b" def test_apple_silicon_64gb_mlx(self) -> None: hw = HardwareInfo( @@ -154,4 +158,4 @@ class TestRecommendModelMlx: gpu=GpuInfo(vendor="apple", name="Apple M2 Max", vram_gb=64.0, count=1), ) result = recommend_model(hw, "mlx") - assert result == "qwen3.5:14b" + assert result == "qwen3.5:27b" diff --git a/tests/intelligence/test_model_catalog_extended.py b/tests/intelligence/test_model_catalog_extended.py index bb5e8d38..f869810b 100644 --- a/tests/intelligence/test_model_catalog_extended.py +++ b/tests/intelligence/test_model_catalog_extended.py @@ -186,26 +186,26 @@ class TestCloudModelSpecs: class TestQwen35ModelSpecs: """Verify Qwen3.5 MoE model entries.""" - def test_qwen35_3b(self) -> None: - spec = _get_spec("qwen3.5:3b") - assert spec.parameter_count_b == 3.0 - assert spec.active_parameter_count_b == 0.6 + def test_qwen35_2b(self) -> None: + spec = _get_spec("qwen3.5:2b") + assert spec.parameter_count_b == 2.0 + assert spec.active_parameter_count_b == 0.4 assert spec.context_length == 131072 assert spec.provider == "alibaba" assert spec.metadata["architecture"] == "moe" for e in ("ollama", "vllm", "llamacpp", "sglang"): assert e in spec.supported_engines - def test_qwen35_8b(self) -> None: - spec = _get_spec("qwen3.5:8b") - assert spec.parameter_count_b == 8.0 - assert spec.active_parameter_count_b == 1.0 + def test_qwen35_9b(self) -> None: + spec = _get_spec("qwen3.5:9b") + assert spec.parameter_count_b == 9.0 + assert spec.active_parameter_count_b == 1.5 assert spec.context_length == 131072 - def test_qwen35_14b(self) -> None: - spec = _get_spec("qwen3.5:14b") - assert spec.parameter_count_b == 14.0 - assert spec.active_parameter_count_b == 2.0 + def test_qwen35_27b(self) -> None: + spec = _get_spec("qwen3.5:27b") + assert spec.parameter_count_b == 27.0 + assert spec.active_parameter_count_b == 3.0 def test_qwen35_35b(self) -> None: spec = _get_spec("qwen3.5:35b") @@ -300,9 +300,9 @@ class TestModelDiscovery: "gpt-oss:120b", "glm-4.7-flash", "trinity-mini", - "qwen3.5:3b", - "qwen3.5:8b", - "qwen3.5:14b", + "qwen3.5:2b", + "qwen3.5:9b", + "qwen3.5:27b", "qwen3.5:35b", "qwen3.5:122b", "qwen3.5:397b", From e5ff7b36957420b72ef1e61ed6cf815816ac039f Mon Sep 17 00:00:00 2001 From: Jon Saad-Falcon <41205309+jonsaadfalcon@users.noreply.github.com> Date: Thu, 26 Mar 2026 12:42:16 -0700 Subject: [PATCH 12/44] Delete MagicMock/load_config().security.audit_log_path directory --- .../4405053488 | Bin 8192 -> 0 bytes .../4424260768 | Bin 8192 -> 0 bytes 2 files changed, 0 insertions(+), 0 deletions(-) delete mode 100644 MagicMock/load_config().security.audit_log_path/4405053488 delete mode 100644 MagicMock/load_config().security.audit_log_path/4424260768 diff --git a/MagicMock/load_config().security.audit_log_path/4405053488 b/MagicMock/load_config().security.audit_log_path/4405053488 deleted file mode 100644 index 93e421ea05d5262d8713fc9cd6a5fdb023a6dddf..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 8192 zcmeI#K}*9h7zW^E2!f!97q8!K%EXI*z^YM(b!e^N@K9oxx;UGTHg4=J{#k#8iBQCr z;(fdy4drX!G|6-8{avW7qO2%$Y3YRRGS1m45o3(I(Q~6N-*427yZipX;_Nv4+IA_v zGh^;TK>z{}fB*y_009U<00Izz!1@b(?D5V0!GM20S^4~?DwV#M+O86{P{vk`ws#(h zAQlt{XQ80ROFG!1M(N4#gW)I^S0bX@Xm}GuW4adOqt*ngb5&WH&)qz?RM<}Z>?^UW zxgv_iW86+;+Dx>0sgl>KFddn+FxH(fnU`vz)v_ZcQ>zP;SUFQ&A1aGwGL_ZTpVM!L zF2rT<5XRK+w|C}OtJ96}ABjW%Z|;8jQ4oLt1Rwwb2tWV=5P$##AOHaftg*m1h_`K% diff --git a/MagicMock/load_config().security.audit_log_path/4424260768 b/MagicMock/load_config().security.audit_log_path/4424260768 deleted file mode 100644 index 93e421ea05d5262d8713fc9cd6a5fdb023a6dddf..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 8192 zcmeI#K}*9h7zW^E2!f!97q8!K%EXI*z^YM(b!e^N@K9oxx;UGTHg4=J{#k#8iBQCr z;(fdy4drX!G|6-8{avW7qO2%$Y3YRRGS1m45o3(I(Q~6N-*427yZipX;_Nv4+IA_v zGh^;TK>z{}fB*y_009U<00Izz!1@b(?D5V0!GM20S^4~?DwV#M+O86{P{vk`ws#(h zAQlt{XQ80ROFG!1M(N4#gW)I^S0bX@Xm}GuW4adOqt*ngb5&WH&)qz?RM<}Z>?^UW zxgv_iW86+;+Dx>0sgl>KFddn+FxH(fnU`vz)v_ZcQ>zP;SUFQ&A1aGwGL_ZTpVM!L zF2rT<5XRK+w|C}OtJ96}ABjW%Z|;8jQ4oLt1Rwwb2tWV=5P$##AOHaftg*m1h_`K% From db2f20e25b2e83aa6b4d2d4888c33db12c082ef6 Mon Sep 17 00:00:00 2001 From: Jon Saad-Falcon <41205309+jonsaadfalcon@users.noreply.github.com> Date: Thu, 26 Mar 2026 12:42:54 -0700 Subject: [PATCH 13/44] Delete instr.md --- instr.md | 417 ------------------------------------------------------- 1 file changed, 417 deletions(-) delete mode 100644 instr.md diff --git a/instr.md b/instr.md deleted file mode 100644 index 4eb6d013..00000000 --- a/instr.md +++ /dev/null @@ -1,417 +0,0 @@ -# Agent Runtime Manual Test Plan - -**Branch:** `main` -**PR Reference:** [#32](https://github.com/open-jarvis/OpenJarvis/pull/32) - ---- - -## Setup - -```bash -git checkout main && git pull -uv sync --extra dev -``` - -Create `~/.openjarvis/config.toml`: - -```toml -[engine] -type = "cloud" - -[intelligence] -default_model = "Qwen/Qwen3.5-35B-A3B" - -[engine.cloud] -provider = "openai" -api_key = "sk-..." -``` - -For every test case, record: **Pass / Fail / Partial / Blocked**, what you actually saw, and screenshots for any UI issues. - ---- - -## Part 1: CLI (`jarvis agents`) - -### 1.1 Commands exist - -| # | Test | Expected | -|---|------|----------| -| 1 | `jarvis agents --help` | Shows all subcommands: `launch`, `start`, `stop`, `run`, `status`, `logs`, `daemon`, `watch`, `recover`, `errors`, `ask`, `instruct`, `messages`, `list`, `info`, `create`, `pause`, `resume`, `delete`, `bind`, `channels`, `search`, `templates`, `tasks` | - -### 1.2 Agent lifecycle: create → run → pause → resume → delete - -| # | Test | Expected | -|---|------|----------| -| 2 | `jarvis agents launch` | Wizard: template list → name/schedule/tools/budget/learning prompts → creates agent, prints ID | -| 3 | `jarvis agents list` | Agent appears, status=`idle` | -| 4 | `jarvis agents status` | Table: name, status dot, schedule, last run, runs=0, cost=$0 | -| 5 | `jarvis agents run ` | Prints progress then "Tick complete. Status: idle, runs: 1" | -| 6 | `jarvis agents status` | runs=1, last run time updated | -| 7 | `jarvis agents pause ` then `status` | Status shows `paused` | -| 8 | `jarvis agents resume ` then `status` | Status back to `idle` | -| 9 | `jarvis agents delete ` then `list` | Agent gone (soft-deleted/archived, not in list) | - -### 1.3 Agent creation variants - -| # | Test | Expected | -|---|------|----------| -| 10 | `jarvis agents create "Test Agent"` | Creates agent by name, prints ID | -| 11 | `jarvis agents create --template ` | Creates from template, inherits template config | -| 12 | `jarvis agents launch` → pick a template | Wizard pre-fills config from template | -| 13 | `jarvis agents launch` → pick "Custom Agent" | Wizard starts with blank config | -| 14 | `jarvis agents templates` | Lists built-in + user templates with descriptions | - -### 1.4 Scheduling - -| # | Test | Expected | -|---|------|----------| -| 15 | Create agent with `schedule_type=interval`, `schedule_value=30` | Created | -| 16 | `jarvis agents start ` | "Agent registered with scheduler" | -| 17 | `jarvis agents stop ` | "Agent deregistered from scheduler" | -| 18 | `jarvis agents daemon` | Starts, prints agent count, blocks. Ctrl+C → "Daemon stopped." clean exit | -| 19 | Create agent with `schedule_type=cron`, `schedule_value="*/5 * * * *"` | Created | -| 20 | `jarvis agents start ` (cron agent) | Registered, next fire time displayed or logged | -| 21 | Create agent with `schedule_type=manual` then `start ` | Agent registered but never auto-fires | - -### 1.5 Interaction: ask / instruct / messages - -| # | Test | Expected | -|---|------|----------| -| 22 | `jarvis agents ask "What is 2+2?"` | Runs tick, prints agent response inline | -| 23 | `jarvis agents messages ` | Shows user→agent ask + agent→user response | -| 24 | `jarvis agents instruct "Focus on ML papers"` | "Instruction queued for next tick" | -| 25 | `jarvis agents messages ` | Queued instruction shows `[queued]`, status=pending | -| 26 | `jarvis agents run ` then `messages ` | Queued message now delivered, status changes from pending | -| 27 | `jarvis agents ask ""` (empty message) | Graceful error or rejection, no crash | -| 28 | `jarvis agents instruct ` with very long message (>1000 chars) | Accepted and stored correctly | - -### 1.6 Error recovery & monitoring - -| # | Test | Expected | -|---|------|----------| -| 29 | `jarvis agents errors` | Lists agents in error/needs_attention/stalled/budget_exceeded (or empty table) | -| 30 | `jarvis agents recover ` (on errored agent) | Restores checkpoint, status → `idle` | -| 31 | `jarvis agents recover ` (on idle agent) | Clear message: "Agent is not in error state" or similar | -| 32 | `jarvis agents logs ` | Recent traces with tick IDs and timestamps | -| 33 | `jarvis agents logs ` | Clear error: "Agent not found" | -| 34 | `jarvis agents watch` (then run a tick in another terminal) | Events stream live: AGENT_TICK_START, AGENT_TICK_END visible. Ctrl+C to stop. | -| 35 | `jarvis agents watch ` | Same, filtered to one agent only | -| 36 | `jarvis agents watch` then Ctrl+C | Clean exit, no traceback, no hanging threads | - -### 1.7 Agent info & inspection - -| # | Test | Expected | -|---|------|----------| -| 37 | `jarvis agents info ` | Shows agent type, status, memory snippet, tasks, channels, config details | -| 38 | `jarvis agents tasks ` | Lists tasks with statuses (or empty state) | -| 39 | `jarvis agents channels ` | Lists channel bindings (or empty state) | -| 40 | `jarvis agents search "keyword"` | Searches across agent traces, returns relevant results | - -### 1.8 Edge cases & invalid input - -| # | Test | Expected | -|---|------|----------| -| 41 | `jarvis agents run ` | Clear error: "Agent not found" — no Python traceback | -| 42 | `jarvis agents pause ` twice | Second pause is no-op or clear message, no crash | -| 43 | `jarvis agents resume ` (already idle) | No-op or clear message, no crash | -| 44 | `jarvis agents run ` while another tick is running | Concurrency guard: "Agent is already running" error | -| 45 | `jarvis agents delete ` then `run ` | Clear error about deleted/archived agent | -| 46 | Create agent with invalid cron expression | Rejected with clear validation error | -| 47 | Create agent with negative budget | Rejected or clamped to 0 | - -### 1.9 CLI aesthetics - -| # | Check | Expected | -|---|-------|----------| -| 48 | `status` table formatting | Columns aligned, readable at 80-char terminal width | -| 49 | Error messages (run with no engine configured) | Clear human-readable message, no Python tracebacks | -| 50 | `launch` wizard prompts | Clear labels, sensible defaults, no confusing jargon | -| 51 | `watch` event stream | Color-coded, event type + agent name visible, timestamps | -| 52 | `list` table with 0 agents | "No agents found" or empty table — not a crash | -| 53 | `list` table with 10+ agents | Table remains readable, no column overflow | -| 54 | All commands with `--help` | Every subcommand has a help string | - ---- - -## Part 2: Web Frontend - -### 2.0 Setup - -```bash -# Terminal 1 # Terminal 2 -uv run jarvis serve cd frontend && npm install && npm run dev -``` - -Open http://localhost:5173, navigate to **Agents** page via sidebar. - -### 2.1 Navigation & routing - -| # | Test | Expected | -|---|------|----------| -| 55 | Click "Agents" in sidebar | AgentsPage renders, URL is `/agents` | -| 56 | Direct navigation to `/agents` | Page loads correctly (no blank screen) | -| 57 | Browser back/forward after visiting agent detail | Navigation works, state preserved | - -### 2.2 List view - -| # | Test | Expected | -|---|------|----------| -| 58 | Page loads with backend running | No console errors, agent list renders | -| 59 | Page loads with backend **down** | User-visible error message (not blank white screen), no console exceptions | -| 60 | Agent cards | Name, color status dot, schedule description, last run time, runs count, cost | -| 61 | "Run Now" button | Triggers tick, card updates (runs count increments, last run time updates) | -| 62 | Pause/Resume button | Toggles status, dot color changes immediately | -| 63 | Agent list auto-refresh | After running a tick via CLI, the web list eventually reflects the updated state | -| 64 | 10+ agents in list | Cards render without performance issues, scroll works | - -### 2.3 Launch wizard - -| # | Test | Expected | -|---|------|----------| -| 65 | Click "Launch Agent" | Modal appears: Step 1 template picker with templates + "Custom Agent" option | -| 66 | Templates load from API | Template cards display with names and descriptions | -| 67 | Select template → Next → Step 2 | Config form: name (pre-filled from template), schedule_type dropdown, schedule_value, tools checkboxes, budget, learning toggle (off) | -| 68 | Select "Custom Agent" → Next → Step 2 | Config form with blank name, no pre-filled values | -| 69 | Next → Step 3 | Review summary of all config values | -| 70 | Click Launch | Agent created, modal closes, new agent appears in list | -| 71 | Back button at Step 2 | Returns to Step 1, template selection preserved | -| 72 | Back button at Step 3 | Returns to Step 2, all form inputs preserved | -| 73 | Launch with empty name | Inline error: "Agent name is required" — modal stays open | -| 74 | Launch with all tools selected | All tools included in review and in created agent config | -| 75 | Click outside modal / press Escape | Modal closes (or stays open — document behavior) | -| 76 | Schedule type = "Manual" | schedule_value input is disabled/hidden | -| 77 | Schedule type = "Cron" | schedule_value placeholder shows cron example | -| 78 | Schedule type = "Interval" | schedule_value placeholder shows seconds example | - -### 2.4 Detail view (click an agent) - -| # | Test | Expected | -|---|------|----------| -| 79 | Click agent card | Detail view opens with tabbed interface | -| 80 | **Overview** tab | Stat cards (Total Runs, Success Rate, Total Cost), config display, channels list, action buttons | -| 81 | **Overview** action buttons | Run Now, Pause, Resume visible and functional | -| 82 | **Interact** tab | Chat message list, textarea, "Immediate" and "Queue" send buttons | -| 83 | Send immediate message | Appears in chat with user styling, agent responds after tick | -| 84 | Send queued message | Shows with "queued" badge, status=pending | -| 85 | Send empty message | Button disabled or graceful rejection — no empty message sent | -| 86 | Rapid-fire send (click Send multiple times quickly) | No duplicate messages, no race condition errors | -| 87 | Chat auto-scroll | New messages scroll into view automatically | -| 88 | **Tasks** tab | Task list with status badges (completed=green, failed=red, active=blue, pending=gray) | -| 89 | **Tasks** tab (no tasks) | Empty state: "No tasks assigned." | -| 90 | **Memory** tab | summary_memory text displayed in readable format | -| 91 | **Memory** tab (no memory) | Empty state: "Agent has no stored memory yet." | -| 92 | **Learning** tab | Toggle switch (read-only, off by default), placeholder text for future events | -| 93 | **Logs** tab | Placeholder / empty state message (not a crash or blank) | -| 94 | Tab switching — rapid clicks | All 6 tabs render instantly, no layout shift, no flash of wrong content | - -### 2.5 Error states - -| # | Test | Expected | -|---|------|----------| -| 95 | Agent in `error` status | Red status dot/badge, "Recover" button visible | -| 96 | Click Recover | Status resets to `idle`, dot turns green | -| 97 | Agent in `needs_attention` status | Amber badge visible | -| 98 | Agent in `budget_exceeded` status | Orange badge visible | -| 99 | Agent in `stalled` status | Yellow badge visible | -| 100 | Backend goes down while page is open | Next refresh/action shows error — not silent failure | -| 101 | Delete agent → confirm it disappears from list | Agent removed from list immediately (or on next refresh) | -| 102 | Delete agent (no confirmation dialog in web) | **Document:** Is instant delete OK or should there be a confirm? | - -### 2.6 Overflow menu - -| # | Test | Expected | -|---|------|----------| -| 103 | Click "..." menu on agent card | Dropdown with Delete + other options | -| 104 | Click Delete from menu | Agent deleted, list updates | -| 105 | Click outside dropdown | Dropdown closes | - -### 2.7 Web aesthetics & UX - -| # | Check | Expected | -|---|-------|----------| -| 106 | Status dot colors | idle=#22c55e, running=#3b82f6, paused=#6b7280, error=#ef4444, needs_attention=#f59e0b, budget_exceeded=#f97316, stalled=#eab308 | -| 107 | Launch wizard spacing/alignment | Modal centered, steps clearly numbered, form inputs aligned, no overlap | -| 108 | Detail view tab switching | Instant, no layout shift or flash | -| 109 | Interact tab chat feel | Messages visually distinct (user=right vs agent=left or different colors), auto-scroll, clear input area | -| 110 | Responsive at 1024px width | No overflow or cut-off content, agent cards reflow | -| 111 | Responsive at 1440px width | Proper use of space, no excessive stretching | -| 112 | Responsive at 768px width (tablet) | Still usable, no broken layout | -| 113 | Empty states | "No agents yet" + CTA button / "No messages" / "No tasks" — not blank white space | -| 114 | Loading states | "Loading agents..." shown during fetch, spinner or skeleton | -| 115 | Page title / browser tab | Meaningful title (not just "Vite App") | -| 116 | Console errors | Zero console errors during normal usage flow | - ---- - -## Part 3: Desktop App - -### 3.0 Setup - -```bash -# Terminal 1 # Terminal 2 -uv run jarvis serve cd desktop && npm install && npm run tauri dev -``` - -Navigate to the **Agents** tab. - -### 3.1 Functionality - -| # | Test | Expected | -|---|------|----------| -| 117 | Left panel: agent list | Status dots, schedule descriptions, last run times | -| 118 | Click agent → right panel | Tabbed detail view (Overview, Interact, Tasks, Memory, Learning, Logs) | -| 119 | No agent selected | Right panel shows "Select an agent to view details" | -| 120 | "Launch Agent" button | Opens wizard, same 3-step flow as web | -| 121 | Launch wizard → Create agent | Agent appears in left panel list | -| 122 | **Overview** tab | Key-value stats (Status, Agent Type, Schedule, Last Run, Total Runs, Total Cost, Budget) + action buttons (Run Now, Pause, Resume, Recover) | -| 123 | **Interact** tab | Chat UI, mode toggle (immediate/queued), Enter shortcut sends message | -| 124 | Send immediate message | Response appears in chat | -| 125 | Send queued message | Shows as pending | -| 126 | **Tasks** tab | Task list with colored status badges + created-at timestamps | -| 127 | **Memory** tab | summary_memory in monospace font | -| 128 | **Learning** tab | Shows enabled/disabled status + placeholder text | -| 129 | **Logs** tab | Placeholder: "Log streaming not yet connected." | -| 130 | Auto-refresh | Agent list refreshes on ~10s interval (verify with CLI-triggered state change) | -| 131 | Delete agent via desktop | Confirmation dialog appears, agent removed on confirm | - -### 3.2 Desktop edge cases - -| # | Test | Expected | -|---|------|----------| -| 132 | Backend not running → open desktop app | Error state shown, not a crash | -| 133 | Backend dies while desktop is open | Graceful degradation on next action/refresh | -| 134 | Selected agent deleted via CLI → desktop refreshes | Selected agent deselects, list updates | - -### 3.3 Desktop aesthetics - -| # | Check | Expected | -|---|-------|----------| -| 135 | Catppuccin color scheme consistent | idle=#a6e3a1, running=#89b4fa, paused=#6c7086, error=#f38ba8, needs_attention=#fab387, stalled=#f9e2af | -| 136 | Left/right panel split | Resizable or fixed at reasonable ratio, no overlap | -| 137 | Tab switching | Smooth, no flicker | -| 138 | Launch wizard modal | Properly overlays content, dismissible with Escape or outside click | -| 139 | Text readability | Font sizes consistent, sufficient contrast against dark background | -| 140 | Window resize | Layout adapts, no overflow or clipping | -| 141 | Status badge consistency with web | Same statuses map to same semantic colors (green=idle, blue=running, etc.) | - ---- - -## Part 4: API Backend (Direct) - -### 4.1 REST endpoint smoke tests - -Run with `uv run jarvis serve` and test via curl or Postman. - -| # | Test | Expected | -|---|------|----------| -| 142 | `GET /v1/managed-agents` | 200, returns `[]` or agent list JSON | -| 143 | `POST /v1/managed-agents` with valid body | 200/201, returns created agent JSON with `id` | -| 144 | `POST /v1/managed-agents` with empty body | 422 or 400 with validation error | -| 145 | `GET /v1/managed-agents/` | 200, returns single agent | -| 146 | `GET /v1/managed-agents/` | 404, returns error JSON | -| 147 | `POST /v1/managed-agents//run` | 200, tick executes | -| 148 | `POST /v1/managed-agents//pause` | 200, status changes to paused | -| 149 | `POST /v1/managed-agents//resume` | 200, status changes to idle | -| 150 | `POST /v1/managed-agents//recover` | 200 if errored, appropriate error if not | -| 151 | `DELETE /v1/managed-agents/` | 200, agent archived | -| 152 | `GET /v1/templates` | 200, returns template list | -| 153 | `POST /v1/templates//instantiate` | 200, creates agent from template | -| 154 | `GET /v1/agents/errors` | 200, returns list of problem agents | -| 155 | `GET /v1/agents/health` | 200, returns health summary | - -### 4.2 Message endpoints - -| # | Test | Expected | -|---|------|----------| -| 156 | `POST /v1/managed-agents//messages` with `{"content":"hi","direction":"user_to_agent","mode":"immediate"}` | 200, message stored | -| 157 | `GET /v1/managed-agents//messages` | 200, returns message list | -| 158 | `POST /v1/managed-agents//messages` with `{"content":"","direction":"user_to_agent","mode":"immediate"}` | 422 or graceful handling | -| 159 | `POST /v1/managed-agents//messages` with `{"content":"cmd","direction":"user_to_agent","mode":"queued"}` | 200, message has status=pending | - -### 4.3 Task & channel endpoints - -| # | Test | Expected | -|---|------|----------| -| 160 | `GET /v1/managed-agents//tasks` | 200, returns task list | -| 161 | `POST /v1/managed-agents//tasks` | 200, creates task | -| 162 | `GET /v1/managed-agents//channels` | 200, returns channel bindings | -| 163 | `GET /v1/managed-agents//state` | 200, returns full agent state | - -### 4.4 WebSocket events - -| # | Test | Expected | -|---|------|----------| -| 164 | Connect to `ws://localhost:8222/v1/agents/events` | Connection established | -| 165 | Trigger a tick → observe WS messages | Receive AGENT_TICK_START and AGENT_TICK_END events | -| 166 | Connect with `?agent_id=` filter | Only events for that agent | -| 167 | Disconnect cleanly | No server error logs | - ---- - -## Part 5: Cross-Platform Consistency - -| # | Test | Expected | -|---|------|----------| -| 168 | Create agent via CLI → check web + desktop | Same name, status, config everywhere | -| 169 | Run tick via CLI → check web + desktop | Run count and last run time update in both UIs | -| 170 | Send message via web Interact → check CLI `messages` | Same content, direction, mode | -| 171 | Pause via desktop → check CLI `status` + web | `paused` everywhere | -| 172 | Delete via web → check CLI `list` + desktop | Gone everywhere | -| 173 | Create via web wizard → check CLI `list` + desktop | Agent visible in all three | -| 174 | Recover via CLI → check web + desktop | Status back to idle in all UIs | -| 175 | Send queued message via CLI `instruct` → check web Interact | Message shows with pending/queued status | -| 176 | Multiple agents created from different clients | All agents appear correctly in all views | - ---- - -## Part 6: Stress & Concurrency - -| # | Test | Expected | -|---|------|----------| -| 177 | Run tick on same agent from two terminals simultaneously | Concurrency guard blocks second tick: "Agent is already running" | -| 178 | Create 20+ agents → check list performance | All clients render list without lag | -| 179 | Rapidly pause/resume same agent | All state transitions correct, no stuck states | -| 180 | Run daemon + manual `run` at same time | No double-ticking, concurrency guard holds | -| 181 | Delete agent while tick is in progress | Tick completes or fails gracefully, agent ends up archived | - ---- - -## Part 7: Deferred Features (Placeholder Verification) - -Confirm these show placeholders (not crashes): - -| # | Feature | CLI | Web | Desktop | -|---|---------|-----|-----|---------| -| 182 | Budget enforcement | `run` still works even if cost > budget | No enforcement, budget is display-only | Same | -| 183 | Stall detection | No automatic stall detection fires | N/A | N/A | -| 184 | Learning event timeline | `Learning` tab shows placeholder text | Same | Same | -| 185 | Logs trace replay | `Logs` tab shows placeholder text | Same | Same | -| 186 | `POST /v1/skills` | N/A | N/A | Returns `"not_implemented"` | -| 187 | `POST /v1/optimize/runs` | N/A | N/A | Returns placeholder `run_id` | -| 188 | `GET /v1/feedback/stats` | N/A | N/A | Returns `{total: 0, mean_score: 0.0}` | - ---- - -## Deliverables - -**1. Test results** — Spreadsheet with columns: #, Status (Pass/Fail/Partial/Blocked), Actual Behavior, Screenshot (for UI issues). - -**2. Bug list** — Each bug: steps to reproduce, expected vs actual, severity (Critical/Major/Minor), screenshot. - -**3. UX & aesthetics feedback** — Is the launch wizard clear? Are status colors distinguishable? Does the Interact tab feel like chat? Is CLI output readable? Are error messages helpful? Is the delete-without-confirm behavior in web acceptable? - -**4. API error handling audit** — Document all cases where the frontend silently swallows errors (currently: agent list fetch, interact tab sends). Recommend which should show user-visible errors. - -**5. Deferred features check** — Confirm placeholder items in Part 7 show graceful stubs (not crashes or blank screens). - ---- - -## Notes - -- Backend (`jarvis serve`) must be running for web and desktop (default port 8222). -- Without an engine configured, `run`/`ask` will error — document whether the error message is clear. -- `daemon` and `watch` block — Ctrl+C to exit. -- Web frontend API client has unused functions (`updateManagedAgent`, `createAgentTask`, `fetchAgentState`, `fetchErrorAgents`) — not a bug, but note for future. -- Desktop API client is missing some endpoints that the web client has (`fetchAgentChannels`, `fetchAgentState`, `fetchErrorAgents`) — may affect feature parity. -- Frontend has **no automated tests** — all testing is manual per this plan. -- Both frontends silently catch API errors (`.catch(() => {})`) — this is a known UX gap to evaluate. From 3b4eeded0e1e86c80e748d5170ba5f84ba3ea796 Mon Sep 17 00:00:00 2001 From: Jon Saad-Falcon <41205309+jonsaadfalcon@users.noreply.github.com> Date: Thu, 26 Mar 2026 13:45:53 -0700 Subject: [PATCH 14/44] fix: normalize TOML arrays for property setters, not just dataclass fields _apply_toml_section only normalized TOML arrays to comma-separated strings for real dataclass fields, but backward-compat property setters like reward_weights also expect string input. When a user's config.toml had an array value for a property-backed attribute, the raw list was passed to the setter which called .split(",") on it, causing: 'list' object has no attribute 'split' This also hardens serve.py against the same issue when reading config.agent.tools. Co-Authored-By: Claude Opus 4.6 (1M context) --- src/openjarvis/cli/serve.py | 14 ++++++++++---- src/openjarvis/core/config.py | 22 ++++++++++++++++------ tests/core/test_config.py | 23 +++++++++++++++++++++++ 3 files changed, 49 insertions(+), 10 deletions(-) diff --git a/src/openjarvis/cli/serve.py b/src/openjarvis/cli/serve.py index bfd819a6..69c1d938 100644 --- a/src/openjarvis/cli/serve.py +++ b/src/openjarvis/cli/serve.py @@ -181,10 +181,16 @@ def serve( _DEFAULT_TOOLS = {"think", "calculator", "web_search"} configured = config.agent.tools if configured: - allowed = { - t.strip() for t in configured.split(",") - if t.strip() - } + if isinstance(configured, list): + allowed = { + t.strip() for t in configured + if isinstance(t, str) and t.strip() + } + else: + allowed = { + t.strip() for t in configured.split(",") + if t.strip() + } else: allowed = _DEFAULT_TOOLS diff --git a/src/openjarvis/core/config.py b/src/openjarvis/core/config.py index d717575b..bce70fa3 100644 --- a/src/openjarvis/core/config.py +++ b/src/openjarvis/core/config.py @@ -1240,8 +1240,9 @@ def _apply_toml_section(target: Any, section: Dict[str, Any]) -> None: """Overlay TOML key/value pairs onto a dataclass instance. Recursively handles nested dicts when the target attribute is itself - a dataclass. Normalises TOML arrays to comma-separated strings when - the target field is annotated as ``str``. + a dataclass. Normalises TOML arrays to comma-separated strings — both + for dataclass fields annotated as ``str`` and for backward-compat + property setters that expect string input. """ for key, value in section.items(): if hasattr(target, key): @@ -1252,10 +1253,19 @@ def _apply_toml_section(target: Any, section: Dict[str, Any]) -> None: else: setattr(target, key, value) else: - # Normalise TOML arrays → comma-separated string when field is str - if isinstance(value, list) and hasattr(target, "__dataclass_fields__"): - field_obj = target.__dataclass_fields__.get(key) - if field_obj is not None and field_obj.type in ("str", str): + # Normalise TOML arrays → comma-separated string. + # Covers both real dataclass fields and backward-compat + # property setters (e.g. reward_weights, default_tools). + if isinstance(value, list): + is_str_field = False + if hasattr(target, "__dataclass_fields__"): + field_obj = target.__dataclass_fields__.get(key) + if field_obj is not None and field_obj.type in ("str", str): + is_str_field = True + elif field_obj is None: + # Property, not a real field — normalise to string + is_str_field = True + if is_str_field: value = ",".join(str(v) for v in value) setattr(target, key, value) diff --git a/tests/core/test_config.py b/tests/core/test_config.py index f4adb5ec..bf4ec66f 100644 --- a/tests/core/test_config.py +++ b/tests/core/test_config.py @@ -458,6 +458,29 @@ class TestApplyTomlSectionListNormalization: assert isinstance(target.enabled, str) assert target.enabled == "code_interpreter,web_search,file_read" + def test_apply_toml_section_list_to_property_setter(self) -> None: + """TOML arrays passed to backward-compat property setters should be + normalized to comma-separated strings, not passed as raw lists.""" + from openjarvis.core.config import _apply_toml_section + + target = LearningConfig() + _apply_toml_section(target, { + "reward_weights": ["accuracy=0.8", "latency=0.2"], + }) + assert target.metrics.accuracy_weight == 0.8 + assert target.metrics.latency_weight == 0.2 + + def test_apply_toml_section_agent_tools_list(self) -> None: + """Agent tools should work as a TOML array.""" + from openjarvis.core.config import _apply_toml_section + + target = AgentConfig() + _apply_toml_section(target, { + "tools": ["web_search", "http_request", "file_read"], + }) + assert isinstance(target.tools, str) + assert target.tools == "web_search,http_request,file_read" + class TestWhatsAppBaileysChannelConfig: def test_defaults(self) -> None: From a212652dc0c1374ae062308b4a84ddc875c784da Mon Sep 17 00:00:00 2001 From: Jon Saad-Falcon <41205309+jonsaadfalcon@users.noreply.github.com> Date: Thu, 26 Mar 2026 15:08:08 -0700 Subject: [PATCH 15/44] =?UTF-8?q?fix:=20agent=20Interact=20tab=20UX=20?= =?UTF-8?q?=E2=80=94=20immediate=20messages,=20ordering,=20scroll?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Backend: - Immediate-mode messages now trigger a background tick so the agent actually processes and responds (previously they were just stored) Frontend (Interact tab): - Reverse message order so newest appear at bottom near the input box - Filter out agent responses with empty content (blank bubbles) - Add "Agent is thinking..." indicator with pulsing dot while processing - Show timestamps instead of raw mode/status labels - Poll for new messages every 3s so responses appear automatically - Only auto-scroll to bottom on initial tab load, not on every poll update (prevents hijacking the user's scroll position) Co-Authored-By: Claude Opus 4.6 (1M context) --- frontend/src/pages/AgentsPage.tsx | 52 +++++++++++++++---- src/openjarvis/server/agent_manager_routes.py | 30 ++++++++++- 2 files changed, 72 insertions(+), 10 deletions(-) diff --git a/frontend/src/pages/AgentsPage.tsx b/frontend/src/pages/AgentsPage.tsx index 702dee16..babe1cca 100644 --- a/frontend/src/pages/AgentsPage.tsx +++ b/frontend/src/pages/AgentsPage.tsx @@ -1166,7 +1166,7 @@ function AgentCard({ // Detail view — Interact tab // --------------------------------------------------------------------------- -function InteractTab({ agentId }: { agentId: string }) { +function InteractTab({ agentId, agentStatus }: { agentId: string; agentStatus: string }) { const [messages, setMessages] = useState([]); const [input, setInput] = useState(''); const [sending, setSending] = useState(false); @@ -1183,10 +1183,19 @@ function InteractTab({ agentId }: { agentId: string }) { useEffect(() => { loadMessages(); + // Poll for new messages while the tab is visible (agent responses + // arrive asynchronously after a background tick completes). + const interval = setInterval(loadMessages, 3000); + return () => clearInterval(interval); }, [loadMessages]); + // Scroll to bottom only on initial load, not on every poll update. + const hasScrolled = useRef(false); useEffect(() => { - bottomRef.current?.scrollIntoView({ behavior: 'smooth' }); + if (!hasScrolled.current && messages.length > 0) { + bottomRef.current?.scrollIntoView({ behavior: 'smooth' }); + hasScrolled.current = true; + } }, [messages]); async function handleSend(mode: 'immediate' | 'queued') { @@ -1203,15 +1212,24 @@ function InteractTab({ agentId }: { agentId: string }) { } } + // Reverse so newest messages appear at the bottom (closest to input). + // Filter out agent responses with empty content. + const displayMessages = [...messages] + .filter((m) => m.direction === 'user_to_agent' || m.content.trim()) + .reverse(); + + const isAgentWorking = agentStatus === 'running'; + const hasPending = messages.some((m) => m.status === 'pending'); + return (
- {messages.length === 0 && ( + {displayMessages.length === 0 && !isAgentWorking && (
No messages yet. Send a message to interact with this agent.
)} - {messages.map((msg) => ( + {displayMessages.map((msg) => (

{msg.content}

-

- {msg.mode} · {msg.status} +

+ {msg.status === 'pending' ? 'sending...' : new Date(msg.created_at * 1000).toLocaleTimeString()}

))} + {/* Progress indicator while agent is working */} + {(isAgentWorking || hasPending) && ( +
+
+
+ + {sending ? 'Sending message...' : 'Agent is thinking...'} +
+
+
+ )}
{/* Input area */} @@ -1792,7 +1826,7 @@ export function AgentsPage() { )} {/* Tab: Interact */} - {detailTab === 'interact' && } + {detailTab === 'interact' && } {/* Tab: Tasks */} {detailTab === 'tasks' && ( diff --git a/src/openjarvis/server/agent_manager_routes.py b/src/openjarvis/server/agent_manager_routes.py index 58dcfe84..d92d51ab 100644 --- a/src/openjarvis/server/agent_manager_routes.py +++ b/src/openjarvis/server/agent_manager_routes.py @@ -597,7 +597,35 @@ def create_agent_manager_router( # Store user message in DB (always, regardless of stream mode) msg = manager.send_message(agent_id, req.content, mode=req.mode) - if not req.stream: + if not req.stream and req.mode != "immediate": + return msg + + if not req.stream and req.mode == "immediate": + # Non-streaming immediate: trigger a background tick so the + # agent processes the message, then return the stored msg. + import threading + + from openjarvis.agents.executor import AgentExecutor + from openjarvis.core.events import get_event_bus + from openjarvis.system import SystemBuilder + + def _immediate_tick(): + try: + executor = AgentExecutor( + manager=manager, event_bus=get_event_bus(), + ) + try: + system = SystemBuilder().build() + executor.set_system(system) + except Exception: + return + executor.execute_tick(agent_id) + except Exception: + pass + + threading.Thread( + target=_immediate_tick, daemon=True, + ).start() return msg # --- Streaming mode: run agent and return SSE response --- From 19fe061696ab020e3466e888286b499c61c23aaa Mon Sep 17 00:00:00 2001 From: Jon Saad-Falcon <41205309+jonsaadfalcon@users.noreply.github.com> Date: Thu, 26 Mar 2026 16:03:51 -0700 Subject: [PATCH 16/44] fix: use server's model for agent ticks, add executor logging, simplify send UI Root cause: _run_tick and _immediate_tick called SystemBuilder().build() which picks the first model from Ollama (qwen3.5:35b) instead of the model the server was started with (e.g. qwen3.5:9b). A 0.6B query was running on a 35B model, causing 5+ minute stalls. Fix: reuse the server's engine/model from app.state via a lightweight system facade instead of rebuilding the full JarvisSystem. Also: - Add detailed logging to AgentExecutor (model, pending messages, timing, content length, errors with tracebacks) - Add logging to immediate tick lifecycle - Remove Queue button from Interact tab (single Send button) - Enter key now sends immediately instead of queueing Co-Authored-By: Claude Opus 4.6 (1M context) --- frontend/src/pages/AgentsPage.tsx | 19 +--- src/openjarvis/agents/executor.py | 47 +++++++++- src/openjarvis/server/agent_manager_routes.py | 93 ++++++++++++++----- 3 files changed, 120 insertions(+), 39 deletions(-) diff --git a/frontend/src/pages/AgentsPage.tsx b/frontend/src/pages/AgentsPage.tsx index babe1cca..f78c28d8 100644 --- a/frontend/src/pages/AgentsPage.tsx +++ b/frontend/src/pages/AgentsPage.tsx @@ -1280,7 +1280,7 @@ function InteractTab({ agentId, agentStatus }: { agentId: string; agentStatus: s onKeyDown={(e) => { if (e.key === 'Enter' && !e.shiftKey) { e.preventDefault(); - handleSend('queued'); + handleSend('immediate'); } }} placeholder="Send a message to this agent..." @@ -1293,23 +1293,8 @@ function InteractTab({ agentId, agentStatus }: { agentId: string; agentStatus: s disabled={sending || !input.trim()} className="flex items-center gap-1.5 px-3 py-1.5 rounded-lg text-sm cursor-pointer font-medium" style={{ background: 'var(--color-accent)', color: '#fff', opacity: sending || !input.trim() ? 0.5 : 1 }} - title="Send immediately (interrupts agent)" > - Immediate - -
diff --git a/src/openjarvis/agents/executor.py b/src/openjarvis/agents/executor.py index d6f7beb2..f7f1b086 100644 --- a/src/openjarvis/agents/executor.py +++ b/src/openjarvis/agents/executor.py @@ -203,6 +203,12 @@ class AgentExecutor: if not model: raise FatalError("No model configured for agent") + logger.info( + "Agent %s [%s]: using model=%s, engine=%s", + agent["name"], agent["id"], + model, type(engine).__name__, + ) + # Optionally override model via router policy router_policy_key = config.get("router_policy") if router_policy_key and self._system: @@ -247,6 +253,16 @@ class AgentExecutor: input_text = f"{input_text}\n\nNew instructions:\n{user_msgs}" for m in pending: self._manager.mark_message_delivered(m["id"]) + logger.info( + "Agent %s: delivering %d pending message(s)", + agent["name"], len(pending), + ) + else: + logger.info( + "Agent %s: no pending messages, running with " + "instruction only", + agent["name"], + ) # Build AgentContext with memory results from FTS5 backend from openjarvis.agents._stubs import AgentContext @@ -298,7 +314,22 @@ class AgentExecutor: pass # Don't break agent tick if memory retrieval fails agent_ctx.memory_results = memory_results - return agent_instance.run(input_text, context=agent_ctx) + logger.info( + "Agent %s: calling agent.run() with %d chars input", + agent["name"], len(input_text), + ) + _t0 = time.time() + result = agent_instance.run(input_text, context=agent_ctx) + _elapsed = time.time() - _t0 + logger.info( + "Agent %s: agent.run() completed in %.1fs, " + "content_len=%d, turns=%d, tokens=%s", + agent["name"], _elapsed, + len(result.content or ""), + result.turns, + result.metadata.get("total_tokens", "?"), + ) + return result def _build_error_detail(self, error: AgentTickError) -> dict[str, Any]: """Build structured error detail for trace metadata.""" @@ -336,6 +367,12 @@ class AgentExecutor: """Update agent state after tick completion or failure.""" if error is None: # Success + logger.info( + "Tick succeeded for agent %s in %.1fs, " + "response_len=%d", + agent_id, duration, + len(result.content or "") if result else 0, + ) self._manager.end_tick(agent_id) self._manager.update_agent(agent_id, total_runs_increment=1) @@ -381,6 +418,10 @@ class AgentExecutor: "status": "ok", }) elif isinstance(error, EscalateError): + logger.warning( + "Tick escalated for agent %s after %.1fs: %s", + agent_id, duration, error, + ) self._manager.end_tick(agent_id) self._manager.update_agent(agent_id, status="needs_attention") self._bus.publish(EventType.AGENT_TICK_ERROR, { @@ -390,6 +431,10 @@ class AgentExecutor: "duration": duration, }) else: + logger.error( + "Tick failed for agent %s after %.1fs: %s", + agent_id, duration, error, exc_info=error, + ) self._manager.end_tick(agent_id) self._manager.update_agent(agent_id, status="error") # Write error detail to summary_memory so frontend can display it diff --git a/src/openjarvis/server/agent_manager_routes.py b/src/openjarvis/server/agent_manager_routes.py index d92d51ab..f2a03386 100644 --- a/src/openjarvis/server/agent_manager_routes.py +++ b/src/openjarvis/server/agent_manager_routes.py @@ -64,6 +64,24 @@ _BROWSER_SUB_TOOLS = { } +class _LightweightSystem: + """Minimal system facade for the executor — avoids rebuilding the + full JarvisSystem (which picks a random model from Ollama).""" + + def __init__(self, engine: Any, model: str, config: Any = None): + self.engine = engine + self.model = model + self.config = config + self.memory_backend = None + + +def _make_lightweight_system( + engine: Any, model: str, config: Any = None, +) -> _LightweightSystem: + """Build a minimal system from the server's existing state.""" + return _LightweightSystem(engine, model, config) + + def _ensure_registries_populated() -> None: """Ensure ToolRegistry and ChannelRegistry are populated. @@ -452,7 +470,7 @@ def create_agent_manager_router( return {"status": "idle"} @agents_router.post("/{agent_id}/run") - async def run_agent(agent_id: str): + async def run_agent(agent_id: str, request: Request): import threading agent = manager.get_agent(agent_id) @@ -473,28 +491,30 @@ def create_agent_manager_router( status_code=409, detail="Agent is already running" ) + # Re-use the server's engine + model so we don't pick a + # random model from Ollama's list. + server_engine = getattr(request.app.state, "engine", None) + server_model = getattr(request.app.state, "model", "") + server_config = getattr(request.app.state, "config", None) + def _run_tick(): try: from openjarvis.agents.executor import AgentExecutor from openjarvis.core.events import get_event_bus - from openjarvis.system import SystemBuilder executor = AgentExecutor( manager=manager, event_bus=get_event_bus(), ) - try: - system = SystemBuilder().build() - executor.set_system(system) - except Exception as build_err: - manager.end_tick(agent_id) - manager.update_agent(agent_id, status="error") - manager.update_summary_memory( - agent_id, - f"ERROR: Failed to build system: {build_err}", - ) - return + system = _make_lightweight_system( + server_engine, server_model, server_config, + ) + executor.set_system(system) executor.execute_tick(agent_id) except Exception as exc: + logger.error( + "Run-tick failed for agent %s: %s", + agent_id, exc, exc_info=True, + ) try: manager.end_tick(agent_id) except Exception: @@ -603,25 +623,56 @@ def create_agent_manager_router( if not req.stream and req.mode == "immediate": # Non-streaming immediate: trigger a background tick so the # agent processes the message, then return the stored msg. + # Re-use the server's existing system (correct model/engine). import threading + import time as _time from openjarvis.agents.executor import AgentExecutor from openjarvis.core.events import get_event_bus - from openjarvis.system import SystemBuilder + + _srv_engine = getattr(request.app.state, "engine", None) + _srv_model = getattr(request.app.state, "model", "") + _srv_config = getattr(request.app.state, "config", None) def _immediate_tick(): + _start = _time.time() + logger.info( + "Immediate tick starting for agent %s " + "(model=%s)", + agent_id, _srv_model, + ) try: executor = AgentExecutor( manager=manager, event_bus=get_event_bus(), ) - try: - system = SystemBuilder().build() - executor.set_system(system) - except Exception: - return + system = _make_lightweight_system( + _srv_engine, _srv_model, _srv_config, + ) + executor.set_system(system) + logger.info( + "Immediate tick: system ready in %.1fs, " + "executing tick for agent %s", + _time.time() - _start, agent_id, + ) executor.execute_tick(agent_id) - except Exception: - pass + logger.info( + "Immediate tick completed for agent %s " + "in %.1fs", + agent_id, _time.time() - _start, + ) + except Exception as exc: + logger.error( + "Immediate tick failed for agent %s: %s", + agent_id, exc, exc_info=True, + ) + try: + manager.end_tick(agent_id) + except Exception: + pass + manager.update_agent(agent_id, status="error") + manager.update_summary_memory( + agent_id, f"ERROR: {exc}", + ) threading.Thread( target=_immediate_tick, daemon=True, From 808ec6eb5ca491160a64f38cc206d14c1507895e Mon Sep 17 00:00:00 2001 From: Jon Saad-Falcon <41205309+jonsaadfalcon@users.noreply.github.com> Date: Thu, 26 Mar 2026 16:17:57 -0700 Subject: [PATCH 17/44] feat: change default startup model to qwen3.5:4b Update STARTUP_MODEL from 2b to 4b for better quality on first launch. Also update preferred_model() to prefer STARTUP_MODEL when it fits, rather than always picking the third-largest model. This gives a consistent default across machines while still falling back to RAM-appropriate sizing. Co-Authored-By: Claude Opus 4.6 (1M context) --- desktop/src-tauri/src/lib.rs | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/desktop/src-tauri/src/lib.rs b/desktop/src-tauri/src/lib.rs index 5f6163e3..84e2a1e1 100644 --- a/desktop/src-tauri/src/lib.rs +++ b/desktop/src-tauri/src/lib.rs @@ -10,7 +10,7 @@ const OLLAMA_PORT: u16 = 11434; const JARVIS_PORT: u16 = 8222; /// Small, fast model pulled at startup so the app opens quickly. -const STARTUP_MODEL: &str = "qwen3.5:2b"; +const STARTUP_MODEL: &str = "qwen3.5:4b"; /// Tiny fallback model if even the startup model can't be pulled. const FALLBACK_MODEL: &str = "qwen3:0.6b"; @@ -86,12 +86,14 @@ fn models_that_fit() -> Vec<&'static str> { .collect() } -/// Pick the third-largest Qwen3.5 model that fits on this machine. -/// This leaves comfortable headroom for the OS / other apps while -/// still providing a capable model. Falls back gracefully when -/// fewer models fit. +/// Pick the default model — prefers STARTUP_MODEL if it fits, otherwise +/// falls back to the third-largest model that fits on this machine. fn preferred_model() -> &'static str { let fitting = models_that_fit(); + // Prefer STARTUP_MODEL when it fits (fast, good quality) + if fitting.contains(&STARTUP_MODEL) { + return STARTUP_MODEL; + } match fitting.len() { 0 => FALLBACK_MODEL, 1 => fitting[0], From 8b358620da580dccc263ec150f070537cf2678d5 Mon Sep 17 00:00:00 2001 From: Jon Saad-Falcon <41205309+jonsaadfalcon@users.noreply.github.com> Date: Thu, 26 Mar 2026 16:47:00 -0700 Subject: [PATCH 18/44] fix: use fresh engine for agent ticks, not wrapped server engine MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The server's engine is wrapped in MultiEngine → InstrumentedEngine → GuardrailsEngine. When reused from a background thread for agent ticks, this chain returns empty content. Create a fresh OllamaEngine for each tick instead, which reliably returns model output. Also fixes Run Now endpoint to use the same lightweight system approach. Co-Authored-By: Claude Opus 4.6 (1M context) --- src/openjarvis/server/agent_manager_routes.py | 20 ++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/src/openjarvis/server/agent_manager_routes.py b/src/openjarvis/server/agent_manager_routes.py index f2a03386..5b7b311e 100644 --- a/src/openjarvis/server/agent_manager_routes.py +++ b/src/openjarvis/server/agent_manager_routes.py @@ -78,7 +78,25 @@ class _LightweightSystem: def _make_lightweight_system( engine: Any, model: str, config: Any = None, ) -> _LightweightSystem: - """Build a minimal system from the server's existing state.""" + """Build a minimal system using a plain engine for the given model. + + The server's ``app.state.engine`` is heavily wrapped + (MultiEngine → InstrumentedEngine → GuardrailsEngine) and can + hang when reused from a background thread. Instead, create a + fresh OllamaEngine pointed at the same backend. + """ + try: + from openjarvis.core.config import load_config + from openjarvis.engine._discovery import get_engine + + cfg = config if config is not None else load_config() + resolved = get_engine(cfg) + if resolved is not None: + _, plain_engine = resolved + return _LightweightSystem(plain_engine, model, cfg) + except Exception: + pass + # Fallback to the (wrapped) server engine return _LightweightSystem(engine, model, config) From aab743a6c7c4754260b4a46c4e611e8760346dad Mon Sep 17 00:00:00 2001 From: Jon Saad-Falcon <41205309+jonsaadfalcon@users.noreply.github.com> Date: Thu, 26 Mar 2026 17:18:48 -0700 Subject: [PATCH 19/44] fix(engine): support Gemini thought_signature for multi-turn tool calling (#139) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Gemini 3.1+ reasoning models require a thought_signature field in function_call parts when replaying conversation history. Without it, the API returns 400 INVALID_ARGUMENT on every multi-turn tool call. Changes: - CloudEngine: capture thought_signature from Gemini responses and store in _thought_sigs dict keyed by tool_call id - CloudEngine: replay thought_signature when building function_call parts for Gemini conversation history - native_openhands: thread thought_signature through via side dict (ToolCall uses slots, can't add dynamic attributes) - Add PinchBench eval configs for Claude Opus 4.6, Gemini 3.1 Pro, Nemotron-3-Super, Qwen 122B, and Qwen 35B Impact: Gemini 3.1 Pro PinchBench score 4% → 78% Co-authored-by: Claude Opus 4.6 (1M context) --- src/openjarvis/agents/native_openhands.py | 14 +++++--- src/openjarvis/engine/cloud.py | 21 ++++++++--- .../evals/configs/pinchbench-claude-opus.toml | 34 ++++++++++++++++++ .../configs/pinchbench-gemini-31-pro.toml | 34 ++++++++++++++++++ .../configs/pinchbench-nemotron3-super.toml | 35 +++++++++++++++++++ .../evals/configs/pinchbench-qwen122b.toml | 35 +++++++++++++++++++ .../evals/configs/pinchbench-qwen35b.toml | 35 +++++++++++++++++++ 7 files changed, 200 insertions(+), 8 deletions(-) create mode 100644 src/openjarvis/evals/configs/pinchbench-claude-opus.toml create mode 100644 src/openjarvis/evals/configs/pinchbench-gemini-31-pro.toml create mode 100644 src/openjarvis/evals/configs/pinchbench-nemotron3-super.toml create mode 100644 src/openjarvis/evals/configs/pinchbench-qwen122b.toml create mode 100644 src/openjarvis/evals/configs/pinchbench-qwen35b.toml diff --git a/src/openjarvis/agents/native_openhands.py b/src/openjarvis/agents/native_openhands.py index 9b518809..811c3047 100644 --- a/src/openjarvis/agents/native_openhands.py +++ b/src/openjarvis/agents/native_openhands.py @@ -287,6 +287,8 @@ class NativeOpenHandsAgent(ToolUsingAgent): openai_tools = ( self._executor.get_openai_tools() if self._tools else [] ) + # Side dict for Gemini thought_signatures (ToolCall uses slots) + _thought_sigs: dict[str, bytes] = {} for _turn in range(self._max_turns): turns += 1 @@ -330,14 +332,18 @@ class NativeOpenHandsAgent(ToolUsingAgent): # --- Native function-calling path (OpenAI, Anthropic, etc.) --- raw_tool_calls = result.get("tool_calls", []) if raw_tool_calls: - native_calls = [ - ToolCall( + native_calls = [] + for i, tc in enumerate(raw_tool_calls): + call = ToolCall( id=tc.get("id", f"call_{turns}_{i}"), name=tc.get("name", ""), arguments=tc.get("arguments", "{}"), ) - for i, tc in enumerate(raw_tool_calls) - ] + # Preserve thought_signature for Gemini reasoning + sig = tc.get("thought_signature") + if sig is not None: + _thought_sigs[call.id] = sig + native_calls.append(call) messages.append( Message( role=Role.ASSISTANT, diff --git a/src/openjarvis/engine/cloud.py b/src/openjarvis/engine/cloud.py index e5324ab6..fa9dd30b 100644 --- a/src/openjarvis/engine/cloud.py +++ b/src/openjarvis/engine/cloud.py @@ -194,6 +194,8 @@ class CloudEngine(InferenceEngine): self._google_client: Any = None self._openrouter_client: Any = None self._minimax_client: Any = None + # Gemini thought_signatures: tool_call_id -> signature bytes + self._thought_sigs: Dict[str, bytes] = {} self._init_clients() def _init_clients(self) -> None: @@ -510,12 +512,17 @@ class CloudEngine(InferenceEngine): args = json.loads(args) except (json.JSONDecodeError, TypeError): args = {"input": args} - parts.append({ + fc_part: Dict[str, Any] = { "function_call": { "name": tc.name, "args": args if isinstance(args, dict) else {}, } - }) + } + # Replay thought_signature for Gemini reasoning models + sig = self._thought_sigs.get(tc.id) + if sig is not None: + fc_part["thought_signature"] = sig + parts.append(fc_part) contents.append({"role": "model", "parts": parts}) elif m.role.value == "assistant": contents.append({"role": "model", "parts": [{"text": m.content}]}) @@ -567,11 +574,17 @@ class CloudEngine(InferenceEngine): fc_args = ( dict(fc.args) if hasattr(fc.args, "items") else {} ) - tool_calls.append({ + tc_dict: Dict[str, Any] = { "id": f"google_{fc.name}", "name": fc.name, "arguments": json.dumps(fc_args), - }) + } + # Preserve thought_signature for Gemini reasoning models + sig = getattr(part, "thought_signature", None) + if sig is not None: + tc_dict["thought_signature"] = sig + self._thought_sigs[tc_dict["id"]] = sig + tool_calls.append(tc_dict) elif hasattr(part, "text") and part.text: text_parts.append(part.text) diff --git a/src/openjarvis/evals/configs/pinchbench-claude-opus.toml b/src/openjarvis/evals/configs/pinchbench-claude-opus.toml new file mode 100644 index 00000000..b175aa59 --- /dev/null +++ b/src/openjarvis/evals/configs/pinchbench-claude-opus.toml @@ -0,0 +1,34 @@ +# PinchBench eval: Claude Opus 4.6 (cloud) +# Agent: native_openhands — all PinchBench-required tools enabled + +[meta] +name = "pinchbench-claude-opus" +description = "PinchBench on claude-opus-4-6" + +[defaults] +temperature = 0.6 +max_tokens = 8192 + +[judge] +model = "claude-opus-4-5" +temperature = 0.0 +engine = "cloud" + +[run] +max_workers = 1 +output_dir = "results/pinchbench-claude-opus/" +seed = 42 + +[[models]] +name = "claude-opus-4-6" +engine = "cloud" + +[[benchmarks]] +name = "pinchbench" +backend = "jarvis-agent" +agent = "native_openhands" +tools = [ + "think", "file_read", "file_write", "web_search", "shell_exec", + "code_interpreter", "browser_navigate", "image_generate", + "calculator", "http_request", "pdf_extract", +] diff --git a/src/openjarvis/evals/configs/pinchbench-gemini-31-pro.toml b/src/openjarvis/evals/configs/pinchbench-gemini-31-pro.toml new file mode 100644 index 00000000..b54619ea --- /dev/null +++ b/src/openjarvis/evals/configs/pinchbench-gemini-31-pro.toml @@ -0,0 +1,34 @@ +# PinchBench eval: Gemini 3.1 Pro Preview (cloud) +# Agent: native_openhands — all PinchBench-required tools enabled + +[meta] +name = "pinchbench-gemini-31-pro" +description = "PinchBench on gemini-3.1-pro-preview" + +[defaults] +temperature = 0.6 +max_tokens = 8192 + +[judge] +model = "claude-opus-4-5" +temperature = 0.0 +engine = "cloud" + +[run] +max_workers = 1 +output_dir = "results/pinchbench-gemini-31-pro/" +seed = 42 + +[[models]] +name = "gemini-3.1-pro-preview" +engine = "cloud" + +[[benchmarks]] +name = "pinchbench" +backend = "jarvis-agent" +agent = "native_openhands" +tools = [ + "think", "file_read", "file_write", "web_search", "shell_exec", + "code_interpreter", "browser_navigate", "image_generate", + "calculator", "http_request", "pdf_extract", +] diff --git a/src/openjarvis/evals/configs/pinchbench-nemotron3-super.toml b/src/openjarvis/evals/configs/pinchbench-nemotron3-super.toml new file mode 100644 index 00000000..4eb09a39 --- /dev/null +++ b/src/openjarvis/evals/configs/pinchbench-nemotron3-super.toml @@ -0,0 +1,35 @@ +# PinchBench eval: NVIDIA-Nemotron-3-Super-120B-A12B-FP8 (vLLM, TP=4) +# Agent: native_openhands — all PinchBench-required tools enabled + +[meta] +name = "pinchbench-nemotron3-super" +description = "PinchBench on NVIDIA-Nemotron-3-Super-120B-A12B-FP8 (vLLM, TP=4)" + +[defaults] +temperature = 0.6 +max_tokens = 8192 + +[judge] +model = "claude-opus-4-5" +temperature = 0.0 +engine = "cloud" + +[run] +max_workers = 1 +output_dir = "results/pinchbench-nemotron3-super/" +seed = 42 + +[[models]] +name = "nemotron" +engine = "vllm" +num_gpus = 4 + +[[benchmarks]] +name = "pinchbench" +backend = "jarvis-agent" +agent = "native_openhands" +tools = [ + "think", "file_read", "file_write", "web_search", "shell_exec", + "code_interpreter", "browser_navigate", "image_generate", + "calculator", "http_request", "pdf_extract", +] diff --git a/src/openjarvis/evals/configs/pinchbench-qwen122b.toml b/src/openjarvis/evals/configs/pinchbench-qwen122b.toml new file mode 100644 index 00000000..796a1671 --- /dev/null +++ b/src/openjarvis/evals/configs/pinchbench-qwen122b.toml @@ -0,0 +1,35 @@ +# PinchBench eval: Qwen3.5-122B-A10B-FP8 (vLLM, TP=4) +# Agent: native_openhands — all PinchBench-required tools enabled + +[meta] +name = "pinchbench-qwen122b" +description = "PinchBench on Qwen/Qwen3.5-122B-A10B-FP8 (vLLM, TP=4)" + +[defaults] +temperature = 0.6 +max_tokens = 8192 + +[judge] +model = "claude-opus-4-5" +temperature = 0.0 +engine = "cloud" + +[run] +max_workers = 1 +output_dir = "results/pinchbench-qwen122b/" +seed = 42 + +[[models]] +name = "Qwen/Qwen3.5-122B-A10B-FP8" +engine = "vllm" +num_gpus = 4 + +[[benchmarks]] +name = "pinchbench" +backend = "jarvis-agent" +agent = "native_openhands" +tools = [ + "think", "file_read", "file_write", "web_search", "shell_exec", + "code_interpreter", "browser_navigate", "image_generate", + "calculator", "http_request", "pdf_extract", +] diff --git a/src/openjarvis/evals/configs/pinchbench-qwen35b.toml b/src/openjarvis/evals/configs/pinchbench-qwen35b.toml new file mode 100644 index 00000000..4af5a649 --- /dev/null +++ b/src/openjarvis/evals/configs/pinchbench-qwen35b.toml @@ -0,0 +1,35 @@ +# PinchBench eval: Qwen3.5-35B-A3B-FP8 (vLLM, TP=4) +# Agent: native_openhands — all PinchBench-required tools enabled + +[meta] +name = "pinchbench-qwen35b" +description = "PinchBench on Qwen/Qwen3.5-35B-A3B-FP8 (vLLM, TP=4)" + +[defaults] +temperature = 0.6 +max_tokens = 8192 + +[judge] +model = "claude-opus-4-5" +temperature = 0.0 +engine = "cloud" + +[run] +max_workers = 1 +output_dir = "results/pinchbench-qwen35b/" +seed = 42 + +[[models]] +name = "Qwen/Qwen3.5-35B-A3B-FP8" +engine = "vllm" +num_gpus = 4 + +[[benchmarks]] +name = "pinchbench" +backend = "jarvis-agent" +agent = "native_openhands" +tools = [ + "think", "file_read", "file_write", "web_search", "shell_exec", + "code_interpreter", "browser_navigate", "image_generate", + "calculator", "http_request", "pdf_extract", +] From 885b2caf996ca46e05586474660eab90a8448e3f Mon Sep 17 00:00:00 2001 From: Jon Saad-Falcon <41205309+jonsaadfalcon@users.noreply.github.com> Date: Thu, 26 Mar 2026 17:23:09 -0700 Subject: [PATCH 20/44] fix: suppress Qwen3.5 thinking mode, retry on empty content, add logging - Append /no_think to system prompt to prevent Qwen3.5 from consuming all tokens on extended thinking and producing empty visible output - Retry once if agent returns empty content - Add debug logging to _make_lightweight_system for engine diagnostics Co-Authored-By: Claude Opus 4.6 (1M context) --- src/openjarvis/agents/executor.py | 17 +++++++++++++- src/openjarvis/server/agent_manager_routes.py | 22 ++++++++++++++----- 2 files changed, 33 insertions(+), 6 deletions(-) diff --git a/src/openjarvis/agents/executor.py b/src/openjarvis/agents/executor.py index f7f1b086..565f22f1 100644 --- a/src/openjarvis/agents/executor.py +++ b/src/openjarvis/agents/executor.py @@ -231,10 +231,15 @@ class AgentExecutor: pass # Fall back to configured model # Construct agent instance (BaseAgent requires engine, model as positional args) + # Append /no_think to suppress Qwen3.5 extended thinking that + # can consume all tokens and produce empty visible output. + sys_prompt = config.get("system_prompt", "") or "" + if sys_prompt and "/no_think" not in sys_prompt: + sys_prompt = sys_prompt.rstrip() + "\n/no_think" agent_instance = agent_cls( engine, model, - system_prompt=config.get("system_prompt"), + system_prompt=sys_prompt or None, tools=[], ) @@ -320,6 +325,16 @@ class AgentExecutor: ) _t0 = time.time() result = agent_instance.run(input_text, context=agent_ctx) + + # Retry once if the model returned empty content (common with + # Qwen3.5 thinking mode consuming all tokens). + if not (result.content or "").strip(): + logger.warning( + "Agent %s: empty content, retrying once", + agent["name"], + ) + result = agent_instance.run(input_text, context=agent_ctx) + _elapsed = time.time() - _t0 logger.info( "Agent %s: agent.run() completed in %.1fs, " diff --git a/src/openjarvis/server/agent_manager_routes.py b/src/openjarvis/server/agent_manager_routes.py index 5b7b311e..919b997a 100644 --- a/src/openjarvis/server/agent_manager_routes.py +++ b/src/openjarvis/server/agent_manager_routes.py @@ -82,8 +82,8 @@ def _make_lightweight_system( The server's ``app.state.engine`` is heavily wrapped (MultiEngine → InstrumentedEngine → GuardrailsEngine) and can - hang when reused from a background thread. Instead, create a - fresh OllamaEngine pointed at the same backend. + return empty content when reused from a background thread. + Instead, create a fresh OllamaEngine pointed at the same backend. """ try: from openjarvis.core.config import load_config @@ -92,11 +92,23 @@ def _make_lightweight_system( cfg = config if config is not None else load_config() resolved = get_engine(cfg) if resolved is not None: - _, plain_engine = resolved + engine_key, plain_engine = resolved + logger.info( + "Lightweight system: fresh %s engine (key=%s), " + "model=%s", + type(plain_engine).__name__, engine_key, model, + ) return _LightweightSystem(plain_engine, model, cfg) - except Exception: - pass + except Exception as exc: + logger.warning( + "Failed to create fresh engine, falling back to " + "server engine: %s", exc, + ) # Fallback to the (wrapped) server engine + logger.warning( + "Using wrapped server engine %s — may return empty content", + type(engine).__name__, + ) return _LightweightSystem(engine, model, config) From 756e28b109cf04b1117592f250dfc5dd9d97257a Mon Sep 17 00:00:00 2001 From: Jon Saad-Falcon <41205309+jonsaadfalcon@users.noreply.github.com> Date: Thu, 26 Mar 2026 17:34:51 -0700 Subject: [PATCH 21/44] fix: create OllamaEngine directly, skip health checks that stall ticks get_engine() probes all engines with health checks, which can load different models and interfere with in-flight Ollama requests, causing intermittent empty responses. Create a plain OllamaEngine directly from config instead. Co-Authored-By: Claude Opus 4.6 (1M context) --- src/openjarvis/server/agent_manager_routes.py | 42 +++++++------------ 1 file changed, 16 insertions(+), 26 deletions(-) diff --git a/src/openjarvis/server/agent_manager_routes.py b/src/openjarvis/server/agent_manager_routes.py index 919b997a..d10da4a0 100644 --- a/src/openjarvis/server/agent_manager_routes.py +++ b/src/openjarvis/server/agent_manager_routes.py @@ -78,37 +78,27 @@ class _LightweightSystem: def _make_lightweight_system( engine: Any, model: str, config: Any = None, ) -> _LightweightSystem: - """Build a minimal system using a plain engine for the given model. + """Build a minimal system with a plain OllamaEngine. The server's ``app.state.engine`` is heavily wrapped - (MultiEngine → InstrumentedEngine → GuardrailsEngine) and can - return empty content when reused from a background thread. - Instead, create a fresh OllamaEngine pointed at the same backend. + (MultiEngine -> InstrumentedEngine -> GuardrailsEngine) and can + return empty content from background threads. Create a fresh + OllamaEngine directly (no health checks or model discovery that + could interfere with in-flight Ollama requests). """ try: - from openjarvis.core.config import load_config - from openjarvis.engine._discovery import get_engine + from openjarvis.engine.ollama import OllamaEngine - cfg = config if config is not None else load_config() - resolved = get_engine(cfg) - if resolved is not None: - engine_key, plain_engine = resolved - logger.info( - "Lightweight system: fresh %s engine (key=%s), " - "model=%s", - type(plain_engine).__name__, engine_key, model, - ) - return _LightweightSystem(plain_engine, model, cfg) - except Exception as exc: - logger.warning( - "Failed to create fresh engine, falling back to " - "server engine: %s", exc, - ) - # Fallback to the (wrapped) server engine - logger.warning( - "Using wrapped server engine %s — may return empty content", - type(engine).__name__, - ) + cfg = config + if cfg is None: + from openjarvis.core.config import load_config + + cfg = load_config() + host = cfg.engine.ollama.host if cfg else "" + plain_engine = OllamaEngine(host=host) if host else OllamaEngine() + return _LightweightSystem(plain_engine, model, cfg) + except Exception: + pass return _LightweightSystem(engine, model, config) From 1531ad656c19eefb7fbf62e6080174d27715f7a4 Mon Sep 17 00:00:00 2001 From: Jon Saad-Falcon Date: Fri, 27 Mar 2026 00:43:22 +0000 Subject: [PATCH 22/44] fix(evals): pipe tool arguments to transcript + multi-session support MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two fixes for the last 4 failing PinchBench tasks: 1. Tool arguments (tasks 05, 07, 13): Arguments were lost in the pipeline — ToolExecutor stored them but system.py stripped them from the tool_results dict, so the LLM judge saw write_file({}) instead of the actual content. Now pipe arguments through: _stubs.py → system.py → scorer transcript. 2. Multi-session tasks (task 22): Parse `sessions` field from task frontmatter, use first session's prompt as record.problem, and execute remaining sessions sequentially within the workspace context in _process_one(). Co-Authored-By: Claude Opus 4.6 (1M context) --- src/openjarvis/evals/core/runner.py | 25 ++++++++++++++++++--- src/openjarvis/evals/datasets/pinchbench.py | 10 ++++++++- src/openjarvis/evals/scorers/pinchbench.py | 2 +- src/openjarvis/system.py | 1 + src/openjarvis/tools/_stubs.py | 1 + 5 files changed, 34 insertions(+), 5 deletions(-) diff --git a/src/openjarvis/evals/core/runner.py b/src/openjarvis/evals/core/runner.py index 9effbad2..9f4d82e6 100644 --- a/src/openjarvis/evals/core/runner.py +++ b/src/openjarvis/evals/core/runner.py @@ -261,10 +261,29 @@ class EvalRunner: **gen_kwargs, ) full = full or {} - # Store tool results for the scorer to build transcripts - record.metadata["tool_results"] = full.get( + all_tool_results = list(full.get( "tool_results", [], - ) + )) + + # Multi-session: execute remaining sessions + sessions = record.metadata.get("sessions", []) + if ( + record.metadata.get("multi_session") + and len(sessions) > 1 + ): + for session in sessions[1:]: + prompt = session.get("prompt", "") + if not prompt: + continue + sfull = self._backend.generate_full( + prompt, **gen_kwargs, + ) + sfull = sfull or {} + all_tool_results.extend( + sfull.get("tool_results", []), + ) + + record.metadata["tool_results"] = all_tool_results # Score INSIDE context so workspace files still exist content = full.get("content", "") is_correct, scoring_meta = self._scorer.score( diff --git a/src/openjarvis/evals/datasets/pinchbench.py b/src/openjarvis/evals/datasets/pinchbench.py index fdcb8fd0..9c084bc4 100644 --- a/src/openjarvis/evals/datasets/pinchbench.py +++ b/src/openjarvis/evals/datasets/pinchbench.py @@ -74,6 +74,8 @@ def _parse_task_markdown(content: str, filename: str = "") -> Dict[str, Any]: "timeout_seconds": frontmatter.get("timeout_seconds", 180), "workspace_files": frontmatter.get("workspace_files", []), "grading_weights": frontmatter.get("grading_weights"), + "multi_session": frontmatter.get("multi_session", False), + "sessions": frontmatter.get("sessions", []), "prompt": sections.get("Prompt", ""), "expected_behavior": sections.get("Expected Behavior", ""), "grading_criteria": sections.get("Grading Criteria", ""), @@ -160,7 +162,11 @@ class PinchBenchDataset(DatasetProvider): self._records = [ EvalRecord( record_id=t["id"], - problem=t["prompt"], + problem=( + t["sessions"][0]["prompt"] + if t.get("multi_session") and t.get("sessions") + else t["prompt"] + ), reference=t["expected_behavior"], category=t["category"], subject=t["name"], @@ -172,6 +178,8 @@ class PinchBenchDataset(DatasetProvider): "timeout_seconds": t["timeout_seconds"], "workspace_files": t["workspace_files"], "pinchbench_repo_dir": str(repo_dir), + "multi_session": t.get("multi_session", False), + "sessions": t.get("sessions", []), }, ) for t in tasks diff --git a/src/openjarvis/evals/scorers/pinchbench.py b/src/openjarvis/evals/scorers/pinchbench.py index 693f3d12..4325012a 100644 --- a/src/openjarvis/evals/scorers/pinchbench.py +++ b/src/openjarvis/evals/scorers/pinchbench.py @@ -121,7 +121,7 @@ def _tool_results_to_transcript( "type": "message", "message": { "role": "assistant", - "content": [{"type": "toolCall", "name": mapped, "params": {}}], + "content": [{"type": "toolCall", "name": mapped, "params": tr.get("arguments", {})}], }, }) transcript.append({ diff --git a/src/openjarvis/system.py b/src/openjarvis/system.py index 2764323a..b2dc1619 100644 --- a/src/openjarvis/system.py +++ b/src/openjarvis/system.py @@ -251,6 +251,7 @@ class JarvisSystem: "tool_name": tr.tool_name, "content": tr.content, "success": tr.success, + "arguments": tr.metadata.get("arguments", {}), } for tr in getattr(result, "tool_results", []) ], diff --git a/src/openjarvis/tools/_stubs.py b/src/openjarvis/tools/_stubs.py index 73a15b27..143ddaa9 100644 --- a/src/openjarvis/tools/_stubs.py +++ b/src/openjarvis/tools/_stubs.py @@ -241,6 +241,7 @@ class ToolExecutor: ) latency = time.time() - t0 result.latency_seconds = latency + result.metadata["arguments"] = params # Auto-detect taints in results if result.success: From 11d1643261f4db9b49eb0137ab23897323f991b5 Mon Sep 17 00:00:00 2001 From: Jon Saad-Falcon <41205309+jonsaadfalcon@users.noreply.github.com> Date: Thu, 26 Mar 2026 17:53:56 -0700 Subject: [PATCH 23/44] feat: live agent progress visibility in Interact tab Add current_activity field to managed agents that the executor updates at each phase of a tick (loading model, delivering messages, generating response, retrying, finalizing). The frontend polls this every 2s and displays the live status instead of static "Agent is thinking...". Backend: - Add current_activity column to managed_agents (migration) - Add _set_activity helper to AgentExecutor - Update activity at: start_tick, model load, message delivery, generation, retry, finalize - Clear activity on end_tick Frontend: - InteractTab polls both messages and agent status in parallel - Shows current_activity text with pulsing indicator - Falls back to "Agent is thinking..." if activity is empty Co-Authored-By: Claude Opus 4.6 (1M context) --- frontend/src/lib/api.ts | 2 ++ frontend/src/pages/AgentsPage.tsx | 29 ++++++++++++++++++----------- src/openjarvis/agents/executor.py | 18 ++++++++++++++++++ src/openjarvis/agents/manager.py | 11 +++++++++-- 4 files changed, 47 insertions(+), 13 deletions(-) diff --git a/frontend/src/lib/api.ts b/frontend/src/lib/api.ts index 201e9c7c..e1678213 100644 --- a/frontend/src/lib/api.ts +++ b/frontend/src/lib/api.ts @@ -304,6 +304,8 @@ export interface ManagedAgent { budget?: number; // Learning learning_enabled?: boolean; + // Live progress + current_activity?: string; } export interface AgentTask { diff --git a/frontend/src/pages/AgentsPage.tsx b/frontend/src/pages/AgentsPage.tsx index f78c28d8..5e31e1c7 100644 --- a/frontend/src/pages/AgentsPage.tsx +++ b/frontend/src/pages/AgentsPage.tsx @@ -1170,24 +1170,31 @@ function InteractTab({ agentId, agentStatus }: { agentId: string; agentStatus: s const [messages, setMessages] = useState([]); const [input, setInput] = useState(''); const [sending, setSending] = useState(false); + const [currentActivity, setCurrentActivity] = useState(''); + const [liveStatus, setLiveStatus] = useState(agentStatus); const bottomRef = useRef(null); - const loadMessages = useCallback(async () => { + const loadData = useCallback(async () => { try { - const msgs = await fetchAgentMessages(agentId); + const [msgs, agent] = await Promise.all([ + fetchAgentMessages(agentId), + fetchManagedAgent(agentId), + ]); setMessages(msgs); + setLiveStatus(agent.status); + setCurrentActivity(agent.current_activity || ''); } catch { // ignore } }, [agentId]); useEffect(() => { - loadMessages(); - // Poll for new messages while the tab is visible (agent responses - // arrive asynchronously after a background tick completes). - const interval = setInterval(loadMessages, 3000); + loadData(); + const interval = setInterval(loadData, 2000); return () => clearInterval(interval); - }, [loadMessages]); + }, [loadData]); + + useEffect(() => { setLiveStatus(agentStatus); }, [agentStatus]); // Scroll to bottom only on initial load, not on every poll update. const hasScrolled = useRef(false); @@ -1204,7 +1211,7 @@ function InteractTab({ agentId, agentStatus }: { agentId: string; agentStatus: s try { await sendAgentMessage(agentId, input.trim(), mode); setInput(''); - await loadMessages(); + await loadData(); } catch { // ignore } finally { @@ -1218,7 +1225,7 @@ function InteractTab({ agentId, agentStatus }: { agentId: string; agentStatus: s .filter((m) => m.direction === 'user_to_agent' || m.content.trim()) .reverse(); - const isAgentWorking = agentStatus === 'running'; + const isAgentWorking = liveStatus === 'running'; const hasPending = messages.some((m) => m.status === 'pending'); return ( @@ -1249,7 +1256,7 @@ function InteractTab({ agentId, agentStatus }: { agentId: string; agentStatus: s ))} - {/* Progress indicator while agent is working */} + {/* Progress indicator with live activity from the executor */} {(isAgentWorking || hasPending) && (
- {sending ? 'Sending message...' : 'Agent is thinking...'} + {sending ? 'Sending message...' : currentActivity || 'Agent is thinking...'}
diff --git a/src/openjarvis/agents/executor.py b/src/openjarvis/agents/executor.py index 565f22f1..f222befb 100644 --- a/src/openjarvis/agents/executor.py +++ b/src/openjarvis/agents/executor.py @@ -47,6 +47,13 @@ class AgentExecutor: """Deferred system injection — called after JarvisSystem is constructed.""" self._system = system + def _set_activity(self, agent_id: str, activity: str) -> None: + """Update the agent's current_activity for progress visibility.""" + try: + self._manager.update_agent(agent_id, current_activity=activity) + except Exception: + pass # Non-critical + def run_ephemeral( self, agent_type: str, @@ -75,6 +82,7 @@ class AgentExecutor: """ try: self._manager.start_tick(agent_id) + self._set_activity(agent_id, "Preparing tick...") except ValueError: logger.warning("Agent %s already running, skipping tick", agent_id) return @@ -208,6 +216,7 @@ class AgentExecutor: agent["name"], agent["id"], model, type(engine).__name__, ) + self._set_activity(agent["id"], f"Loading model {model}...") # Optionally override model via router policy router_policy_key = config.get("router_policy") @@ -262,6 +271,10 @@ class AgentExecutor: "Agent %s: delivering %d pending message(s)", agent["name"], len(pending), ) + self._set_activity( + agent["id"], + f"Delivering {len(pending)} message(s)...", + ) else: logger.info( "Agent %s: no pending messages, running with " @@ -319,6 +332,7 @@ class AgentExecutor: pass # Don't break agent tick if memory retrieval fails agent_ctx.memory_results = memory_results + self._set_activity(agent["id"], "Generating response...") logger.info( "Agent %s: calling agent.run() with %d chars input", agent["name"], len(input_text), @@ -329,6 +343,9 @@ class AgentExecutor: # Retry once if the model returned empty content (common with # Qwen3.5 thinking mode consuming all tokens). if not (result.content or "").strip(): + self._set_activity( + agent["id"], "Retrying (empty response)...", + ) logger.warning( "Agent %s: empty content, retrying once", agent["name"], @@ -380,6 +397,7 @@ class AgentExecutor: duration: float, ) -> None: """Update agent state after tick completion or failure.""" + self._set_activity(agent_id, "Finalizing...") if error is None: # Success logger.info( diff --git a/src/openjarvis/agents/manager.py b/src/openjarvis/agents/manager.py index d197183d..5d5b331e 100644 --- a/src/openjarvis/agents/manager.py +++ b/src/openjarvis/agents/manager.py @@ -111,6 +111,7 @@ class AgentManager: "ALTER TABLE managed_agents ADD COLUMN last_run_at REAL", "ALTER TABLE managed_agents ADD COLUMN last_activity_at REAL", "ALTER TABLE managed_agents ADD COLUMN stall_retries INTEGER DEFAULT 0", + "ALTER TABLE managed_agents ADD COLUMN current_activity TEXT DEFAULT ''", ] for migration in _MIGRATIONS: try: @@ -160,7 +161,7 @@ class AgentManager: def update_agent(self, agent_id: str, **kwargs: Any) -> Dict[str, Any]: sets: List[str] = [] vals: List[Any] = [] - for key in ("name", "agent_type", "status"): + for key in ("name", "agent_type", "status", "current_activity"): if key in kwargs: sets.append(f"{key} = ?") vals.append(kwargs[key]) @@ -222,7 +223,12 @@ class AgentManager: self._set_status(agent_id, "running") def end_tick(self, agent_id: str) -> None: - self._set_status(agent_id, "idle") + self._conn.execute( + "UPDATE managed_agents SET status = 'idle', " + "current_activity = '', updated_at = ? WHERE id = ?", + (time.time(), agent_id), + ) + self._conn.commit() # ── Checkpoints ─────────────────────────────────────────────── @@ -638,6 +644,7 @@ class AgentManager: "last_run_at": row["last_run_at"], "last_activity_at": row["last_activity_at"], "stall_retries": row["stall_retries"] or 0, + "current_activity": row["current_activity"] or "", } @staticmethod From 4d05475ec4aad571e8fd2eb435704ed6b718bbf4 Mon Sep 17 00:00:00 2001 From: krypticmouse Date: Fri, 27 Mar 2026 01:00:38 +0000 Subject: [PATCH 24/44] feat: enhance security scan with DNS check, JSON output, Rich UI, and API endpoint Incorporates the useful new features from PR #135 (by @gridworks) on top of the existing PrivacyScanner implementation: - Add DNS configuration check (macOS, via scutil --dns) - Add --json flag to `jarvis scan` for machine-readable output - Add --no-scan flag to `jarvis init` to skip the post-init audit - Expand remote-access process list (ngrok, tailscaled, cloudflared, ZeroTier) - Upgrade `jarvis scan` output from plain text to Rich table - Add GET /v1/security/scan API endpoint - Add tests for all new features (30 tests, all passing) Closes #133 Co-Authored-By: gridworks <5502067+gridworks@users.noreply.github.com> Co-Authored-By: Claude Opus 4.6 (1M context) --- src/openjarvis/cli/init_cmd.py | 11 +- src/openjarvis/cli/scan_cmd.py | 190 +++++++++++++++++++++++++++----- src/openjarvis/server/routes.py | 182 +++++++++++++++++++++--------- tests/cli/test_scan.py | 155 ++++++++++++++++++++++---- 4 files changed, 434 insertions(+), 104 deletions(-) diff --git a/src/openjarvis/cli/init_cmd.py b/src/openjarvis/cli/init_cmd.py index d93858e1..98b9da85 100644 --- a/src/openjarvis/cli/init_cmd.py +++ b/src/openjarvis/cli/init_cmd.py @@ -236,6 +236,13 @@ def _do_download(engine: str, model: str, spec, console: Console) -> None: @click.option( "--no-download", is_flag=True, default=False, help="Skip the model download prompt." ) +@click.option( + "--no-scan", + "skip_scan", + is_flag=True, + default=False, + help="Skip the post-init security environment audit.", +) @click.option( "--host", default=None, @@ -247,6 +254,7 @@ def init( full_config: bool = False, engine: Optional[str] = None, no_download: bool = False, + skip_scan: bool = False, host: Optional[str] = None, ) -> None: """Detect hardware and generate ~/.openjarvis/config.toml.""" @@ -405,7 +413,8 @@ def init( if click.confirm(prompt, default=True): _do_download(selected_engine, model, spec, console) - _quick_privacy_check(console) + if not skip_scan: + _quick_privacy_check(console) console.print() console.print( Panel( diff --git a/src/openjarvis/cli/scan_cmd.py b/src/openjarvis/cli/scan_cmd.py index 6ff2e210..a29571f7 100644 --- a/src/openjarvis/cli/scan_cmd.py +++ b/src/openjarvis/cli/scan_cmd.py @@ -19,11 +19,24 @@ _CLOUD_SYNC_PROCS = ["Dropbox", "OneDrive", "Google Drive", "iCloudDrive"] # Screen-recording / remote-access processes (macOS). _SCREEN_RECORDING_PROCS = [ - "TeamViewer", "AnyDesk", "ScreenConnect", "vncviewer", "Vine" + "TeamViewer", + "AnyDesk", + "ScreenConnect", + "vncviewer", + "Vine", ] -# Remote-access processes (Linux). -_REMOTE_ACCESS_PROCS = ["xrdp", "x11vnc", "vncserver", "AnyDesk"] +# Remote-access processes (cross-platform). +_REMOTE_ACCESS_PROCS = [ + "xrdp", + "x11vnc", + "vncserver", + "AnyDesk", + "ngrok", + "tailscaled", + "cloudflared", + "ZeroTier", +] # --------------------------------------------------------------------------- @@ -295,12 +308,83 @@ class PrivacyScanner: ) def check_remote_access(self) -> ScanResult: - """Check for running remote-access processes (Linux).""" + """Check for running remote-access / tunneling processes.""" return self._check_processes( names=_REMOTE_ACCESS_PROCS, check_name="Remote Access", warn_msg="{name} is running — system may be accessible remotely.", - platform="linux", + platform="all", + ) + + def check_dns(self) -> ScanResult: + """Check DNS configuration for encrypted resolvers (macOS).""" + if sys.platform != "darwin": + return ScanResult( + name="DNS Configuration", + status="skip", + message="DNS check not yet implemented for this platform.", + platform="darwin", + ) + try: + proc = self._run(["scutil", "--dns"]) + output = proc.stdout + except Exception: + return ScanResult( + name="DNS Configuration", + status="skip", + message="scutil command not available.", + platform="darwin", + ) + + _DOH_INDICATORS = [ + "dns-over-https", + "dns-over-tls", + "encrypted", + "doh", + "dot", + ] + if any(ind in output.lower() for ind in _DOH_INDICATORS): + return ScanResult( + name="DNS Configuration", + status="ok", + message="Encrypted DNS (DoH/DoT) appears to be active.", + platform="darwin", + ) + + # Extract nameserver IPs + nameservers: list[str] = [] + for line in output.splitlines(): + stripped = line.strip() + if stripped.startswith("nameserver["): + parts = stripped.split(":", 1) + if len(parts) == 2: + nameservers.append(parts[1].strip()) + + if not nameservers: + return ScanResult( + name="DNS Configuration", + status="ok", + message="Could not parse DNS nameservers from scutil output.", + platform="darwin", + ) + + _PLAIN_DNS = {"8.8.8.8", "8.8.4.4", "1.1.1.1", "1.0.0.1", "9.9.9.9"} + plain = [ns for ns in nameservers if ns in _PLAIN_DNS] + if plain: + return ScanResult( + name="DNS Configuration", + status="warn", + message=( + f"Plain DNS in use ({', '.join(plain)}). " + "Queries may be visible to your ISP or resolver." + ), + platform="darwin", + ) + return ScanResult( + name="DNS Configuration", + status="ok", + message=f"DNS resolvers: {', '.join(nameservers[:3])}.", + platform="darwin", ) # -- Orchestration ------------------------------------------------------- @@ -315,6 +399,7 @@ class PrivacyScanner: self.check_luks, self.check_screen_recording, self.check_remote_access, + self.check_dns, ] def run_all(self) -> list[ScanResult]: @@ -350,37 +435,88 @@ class PrivacyScanner: # CLI command # --------------------------------------------------------------------------- -_STATUS_ICONS = {"ok": "✓", "warn": "!", "fail": "✗", "skip": "-"} +_RICH_ICONS = { + "ok": "[green]\u2713[/green]", + "warn": "[yellow]![/yellow]", + "fail": "[red]\u2717[/red]", + "skip": "[dim]-[/dim]", +} + + +def _render_results(results: List[ScanResult]) -> None: + """Render scan results as a Rich table.""" + from rich.console import Console + from rich.table import Table + + console = Console() + console.print() + console.print("[bold]OpenJarvis Security Scan[/bold]") + console.print() + + table = Table(show_header=True, header_style="bold", show_lines=True) + table.add_column("", width=3, justify="center") + table.add_column("Check") + table.add_column("Finding") + + for r in results: + icon = _RICH_ICONS.get(r.status, "?") + style = {"ok": "green", "warn": "yellow", "fail": "red"}.get(r.status, "white") + table.add_row(icon, r.name, f"[{style}]{r.message}[/{style}]") + + console.print(table) + + ok_count = sum(1 for r in results if r.status == "ok") + warn_count = sum(1 for r in results if r.status == "warn") + fail_count = sum(1 for r in results if r.status == "fail") + + console.print() + parts = [f"[green]{ok_count} ok[/green]"] + if warn_count: + parts.append(f"[yellow]{warn_count} warning(s)[/yellow]") + if fail_count: + parts.append(f"[red]{fail_count} issue(s)[/red]") + console.print(" " + ", ".join(parts)) + console.print() + + if fail_count: + console.print( + "[red bold]Action required:[/red bold] address critical findings " + "before storing sensitive data with OpenJarvis." + ) + console.print() + elif warn_count: + console.print( + "[yellow]Review warnings above[/yellow] — they may not block " + "usage but could affect your privacy posture." + ) + console.print() @click.command() @click.option("--quick", is_flag=True, default=False, help="Run only critical checks.") -def scan(quick: bool) -> None: +@click.option("--json", "as_json", is_flag=True, default=False, help="Output as JSON.") +def scan(quick: bool, as_json: bool) -> None: """Audit your environment for privacy and security risks.""" scanner = PrivacyScanner() results: List[ScanResult] = scanner.run_quick() if quick else scanner.run_all() + if as_json: + import json as json_mod + + output = [ + { + "name": r.name, + "status": r.status, + "message": r.message, + "platform": r.platform, + } + for r in results + ] + click.echo(json_mod.dumps(output, indent=2)) + return + if not results: click.echo("No applicable checks for this platform.") return - warnings = 0 - failures = 0 - for r in results: - icon = _STATUS_ICONS.get(r.status, "?") - click.echo(f" [{icon}] {r.name}: {r.message}") - if r.status == "warn": - warnings += 1 - elif r.status == "fail": - failures += 1 - - click.echo("") - parts = [] - if warnings: - parts.append(f"{warnings} warning(s)") - if failures: - parts.append(f"{failures} issue(s)") - if parts: - click.echo("Summary: " + ", ".join(parts) + ".") - else: - click.echo("Summary: all checks passed.") + _render_results(results) diff --git a/src/openjarvis/server/routes.py b/src/openjarvis/server/routes.py index 50fb645b..a615d7d0 100644 --- a/src/openjarvis/server/routes.py +++ b/src/openjarvis/server/routes.py @@ -32,12 +32,14 @@ def _to_messages(chat_messages) -> list[Message]: messages = [] for m in chat_messages: role = Role(m.role) if m.role in {r.value for r in Role} else Role.USER - messages.append(Message( - role=role, - content=m.content or "", - name=m.name, - tool_call_id=m.tool_call_id, - )) + messages.append( + Message( + role=role, + content=m.content or "", + name=m.name, + tool_call_id=m.tool_call_id, + ) + ) return messages @@ -75,7 +77,10 @@ async def chat_completions(request_body: ChatCompletionRequest, request: Request max_context_tokens=config.memory.context_max_tokens, ) enriched = inject_context( - query_text, messages, memory_backend, config=ctx_cfg, + query_text, + messages, + memory_backend, + config=ctx_cfg, ) # Rebuild request messages from enriched Message objects if len(enriched) > len(messages): @@ -83,16 +88,19 @@ async def chat_completions(request_body: ChatCompletionRequest, request: Request new_msgs = [] for msg in enriched: - new_msgs.append(ChatMessage( - role=msg.role.value, - content=msg.content, - name=msg.name, - tool_call_id=getattr(msg, "tool_call_id", None), - )) + new_msgs.append( + ChatMessage( + role=msg.role.value, + content=msg.content, + name=msg.name, + tool_call_id=getattr(msg, "tool_call_id", None), + ) + ) request_body.messages = new_msgs except Exception: logging.getLogger("openjarvis.server").debug( - "Memory context injection failed", exc_info=True, + "Memory context injection failed", + exc_info=True, ) # Run complexity analysis on the last user message @@ -111,7 +119,8 @@ async def chat_completions(request_body: ChatCompletionRequest, request: Request cr = score_complexity(query_text_for_complexity) suggested = adjust_tokens_for_model( - cr.suggested_max_tokens, model, + cr.suggested_max_tokens, + model, ) complexity_info = ComplexityInfo( score=cr.score, @@ -124,7 +133,8 @@ async def chat_completions(request_body: ChatCompletionRequest, request: Request request_body.max_tokens = suggested except Exception: logging.getLogger("openjarvis.server").debug( - "Complexity analysis failed", exc_info=True, + "Complexity analysis failed", + exc_info=True, ) if request_body.stream: @@ -143,13 +153,19 @@ async def chat_completions(request_body: ChatCompletionRequest, request: Request bus = getattr(request.app.state, "bus", None) return _handle_direct( - engine, model, request_body, - bus=bus, complexity_info=complexity_info, + engine, + model, + request_body, + bus=bus, + complexity_info=complexity_info, ) def _handle_direct( - engine, model: str, req: ChatCompletionRequest, bus=None, + engine, + model: str, + req: ChatCompletionRequest, + bus=None, complexity_info=None, ) -> ChatCompletionResponse: """Direct engine call without agent.""" @@ -161,8 +177,12 @@ def _handle_direct( from openjarvis.telemetry.wrapper import instrumented_generate result = instrumented_generate( - engine, messages, model=model, bus=bus, - temperature=req.temperature, max_tokens=req.max_tokens, + engine, + messages, + model=model, + bus=bus, + temperature=req.temperature, + max_tokens=req.max_tokens, **kwargs, ) else: @@ -194,10 +214,12 @@ def _handle_direct( return ChatCompletionResponse( model=model, - choices=[Choice( - message=choice_msg, - finish_reason=result.get("finish_reason", "stop"), - )], + choices=[ + Choice( + message=choice_msg, + finish_reason=result.get("finish_reason", "stop"), + ) + ], usage=UsageInfo( prompt_tokens=usage.get("prompt_tokens", 0), completion_tokens=usage.get("completion_tokens", 0), @@ -208,7 +230,9 @@ def _handle_direct( def _handle_agent( - agent, model: str, req: ChatCompletionRequest, + agent, + model: str, + req: ChatCompletionRequest, complexity_info=None, ) -> ChatCompletionResponse: """Run through agent.""" @@ -241,10 +265,12 @@ def _handle_agent( return ChatCompletionResponse( model=model, - choices=[Choice( - message=ChoiceMessage(role="assistant", content=result.content), - finish_reason="stop", - )], + choices=[ + Choice( + message=ChoiceMessage(role="assistant", content=result.content), + finish_reason="stop", + ) + ], usage=usage, complexity=complexity_info, ) @@ -258,7 +284,9 @@ async def _handle_agent_stream(agent, bus, model, req): async def _handle_stream( - engine, model: str, req: ChatCompletionRequest, + engine, + model: str, + req: ChatCompletionRequest, complexity_info=None, ): """Stream response using SSE format.""" @@ -270,9 +298,11 @@ async def _handle_stream( first_chunk = ChatCompletionChunk( id=chunk_id, model=model, - choices=[StreamChoice( - delta=DeltaMessage(role="assistant"), - )], + choices=[ + StreamChoice( + delta=DeltaMessage(role="assistant"), + ) + ], ) yield f"data: {first_chunk.model_dump_json()}\n\n" @@ -287,27 +317,34 @@ async def _handle_stream( chunk = ChatCompletionChunk( id=chunk_id, model=model, - choices=[StreamChoice( - delta=DeltaMessage(content=token), - )], + choices=[ + StreamChoice( + delta=DeltaMessage(content=token), + ) + ], ) yield f"data: {chunk.model_dump_json()}\n\n" except Exception as exc: # Surface errors as a content chunk so the frontend can # display them instead of silently failing. import logging + logging.getLogger("openjarvis.server").error( - "Stream error: %s", exc, exc_info=True, + "Stream error: %s", + exc, + exc_info=True, ) error_chunk = ChatCompletionChunk( id=chunk_id, model=model, - choices=[StreamChoice( - delta=DeltaMessage( - content=f"\n\nError during generation: {exc}", - ), - finish_reason="stop", - )], + choices=[ + StreamChoice( + delta=DeltaMessage( + content=f"\n\nError during generation: {exc}", + ), + finish_reason="stop", + ) + ], ) yield f"data: {error_chunk.model_dump_json()}\n\n" yield "data: [DONE]\n\n" @@ -315,13 +352,16 @@ async def _handle_stream( # Send finish chunk with usage data if available import json as _json + finish_data = ChatCompletionChunk( id=chunk_id, model=model, - choices=[StreamChoice( - delta=DeltaMessage(), - finish_reason="stop", - )], + choices=[ + StreamChoice( + delta=DeltaMessage(), + finish_reason="stop", + ) + ], ) finish_dict = _json.loads(finish_data.model_dump_json()) @@ -456,18 +496,22 @@ async def savings(request: Request): # Exclude cloud model tokens from savings — only local # inference counts toward cost savings. _cloud_prefixes = ( - "gpt-", "o1-", "o3-", "o4-", - "claude-", "gemini-", "openrouter/", + "gpt-", + "o1-", + "o3-", + "o4-", + "claude-", + "gemini-", + "openrouter/", ) local_models = [ - m for m in summary.per_model + m + for m in summary.per_model if not any(m.model_id.startswith(p) for p in _cloud_prefixes) ] result = compute_savings( prompt_tokens=sum(m.prompt_tokens for m in local_models), - completion_tokens=sum( - m.completion_tokens for m in local_models - ), + completion_tokens=sum(m.completion_tokens for m in local_models), total_calls=sum(m.call_count for m in local_models), session_start=session_start if session_start else 0.0, prompt_tokens_evaluated=sum( @@ -557,7 +601,8 @@ async def channel_send(request: Request): if not channel_name or not content: raise HTTPException( - status_code=400, detail="'channel' and 'content' are required", + status_code=400, + detail="'channel' and 'content' are required", ) ok = bridge.send(channel_name, content, conversation_id=conversation_id) @@ -575,4 +620,31 @@ async def channel_status(request: Request): return {"status": bridge.status().value} +# --------------------------------------------------------------------------- +# Security scan endpoint +# --------------------------------------------------------------------------- + + +@router.get("/v1/security/scan") +async def security_scan(): + """Run a read-only security environment audit and return findings.""" + from openjarvis.cli.scan_cmd import PrivacyScanner + + scanner = PrivacyScanner() + results = scanner.run_all() + return { + "has_warnings": any(r.status == "warn" for r in results), + "has_failures": any(r.status == "fail" for r in results), + "findings": [ + { + "name": r.name, + "status": r.status, + "message": r.message, + "platform": r.platform, + } + for r in results + ], + } + + __all__ = ["router"] diff --git a/tests/cli/test_scan.py b/tests/cli/test_scan.py index 536439d8..41370832 100644 --- a/tests/cli/test_scan.py +++ b/tests/cli/test_scan.py @@ -31,9 +31,7 @@ def _make_proc( class TestScanResultDataclass: def test_fields_exist(self) -> None: - r = ScanResult( - name="test", status="ok", message="all good", platform="all" - ) + r = ScanResult(name="test", status="ok", message="all good", platform="all") assert r.name == "test" assert r.status == "ok" assert r.message == "all good" @@ -246,15 +244,9 @@ class TestPlatformFiltering: other_plat = "linux" if current_plat == "darwin" else "darwin" with patch.object(scanner, "_get_all_checks") as mock_get: - darwin_ck = MagicMock( - return_value=ScanResult("d", "ok", "msg", "darwin") - ) - linux_ck = MagicMock( - return_value=ScanResult("l", "ok", "msg", "linux") - ) - all_ck = MagicMock( - return_value=ScanResult("a", "ok", "msg", "all") - ) + darwin_ck = MagicMock(return_value=ScanResult("d", "ok", "msg", "darwin")) + linux_ck = MagicMock(return_value=ScanResult("l", "ok", "msg", "linux")) + all_ck = MagicMock(return_value=ScanResult("a", "ok", "msg", "all")) mock_get.return_value = [darwin_ck, linux_ck, all_ck] results = scanner.run_all() @@ -275,23 +267,19 @@ class TestRemoteAccess: def test_no_remote_access(self) -> None: scanner = PrivacyScanner() with patch.object(scanner, "_run") as mock_run: - mock_run.return_value = CompletedProcess( - [], 1, stdout="", stderr="" - ) + mock_run.return_value = CompletedProcess([], 1, stdout="", stderr="") result = scanner.check_remote_access() assert result.status == "ok" def test_xrdp_running(self) -> None: scanner = PrivacyScanner() with patch.object(scanner, "_run") as mock_run: + def side_effect(cmd, **kw): if any("xrdp" in str(c) for c in cmd): - return CompletedProcess( - cmd, 0, stdout="12345", stderr="" - ) - return CompletedProcess( - cmd, 1, stdout="", stderr="" - ) + return CompletedProcess(cmd, 0, stdout="12345", stderr="") + return CompletedProcess(cmd, 1, stdout="", stderr="") + mock_run.side_effect = side_effect result = scanner.check_remote_access() assert result.status == "warn" @@ -348,3 +336,128 @@ class TestRunQuick: names = {r.name for r in results} assert "Network Exposure" not in names assert "Screen Recording" not in names + + +# --------------------------------------------------------------------------- +# TestDNS +# --------------------------------------------------------------------------- + + +class TestDNS: + """Tests for check_dns (macOS).""" + + def test_encrypted_dns_detected(self) -> None: + scanner = PrivacyScanner() + scutil_out = ( + "resolver #1\n nameserver[0] : 127.0.0.1\n flags : dns-over-https\n" + ) + with ( + patch("sys.platform", "darwin"), + patch("subprocess.run", return_value=_make_proc(stdout=scutil_out)), + ): + result = scanner.check_dns() + assert result.status == "ok" + assert "Encrypted" in result.message or "DoH" in result.message + + def test_plain_dns_detected(self) -> None: + scanner = PrivacyScanner() + scutil_out = ( + "resolver #1\n nameserver[0] : 8.8.8.8\n nameserver[1] : 8.8.4.4\n" + ) + with ( + patch("sys.platform", "darwin"), + patch("subprocess.run", return_value=_make_proc(stdout=scutil_out)), + ): + result = scanner.check_dns() + assert result.status == "warn" + assert "8.8.8.8" in result.message + + def test_private_dns(self) -> None: + scanner = PrivacyScanner() + scutil_out = "resolver #1\n nameserver[0] : 192.168.1.1\n" + with ( + patch("sys.platform", "darwin"), + patch("subprocess.run", return_value=_make_proc(stdout=scutil_out)), + ): + result = scanner.check_dns() + assert result.status == "ok" + assert "192.168.1.1" in result.message + + def test_skip_on_linux(self) -> None: + scanner = PrivacyScanner() + with patch("sys.platform", "linux"): + result = scanner.check_dns() + assert result.status == "skip" + + def test_scutil_not_available(self) -> None: + scanner = PrivacyScanner() + with ( + patch("sys.platform", "darwin"), + patch("subprocess.run", side_effect=FileNotFoundError), + ): + result = scanner.check_dns() + assert result.status == "skip" + + +# --------------------------------------------------------------------------- +# TestExpandedRemoteAccess +# --------------------------------------------------------------------------- + + +class TestExpandedRemoteAccess: + """Verify expanded remote-access process list.""" + + def test_ngrok_detected(self) -> None: + scanner = PrivacyScanner() + with patch.object(scanner, "_run") as mock_run: + + def side_effect(cmd, **kw): + if any("ngrok" in str(c) for c in cmd): + return CompletedProcess(cmd, 0, stdout="99999", stderr="") + return CompletedProcess(cmd, 1, stdout="", stderr="") + + mock_run.side_effect = side_effect + result = scanner.check_remote_access() + assert result.status == "warn" + assert "ngrok" in result.message + + def test_tailscaled_detected(self) -> None: + scanner = PrivacyScanner() + with patch.object(scanner, "_run") as mock_run: + + def side_effect(cmd, **kw): + if any("tailscaled" in str(c) for c in cmd): + return CompletedProcess(cmd, 0, stdout="88888", stderr="") + return CompletedProcess(cmd, 1, stdout="", stderr="") + + mock_run.side_effect = side_effect + result = scanner.check_remote_access() + assert result.status == "warn" + + +# --------------------------------------------------------------------------- +# TestJsonOutput +# --------------------------------------------------------------------------- + + +class TestJsonOutput: + """Test --json output flag.""" + + def test_json_output_structure(self) -> None: + from click.testing import CliRunner + + from openjarvis.cli.scan_cmd import scan + + runner = CliRunner() + with patch( + "openjarvis.cli.scan_cmd.PrivacyScanner.run_all", + return_value=[ + ScanResult("Test", "ok", "all good", "all"), + ], + ): + result = runner.invoke(scan, ["--json"]) + assert result.exit_code == 0 + data = json.loads(result.output) + assert isinstance(data, list) + assert data[0]["name"] == "Test" + assert data[0]["status"] == "ok" From 8590bc514067c393d1c9fe02d6d504ffc97d5da4 Mon Sep 17 00:00:00 2001 From: Jon Saad-Falcon <41205309+jonsaadfalcon@users.noreply.github.com> Date: Thu, 26 Mar 2026 18:19:58 -0700 Subject: [PATCH 25/44] fix: disable Ollama thinking by default, fix token tracking Root cause of empty responses: Qwen3.5's extended thinking mode consumes all tokens (4096) on hidden tags, leaving zero visible content. The /no_think text tag was unreliable. Fix: pass think=false in the Ollama API payload, which properly disables thinking at the API level. Drops token usage from ~4096 to ~5-200 per response and eliminates empty content. Also: - Fix token tracking to read total_tokens from metadata (was looking for tokens_used which is never set) - Remove the /no_think system prompt hack (superseded by API param) Co-Authored-By: Claude Opus 4.6 (1M context) --- src/openjarvis/agents/executor.py | 13 ++++++------- src/openjarvis/engine/ollama.py | 7 +++++++ 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/src/openjarvis/agents/executor.py b/src/openjarvis/agents/executor.py index f222befb..4ad52e2a 100644 --- a/src/openjarvis/agents/executor.py +++ b/src/openjarvis/agents/executor.py @@ -240,15 +240,10 @@ class AgentExecutor: pass # Fall back to configured model # Construct agent instance (BaseAgent requires engine, model as positional args) - # Append /no_think to suppress Qwen3.5 extended thinking that - # can consume all tokens and produce empty visible output. - sys_prompt = config.get("system_prompt", "") or "" - if sys_prompt and "/no_think" not in sys_prompt: - sys_prompt = sys_prompt.rstrip() + "\n/no_think" agent_instance = agent_cls( engine, model, - system_prompt=sys_prompt or None, + system_prompt=config.get("system_prompt"), tools=[], ) @@ -411,7 +406,11 @@ class AgentExecutor: # Accumulate budget metrics from AgentResult metadata if result: - tokens = result.metadata.get("tokens_used", 0) + tokens = ( + result.metadata.get("total_tokens") + or result.metadata.get("tokens_used") + or 0 + ) cost = result.metadata.get("cost", 0.0) budget_kwargs: dict[str, Any] = {"stall_retries": 0} if tokens > 0: diff --git a/src/openjarvis/engine/ollama.py b/src/openjarvis/engine/ollama.py index a757293e..c6fcf489 100644 --- a/src/openjarvis/engine/ollama.py +++ b/src/openjarvis/engine/ollama.py @@ -75,6 +75,13 @@ class OllamaEngine(InferenceEngine): "num_ctx": kwargs.get("num_ctx", 8192), }, } + # Disable extended thinking by default (Qwen3.5 etc.). + # When enabled, thinking tokens consume the entire budget and + # the visible content comes back empty. + if "think" not in kwargs: + payload["think"] = False + elif kwargs["think"] is not None: + payload["think"] = kwargs["think"] # Pass tools if provided tools = kwargs.get("tools") if tools: From c1e02a236af7f1c0e05aa0130cc70754f6929015 Mon Sep 17 00:00:00 2001 From: Jon Saad-Falcon <41205309+jonsaadfalcon@users.noreply.github.com> Date: Thu, 26 Mar 2026 18:32:25 -0700 Subject: [PATCH 26/44] feat: show model in agent overview, split tokens, editable model Overview tab: - Show Intelligence (model name) with click-to-change dropdown - Split "Total Tokens" into "Input Tokens" and "Output Tokens" - Model can be switched for existing agents via dropdown Backend: - Add input_tokens/output_tokens columns to managed_agents - Track prompt_tokens and completion_tokens separately in executor - Disable Ollama thinking by default (think:false) to prevent empty responses from token exhaustion Co-Authored-By: Claude Opus 4.6 (1M context) --- frontend/src/lib/api.ts | 2 + frontend/src/pages/AgentsPage.tsx | 91 ++++++++++++++++++++++++++----- src/openjarvis/agents/executor.py | 8 +++ src/openjarvis/agents/manager.py | 12 ++++ 4 files changed, 98 insertions(+), 15 deletions(-) diff --git a/frontend/src/lib/api.ts b/frontend/src/lib/api.ts index e1678213..ceb3614e 100644 --- a/frontend/src/lib/api.ts +++ b/frontend/src/lib/api.ts @@ -296,6 +296,8 @@ export interface ManagedAgent { total_runs?: number; total_cost?: number; total_tokens?: number; + input_tokens?: number; + output_tokens?: number; last_run_at?: number | null; // Schedule schedule_type?: string; diff --git a/frontend/src/pages/AgentsPage.tsx b/frontend/src/pages/AgentsPage.tsx index 5e31e1c7..47700a41 100644 --- a/frontend/src/pages/AgentsPage.tsx +++ b/frontend/src/pages/AgentsPage.tsx @@ -20,6 +20,8 @@ import { fetchManagedAgent, fetchAvailableTools, saveToolCredentials, + fetchModels, + updateManagedAgent, } from '../lib/api'; import type { AgentTask, ChannelBinding, AgentTemplate, AgentMessage, ManagedAgent, LearningLogEntry, AgentTrace, ToolInfo } from '../lib/api'; import { @@ -1162,6 +1164,79 @@ function AgentCard({ ); } +// --------------------------------------------------------------------------- +// Detail view — Configuration grid with editable model +// --------------------------------------------------------------------------- + +function AgentConfigGrid({ agent, onAgentUpdated }: { agent: ManagedAgent; onAgentUpdated: () => void }) { + const [editingModel, setEditingModel] = useState(false); + const [models, setModels] = useState([]); + const currentModel = (agent.config?.model as string) || '(default)'; + + async function startEditing() { + try { + const fetched = await fetchModels(); + setModels(fetched.map((m) => m.id)); + } catch { + // ignore + } + setEditingModel(true); + } + + async function changeModel(newModel: string) { + try { + const newConfig = { ...(agent.config || {}), model: newModel }; + await updateManagedAgent(agent.id, { config: newConfig }); + onAgentUpdated(); + } catch { + // ignore + } + setEditingModel(false); + } + + const rows: [string, React.ReactNode][] = [ + ['Intelligence', editingModel ? ( + + ) : ( + + {currentModel} + + )], + ['Agent Type', {agent.agent_type}], + ['Schedule', {formatSchedule(agent.schedule_type, agent.schedule_value)}], + ['Last Run', {formatRelativeTime(agent.last_run_at)}], + ['Budget', {agent.budget ? formatCost(agent.budget) : 'Unlimited'}], + ['Learning', {agent.learning_enabled ? 'Enabled' : 'Disabled'}], + ['Input Tokens', {String(agent.input_tokens ?? 0)}], + ['Output Tokens', {String(agent.output_tokens ?? 0)}], + ]; + + return ( +
+ {rows.map(([label, value]) => ( +
+ {label} + {value} +
+ ))} +
+ ); +} + // --------------------------------------------------------------------------- // Detail view — Interact tab // --------------------------------------------------------------------------- @@ -1776,21 +1851,7 @@ export function AgentsPage() {

Configuration

-
- {[ - ['Agent Type', selectedAgent.agent_type], - ['Schedule', formatSchedule(selectedAgent.schedule_type, selectedAgent.schedule_value)], - ['Last Run', formatRelativeTime(selectedAgent.last_run_at)], - ['Budget', selectedAgent.budget ? formatCost(selectedAgent.budget) : 'Unlimited'], - ['Learning', selectedAgent.learning_enabled ? 'Enabled' : 'Disabled'], - ['Total Tokens', String(selectedAgent.total_tokens ?? 0)], - ].map(([k, v]) => ( -
- {k} - {v} -
- ))} -
+
ID: {selectedAgent.id} diff --git a/src/openjarvis/agents/executor.py b/src/openjarvis/agents/executor.py index 4ad52e2a..c7e34342 100644 --- a/src/openjarvis/agents/executor.py +++ b/src/openjarvis/agents/executor.py @@ -411,10 +411,18 @@ class AgentExecutor: or result.metadata.get("tokens_used") or 0 ) + in_tokens = result.metadata.get("prompt_tokens", 0) + out_tokens = result.metadata.get( + "completion_tokens", 0, + ) cost = result.metadata.get("cost", 0.0) budget_kwargs: dict[str, Any] = {"stall_retries": 0} if tokens > 0: budget_kwargs["total_tokens_increment"] = tokens + if in_tokens > 0: + budget_kwargs["input_tokens_increment"] = in_tokens + if out_tokens > 0: + budget_kwargs["output_tokens_increment"] = out_tokens if cost > 0: budget_kwargs["total_cost_increment"] = cost self._manager.update_agent(agent_id, **budget_kwargs) diff --git a/src/openjarvis/agents/manager.py b/src/openjarvis/agents/manager.py index 5d5b331e..ba11f1ff 100644 --- a/src/openjarvis/agents/manager.py +++ b/src/openjarvis/agents/manager.py @@ -112,6 +112,8 @@ class AgentManager: "ALTER TABLE managed_agents ADD COLUMN last_activity_at REAL", "ALTER TABLE managed_agents ADD COLUMN stall_retries INTEGER DEFAULT 0", "ALTER TABLE managed_agents ADD COLUMN current_activity TEXT DEFAULT ''", + "ALTER TABLE managed_agents ADD COLUMN input_tokens INTEGER DEFAULT 0", + "ALTER TABLE managed_agents ADD COLUMN output_tokens INTEGER DEFAULT 0", ] for migration in _MIGRATIONS: try: @@ -182,6 +184,14 @@ class AgentManager: if total_tokens_increment: sets.append("total_tokens = total_tokens + ?") vals.append(total_tokens_increment) + input_tokens_increment = kwargs.get("input_tokens_increment", 0) + if input_tokens_increment: + sets.append("input_tokens = input_tokens + ?") + vals.append(input_tokens_increment) + output_tokens_increment = kwargs.get("output_tokens_increment", 0) + if output_tokens_increment: + sets.append("output_tokens = output_tokens + ?") + vals.append(output_tokens_increment) if "last_activity_at" in kwargs: sets.append("last_activity_at = ?") vals.append(kwargs["last_activity_at"]) @@ -645,6 +655,8 @@ class AgentManager: "last_activity_at": row["last_activity_at"], "stall_retries": row["stall_retries"] or 0, "current_activity": row["current_activity"] or "", + "input_tokens": row["input_tokens"] or 0, + "output_tokens": row["output_tokens"] or 0, } @staticmethod From 63ae9421169c4f7daeab5909b7a75d0782a65a6d Mon Sep 17 00:00:00 2001 From: krypticmouse Date: Fri, 27 Mar 2026 01:32:52 +0000 Subject: [PATCH 27/44] feat: add Codex cloud engine for ChatGPT Plus/Pro subscribers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds `codex/` prefixed model support using the OpenAI Responses API — the same protocol used by zeroclaw and other Codex-compatible tools. Live-tested against gpt-5-mini-2025-08-07 with: - Generate (non-streaming): confirmed working - System prompt → instructions mapping: confirmed working - SSE streaming: confirmed working (9 chunks) - End-to-end via `jarvis ask`: confirmed working Implementation: - Default endpoint: api.openai.com/v1/responses (standard API key) - Override via OPENAI_CODEX_BASE_URL for ChatGPT OAuth tokens (e.g. chatgpt.com/backend-api/codex) - Auth via OPENAI_CODEX_API_KEY env var - Responses API format: input array, instructions field, output_text extraction - Handles reasoning+message output blocks correctly - SSE streaming parses response.output_text.delta events Models: codex/gpt-4o, codex/gpt-4o-mini, codex/o3-mini, codex/gpt-5-mini, codex/gpt-5-mini-2025-08-07 Usage: export OPENAI_CODEX_API_KEY="your-api-key-or-oauth-token" jarvis ask "Hello" --model codex/gpt-5-mini-2025-08-07 Closes #134 Co-Authored-By: Claude Opus 4.6 (1M context) --- src/openjarvis/engine/cloud.py | 310 +++++++++++++++---- src/openjarvis/intelligence/model_catalog.py | 83 +++++ tests/engine/test_cloud.py | 184 ++++++++++- 3 files changed, 520 insertions(+), 57 deletions(-) diff --git a/src/openjarvis/engine/cloud.py b/src/openjarvis/engine/cloud.py index fa9dd30b..dc9a99cd 100644 --- a/src/openjarvis/engine/cloud.py +++ b/src/openjarvis/engine/cloud.py @@ -8,6 +8,8 @@ import time from collections.abc import AsyncIterator, Sequence from typing import Any, Dict, List +import httpx + from openjarvis.core.registry import EngineRegistry from openjarvis.core.types import Message from openjarvis.engine._base import ( @@ -46,7 +48,12 @@ PRICING: Dict[str, tuple[float, float]] = { # Well-known model IDs per provider _OPENAI_MODELS = [ - "gpt-4o", "gpt-4o-mini", "gpt-5", "gpt-5.4", "gpt-5-mini", "o3-mini", + "gpt-4o", + "gpt-4o-mini", + "gpt-5", + "gpt-5.4", + "gpt-5-mini", + "o3-mini", ] _ANTHROPIC_MODELS = [ "claude-sonnet-4-20250514", @@ -85,6 +92,16 @@ _OPENROUTER_POPULAR = [ "openrouter/qwen/qwen3-235b-a22b", ] +# Codex models — prefixed with "codex/" for ChatGPT Plus/Pro subscribers. +# Uses the Responses API at chatgpt.com, not the standard OpenAI API. +_CODEX_MODELS = [ + "codex/gpt-4o", + "codex/gpt-4o-mini", + "codex/o3-mini", + "codex/gpt-5-mini", + "codex/gpt-5-mini-2025-08-07", +] + def _is_minimax_model(model: str) -> bool: return model.lower().startswith("minimax") @@ -94,6 +111,10 @@ def _is_openrouter_model(model: str) -> bool: return model.startswith("openrouter/") +def _is_codex_model(model: str) -> bool: + return model.startswith("codex/") + + def _is_anthropic_model(model: str) -> bool: return "claude" in model.lower() and not _is_openrouter_model(model) @@ -159,11 +180,13 @@ def _convert_tools_to_anthropic( result = [] for tool in openai_tools: func = tool.get("function", {}) - result.append({ - "name": func.get("name", ""), - "description": func.get("description", ""), - "input_schema": func.get("parameters", {}), - }) + result.append( + { + "name": func.get("name", ""), + "description": func.get("description", ""), + "input_schema": func.get("parameters", {}), + } + ) return result @@ -174,11 +197,13 @@ def _convert_tools_to_google( declarations = [] for tool in openai_tools: func = tool.get("function", {}) - declarations.append({ - "name": func.get("name", ""), - "description": func.get("description", ""), - "parameters": func.get("parameters", {}), - }) + declarations.append( + { + "name": func.get("name", ""), + "description": func.get("description", ""), + "parameters": func.get("parameters", {}), + } + ) return declarations @@ -194,6 +219,7 @@ class CloudEngine(InferenceEngine): self._google_client: Any = None self._openrouter_client: Any = None self._minimax_client: Any = None + self._codex_client: Any = None # Gemini thought_signatures: tool_call_id -> signature bytes self._thought_sigs: Dict[str, bytes] = {} self._init_clients() @@ -202,22 +228,24 @@ class CloudEngine(InferenceEngine): if os.environ.get("OPENAI_API_KEY"): try: import openai + self._openai_client = openai.OpenAI() except ImportError: pass if os.environ.get("ANTHROPIC_API_KEY"): try: import anthropic + self._anthropic_client = anthropic.Anthropic() except ImportError: pass - gemini_key = ( - os.environ.get("GEMINI_API_KEY") - or os.environ.get("GOOGLE_API_KEY") + gemini_key = os.environ.get("GEMINI_API_KEY") or os.environ.get( + "GOOGLE_API_KEY" ) if gemini_key: try: from google import genai + self._google_client = genai.Client(api_key=gemini_key) except ImportError: pass @@ -225,6 +253,7 @@ class CloudEngine(InferenceEngine): if openrouter_key: try: import openai + self._openrouter_client = openai.OpenAI( base_url="https://openrouter.ai/api/v1", api_key=openrouter_key, @@ -235,12 +264,125 @@ class CloudEngine(InferenceEngine): if minimax_key: try: import openai + self._minimax_client = openai.OpenAI( base_url="https://api.minimax.io/v1", api_key=minimax_key, ) except ImportError: pass + # Codex — uses the OpenAI Responses API. + # Supports both standard API keys (api.openai.com) and ChatGPT + # OAuth tokens (chatgpt.com) via OPENAI_CODEX_BASE_URL override. + codex_token = os.environ.get("OPENAI_CODEX_API_KEY") + if codex_token: + codex_url = os.environ.get( + "OPENAI_CODEX_BASE_URL", + "https://api.openai.com/v1", + ).rstrip("/") + if not codex_url.endswith("/responses"): + codex_url += "/responses" + self._codex_client = { + "token": codex_token, + "url": codex_url, + } + + @staticmethod + def _codex_build_input( + messages: Sequence[Message], + ) -> tuple[str, List[Dict[str, Any]]]: + """Convert Message list to Codex Responses API format. + + Returns (system_instructions, input_messages). + """ + instructions = "" + input_msgs: List[Dict[str, Any]] = [] + for m in messages: + if m.role.value == "system": + instructions = m.content + elif m.role.value in ("user", "assistant"): + input_msgs.append( + { + "role": m.role.value, + "content": [{"type": "input_text", "text": m.content}], + } + ) + return instructions, input_msgs + + def _generate_codex( + self, + messages: Sequence[Message], + *, + model: str, + temperature: float, + max_tokens: int, + **kwargs: Any, + ) -> Dict[str, Any]: + """Generate via Codex Responses API (ChatGPT Plus/Pro).""" + if self._codex_client is None: + raise EngineConnectionError( + "Codex client not available — set OPENAI_CODEX_API_KEY" + ) + actual_model = model.removeprefix("codex/") + instructions, input_msgs = self._codex_build_input(messages) + + body: Dict[str, Any] = { + "model": actual_model, + "input": input_msgs, + "store": False, + "stream": False, + } + if instructions: + body["instructions"] = instructions + + headers = { + "Authorization": f"Bearer {self._codex_client['token']}", + "Content-Type": "application/json", + "OpenAI-Beta": "responses=experimental", + } + + t0 = time.monotonic() + resp = httpx.post( + self._codex_client["url"], + json=body, + headers=headers, + timeout=120.0, + ) + elapsed = time.monotonic() - t0 + resp.raise_for_status() + data = resp.json() + + # Extract text from Responses API output. + # The output array contains items of type "reasoning" and "message"; + # we want the "message" item's content blocks. + content = data.get("output_text", "") + if not content: + for item in data.get("output", []): + if item.get("type") not in ("message", None): + continue + for block in item.get("content", []): + if block.get("type") == "output_text": + content = block.get("text", "") + break + if content: + break + + usage_data = data.get("usage", {}) + prompt_tokens = usage_data.get("input_tokens", 0) + completion_tokens = usage_data.get("output_tokens", 0) + + return { + "content": content, + "usage": { + "prompt_tokens": prompt_tokens, + "completion_tokens": completion_tokens, + "total_tokens": prompt_tokens + completion_tokens, + }, + "model": actual_model, + "finish_reason": "stop", + "cost_usd": 0.0, + "ttft": elapsed, + } def _generate_openai( self, @@ -359,10 +501,12 @@ class CloudEngine(InferenceEngine): ): chat_msgs[-1]["content"].append(tool_result_block) else: - chat_msgs.append({ - "role": "user", - "content": [tool_result_block], - }) + chat_msgs.append( + { + "role": "user", + "content": [tool_result_block], + } + ) elif m.role.value == "assistant" and m.tool_calls: # Convert assistant messages with tool_calls to Anthropic # content blocks (text + tool_use) @@ -376,12 +520,14 @@ class CloudEngine(InferenceEngine): args = json.loads(args) except (json.JSONDecodeError, TypeError): args = {"input": args} - content_blocks.append({ - "type": "tool_use", - "id": tc.id, - "name": tc.name, - "input": args if isinstance(args, dict) else {}, - }) + content_blocks.append( + { + "type": "tool_use", + "id": tc.id, + "name": tc.name, + "input": args if isinstance(args, dict) else {}, + } + ) chat_msgs.append({"role": "assistant", "content": content_blocks}) else: chat_msgs.append({"role": m.role.value, "content": m.content}) @@ -428,13 +574,15 @@ class CloudEngine(InferenceEngine): tool_calls: list[Dict[str, Any]] = [] for block in resp.content: if getattr(block, "type", None) == "tool_use": - tool_calls.append({ - "id": block.id, - "name": block.name, - "arguments": json.dumps(block.input) - if isinstance(block.input, dict) - else str(block.input), - }) + tool_calls.append( + { + "id": block.id, + "name": block.name, + "arguments": json.dumps(block.input) + if isinstance(block.input, dict) + else str(block.input), + } + ) elif hasattr(block, "text"): content_parts.append(block.text) @@ -571,9 +719,7 @@ class CloudEngine(InferenceEngine): for part in parts: if hasattr(part, "function_call") and part.function_call: fc = part.function_call - fc_args = ( - dict(fc.args) if hasattr(fc.args, "items") else {} - ) + fc_args = dict(fc.args) if hasattr(fc.args, "items") else {} tc_dict: Dict[str, Any] = { "id": f"google_{fc.name}", "name": fc.name, @@ -598,12 +744,8 @@ class CloudEngine(InferenceEngine): content = "" um = resp.usage_metadata - prompt_tokens = ( - getattr(um, "prompt_token_count", 0) if um else 0 - ) - completion_tokens = ( - getattr(um, "candidates_token_count", 0) if um else 0 - ) + prompt_tokens = getattr(um, "prompt_token_count", 0) if um else 0 + completion_tokens = getattr(um, "candidates_token_count", 0) if um else 0 result: Dict[str, Any] = { "content": content, @@ -732,6 +874,8 @@ class CloudEngine(InferenceEngine): max_tokens=max_tokens, **kwargs, ) + if _is_codex_model(model): + return self._generate_codex(messages, **kw) if _is_openrouter_model(model): return self._generate_openrouter(messages, **kw) if _is_minimax_model(model): @@ -757,32 +901,81 @@ class CloudEngine(InferenceEngine): max_tokens=max_tokens, **kwargs, ) - if _is_openrouter_model(model): - async for token in self._stream_openrouter( - messages, **kw - ): + if _is_codex_model(model): + async for token in self._stream_codex(messages, **kw): + yield token + elif _is_openrouter_model(model): + async for token in self._stream_openrouter(messages, **kw): yield token elif _is_minimax_model(model): - async for token in self._stream_minimax( - messages, **kw - ): + async for token in self._stream_minimax(messages, **kw): yield token elif _is_anthropic_model(model): - async for token in self._stream_anthropic( - messages, **kw - ): + async for token in self._stream_anthropic(messages, **kw): yield token elif _is_google_model(model): - async for token in self._stream_google( - messages, **kw - ): + async for token in self._stream_google(messages, **kw): yield token else: - async for token in self._stream_openai( - messages, **kw - ): + async for token in self._stream_openai(messages, **kw): yield token + async def _stream_codex( + self, + messages: Sequence[Message], + *, + model: str, + temperature: float, + max_tokens: int, + **kwargs: Any, + ) -> AsyncIterator[str]: + """Stream via Codex Responses API (SSE).""" + if self._codex_client is None: + raise EngineConnectionError("Codex client not available") + actual_model = model.removeprefix("codex/") + instructions, input_msgs = self._codex_build_input(messages) + + body: Dict[str, Any] = { + "model": actual_model, + "input": input_msgs, + "store": False, + "stream": True, + } + if instructions: + body["instructions"] = instructions + + headers = { + "Authorization": f"Bearer {self._codex_client['token']}", + "Content-Type": "application/json", + "Accept": "text/event-stream", + "OpenAI-Beta": "responses=experimental", + } + + async with httpx.AsyncClient() as client: + async with client.stream( + "POST", + self._codex_client["url"], + json=body, + headers=headers, + timeout=120.0, + ) as resp: + resp.raise_for_status() + async for line in resp.aiter_lines(): + if not line.startswith("data: "): + continue + payload = line[6:] + if payload == "[DONE]": + break + try: + event = json.loads(payload) + except (json.JSONDecodeError, ValueError): + continue + etype = event.get("type", "") + if etype == "response.output_text.delta": + delta = event.get("delta", "") + if delta: + yield delta + async def _stream_openai( self, messages: Sequence[Message], @@ -940,6 +1133,8 @@ class CloudEngine(InferenceEngine): models.extend(_OPENROUTER_POPULAR) if self._minimax_client is not None: models.extend(_MINIMAX_MODELS) + if self._codex_client is not None: + models.extend(_CODEX_MODELS) return models def health(self) -> bool: @@ -949,6 +1144,7 @@ class CloudEngine(InferenceEngine): or self._google_client is not None or self._openrouter_client is not None or self._minimax_client is not None + or self._codex_client is not None ) def close(self) -> None: @@ -970,6 +1166,8 @@ class CloudEngine(InferenceEngine): if hasattr(self._minimax_client, "close"): self._minimax_client.close() self._minimax_client = None + if self._codex_client is not None: + self._codex_client = None __all__ = ["CloudEngine", "PRICING", "_annotate_anthropic_cache", "estimate_cost"] diff --git a/src/openjarvis/intelligence/model_catalog.py b/src/openjarvis/intelligence/model_catalog.py index af561c6a..9aed067a 100644 --- a/src/openjarvis/intelligence/model_catalog.py +++ b/src/openjarvis/intelligence/model_catalog.py @@ -650,6 +650,89 @@ BUILTIN_MODELS: List[ModelSpec] = [ }, ), # ----------------------------------------------------------------------- + # Cloud models — OpenAI Codex (ChatGPT Plus/Pro subscription) + # ----------------------------------------------------------------------- + ModelSpec( + model_id="codex/gpt-4o", + name="GPT-4o (Codex)", + parameter_count_b=0.0, + context_length=128000, + supported_engines=("cloud",), + provider="openai-codex", + requires_api_key=True, + metadata={ + "architecture": "proprietary", + "auth": "OPENAI_CODEX_API_KEY", + "pricing_input": 0.0, + "pricing_output": 0.0, + "url": "https://platform.openai.com/docs/models/gpt-4o", + }, + ), + ModelSpec( + model_id="codex/gpt-4o-mini", + name="GPT-4o Mini (Codex)", + parameter_count_b=0.0, + context_length=128000, + supported_engines=("cloud",), + provider="openai-codex", + requires_api_key=True, + metadata={ + "architecture": "proprietary", + "auth": "OPENAI_CODEX_API_KEY", + "pricing_input": 0.0, + "pricing_output": 0.0, + "url": "https://platform.openai.com/docs/models/gpt-4o-mini", + }, + ), + ModelSpec( + model_id="codex/o3-mini", + name="o3-mini (Codex)", + parameter_count_b=0.0, + context_length=200000, + supported_engines=("cloud",), + provider="openai-codex", + requires_api_key=True, + metadata={ + "architecture": "proprietary", + "auth": "OPENAI_CODEX_API_KEY", + "pricing_input": 0.0, + "pricing_output": 0.0, + "url": "https://platform.openai.com/docs/models", + }, + ), + ModelSpec( + model_id="codex/gpt-5-mini", + name="GPT-5 Mini (Codex)", + parameter_count_b=0.0, + context_length=400000, + supported_engines=("cloud",), + provider="openai-codex", + requires_api_key=True, + metadata={ + "architecture": "proprietary", + "auth": "OPENAI_CODEX_API_KEY", + "pricing_input": 0.0, + "pricing_output": 0.0, + "url": "https://platform.openai.com/docs/models", + }, + ), + ModelSpec( + model_id="codex/gpt-5-mini-2025-08-07", + name="GPT-5 Mini 2025-08-07 (Codex)", + parameter_count_b=0.0, + context_length=400000, + supported_engines=("cloud",), + provider="openai-codex", + requires_api_key=True, + metadata={ + "architecture": "proprietary", + "auth": "OPENAI_CODEX_API_KEY", + "pricing_input": 0.0, + "pricing_output": 0.0, + "url": "https://platform.openai.com/docs/models", + }, + ), + # ----------------------------------------------------------------------- # Cloud models — Anthropic # ----------------------------------------------------------------------- ModelSpec( diff --git a/tests/engine/test_cloud.py b/tests/engine/test_cloud.py index 81c908a1..0f272f24 100644 --- a/tests/engine/test_cloud.py +++ b/tests/engine/test_cloud.py @@ -9,7 +9,11 @@ import pytest from openjarvis.core.registry import EngineRegistry from openjarvis.core.types import Message, Role -from openjarvis.engine.cloud import CloudEngine, estimate_cost +from openjarvis.engine.cloud import ( + CloudEngine, + _is_codex_model, + estimate_cost, +) class TestEstimateCost: @@ -108,3 +112,181 @@ class TestCloudEngineGenerate: assert result["content"] == "Greetings!" assert result["usage"]["prompt_tokens"] == 12 assert result["usage"]["completion_tokens"] == 8 + + +# --------------------------------------------------------------------------- +# Codex provider support (OpenAI Responses API) +# --------------------------------------------------------------------------- + + +class TestCodexModelDetection: + def test_is_codex_model(self) -> None: + assert _is_codex_model("codex/gpt-4o") is True + assert _is_codex_model("codex/gpt-5-mini") is True + assert _is_codex_model("codex/gpt-5-mini-2025-08-07") is True + + def test_not_codex_model(self) -> None: + assert _is_codex_model("gpt-4o") is False + assert _is_codex_model("openrouter/openai/gpt-4o") is False + + +class TestCodexClientInit: + def test_health_with_codex_key(self, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.delenv("OPENAI_API_KEY", raising=False) + monkeypatch.delenv("ANTHROPIC_API_KEY", raising=False) + monkeypatch.setenv("OPENAI_CODEX_API_KEY", "test-token") + engine = CloudEngine() + assert engine.health() is True + assert engine._codex_client is not None + assert engine._codex_client["token"] == "test-token" + assert "responses" in engine._codex_client["url"] + + def test_custom_codex_base_url(self, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.delenv("OPENAI_API_KEY", raising=False) + monkeypatch.delenv("ANTHROPIC_API_KEY", raising=False) + monkeypatch.setenv("OPENAI_CODEX_API_KEY", "test-token") + monkeypatch.setenv("OPENAI_CODEX_BASE_URL", "http://localhost:9999") + engine = CloudEngine() + assert engine._codex_client["url"] == "http://localhost:9999/responses" + + def test_list_models_includes_codex(self, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.delenv("OPENAI_API_KEY", raising=False) + monkeypatch.delenv("ANTHROPIC_API_KEY", raising=False) + monkeypatch.setenv("OPENAI_CODEX_API_KEY", "test-token") + engine = CloudEngine() + models = engine.list_models() + assert "codex/gpt-4o" in models + assert "codex/gpt-5-mini" in models + assert "codex/gpt-5-mini-2025-08-07" in models + + def test_no_codex_key_means_no_codex(self, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.delenv("OPENAI_API_KEY", raising=False) + monkeypatch.delenv("ANTHROPIC_API_KEY", raising=False) + monkeypatch.delenv("OPENAI_CODEX_API_KEY", raising=False) + engine = CloudEngine() + assert engine._codex_client is None + assert "codex/gpt-4o" not in engine.list_models() + + +class TestCodexGenerate: + def test_generate_codex_uses_responses_api( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + monkeypatch.delenv("OPENAI_API_KEY", raising=False) + monkeypatch.delenv("ANTHROPIC_API_KEY", raising=False) + + fake_response = mock.MagicMock() + fake_response.status_code = 200 + fake_response.json.return_value = { + "output_text": "Codex response!", + "usage": {"input_tokens": 10, "output_tokens": 5}, + } + fake_response.raise_for_status = mock.MagicMock() + + engine = CloudEngine() + engine._codex_client = { + "token": "test-token", + "url": "https://api.openai.com/v1/responses", + } + + with mock.patch( + "openjarvis.engine.cloud.httpx.post", + return_value=fake_response, + ) as mock_post: + result = engine.generate( + [Message(role=Role.USER, content="Hi")], + model="codex/gpt-5-mini-2025-08-07", + ) + + assert result["content"] == "Codex response!" + assert result["model"] == "gpt-5-mini-2025-08-07" + assert result["usage"]["prompt_tokens"] == 10 + assert result["usage"]["completion_tokens"] == 5 + + # Verify correct Responses API request format + call_kwargs = mock_post.call_args + sent_body = call_kwargs.kwargs["json"] + assert sent_body["model"] == "gpt-5-mini-2025-08-07" + assert sent_body["stream"] is False + assert "input" in sent_body # Responses API format + assert "messages" not in sent_body # NOT chat completions + + # Verify correct headers + sent_headers = call_kwargs.kwargs["headers"] + assert sent_headers["Authorization"] == "Bearer test-token" + assert sent_headers["OpenAI-Beta"] == "responses=experimental" + + def test_generate_codex_extracts_from_output_blocks( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + """Fallback extraction from output[].content[] blocks.""" + monkeypatch.delenv("OPENAI_API_KEY", raising=False) + monkeypatch.delenv("ANTHROPIC_API_KEY", raising=False) + + fake_response = mock.MagicMock() + fake_response.json.return_value = { + "output": [{"content": [{"type": "output_text", "text": "From blocks!"}]}], + "usage": {"input_tokens": 5, "output_tokens": 3}, + } + fake_response.raise_for_status = mock.MagicMock() + + engine = CloudEngine() + engine._codex_client = { + "token": "t", + "url": "https://api.openai.com/v1/responses", + } + + with mock.patch( + "openjarvis.engine.cloud.httpx.post", + return_value=fake_response, + ): + result = engine.generate( + [Message(role=Role.USER, content="Hi")], + model="codex/gpt-4o", + ) + assert result["content"] == "From blocks!" + + def test_generate_codex_passes_system_as_instructions( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + monkeypatch.delenv("OPENAI_API_KEY", raising=False) + monkeypatch.delenv("ANTHROPIC_API_KEY", raising=False) + + fake_response = mock.MagicMock() + fake_response.json.return_value = { + "output_text": "ok", + "usage": {}, + } + fake_response.raise_for_status = mock.MagicMock() + + engine = CloudEngine() + engine._codex_client = { + "token": "t", + "url": "https://api.openai.com/v1/responses", + } + + with mock.patch( + "openjarvis.engine.cloud.httpx.post", + return_value=fake_response, + ) as mock_post: + engine.generate( + [ + Message(role=Role.SYSTEM, content="Be helpful"), + Message(role=Role.USER, content="Hi"), + ], + model="codex/gpt-4o", + ) + + sent_body = mock_post.call_args.kwargs["json"] + assert sent_body["instructions"] == "Be helpful" + # System message should NOT appear in input messages + roles = [m["role"] for m in sent_body["input"]] + assert "system" not in roles + + def test_codex_close(self, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.delenv("OPENAI_API_KEY", raising=False) + monkeypatch.delenv("ANTHROPIC_API_KEY", raising=False) + engine = CloudEngine() + engine._codex_client = {"token": "t", "url": "http://test"} + engine.close() + assert engine._codex_client is None From fc369b9ab057fc50a24b6003e55367a3c4d612a6 Mon Sep 17 00:00:00 2001 From: Jon Saad-Falcon <41205309+jonsaadfalcon@users.noreply.github.com> Date: Thu, 26 Mar 2026 18:59:40 -0700 Subject: [PATCH 28/44] fix: use Claude Opus 4.6 as sole baseline for leaderboard savings Dollar savings were previously summed across all three cloud providers (GPT-5.3 + Claude Opus 4.6 + Gemini 3.1 Pro), inflating the reported number by ~3x. Now uses only Claude Opus 4.6 pricing as the baseline, with an asterisk footnote on the leaderboard explaining this. Co-Authored-By: Claude Opus 4.6 (1M context) --- docs/leaderboard.md | 6 +++--- frontend/src/App.tsx | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/docs/leaderboard.md b/docs/leaderboard.md index f676dfb1..ffc8ef0e 100644 --- a/docs/leaderboard.md +++ b/docs/leaderboard.md @@ -16,7 +16,7 @@ See how the OpenJarvis community saves money, energy, and compute by running AI
—
-
Total Saved
+
Total Saved*
—
@@ -35,7 +35,7 @@ See how the OpenJarvis community saves money, energy, and compute by running AI # Name - $ Saved + $ Saved* Energy (Wh) FLOPs Requests @@ -55,5 +55,5 @@ See how the OpenJarvis community saves money, energy, and compute by running AI

-*Dollar savings estimates assume local open-source models (e.g. Qwen, Nemotron, Kimi) produce roughly the same number of tokens per request, on average, as closed-source cloud models. +*Dollar savings estimated vs. Claude Opus 4.6 API pricing ($5/1M input, $25/1M output tokens). Assumes local open-source models produce roughly the same number of tokens per request as cloud models.

diff --git a/frontend/src/App.tsx b/frontend/src/App.tsx index f619379a..e627931f 100644 --- a/frontend/src/App.tsx +++ b/frontend/src/App.tsx @@ -67,10 +67,10 @@ export default function App() { .then((data) => { setSavings(data); if (optInEnabled && optInDisplayName && data) { - const dollarSavings = data.per_provider.reduce( - (sum, p) => sum + p.total_cost, - 0, + const claudeEntry = data.per_provider.find( + (p) => p.provider === 'claude-opus-4.6', ); + const dollarSavings = claudeEntry ? claudeEntry.total_cost : 0; const energySaved = data.per_provider.reduce( (sum, p) => sum + (p.energy_wh || 0), 0, From c807666bd5352689e6addad92a541373ee78ad58 Mon Sep 17 00:00:00 2001 From: Jon Saad-Falcon <41205309+jonsaadfalcon@users.noreply.github.com> Date: Thu, 26 Mar 2026 19:00:39 -0700 Subject: [PATCH 29/44] fix: cost format $0.0000, compact overview layout - Remove cent sign from cost display, use $X.XXXX format - Compact stat cards (horizontal icon+value layout) - Tighter config grid spacing with bolder labels - Reduce padding and gaps throughout overview tab Co-Authored-By: Claude Opus 4.6 (1M context) --- frontend/src/pages/AgentsPage.tsx | 58 ++++++++++--------------------- 1 file changed, 19 insertions(+), 39 deletions(-) diff --git a/frontend/src/pages/AgentsPage.tsx b/frontend/src/pages/AgentsPage.tsx index 47700a41..71f37e3f 100644 --- a/frontend/src/pages/AgentsPage.tsx +++ b/frontend/src/pages/AgentsPage.tsx @@ -101,8 +101,7 @@ function StatusDot({ status }: { status: string }) { function formatCost(cost?: number): string { if (cost === undefined || cost === null) return '—'; - if (cost < 0.01) return `$${(cost * 100).toFixed(2)}¢`; - return `$${cost.toFixed(3)}`; + return `$${cost.toFixed(4)}`; } function formatRelativeTime(ts?: number | null): string { @@ -1226,10 +1225,10 @@ function AgentConfigGrid({ agent, onAgentUpdated }: { agent: ManagedAgent; onAge ]; return ( -
+
{rows.map(([label, value]) => ( -
- {label} +
+ {label} {value}
))} @@ -1802,57 +1801,38 @@ export function AgentsPage() { {/* Tab: Overview */} {detailTab === 'overview' && ( -
- {/* Stat cards */} -
+
+ {/* Stat cards — compact row */} +
{[ - { - label: 'Total Runs', - value: String(selectedAgent.total_runs ?? 0), - icon: Activity, - color: '#3b82f6', - }, - { - label: 'Success Rate', - value: successRate !== null ? `${successRate}%` : '—', - icon: Zap, - color: '#22c55e', - }, - { - label: 'Total Cost', - value: formatCost(selectedAgent.total_cost), - icon: DollarSign, - color: '#f59e0b', - }, + { label: 'Total Runs', value: String(selectedAgent.total_runs ?? 0), icon: Activity, color: '#3b82f6' }, + { label: 'Success Rate', value: successRate !== null ? `${successRate}%` : '—', icon: Zap, color: '#22c55e' }, + { label: 'Total Cost', value: formatCost(selectedAgent.total_cost), icon: DollarSign, color: '#f59e0b' }, ].map(({ label, value, icon: Icon, color }) => (
-
- - - {label} - + +
+

{value}

+

{label}

-

- {value} -

))}
- {/* Config display */} + {/* Config display — tighter spacing */}
-

+

Configuration

-
+
ID: {selectedAgent.id} From d7d0125454dc8f5b2aa1c9dcf58c2f0dd00db682 Mon Sep 17 00:00:00 2001 From: Jon Saad-Falcon <41205309+jonsaadfalcon@users.noreply.github.com> Date: Fri, 27 Mar 2026 09:13:31 -0700 Subject: [PATCH 30/44] fix: add SQL migration to recompute historical leaderboard savings Companion to #143 which fixed the frontend to use Claude Opus 4.6 as the sole baseline. This migration recomputes existing Supabase rows using the exact closed-form: new = T/3.8M + 10*old/19, derived from the original triple-provider formula. Co-Authored-By: Claude Opus 4.6 (1M context) --- scripts/migrate_savings_single_provider.sql | 37 +++++++++++++++++++++ 1 file changed, 37 insertions(+) create mode 100644 scripts/migrate_savings_single_provider.sql diff --git a/scripts/migrate_savings_single_provider.sql b/scripts/migrate_savings_single_provider.sql new file mode 100644 index 00000000..e6e42eb8 --- /dev/null +++ b/scripts/migrate_savings_single_provider.sql @@ -0,0 +1,37 @@ +-- Migration: Recompute dollar_savings to use only Claude Opus 4.6 pricing +-- ----------------------------------------------------------------------- +-- Previously the frontend summed hypothetical costs across all 3 cloud +-- providers (GPT-5.3 + Claude Opus 4.6 + Gemini 3.1 Pro). This +-- migration recalculates dollar_savings using Claude Opus 4.6 only. +-- +-- Derivation +-- ---------- +-- Let P = prompt_tokens, C = completion_tokens, T = total_tokens = P + C. +-- +-- old = (P/1M)*(2+5+2) + (C/1M)*(10+25+12) = (P/1M)*9 + (C/1M)*47 +-- new = (P/1M)*5 + (C/1M)*25 +-- +-- Solving the system {T = P + C, old = 9P/1M + 47C/1M} for P and C and +-- substituting into the "new" formula gives: +-- +-- new = T / 3_800_000 + 10 * old / 19 +-- +-- Run this in the Supabase SQL Editor (Dashboard > SQL Editor). +-- ----------------------------------------------------------------------- + +BEGIN; + +-- Preview the changes first (uncomment the SELECT, comment the UPDATE) +-- SELECT +-- display_name, +-- dollar_savings AS old_savings, +-- total_tokens / 3800000.0 + 10.0 * dollar_savings / 19.0 AS new_savings +-- FROM savings_entries +-- WHERE dollar_savings > 0 +-- ORDER BY dollar_savings DESC; + +UPDATE savings_entries +SET dollar_savings = total_tokens / 3800000.0 + 10.0 * dollar_savings / 19.0 +WHERE dollar_savings > 0; + +COMMIT; From 0ce10bd97b419326cd500c9ea31babccbc7b8f37 Mon Sep 17 00:00:00 2001 From: krypticmouse Date: Fri, 27 Mar 2026 16:17:17 +0000 Subject: [PATCH 31/44] docs: add macOS installation guide for llama.cpp MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Comprehensive step-by-step guide covering Homebrew, uv, Rust, llama.cpp, model download, Python 3.12 pin (PyO3 compat), and common pitfalls. Cherry-picked from PR #131 by @gridworks — cleaned up to include only the docs content (removed duplicate files, binary artifacts, and unrelated lockfile changes from the original PR). Co-Authored-By: gridworks <5502067+gridworks@users.noreply.github.com> Co-Authored-By: Claude Opus 4.6 (1M context) --- docs/getting-started/macos.md | 421 ++++++++++++++++++++++++++++++++++ mkdocs.yml | 1 + 2 files changed, 422 insertions(+) create mode 100644 docs/getting-started/macos.md diff --git a/docs/getting-started/macos.md b/docs/getting-started/macos.md new file mode 100644 index 00000000..67a6fe1b --- /dev/null +++ b/docs/getting-started/macos.md @@ -0,0 +1,421 @@ +--- +title: macOS Installation Guide +description: Complete step-by-step guide to installing OpenJarvis on macOS with llama.cpp, including common pitfalls and fixes +search: + boost: 2 +--- + +# macOS Installation Guide + +This guide walks through a complete OpenJarvis installation on macOS using **llama.cpp** as +the inference engine. It covers every step from scratch — including pitfalls not documented +elsewhere — and is suitable for both Apple Silicon and Intel Macs. + +!!! tip "Prefer Ollama?" + If you want the fastest possible setup, use [Ollama](installation.md#ollama-recommended) + instead. This guide is for users who want to run GGUF models directly with llama.cpp, + or who want a deeper understanding of the full stack. + +--- + +## What You'll Install + +| Tool | Purpose | +|------|---------| +| Homebrew | macOS package manager — installs everything else | +| uv | Python version and dependency manager | +| Git | Clones the OpenJarvis repo | +| Node.js | Required for the browser UI | +| Rust | Compiles the OpenJarvis security and memory extension | +| llama.cpp | Local inference engine that runs GGUF model files | +| OpenJarvis | The framework itself | +| A GGUF model | The actual AI model (downloaded separately) | + +--- + +## Step-by-Step Installation + +### Step 1 — Install Homebrew + +Homebrew is the standard macOS package manager. Everything else in this guide is installed +through it. + +```bash +/bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)" +``` + +If you already have Homebrew, skip this step. + +--- + +### Step 2 — Install uv + +`uv` replaces pip, virtualenv, and pyenv in one tool. OpenJarvis uses it to manage Python +versions, virtual environments, and project dependencies. + +```bash +brew install uv +``` + +--- + +### Step 3 — Install Git + +Git is used to clone the OpenJarvis source code. It may already be present if you have +Xcode Command Line Tools installed. + +```bash +brew install git +``` + +--- + +### Step 4 — Install Node.js + +Node.js is required to build and run the browser frontend. Without it you can still use +the CLI, but not the web UI. + +```bash +brew install node +``` + +--- + +### Step 5 — Install Rust + +OpenJarvis includes a Rust extension that provides security scanning, memory indexing, +rate limiting, and tool execution. It must be compiled from source. + +```bash +curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh +``` + +After the installer finishes, reload your shell so `rustc` is available: + +```bash +source "$HOME/.cargo/env" +``` + +Verify: + +```bash +rustc --version +``` + +--- + +### Step 6 — Install llama.cpp + +llama.cpp is the inference engine that loads and runs GGUF model files. It is not a model +itself — think of it as a media player and the `.gguf` file as the content. + +```bash +brew install llama.cpp +``` + +--- + +### Step 7 — Clone the OpenJarvis repo + +Run this from your home directory or any neutral parent folder. + +```bash +cd ~ +git clone https://github.com/open-jarvis/OpenJarvis.git +cd OpenJarvis +``` + +!!! warning "Do not clone from inside an existing OpenJarvis folder" + A common mistake is running `git clone` while already inside the repo, creating deeply + nested duplicates (`OpenJarvis/OpenJarvis/OpenJarvis`). Always clone from `~` or a + neutral parent directory. + +--- + +### Step 8 — Pin Python to 3.12 + +!!! warning "Critical step — do not skip" + OpenJarvis requires Python 3.10–3.13. Its Rust extension uses PyO3, which does not yet + support Python 3.14. If `uv` has Python 3.14 available, it will use it by default, + causing the Rust extension build to fail silently and resulting in ~250 test failures + with `ModuleNotFoundError: No module named 'openjarvis_rust'`. + +Pin the project to Python 3.12: + +```bash +echo "3.12" > .python-version +uv python install 3.12 +rm -rf .venv +uv venv +``` + +**Restart your terminal**, then verify: + +```bash +uv run python --version +# Must show: Python 3.12.x +``` + +!!! tip "Why restart the terminal?" + Without restarting, the shell may still reference the old virtual environment. This is + the most common reason the version pin appears not to work. + +--- + +### Step 9 — Install Python dependencies + +```bash +uv sync --extra dev --extra server +``` + +The `--extra server` flag adds the FastAPI backend required for the browser UI. + +--- + +### Step 10 — Build the Rust extension + +This compiles the Rust extension and installs it into the virtual environment. It provides +security scanning, memory indexing, MCP tool execution, and rate limiting. This step takes +a few minutes on first run. + +```bash +uv run maturin develop -m rust/crates/openjarvis-python/Cargo.toml +``` + +Verify it built correctly: + +```bash +uv run python -c "import openjarvis_rust; print('Rust extension OK')" +``` + +--- + +### Step 11 — Install frontend dependencies + +```bash +cd frontend && npm install && cd .. +``` + +--- + +### Step 12 — Download a model + +OpenJarvis needs a GGUF model file to run inference. First install the Hugging Face CLI, +then download your chosen model. + +```bash +uv tool install huggingface_hub +``` + +!!! note "The CLI command is `hf`, not `huggingface-cli`" + When installed via `uv tool`, the Hugging Face CLI is invoked as `hf`. + +=== "Qwen3 4B (~2.5 GB)" + + Faster, lower RAM requirement. Good for most everyday tasks. + + ```bash + hf download bartowski/Qwen_Qwen3-4B-GGUF \ + --include "Qwen_Qwen3-4B-Q4_K_M.gguf" \ + --local-dir ~/models + ``` + +=== "Qwen3 8B (~4.7 GB)" + + Better reasoning and instruction following. Requires more RAM. + + ```bash + hf download bartowski/Qwen_Qwen3-8B-GGUF \ + --include "Qwen_Qwen3-8B-Q4_K_M.gguf" \ + --local-dir ~/models + ``` + +!!! warning "Use the `Qwen_` prefix" + bartowski's Qwen3 repos use the `Qwen_` prefix (e.g. `Qwen_Qwen3-4B-GGUF`). Using + the shorter name without the prefix returns a "repository not found" error. + +!!! tip "Apple Silicon vs Intel" + On Apple Silicon, both models benefit from Metal GPU acceleration when using the MLX + engine. On Intel, inference runs on CPU — the 4B model is recommended for speed. + +--- + +### Step 13 — Configure OpenJarvis + +Run the init command to detect your hardware and generate a config file: + +```bash +uv run jarvis init +``` + +Then open the config and set the default model to match the filename you downloaded: + +```bash +nano ~/.openjarvis/config.toml +``` + +Find the `default_model` line and update it, for example: + +```toml +default_model = "Qwen_Qwen3-4B-Q4_K_M.gguf" +``` + +--- + +### Step 14 — Verify the installation + +```bash +uv run jarvis doctor +``` + +A healthy setup looks like this: + +``` +✓ Python version 3.12.x +✓ Config file ~/.openjarvis/config.toml +✓ Config parsing Config loaded successfully +✓ Engine: llamacpp Reachable +✓ Models: llamacpp Qwen_Qwen3-4B-Q4_K_M.gguf +✓ Default model Qwen_Qwen3-4B-Q4_K_M.gguf (on llamacpp) +``` + +!!! note "Warnings for other engines are normal" + The `!` warnings for engines like `ollama`, `vllm`, and `lmstudio` simply mean those + backends are not running. You only need `llamacpp` to be reachable. + +--- + +## Running OpenJarvis + +### CLI + +Start llama-server in one terminal, then run queries in another: + +```bash +# Terminal 1 — start the inference engine +llama-server -m ~/models/Qwen_Qwen3-4B-Q4_K_M.gguf -c 4096 -t 8 + +# Terminal 2 — ask a question +cd ~/OpenJarvis +uv run jarvis ask "What is the capital of France?" +``` + +### Browser UI + +```bash +# Terminal 1 — inference engine +llama-server -m ~/models/Qwen_Qwen3-4B-Q4_K_M.gguf -c 4096 -t 8 + +# Terminal 2 — backend +cd ~/OpenJarvis && uv run jarvis serve --port 8000 + +# Terminal 3 — frontend +cd ~/OpenJarvis/frontend && npm run dev +``` + +Then open [http://localhost:5173](http://localhost:5173). + +### Skip typing `uv run` every time + +Activate the virtual environment for your current terminal session: + +```bash +source ~/OpenJarvis/.venv/bin/activate +``` + +Your prompt will show `(openjarvis)` when active, and you can type `jarvis ask "..."` directly. + +--- + +## Performance Tips + +These tips apply when using llama.cpp for CPU inference. + +| Flag | Effect | +|------|--------| +| `-c 4096` | Reduces context window from the 32,768 default, freeing RAM for faster inference | +| `-t 8` | Uses all available CPU threads (default is only 4) — adjust to your machine's thread count | +| `Q4_K_M` quantization | Best balance of size, speed, and quality for CPU inference | + +On Apple Silicon, switching to the [MLX engine](../architecture/engine.md) gives +significantly better performance than llama.cpp for most models. + +--- + +## Common Errors + +### `No such file or directory` when loading model + +The path `path/to/model.gguf` in examples is a placeholder. Replace it with your actual +model path, e.g.: + +```bash +llama-server -m ~/models/Qwen_Qwen3-4B-Q4_K_M.gguf +``` + +--- + +### `No module named 'openjarvis_rust'` + +The Rust extension did not build correctly, or was built against the wrong Python version. + +1. Confirm Python 3.12 is active: `uv run python --version` +2. Rebuild: `uv run maturin develop -m rust/crates/openjarvis-python/Cargo.toml` + +If the version shows 3.14, go back to [Step 8](#step-8--pin-python-to-312). + +--- + +### `PyO3 version error — Python 3.14 too new` + +``` +error: the configured Python interpreter version (3.14) is newer than +PyO3's maximum supported version (3.13) +``` + +PyO3 0.23.5 supports Python up to 3.13. Follow [Step 8](#step-8--pin-python-to-312) to +pin to 3.12, then delete `.venv`, recreate it, and restart your terminal before retrying. + +--- + +### `Repository not found` when downloading model + +bartowski's Qwen3 repos use the `Qwen_` prefix. Use: + +``` +bartowski/Qwen_Qwen3-4B-GGUF ✓ +bartowski/Qwen3-4B-GGUF ✗ +``` + +--- + +### `No inference engine available` + +llama-server is not running. Start it in a separate terminal before running any `jarvis` +commands, and wait until you see `model loaded` in the output. + +--- + +### Python version still shows 3.14 after recreating the venv + +Close the terminal completely and reopen it. The old venv path is cached in the shell +environment and persists across commands until the session ends. + +--- + +### `zsh: command not found: huggingface-cli` + +When installed via `uv tool`, the CLI is invoked as `hf`, not `huggingface-cli`: + +```bash +hf download ... # ✓ +huggingface-cli download ... # ✗ +``` + +--- + +## Next Steps + +- [Quick Start](quickstart.md) — Run your first query and explore agents and tools +- [Configuration](configuration.md) — Customize engine hosts, model routing, memory, and more +- [Architecture](../architecture/overview.md) — Understand how OpenJarvis is structured diff --git a/mkdocs.yml b/mkdocs.yml index 73869262..c8b179f9 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -143,6 +143,7 @@ nav: - Home: index.md - Getting Started: - Installation: getting-started/installation.md + - macOS Guide: getting-started/macos.md - Quick Start: getting-started/quickstart.md - Code Snippets: getting-started/snippets.md - Configuration: getting-started/configuration.md From 77b5567e47f846c6693218f41d0de9dd78849b46 Mon Sep 17 00:00:00 2001 From: Jon Saad-Falcon <41205309+jonsaadfalcon@users.noreply.github.com> Date: Fri, 27 Mar 2026 10:58:36 -0700 Subject: [PATCH 32/44] fix: recompute leaderboard energy/FLOPs from tokens, cap dollar savings Energy and FLOPs are now derived from total_tokens on the leaderboard display using Claude Opus 4.6 constants, rather than trusting submitted DB values. This prevents gaming (e.g. TotallyNoire submitting 14B Wh from 15M tokens). Dollar savings are clamped at the theoretical max ($25/1M tokens) on both the leaderboard and frontend submission side. Co-Authored-By: Claude Opus 4.6 (1M context) --- docs/javascripts/leaderboard.js | 42 +++++++++++++++++++++------------ frontend/src/App.tsx | 5 +++- 2 files changed, 31 insertions(+), 16 deletions(-) diff --git a/docs/javascripts/leaderboard.js b/docs/javascripts/leaderboard.js index 742b398a..093a3022 100644 --- a/docs/javascripts/leaderboard.js +++ b/docs/javascripts/leaderboard.js @@ -9,20 +9,27 @@ var allRows = []; var currentPage = 0; - // No-KV-cache formula: FLOPs = params_b * 1e9 * N * (N+1) - // Reference values only (GPT-5.3 constants) — recompute functions below - // are defined but not called; the leaderboard displays database values directly. - var DEFAULT_PARAMS_B = 137; - var ENERGY_WH_PER_FLOP = 0.4 / (1000 * 3e12); + // Claude Opus 4.6 constants — recompute energy and FLOPs from + // total_tokens so submitted values cannot be gamed. + var CLAUDE_PARAMS_B = 137; + var CLAUDE_FLOPS_PER_TOKEN = 4.0e12; + var CLAUDE_WH_PER_FLOP = 0.5 / (1000 * CLAUDE_FLOPS_PER_TOKEN); + // Max possible $/token: all tokens are output at $25/1M + var MAX_DOLLAR_PER_TOKEN = 25.0 / 1e6; - function recomputeFlopsNoKvCache(totalTokens) { + function recomputeFlops(totalTokens) { var n = Number(totalTokens) || 0; if (n <= 0) return 0; - return DEFAULT_PARAMS_B * 1e9 * n * (n + 1); + return 2.0 * CLAUDE_PARAMS_B * 1e9 * n; } - function recomputeEnergyNoKvCache(flops) { - return flops * ENERGY_WH_PER_FLOP; + function recomputeEnergy(totalTokens) { + return recomputeFlops(totalTokens) * CLAUDE_WH_PER_FLOP; + } + + function clampDollars(dollarSavings, totalTokens) { + var max = (Number(totalTokens) || 0) * MAX_DOLLAR_PER_TOKEN; + return Math.min(Number(dollarSavings) || 0, max); } function escapeHtml(s) { @@ -58,15 +65,19 @@ var medal = rank === 1 ? "\uD83E\uDD47" : rank === 2 ? "\uD83E\uDD48" : rank === 3 ? "\uD83E\uDD49" : ""; var row = pageRows[j]; + var tokens = Number(row.total_tokens || 0); + var dollars = clampDollars(row.dollar_savings, tokens); + var flops = recomputeFlops(tokens); + var energyWh = recomputeEnergy(tokens); html += "" + '' + (medal || rank) + "" + '' + escapeHtml(row.display_name) + "" + - '$' + Number(row.dollar_savings || 0).toFixed(4) + "" + - '' + Number(row.energy_wh_saved || 0).toFixed(2) + "" + - '' + fmtLarge(Number(row.flops_saved || 0)) + "" + + '$' + dollars.toFixed(4) + "" + + '' + energyWh.toFixed(2) + "" + + '' + fmtLarge(flops) + "" + '' + Number(row.total_calls || 0).toLocaleString() + "" + - '' + Number(row.total_tokens || 0).toLocaleString() + "" + + '' + tokens.toLocaleString() + "" + ""; } tbody.innerHTML = html; @@ -141,9 +152,10 @@ var totalRequests = 0; var totalTokens = 0; for (var i = 0; i < rows.length; i++) { - totalDollars += Number(rows[i].dollar_savings || 0); + var t = Number(rows[i].total_tokens || 0); + totalDollars += clampDollars(rows[i].dollar_savings, t); totalRequests += Number(rows[i].total_calls || 0); - totalTokens += Number(rows[i].total_tokens || 0); + totalTokens += t; } var elMembers = document.getElementById("stat-members"); diff --git a/frontend/src/App.tsx b/frontend/src/App.tsx index e627931f..c6ccf928 100644 --- a/frontend/src/App.tsx +++ b/frontend/src/App.tsx @@ -71,6 +71,9 @@ export default function App() { (p) => p.provider === 'claude-opus-4.6', ); const dollarSavings = claudeEntry ? claudeEntry.total_cost : 0; + // Cap at theoretical max ($25/1M output tokens) to prevent spoofed values + const maxDollars = (data.total_tokens * 25) / 1_000_000; + const clampedDollars = Math.min(dollarSavings, maxDollars); const energySaved = data.per_provider.reduce( (sum, p) => sum + (p.energy_wh || 0), 0, @@ -85,7 +88,7 @@ export default function App() { email: optInEmail, total_calls: data.total_calls, total_tokens: data.total_tokens, - dollar_savings: dollarSavings, + dollar_savings: clampedDollars, energy_wh_saved: energySaved, flops_saved: flopsSaved, token_counting_version: data.token_counting_version ?? 1, From b39dbedcc46307c43f8b630fa3a3baea042efb60 Mon Sep 17 00:00:00 2001 From: krypticmouse Date: Fri, 27 Mar 2026 18:09:54 +0000 Subject: [PATCH 33/44] feat: fix external MCP server integration and add streaming tool-call support MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Rebased and cleaned-up version of PR #113 by @mricharz, resolved against current main (including Codex engine, Gemini thought_signature, and agent manager fixes merged since the original PR). MCP Transport & Client: - StreamableHTTPTransport with session tracking, SSE parsing, timeouts - MCPClient.initialize() sends proper MCP handshake (protocolVersion, capabilities, clientInfo) + notifications/initialized - Fix StdioTransport constructor: command=[command] + args - MCPRequest.to_dict() with notification support (id=None) External MCP Discovery: - _discover_external_mcp supports both url (HTTP) and command (stdio) - Per-server include_tools / exclude_tools filtering - MCP clients persisted on JarvisSystem for runtime lifetime Streaming Tool-Call Support (stream_full): - StreamChunk dataclass in _stubs.py (content, tool_calls, finish_reason, usage) - Default stream_full() on InferenceEngine ABC wraps stream() for backward compat - _OpenAICompatibleEngine.stream_full() with SSE parsing - CloudEngine: _stream_full_openai (OpenAI/OpenRouter/MiniMax/Codex routing) _stream_full_anthropic (event-based → OpenAI delta format) - InstrumentedEngine, MultiEngine: stream_full delegation - GuardrailsEngine: stream_full with post-hoc security scanning (FIXED: original PR bypassed output scanning — now accumulates and scans like stream()) Other improvements: - _prepare_anthropic_messages() extracted to eliminate duplication - Default tool_choice=auto when tools are provided (OpenAI compat engines) - MCP tool injection into managed agent streaming path - Documentation: docs/user-guide/mcp-external-servers.md Tests: ~59 new tests across 8 test files, all passing. Closes PR #113 Co-Authored-By: mricharz Co-Authored-By: Claude Opus 4.6 (1M context) --- docs/user-guide/agents.md | 31 +- docs/user-guide/mcp-external-servers.md | 158 +++++ docs/user-guide/tools.md | 2 +- mkdocs.yml | 3 + src/openjarvis/engine/_openai_compat.py | 75 ++- src/openjarvis/engine/_stubs.py | 41 +- src/openjarvis/engine/cloud.py | 295 ++++++-- src/openjarvis/engine/multi.py | 32 +- src/openjarvis/mcp/__init__.py | 2 + src/openjarvis/mcp/client.py | 35 +- src/openjarvis/mcp/protocol.py | 31 +- src/openjarvis/mcp/server.py | 33 +- src/openjarvis/mcp/transport.py | 148 +++- src/openjarvis/security/guardrails.py | 77 ++- src/openjarvis/server/agent_manager_routes.py | 634 +++++++++++++----- src/openjarvis/server/stream_bridge.py | 136 ++-- src/openjarvis/system.py | 63 +- .../telemetry/instrumented_engine.py | 111 ++- tests/engine/test_cloud_stream_full.py | 466 +++++++++++++ tests/engine/test_engine_wrappers.py | 142 ++++ tests/engine/test_stream_full.py | 250 +++++++ tests/mcp/test_client_extended.py | 176 +++++ tests/mcp/test_discovery.py | 195 ++++++ tests/mcp/test_streamable_http_transport.py | 142 ++++ tests/mcp/test_transport.py | 116 +++- tests/server/test_agent_manager_routes.py | 118 ++-- tests/server/test_mcp_tools_cache.py | 154 +++++ 27 files changed, 3228 insertions(+), 438 deletions(-) create mode 100644 docs/user-guide/mcp-external-servers.md create mode 100644 tests/engine/test_cloud_stream_full.py create mode 100644 tests/engine/test_engine_wrappers.py create mode 100644 tests/engine/test_stream_full.py create mode 100644 tests/mcp/test_client_extended.py create mode 100644 tests/mcp/test_discovery.py create mode 100644 tests/mcp/test_streamable_http_transport.py create mode 100644 tests/server/test_mcp_tools_cache.py diff --git a/docs/user-guide/agents.md b/docs/user-guide/agents.md index 9f21a46f..2a988f54 100644 --- a/docs/user-guide/agents.md +++ b/docs/user-guide/agents.md @@ -626,7 +626,20 @@ These events enable the telemetry and trace systems to record detailed interacti ## Managed Agent Streaming -The Managed Agent API (`/v1/managed-agents/{id}/messages`) supports real-time SSE streaming. Send a message with `stream: true` to receive the agent's response as a Server-Sent Events stream instead of the default asynchronous queue mode. +The Managed Agent API (`/v1/managed-agents/{id}/messages`) supports **real LLM token streaming** via SSE. Send a message with `stream: true` to receive the model's response tokens as they are generated, rather than waiting for the full response. + +### How It Works + +The streaming endpoint calls `engine.stream_full()` directly, which yields `StreamChunk` objects containing content tokens, tool-call fragments, and finish reasons. This provides genuine token-by-token streaming from the LLM -- not a post-hoc word replay. + +For multi-turn tool-calling agents, the streaming loop automatically: + +1. Yields content tokens to the client as they arrive. +2. Accumulates tool-call fragments (OpenAI sends these incrementally). +3. Executes tools when `finish_reason="tool_calls"` is received. +4. Emits tool results as named SSE events (`event: tool_result`). +5. Feeds results back to the LLM for the next turn. +6. Repeats until the model produces a final text response or `max_turns` is reached. ### Streaming Messages @@ -639,19 +652,21 @@ curl -N -X POST http://localhost:8000/v1/managed-agents/{id}/messages \ The response follows the OpenAI SSE format: 1. **Content chunks** -- `data: {"choices": [{"delta": {"content": "token"}}]}` -2. **Tool results** (if the agent used tools) -- `event: tool_results\ndata: {"results": [...]}` -3. **Final chunk** -- `data: {"choices": [{"delta": {}, "finish_reason": "stop"}]}` -4. **Done sentinel** -- `data: [DONE]` +2. **Tool calls** (if the model requests tool use) -- `event: tool_calls\ndata: {"calls": [{"tool_name": "...", "arguments": "..."}]}` +3. **Tool results** -- `event: tool_result\ndata: {"tool_name": "...", "output": "..."}` +4. **Final chunk** -- `data: {"choices": [{"delta": {}, "finish_reason": "stop"}]}` +5. **Done sentinel** -- `data: [DONE]` When `stream: false` (the default), the endpoint behaves exactly as before -- the message is queued and the agent must be triggered separately via `/run`. ### Behavior Details -- The user message is always stored in the database before the agent runs. -- After streaming completes, the full agent response is persisted as an `agent_to_user` message. -- The agent is instantiated from the managed agent's stored `agent_type` and `config`. -- Conversation history from prior messages is automatically loaded as context. +- The user message is always stored in the database before streaming starts. +- After streaming completes, the full collected response is persisted as an `agent_to_user` message. +- Conversation history from prior messages is automatically loaded as LLM context. +- The engine's `stream_full()` method is used for real token streaming. Engines that do not override it fall back to the default implementation which wraps the plain `stream()` method. - If the engine is not available on the server, a `503` error is returned. +- Tool execution during streaming uses the `ToolRegistry` to find and instantiate tools. ### Python Example diff --git a/docs/user-guide/mcp-external-servers.md b/docs/user-guide/mcp-external-servers.md new file mode 100644 index 00000000..7fd316e0 --- /dev/null +++ b/docs/user-guide/mcp-external-servers.md @@ -0,0 +1,158 @@ +# External MCP Server Integration + +OpenJarvis can extend agent capabilities by connecting to external [Model Context Protocol (MCP)](https://modelcontextprotocol.io/) servers. This allows agents to use tools provided by services like Home Assistant, databases, custom APIs, or any MCP-compatible server -- without writing custom tool code. + +## How It Works + +When OpenJarvis starts, it reads the `[tools.mcp]` section in `config.toml`. For each configured server, it: + +1. Opens a connection using the appropriate transport (Streamable HTTP or stdio). +2. Performs the MCP initialize handshake (protocol version negotiation and `initialized` notification). +3. Discovers available tools via `tools/list`. +4. Wraps each discovered tool as a standard `BaseTool` so agents can call them like any built-in tool. + +If a server is unreachable or returns an error, OpenJarvis logs a warning and continues loading the remaining servers. One broken server does not prevent other tools from being available. + +## Configuration + +External MCP servers are configured in `config.toml` under `[tools.mcp]`: + +```toml +[tools.mcp] +enabled = true +servers = '[{"name": "homeassistant", "url": "http://172.16.3.1:9583/private_abc123"}]' +``` + +The `servers` value is a **JSON-encoded string** containing an array of server objects. Each object defines one external MCP server. + +!!! note + The value must be a JSON string (with single-quote TOML delimiters around it), not a native TOML array. This is because the configuration system passes it through as a single string field. + +## Server Config Schema + +Each server object supports the following fields: + +| Field | Type | Required | Description | +|------------------|----------------|----------|----------------------------------------------------------| +| `name` | string | No | Human-readable name used in log messages. Defaults to ``. | +| `url` | string | No* | URL for Streamable HTTP transport. | +| `command` | string | No* | Command to launch a stdio-based MCP server. | +| `args` | list of strings| No | Arguments passed to the stdio command. | +| `include_tools` | list of strings| No | Whitelist of tool names to import. Only these tools are loaded. | +| `exclude_tools` | list of strings| No | Blacklist of tool names to skip. All other tools are loaded. | + +*Either `url` or `command` must be provided. If neither is set, the server is skipped with a warning. + +When both `include_tools` and `exclude_tools` are specified, the whitelist is applied first, then the blacklist filters the result. + +## Examples + +### Home Assistant via Streamable HTTP + +Connect to the [ha-mcp](https://github.com/tevonsb/ha-mcp) Home Assistant add-on: + +```toml +[tools.mcp] +enabled = true +servers = '[{"name": "homeassistant", "url": "http://172.16.3.1:9583/private_abc123"}]' +``` + +This discovers all HA tools (entity control, automations, history, etc.) and makes them available to agents. + +### Stdio Server + +Launch a local MCP server as a subprocess: + +```toml +[tools.mcp] +enabled = true +servers = '[{"name": "myserver", "command": "python", "args": ["-m", "my_mcp_server"]}]' +``` + +OpenJarvis starts the process automatically, communicates via JSON-RPC over stdin/stdout, and terminates it on shutdown. + +### Multiple Servers + +```toml +[tools.mcp] +enabled = true +servers = '[{"name": "homeassistant", "url": "http://172.16.3.1:9583/private_abc123"}, {"name": "database", "command": "db-mcp-server", "args": ["--db", "postgres://localhost/mydb"]}]' +``` + +### Tool Filtering + +When a server exposes many tools but you only need a few, use `include_tools` to whitelist: + +```toml +[tools.mcp] +enabled = true +servers = '[{"name": "ha", "url": "http://172.16.3.1:9583/private_abc123", "include_tools": ["hassTurnOn", "hassTurnOff", "hassGetState"]}]' +``` + +To load everything except specific tools, use `exclude_tools`: + +```toml +[tools.mcp] +enabled = true +servers = '[{"name": "ha", "url": "http://172.16.3.1:9583/private_abc123", "exclude_tools": ["hassCreateBackup", "hassDeleteBackup"]}]' +``` + +## Transport Types + +### Streamable HTTP + +Used when the `url` field is set. The transport sends JSON-RPC requests as HTTP POST to the given URL using `httpx`. It tracks the `Mcp-Session-Id` header across requests as required by the MCP Streamable HTTP specification. + +**When to use:** Remote MCP servers, services running as HTTP endpoints (e.g., Home Assistant MCP add-on, cloud-hosted MCP servers). + +**Connection parameters:** + +- Connect timeout: 10 seconds +- Request timeout: 60 seconds + +### Stdio + +Used when the `command` field is set. OpenJarvis spawns the command as a subprocess and communicates via JSON-RPC lines on stdin/stdout. + +**When to use:** Local MCP servers distributed as CLI tools, development/testing, servers that require filesystem access on the same machine. + +!!! info "SSETransport alias" + `SSETransport` is provided as a backward-compatible alias for `StreamableHTTPTransport`. Both refer to the same implementation. + +## Error Handling + +OpenJarvis handles MCP server failures gracefully: + +- **Server unreachable:** A warning is logged and the server is skipped. All other servers and built-in tools continue to load normally. +- **Timeout:** HTTP requests time out after 60 seconds. The server is skipped with a warning. +- **Invalid config:** If the `servers` JSON is malformed or a server entry has neither `url` nor `command`, a warning is logged and that entry is skipped. +- **Tool discovery failure:** If `tools/list` fails on a server, the error is caught and the server is skipped. +- **Runtime tool call failure:** If a tool call to an external MCP server fails at runtime, it returns a `ToolResult` with `success=False` and the error message. + +No single server failure causes OpenJarvis to crash or prevents other tools from working. + +## Troubleshooting + +### Server not discovered + +1. Check that `[tools.mcp]` has `enabled = true`. +2. Verify the `servers` JSON is valid. A common mistake is using TOML arrays instead of a JSON string. +3. Check the OpenJarvis logs for warnings like `Failed to discover external MCP tools`. + +### Connection refused / timeout + +1. Verify the server is running and reachable from the OpenJarvis host: `curl -v http://host:port/`. +2. Check firewall rules between the OpenJarvis container and the MCP server. +3. For Docker deployments, ensure both containers are on the same network or use host IPs. + +### Tools not appearing + +1. Run with debug logging to see which tools were discovered. +2. Check if `include_tools` or `exclude_tools` filters are too restrictive. +3. Verify the MCP server actually exposes tools via `tools/list` (some servers only expose resources or prompts). + +### Stdio server crashes immediately + +1. Test the command manually: `python -m my_mcp_server` should start and wait for input on stdin. +2. Check stderr output in the OpenJarvis logs for error messages from the subprocess. +3. Ensure all dependencies for the MCP server are installed in the same environment. diff --git a/docs/user-guide/tools.md b/docs/user-guide/tools.md index 0e3876fd..f22bcd23 100644 --- a/docs/user-guide/tools.md +++ b/docs/user-guide/tools.md @@ -204,7 +204,7 @@ All built-in tools are registered via `@ToolRegistry.register()` and are availab | **Scheduler** | `pause_scheduled_task` | Pause an active scheduled task | | **Scheduler** | `resume_scheduled_task` | Resume a paused scheduled task | | **Scheduler** | `cancel_scheduled_task` | Cancel a scheduled task permanently | -| **Integration** | `mcp_adapter` | Bridge to external MCP tool servers | +| **Integration** | `mcp_adapter` | Bridge to external MCP tool servers (see [External MCP Servers](mcp-external-servers.md)) | --- diff --git a/mkdocs.yml b/mkdocs.yml index c8b179f9..a90a5e9c 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -164,5 +164,8 @@ nav: - Security: architecture/security.md - Design Principles: architecture/design-principles.md - API Reference: api-reference/ + - User Guide: + - Tools: user-guide/tools.md + - External MCP Servers: user-guide/mcp-external-servers.md - Leaderboard: leaderboard.md - Roadmap: development/roadmap.md diff --git a/src/openjarvis/engine/_openai_compat.py b/src/openjarvis/engine/_openai_compat.py index 1fc71e42..b9751452 100644 --- a/src/openjarvis/engine/_openai_compat.py +++ b/src/openjarvis/engine/_openai_compat.py @@ -16,6 +16,7 @@ from openjarvis.engine._base import ( estimate_prompt_tokens, messages_to_dicts, ) +from openjarvis.engine._stubs import StreamChunk logger = logging.getLogger(__name__) @@ -50,6 +51,9 @@ class _OpenAICompatibleEngine(InferenceEngine): "stream": False, **kwargs, } + # Default to tool_choice=auto when tools are provided + if "tools" in payload and "tool_choice" not in payload: + payload["tool_choice"] = "auto" try: url = f"{self._api_prefix}/chat/completions" resp = self._client.post(url, json=payload) @@ -122,6 +126,9 @@ class _OpenAICompatibleEngine(InferenceEngine): "stream": True, **kwargs, } + # Default to tool_choice=auto when tools are provided + if "tools" in payload and "tool_choice" not in payload: + payload["tool_choice"] = "auto" try: url = f"{self._api_prefix}/chat/completions" with self._client.stream("POST", url, json=payload) as resp: @@ -129,7 +136,7 @@ class _OpenAICompatibleEngine(InferenceEngine): for line in resp.iter_lines(): if not line.startswith("data:"): continue - data_str = line[len("data:"):].strip() + data_str = line[len("data:") :].strip() if data_str == "[DONE]": break try: @@ -145,16 +152,74 @@ class _OpenAICompatibleEngine(InferenceEngine): f"{self.engine_id} engine not reachable at {self._host}" ) from exc + async def stream_full( + self, + messages: Sequence[Message], + *, + model: str, + temperature: float = 0.7, + max_tokens: int = 1024, + **kwargs: Any, + ) -> AsyncIterator["StreamChunk"]: + """Yield StreamChunks with content, tool_calls, and finish_reason.""" + msg_dicts = messages_to_dicts(messages) + payload: Dict[str, Any] = { + "model": model, + "messages": msg_dicts, + "temperature": temperature, + "max_tokens": max_tokens, + "stream": True, + **kwargs, + } + if "tools" in payload and "tool_choice" not in payload: + payload["tool_choice"] = "auto" + try: + url = f"{self._api_prefix}/chat/completions" + with self._client.stream("POST", url, json=payload) as resp: + resp.raise_for_status() + for line in resp.iter_lines(): + if not line.startswith("data:"): + continue + data_str = line[len("data:") :].strip() + if data_str == "[DONE]": + break + try: + chunk = json.loads(data_str) + except json.JSONDecodeError: + continue + choice = chunk.get("choices", [{}])[0] + delta = choice.get("delta", {}) + finish = choice.get("finish_reason") + content = delta.get("content") + tool_calls = delta.get("tool_calls") + usage = chunk.get("usage") + + if content or tool_calls or finish or usage: + yield StreamChunk( + content=content, + tool_calls=tool_calls, + finish_reason=finish, + usage=usage, + ) + except (httpx.ConnectError, httpx.TimeoutException) as exc: + raise EngineConnectionError( + f"{self.engine_id} engine not reachable at {self._host}" + ) from exc + def list_models(self) -> List[str]: try: resp = self._client.get(f"{self._api_prefix}/models") resp.raise_for_status() except ( - httpx.ConnectError, httpx.TimeoutException, httpx.HTTPStatusError, + httpx.ConnectError, + httpx.TimeoutException, + httpx.HTTPStatusError, ) as exc: logger.warning( "Failed to list models from %s at %s: %s", - self.engine_id, self._host, exc, + self.engine_id, + self._host, + exc, ) return [] data = resp.json() @@ -167,7 +232,9 @@ class _OpenAICompatibleEngine(InferenceEngine): except Exception as exc: logger.debug( "%s health check failed at %s: %s", - self.engine_id, self._host, exc, + self.engine_id, + self._host, + exc, ) return False diff --git a/src/openjarvis/engine/_stubs.py b/src/openjarvis/engine/_stubs.py index fb7feb64..04604c0e 100644 --- a/src/openjarvis/engine/_stubs.py +++ b/src/openjarvis/engine/_stubs.py @@ -14,6 +14,21 @@ from typing import Any, Dict, List, Optional, Sequence from openjarvis.core.types import Message +@dataclass(slots=True) +class StreamChunk: + """A single chunk from a streaming LLM response. + + Used by ``stream_full()`` to yield rich streaming data including + tool_calls fragments and finish_reason, unlike ``stream()`` which + only yields plain content strings. + """ + + content: Optional[str] = None + tool_calls: Optional[List[Dict[str, Any]]] = None + finish_reason: Optional[str] = None + usage: Optional[Dict[str, Any]] = None + + @dataclass(slots=True) class ResponseFormat: """Structured output configuration for inference engines. @@ -65,6 +80,30 @@ class InferenceEngine(ABC): # NOTE: must contain a yield to satisfy the type checker yield "" # pragma: no cover + async def stream_full( + self, + messages: Sequence[Message], + *, + model: str, + temperature: float = 0.7, + max_tokens: int = 1024, + **kwargs: Any, + ) -> AsyncIterator["StreamChunk"]: + """Yield full StreamChunks including tool_calls and finish_reason. + + Default implementation wraps ``stream()`` for backward compatibility. + Engines with native tool-call streaming should override this. + """ + async for token in self.stream( + messages, + model=model, + temperature=temperature, + max_tokens=max_tokens, + **kwargs, + ): + yield StreamChunk(content=token) + yield StreamChunk(finish_reason="stop") + @abstractmethod def list_models(self) -> List[str]: """Return identifiers of models available on this engine.""" @@ -80,4 +119,4 @@ class InferenceEngine(ABC): """Optional warm-up hook called before the first request.""" -__all__ = ["InferenceEngine", "ResponseFormat"] +__all__ = ["InferenceEngine", "ResponseFormat", "StreamChunk"] diff --git a/src/openjarvis/engine/cloud.py b/src/openjarvis/engine/cloud.py index dc9a99cd..d58b131f 100644 --- a/src/openjarvis/engine/cloud.py +++ b/src/openjarvis/engine/cloud.py @@ -6,7 +6,7 @@ import json import os import time from collections.abc import AsyncIterator, Sequence -from typing import Any, Dict, List +from typing import Any, Dict, List, Tuple import httpx @@ -17,6 +17,7 @@ from openjarvis.engine._base import ( InferenceEngine, messages_to_dicts, ) +from openjarvis.engine._stubs import StreamChunk # Pricing per million tokens (input, output) PRICING: Dict[str, tuple[float, float]] = { @@ -287,6 +288,61 @@ class CloudEngine(InferenceEngine): "url": codex_url, } + def _prepare_anthropic_messages( + self, + messages: Sequence[Message], + ) -> Tuple[str, List[Dict[str, Any]]]: + """Extract system text and convert messages to Anthropic format.""" + system_text = "" + chat_msgs: List[Dict[str, Any]] = [] + for m in messages: + if m.role.value == "system": + system_text = m.content + elif m.role.value == "tool": + tool_result_block = { + "type": "tool_result", + "tool_use_id": m.tool_call_id or "", + "content": m.content, + } + if ( + chat_msgs + and chat_msgs[-1]["role"] == "user" + and isinstance(chat_msgs[-1]["content"], list) + and chat_msgs[-1]["content"] + and chat_msgs[-1]["content"][-1].get("type") == "tool_result" + ): + chat_msgs[-1]["content"].append(tool_result_block) + else: + chat_msgs.append( + { + "role": "user", + "content": [tool_result_block], + } + ) + elif m.role.value == "assistant" and m.tool_calls: + content_blocks: List[Dict[str, Any]] = [] + if m.content: + content_blocks.append({"type": "text", "text": m.content}) + for tc in m.tool_calls: + args = tc.arguments + if isinstance(args, str): + try: + args = json.loads(args) + except (json.JSONDecodeError, TypeError): + args = {"input": args} + content_blocks.append( + { + "type": "tool_use", + "id": tc.id, + "name": tc.name, + "input": args if isinstance(args, dict) else {}, + } + ) + chat_msgs.append({"role": "assistant", "content": content_blocks}) + else: + chat_msgs.append({"role": m.role.value, "content": m.content}) + return system_text, chat_msgs + @staticmethod def _codex_build_input( messages: Sequence[Message], @@ -477,60 +533,7 @@ class CloudEngine(InferenceEngine): "ANTHROPIC_API_KEY and install " "openjarvis[inference-cloud]" ) - # Separate system message and convert to Anthropic message format - system_text = "" - chat_msgs: List[Dict[str, Any]] = [] - for m in messages: - if m.role.value == "system": - system_text = m.content - elif m.role.value == "tool": - # Anthropic expects tool results as role="user" with - # tool_result content blocks - tool_result_block = { - "type": "tool_result", - "tool_use_id": m.tool_call_id or "", - "content": m.content, - } - # Merge consecutive tool results into a single user message - if ( - chat_msgs - and chat_msgs[-1]["role"] == "user" - and isinstance(chat_msgs[-1]["content"], list) - and chat_msgs[-1]["content"] - and chat_msgs[-1]["content"][-1].get("type") == "tool_result" - ): - chat_msgs[-1]["content"].append(tool_result_block) - else: - chat_msgs.append( - { - "role": "user", - "content": [tool_result_block], - } - ) - elif m.role.value == "assistant" and m.tool_calls: - # Convert assistant messages with tool_calls to Anthropic - # content blocks (text + tool_use) - content_blocks: List[Dict[str, Any]] = [] - if m.content: - content_blocks.append({"type": "text", "text": m.content}) - for tc in m.tool_calls: - args = tc.arguments - if isinstance(args, str): - try: - args = json.loads(args) - except (json.JSONDecodeError, TypeError): - args = {"input": args} - content_blocks.append( - { - "type": "tool_use", - "id": tc.id, - "name": tc.name, - "input": args if isinstance(args, dict) else {}, - } - ) - chat_msgs.append({"role": "assistant", "content": content_blocks}) - else: - chat_msgs.append({"role": m.role.value, "content": m.content}) + system_text, chat_msgs = self._prepare_anthropic_messages(messages) create_kwargs: Dict[str, Any] = { "model": model, "messages": chat_msgs, @@ -1121,6 +1124,188 @@ class CloudEngine(InferenceEngine): if delta and delta.content: yield delta.content + # -- stream_full: rich streaming with tool_calls support ---------------- + + async def _stream_full_openai( + self, + messages: Sequence[Message], + *, + model: str, + temperature: float, + max_tokens: int, + **kwargs: Any, + ) -> AsyncIterator[StreamChunk]: + """Yield StreamChunks from an OpenAI-compatible streaming response. + + Works for OpenAI, OpenRouter, MiniMax, and Codex. + """ + if _is_codex_model(model): + # Codex uses Responses API — fall back to base stream_full wrapper + async for chunk in super().stream_full( + messages, + model=model, + temperature=temperature, + max_tokens=max_tokens, + **kwargs, + ): + yield chunk + return + if _is_openrouter_model(model): + client = self._openrouter_client + if client is None: + raise EngineConnectionError("OpenRouter client not available") + actual_model = model.removeprefix("openrouter/") + create_kwargs: Dict[str, Any] = { + "model": actual_model, + "messages": messages_to_dicts(messages), + "max_tokens": max_tokens, + "temperature": temperature, + "stream": True, + **kwargs, + } + elif _is_minimax_model(model): + client = self._minimax_client + if client is None: + raise EngineConnectionError("MiniMax client not available") + temperature = max(temperature, 0.01) + temperature = min(temperature, 1.0) + create_kwargs = { + "model": model, + "messages": messages_to_dicts(messages), + "max_tokens": max_tokens, + "temperature": temperature, + "stream": True, + **kwargs, + } + else: + client = self._openai_client + if client is None: + raise EngineConnectionError("OpenAI client not available") + create_kwargs = { + "model": model, + "messages": messages_to_dicts(messages), + "max_completion_tokens": max_tokens, + "stream": True, + **kwargs, + } + if not _is_openai_reasoning_model(model): + create_kwargs["temperature"] = temperature + resp = client.chat.completions.create(**create_kwargs) + for chunk in resp: + choice = chunk.choices[0] if chunk.choices else None + if not choice: + continue + delta = choice.delta + content = delta.content if delta else None + tool_calls = None + if delta and delta.tool_calls: + tool_calls = [ + { + "index": tc.index, + "id": tc.id or "", + "function": { + "name": (tc.function.name or "") if tc.function else "", + "arguments": ( + (tc.function.arguments or "") if tc.function else "" + ), + }, + } + for tc in delta.tool_calls + ] + finish = choice.finish_reason + if content or tool_calls or finish: + yield StreamChunk( + content=content, + tool_calls=tool_calls, + finish_reason=finish, + ) + + async def _stream_full_anthropic( + self, + messages: Sequence[Message], + *, + model: str, + temperature: float, + max_tokens: int, + **kwargs: Any, + ) -> AsyncIterator[StreamChunk]: + """Yield StreamChunks from an Anthropic streaming response.""" + if self._anthropic_client is None: + raise EngineConnectionError("Anthropic client not available") + system_text, chat_msgs = self._prepare_anthropic_messages(messages) + create_kwargs: Dict[str, Any] = { + "model": model, + "messages": chat_msgs, + "temperature": temperature, + "max_tokens": max_tokens, + } + if system_text: + create_kwargs["system"] = system_text + raw_tools = kwargs.pop("tools", None) + if raw_tools: + create_kwargs["tools"] = _convert_tools_to_anthropic(raw_tools) + kwargs.pop("tool_choice", None) + + with self._anthropic_client.messages.stream(**create_kwargs) as stream: + tool_index = -1 + for event in stream: + if event.type == "content_block_start": + block = event.content_block + if block.type == "tool_use": + tool_index += 1 + yield StreamChunk( + tool_calls=[ + { + "index": tool_index, + "id": block.id, + "function": {"name": block.name, "arguments": ""}, + } + ] + ) + elif event.type == "content_block_delta": + delta = event.delta + if delta.type == "text_delta": + yield StreamChunk(content=delta.text) + elif delta.type == "input_json_delta": + yield StreamChunk( + tool_calls=[ + { + "index": tool_index, + "function": {"arguments": delta.partial_json}, + } + ] + ) + elif event.type == "message_delta": + stop_reason = event.delta.stop_reason + finish = "tool_calls" if stop_reason == "tool_use" else "stop" + yield StreamChunk(finish_reason=finish) + + async def stream_full( + self, + messages: Sequence[Message], + *, + model: str, + temperature: float = 0.7, + max_tokens: int = 1024, + **kwargs: Any, + ) -> AsyncIterator[StreamChunk]: + """Yield StreamChunks with content, tool_calls, and finish_reason.""" + kw = dict( + model=model, + temperature=temperature, + max_tokens=max_tokens, + **kwargs, + ) + if _is_anthropic_model(model): + async for chunk in self._stream_full_anthropic(messages, **kw): + yield chunk + elif _is_google_model(model): + async for chunk in super().stream_full(messages, **kw): + yield chunk + else: + async for chunk in self._stream_full_openai(messages, **kw): + yield chunk + def list_models(self) -> List[str]: models: List[str] = [] if self._openai_client is not None: diff --git a/src/openjarvis/engine/multi.py b/src/openjarvis/engine/multi.py index f3b9a42a..7fa48f14 100644 --- a/src/openjarvis/engine/multi.py +++ b/src/openjarvis/engine/multi.py @@ -6,7 +6,9 @@ import logging from collections.abc import AsyncIterator, Sequence from typing import Any, Dict, List +from openjarvis.core.types import Message from openjarvis.engine._base import InferenceEngine +from openjarvis.engine._stubs import StreamChunk logger = logging.getLogger(__name__) @@ -65,7 +67,7 @@ class MultiEngine(InferenceEngine): def generate( self, - messages: Sequence[Any], + messages: Sequence[Message], *, model: str, temperature: float = 0.7, @@ -73,13 +75,16 @@ class MultiEngine(InferenceEngine): **kwargs: Any, ) -> Dict[str, Any]: return self._engine_for(model).generate( - messages, model=model, temperature=temperature, - max_tokens=max_tokens, **kwargs, + messages, + model=model, + temperature=temperature, + max_tokens=max_tokens, + **kwargs, ) async def stream( self, - messages: Sequence[Any], + messages: Sequence[Message], *, model: str, temperature: float = 0.7, @@ -87,11 +92,26 @@ class MultiEngine(InferenceEngine): **kwargs: Any, ) -> AsyncIterator[str]: async for token in self._engine_for(model).stream( - messages, model=model, temperature=temperature, - max_tokens=max_tokens, **kwargs, + messages, + model=model, + temperature=temperature, + max_tokens=max_tokens, + **kwargs, ): yield token + async def stream_full( + self, + messages: Sequence[Message], + *, + model: str, + **kwargs: Any, + ) -> AsyncIterator["StreamChunk"]: + """Delegate stream_full() to the engine that owns the model.""" + engine = self._engine_for(model) + async for chunk in engine.stream_full(messages, model=model, **kwargs): + yield chunk + def list_models(self) -> List[str]: self._refresh_map() return list(self._model_map.keys()) diff --git a/src/openjarvis/mcp/__init__.py b/src/openjarvis/mcp/__init__.py index cad02d97..6fe21da9 100644 --- a/src/openjarvis/mcp/__init__.py +++ b/src/openjarvis/mcp/__init__.py @@ -8,6 +8,7 @@ from openjarvis.mcp.transport import ( MCPTransport, SSETransport, StdioTransport, + StreamableHTTPTransport, ) __all__ = [ @@ -21,4 +22,5 @@ __all__ = [ "InProcessTransport", "SSETransport", "StdioTransport", + "StreamableHTTPTransport", ] diff --git a/src/openjarvis/mcp/client.py b/src/openjarvis/mcp/client.py index 95cd7e42..c37a04a0 100644 --- a/src/openjarvis/mcp/client.py +++ b/src/openjarvis/mcp/client.py @@ -47,13 +47,36 @@ class MCPClient: def initialize(self) -> Dict[str, Any]: """Perform the MCP initialize handshake. + Sends the required client info and protocol version, then + confirms with a ``notifications/initialized`` notification + as required by the MCP specification. + Returns the server capabilities. """ - response = self._send("initialize") + params = { + "protocolVersion": "2025-03-26", + "capabilities": {}, + "clientInfo": {"name": "openjarvis", "version": "0.1.0"}, + } + response = self._send("initialize", params) self._initialized = True self._capabilities = response.result.get("capabilities", {}) + # Send the required initialized notification per MCP spec + self.notify("notifications/initialized") return response.result + def notify(self, method: str, params: Dict[str, Any] | None = None) -> None: + """Send a JSON-RPC notification (no response expected). + + Per JSON-RPC 2.0 spec, notifications omit the ``id`` field entirely. + """ + request = MCPRequest( + method=method, + params=params or {}, + id=None, # None → no id field in JSON (notification) + ) + self._transport.send_notification(request) + def list_tools(self) -> List[ToolSpec]: """Discover available tools from the server. @@ -71,7 +94,9 @@ class MCPClient: ] def call_tool( - self, name: str, arguments: Dict[str, Any] | None = None, + self, + name: str, + arguments: Dict[str, Any] | None = None, ) -> Dict[str, Any]: """Call a tool on the server. @@ -87,5 +112,11 @@ class MCPClient: """Close the transport connection.""" self._transport.close() + def __enter__(self) -> MCPClient: + return self + + def __exit__(self, *exc: Any) -> None: + self.close() + __all__ = ["MCPClient"] diff --git a/src/openjarvis/mcp/protocol.py b/src/openjarvis/mcp/protocol.py index 1b97a9ad..fe8dd99f 100644 --- a/src/openjarvis/mcp/protocol.py +++ b/src/openjarvis/mcp/protocol.py @@ -16,23 +16,34 @@ INTERNAL_ERROR = -32603 @dataclass class MCPRequest: - """JSON-RPC 2.0 request message.""" + """JSON-RPC 2.0 request message. + + Set *id* to ``None`` to create a JSON-RPC **notification** (no ``id`` + field will appear in the serialized output, and no response is expected). + """ method: str params: Dict[str, Any] = field(default_factory=dict) - id: int | str = 0 + id: Optional[int | str] = 0 jsonrpc: str = "2.0" + def to_dict(self) -> Dict[str, Any]: + """Return a dict suitable for JSON serialization. + + Omits the ``id`` key when it is ``None`` (notification). + """ + obj: Dict[str, Any] = { + "jsonrpc": self.jsonrpc, + "method": self.method, + "params": self.params, + } + if self.id is not None: + obj["id"] = self.id + return obj + def to_json(self) -> str: """Serialize to JSON string.""" - return json.dumps( - { - "jsonrpc": self.jsonrpc, - "id": self.id, - "method": self.method, - "params": self.params, - } - ) + return json.dumps(self.to_dict()) @classmethod def from_json(cls, data: str) -> MCPRequest: diff --git a/src/openjarvis/mcp/server.py b/src/openjarvis/mcp/server.py index 1988a9f4..25500372 100644 --- a/src/openjarvis/mcp/server.py +++ b/src/openjarvis/mcp/server.py @@ -70,31 +70,37 @@ class MCPServer: # Built-in API tools try: from openjarvis.tools.calculator import CalculatorTool + _tool_classes.append(CalculatorTool) except ImportError: pass try: from openjarvis.tools.think import ThinkTool + _tool_classes.append(ThinkTool) except ImportError: pass try: from openjarvis.tools.file_read import FileReadTool + _tool_classes.append(FileReadTool) except ImportError: pass try: from openjarvis.tools.web_search import WebSearchTool + _tool_classes.append(WebSearchTool) except ImportError: pass try: from openjarvis.tools.code_interpreter import CodeInterpreterTool + _tool_classes.append(CodeInterpreterTool) except ImportError: pass try: from openjarvis.tools.repl import ReplTool + _tool_classes.append(ReplTool) except ImportError: pass @@ -107,10 +113,15 @@ class MCPServer: MemorySearchTool, MemoryStoreTool, ) - _tool_classes.extend([ - MemoryStoreTool, MemoryRetrieveTool, - MemorySearchTool, MemoryIndexTool, - ]) + + _tool_classes.extend( + [ + MemoryStoreTool, + MemoryRetrieveTool, + MemorySearchTool, + MemoryIndexTool, + ] + ) except ImportError: pass @@ -121,15 +132,21 @@ class MCPServer: ChannelSendTool, ChannelStatusTool, ) - _tool_classes.extend([ - ChannelSendTool, ChannelListTool, ChannelStatusTool, - ]) + + _tool_classes.extend( + [ + ChannelSendTool, + ChannelListTool, + ChannelStatusTool, + ] + ) except ImportError: pass # LM tool (needs engine/model — instantiate with None) try: from openjarvis.tools.llm_tool import LLMTool + _tool_classes.append(LLMTool) except ImportError: pass @@ -137,6 +154,7 @@ class MCPServer: # Retrieval tool (needs backend — instantiate with None) try: from openjarvis.tools.retrieval import RetrievalTool + _tool_classes.append(RetrievalTool) except ImportError: pass @@ -150,6 +168,7 @@ class MCPServer: # Also check ToolRegistry for any user-registered tools try: from openjarvis.core.registry import ToolRegistry + known_names = {t.spec.name for t in tools} for key in ToolRegistry.keys(): if key not in known_names: diff --git a/src/openjarvis/mcp/transport.py b/src/openjarvis/mcp/transport.py index 52c7dab3..2c40bacc 100644 --- a/src/openjarvis/mcp/transport.py +++ b/src/openjarvis/mcp/transport.py @@ -2,10 +2,9 @@ from __future__ import annotations -import json import subprocess from abc import ABC, abstractmethod -from typing import TYPE_CHECKING, List, Optional +from typing import TYPE_CHECKING, Any, List, Optional from openjarvis.mcp.protocol import MCPRequest, MCPResponse @@ -20,6 +19,15 @@ class MCPTransport(ABC): def send(self, request: MCPRequest) -> MCPResponse: """Send a request and return the response.""" + def send_notification(self, request: MCPRequest) -> None: + """Send a JSON-RPC notification (no response expected). + + The default implementation delegates to :meth:`send` and discards the + response. Transports may override this when the server returns no + body for notifications (e.g. HTTP 202 Accepted). + """ + self.send(request) + @abstractmethod def close(self) -> None: """Release transport resources.""" @@ -88,30 +96,133 @@ class StdioTransport(MCPTransport): self._process = None -class SSETransport(MCPTransport): - """JSON-RPC over HTTP with Server-Sent Events. +class StreamableHTTPTransport(MCPTransport): + """MCP Streamable HTTP transport (JSON-RPC over HTTP). - Sends requests via HTTP POST and reads SSE responses. + Uses a persistent ``httpx.Client`` session, tracks the + ``Mcp-Session-Id`` header, and sends the ``Accept`` header + required by the MCP Streamable HTTP specification. """ - def __init__(self, url: str) -> None: - self._url = url - - def send(self, request: MCPRequest) -> MCPResponse: - """Send request via HTTP POST.""" + def __init__( + self, + url: str, + *, + connect_timeout: float = 10.0, + request_timeout: float = 60.0, + ) -> None: import httpx - response = httpx.post( - self._url, - json=json.loads(request.to_json()), - headers={"Content-Type": "application/json"}, - timeout=30.0, + self._url = url + self._session_id: Optional[str] = None + self._client = httpx.Client( + timeout=httpx.Timeout( + connect=connect_timeout, + read=request_timeout, + write=request_timeout, + pool=connect_timeout, + ), ) - response.raise_for_status() - return MCPResponse.from_json(response.text) + + def _safe_url(self) -> str: + """Return scheme://host:port without path or query (avoids leaking tokens).""" + from urllib.parse import urlparse + + parsed = urlparse(self._url) + return f"{parsed.scheme}://{parsed.netloc}" + + def _build_headers(self) -> dict: + """Build common request headers.""" + headers = { + "Content-Type": "application/json", + "Accept": "application/json, text/event-stream", + } + if self._session_id is not None: + headers["Mcp-Session-Id"] = self._session_id + return headers + + def _post(self, request: MCPRequest) -> Any: + """Post a request and return the raw httpx response.""" + import httpx + + headers = self._build_headers() + try: + response = self._client.post( + self._url, + json=request.to_dict(), + headers=headers, + ) + response.raise_for_status() + except httpx.ConnectError as exc: + raise RuntimeError( + f"Failed to connect to MCP server at {self._safe_url()}: {exc}" + ) from exc + except httpx.TimeoutException as exc: + raise RuntimeError( + f"Timeout communicating with MCP server at {self._safe_url()}: {exc}" + ) from exc + except httpx.HTTPStatusError as exc: + raise RuntimeError( + f"MCP server at {self._safe_url()} returned HTTP " + f"{exc.response.status_code}" + ) from exc + + # Track session id from the first response + new_session_id = response.headers.get("mcp-session-id") + if new_session_id is not None: + self._session_id = new_session_id + return response + + @staticmethod + def _extract_json_from_sse(text: str) -> str: + """Extract JSON payload from an SSE response body. + + MCP Streamable HTTP servers may respond with ``text/event-stream`` + instead of ``application/json``. In that case the body looks like:: + + event: message + data: {"jsonrpc":"2.0", ...} + + This helper finds the last ``data:`` line and returns its content, + which is the actual JSON-RPC response. + """ + last_data = "" + for line in text.splitlines(): + if line.startswith("data:"): + last_data = line[len("data:") :].strip() + if not last_data: + raise RuntimeError( + "SSE response contained no 'data:' lines" + " — cannot extract JSON-RPC payload" + ) + return last_data + + def send(self, request: MCPRequest) -> MCPResponse: + """Send request via HTTP POST following the MCP Streamable HTTP spec. + + Handles both ``application/json`` and ``text/event-stream`` responses + as allowed by the MCP Streamable HTTP specification. + """ + response = self._post(request) + content_type = response.headers.get("content-type", "") + body = response.text + if "text/event-stream" in content_type or body.lstrip().startswith("event:"): + body = self._extract_json_from_sse(body) + return MCPResponse.from_json(body) + + def send_notification(self, request: MCPRequest) -> None: + """Send a notification — accept any 2xx, don't parse the body.""" + # Track session id but don't try to parse a JSON-RPC response. + # Servers may return 202 Accepted with an empty body. + self._post(request) def close(self) -> None: - """No persistent connection to close.""" + """Close the underlying httpx client.""" + self._client.close() + + +# Backward-compatible alias +SSETransport = StreamableHTTPTransport __all__ = [ @@ -119,4 +230,5 @@ __all__ = [ "MCPTransport", "SSETransport", "StdioTransport", + "StreamableHTTPTransport", ] diff --git a/src/openjarvis/security/guardrails.py b/src/openjarvis/security/guardrails.py index aa7f4dc1..5d16883e 100644 --- a/src/openjarvis/security/guardrails.py +++ b/src/openjarvis/security/guardrails.py @@ -7,7 +7,7 @@ from typing import Any, Dict, List, Optional, Sequence from openjarvis.core.events import EventBus, EventType from openjarvis.core.types import Message -from openjarvis.engine._stubs import InferenceEngine +from openjarvis.engine._stubs import InferenceEngine, StreamChunk from openjarvis.security._stubs import BaseScanner from openjarvis.security.scanner import PIIScanner, SecretScanner from openjarvis.security.types import RedactionMode, ScanResult @@ -50,10 +50,14 @@ class GuardrailsEngine(InferenceEngine): bus: Optional[EventBus] = None, ) -> None: self._engine = engine - self._scanners: List[BaseScanner] = scanners if scanners is not None else [ - SecretScanner(), - PIIScanner(), - ] + self._scanners: List[BaseScanner] = ( + scanners + if scanners is not None + else [ + SecretScanner(), + PIIScanner(), + ] + ) self._mode = mode self._scan_input = scan_input self._scan_output = scan_output @@ -180,7 +184,9 @@ class GuardrailsEngine(InferenceEngine): processed[i] = Message( role=msg.role, content=self._handle_findings( - msg.content, result, "input", + msg.content, + result, + "input", ), name=msg.name, tool_calls=msg.tool_calls, @@ -191,8 +197,11 @@ class GuardrailsEngine(InferenceEngine): # Call wrapped engine response = self._engine.generate( - messages, model=model, temperature=temperature, - max_tokens=max_tokens, **kwargs, + messages, + model=model, + temperature=temperature, + max_tokens=max_tokens, + **kwargs, ) # Scan output @@ -219,8 +228,11 @@ class GuardrailsEngine(InferenceEngine): """Yield tokens in real-time, scan accumulated output post-hoc.""" accumulated = [] async for token in self._engine.stream( - messages, model=model, temperature=temperature, - max_tokens=max_tokens, **kwargs, + messages, + model=model, + temperature=temperature, + max_tokens=max_tokens, + **kwargs, ): accumulated.append(token) yield token @@ -248,6 +260,51 @@ class GuardrailsEngine(InferenceEngine): }, ) + async def stream_full( + self, + messages: Sequence[Message], + *, + model: str, + temperature: float = 0.7, + max_tokens: int = 1024, + **kwargs: Any, + ) -> AsyncIterator["StreamChunk"]: + """Delegate to wrapped engine, scan accumulated output post-hoc.""" + accumulated: list[str] = [] + async for chunk in self._engine.stream_full( + messages, + model=model, + temperature=temperature, + max_tokens=max_tokens, + **kwargs, + ): + if chunk.content: + accumulated.append(chunk.content) + yield chunk + + # Post-hoc scan of accumulated output + if self._scan_output: + full_output = "".join(accumulated) + if full_output: + result = self._scan_text(full_output) + if not result.clean and self._bus: + finding_dicts = [ + { + "pattern": f.pattern_name, + "threat": f.threat_level.value, + "description": f.description, + } + for f in result.findings + ] + self._bus.publish( + EventType.SECURITY_ALERT, + { + "direction": "output", + "findings": finding_dicts, + "mode": "stream_full_post_hoc", + }, + ) + def list_models(self) -> List[str]: """Delegate to wrapped engine.""" return self._engine.list_models() diff --git a/src/openjarvis/server/agent_manager_routes.py b/src/openjarvis/server/agent_manager_routes.py index d10da4a0..11c5404b 100644 --- a/src/openjarvis/server/agent_manager_routes.py +++ b/src/openjarvis/server/agent_manager_routes.py @@ -59,8 +59,12 @@ class FeedbackRequest(BaseModel): _BROWSER_SUB_TOOLS = { - "browser_navigate", "browser_click", "browser_type", - "browser_screenshot", "browser_extract", "browser_axtree", + "browser_navigate", + "browser_click", + "browser_type", + "browser_screenshot", + "browser_extract", + "browser_axtree", } @@ -76,7 +80,9 @@ class _LightweightSystem: def _make_lightweight_system( - engine: Any, model: str, config: Any = None, + engine: Any, + model: str, + config: Any = None, ) -> _LightweightSystem: """Build a minimal system with a plain OllamaEngine. @@ -135,9 +141,8 @@ def _ensure_registries_populated() -> None: # If registries are still empty, reload individual submodules from sys.modules if not ChannelRegistry.keys(): for mod_name in list(sys.modules): - if ( - mod_name.startswith("openjarvis.channels.") - and not mod_name.endswith("_stubs") + if mod_name.startswith("openjarvis.channels.") and not mod_name.endswith( + "_stubs" ): try: importlib.reload(sys.modules[mod_name]) @@ -192,59 +197,203 @@ def build_tools_list() -> List[Dict[str, Any]]: except Exception: spec = None cred_keys = TOOL_CREDENTIALS.get(name, []) - items.append({ - "name": name, - "description": spec.description if spec else "", - "category": spec.category if spec else "", - "source": "tool", - "requires_credentials": len(cred_keys) > 0, - "credential_keys": cred_keys, - "configured": ( - all(bool(os.environ.get(k)) for k in cred_keys) - if cred_keys else True - ), - }) + items.append( + { + "name": name, + "description": spec.description if spec else "", + "category": spec.category if spec else "", + "source": "tool", + "requires_credentials": len(cred_keys) > 0, + "credential_keys": cred_keys, + "configured": ( + all(bool(os.environ.get(k)) for k in cred_keys) + if cred_keys + else True + ), + } + ) except Exception: pass try: if any(ToolRegistry.contains(n) for n in _BROWSER_SUB_TOOLS): - items.append({ - "name": "browser", - "description": ( - "Web browser automation" - " (navigate, click, type, screenshot, extract)" - ), - "category": "browser", - "source": "tool", - "requires_credentials": False, - "credential_keys": [], - "configured": True, - }) + items.append( + { + "name": "browser", + "description": ( + "Web browser automation" + " (navigate, click, type, screenshot, extract)" + ), + "category": "browser", + "source": "tool", + "requires_credentials": False, + "credential_keys": [], + "configured": True, + } + ) except Exception: pass try: for name, _cls in ChannelRegistry.items(): cred_keys = TOOL_CREDENTIALS.get(name, []) - items.append({ - "name": name, - "description": f"{name.replace('_', ' ').title()} messaging channel", - "category": "communication", - "source": "channel", - "requires_credentials": len(cred_keys) > 0, - "credential_keys": cred_keys, - "configured": ( - all(bool(os.environ.get(k)) for k in cred_keys) - if cred_keys else True - ), - }) + items.append( + { + "name": name, + "description": f"{name.replace('_', ' ').title()} messaging channel", + "category": "communication", + "source": "channel", + "requires_credentials": len(cred_keys) > 0, + "credential_keys": cred_keys, + "configured": ( + all(bool(os.environ.get(k)) for k in cred_keys) + if cred_keys + else True + ), + } + ) except Exception: pass return items +def _merge_tool_call_fragments( + accumulated: Dict[int, Dict[str, Any]], + fragments: List[Dict[str, Any]], +) -> None: + """Merge incremental tool_call delta fragments into accumulated state. + + OpenAI-compatible APIs send tool_calls as incremental fragments keyed + by ``index``. Each fragment may contain partial ``function.name`` and/or + ``function.arguments`` strings that must be concatenated. + """ + for frag in fragments: + idx = frag.get("index", 0) + if idx not in accumulated: + accumulated[idx] = { + "id": frag.get("id", ""), + "type": "function", + "function": {"name": "", "arguments": ""}, + } + entry = accumulated[idx] + if frag.get("id"): + entry["id"] = frag["id"] + fn = frag.get("function", {}) + if fn.get("name"): + entry["function"]["name"] += fn["name"] + if fn.get("arguments"): + entry["function"]["arguments"] += fn["arguments"] + + +def _get_mcp_tools(app_state: Any) -> Tuple[List[Dict[str, Any]], Dict[str, Any]]: + """Return (openai_tools_list, mcp_adapters_by_name). + + Lazily discovers MCP tools from config and caches them on ``app_state`` + so that subsequent requests reuse the same connections. + """ + cached = getattr(app_state, "_mcp_tools_cache", None) + if cached is not None: + return cached + + import json as _json + + from openjarvis.core.config import load_config + + openai_tools: List[Dict[str, Any]] = [] + adapters_by_name: Dict[str, Any] = {} + + try: + app_config = load_config() + except Exception as exc: + logger.warning("Failed to load config for MCP discovery: %s", exc) + return openai_tools, adapters_by_name + + if not app_config.tools.mcp.enabled or not app_config.tools.mcp.servers: + return openai_tools, adapters_by_name + + from openjarvis.mcp.client import MCPClient + from openjarvis.mcp.transport import StdioTransport, StreamableHTTPTransport + from openjarvis.tools.mcp_adapter import MCPToolProvider + + # Keep clients alive so transports persist for tool calls at runtime + mcp_clients: list = getattr(app_state, "_mcp_clients", []) + + try: + server_list = _json.loads(app_config.tools.mcp.servers) + except (_json.JSONDecodeError, TypeError) as exc: + logger.warning("Failed to parse MCP server config: %s", exc) + return openai_tools, adapters_by_name + + if not isinstance(server_list, list): + return openai_tools, adapters_by_name + + for server_cfg in server_list: + cfg = _json.loads(server_cfg) if isinstance(server_cfg, str) else server_cfg + name = cfg.get("name", "") + url = cfg.get("url") + command = cfg.get("command", "") + args = cfg.get("args", []) + + try: + if url: + transport = StreamableHTTPTransport(url=url) + elif command: + transport = StdioTransport(command=[command] + args) + else: + logger.warning( + "MCP server '%s' has neither 'url' nor 'command' — skipping", + name, + ) + continue + + client = MCPClient(transport) + client.initialize() + mcp_clients.append(client) + + provider = MCPToolProvider(client) + discovered = provider.discover() + + # Per-server tool filtering + include_tools = set(cfg.get("include_tools", [])) + exclude_tools = set(cfg.get("exclude_tools", [])) + if include_tools: + discovered = [t for t in discovered if t.spec.name in include_tools] + if exclude_tools: + discovered = [t for t in discovered if t.spec.name not in exclude_tools] + + for adapter in discovered: + spec = adapter.spec + openai_tools.append( + { + "type": "function", + "function": { + "name": spec.name, + "description": spec.description, + "parameters": spec.parameters, + }, + } + ) + adapters_by_name[spec.name] = adapter + + logger.info( + "Discovered %d MCP tools from server '%s'", + len(discovered), + name, + ) + except Exception as exc: + logger.warning( + "Failed to discover MCP tools from '%s': %s", + name, + exc, + ) + + app_state._mcp_clients = mcp_clients + if openai_tools: + app_state._mcp_tools_cache = (openai_tools, adapters_by_name) + return openai_tools, adapters_by_name + + async def _stream_managed_agent( *, manager: AgentManager, @@ -253,146 +402,263 @@ async def _stream_managed_agent( message_id: str, engine: Any, bus: Any, + app_state: Any = None, ) -> StreamingResponse: - """Run a managed agent and stream the response as SSE. + """Run a managed agent with real LLM token streaming via SSE. - Instantiates the agent from its stored config, builds conversation - context from message history, executes the agent in a background - thread, and yields SSE-formatted chunks. After completion the - full response is persisted via ``manager.store_agent_response()``. + Uses ``engine.stream_full()`` to yield tokens as they arrive from the + LLM. Supports multi-turn tool-calling: when the model emits tool_calls, + they are executed and the results fed back for the next turn. """ - import asyncio import json import uuid - from openjarvis.agents._stubs import AgentContext - from openjarvis.core.registry import AgentRegistry from openjarvis.core.types import Message, Role agent_id = agent_record["id"] config = agent_record.get("config", {}) - agent_type = agent_record.get("agent_type", "orchestrator") model = config.get("model", getattr(engine, "_model", "")) + system_prompt = config.get("system_prompt") + temperature = config.get("temperature", 0.7) + max_tokens = config.get("max_tokens", 1024) + max_turns = config.get("max_turns", 10) - # Resolve the agent class from registry - agent_cls = AgentRegistry.get(agent_type) - if agent_cls is None: - # Fallback to orchestrator if the type is not registered - agent_cls = AgentRegistry.get("orchestrator") - if agent_cls is None: - raise HTTPException( - status_code=500, detail=f"Agent type '{agent_type}' not found in registry", - ) + # Build conversation messages from history + current input + llm_messages: List[Message] = [] + if system_prompt: + llm_messages.append(Message(role=Role.SYSTEM, content=system_prompt)) - # Build agent constructor kwargs from config - agent_kwargs: Dict[str, Any] = { - "engine": engine, - "model": model, - } - if bus is not None: - agent_kwargs["bus"] = bus - if config.get("system_prompt"): - agent_kwargs["system_prompt"] = config["system_prompt"] - if config.get("temperature") is not None: - agent_kwargs["temperature"] = config["temperature"] - if config.get("max_tokens") is not None: - agent_kwargs["max_tokens"] = config["max_tokens"] - if config.get("max_turns") is not None: - agent_kwargs["max_turns"] = config["max_turns"] - - try: - agent = agent_cls(**agent_kwargs) - except TypeError as exc: - logger.warning( - "Agent instantiation failed with all kwargs, retrying minimal: %s", - exc, - ) - agent = agent_cls(engine=engine, model=model) - - # Build conversation context from existing messages - ctx = AgentContext() - messages = manager.list_messages(agent_id, limit=50) - # Messages come in DESC order, reverse for chronological - for m in reversed(messages): - # Skip the message we just stored (it will be the input) + # Load prior conversation context (DESC order, reverse for chronological) + history = manager.list_messages(agent_id, limit=50) + for m in reversed(history): if m["id"] == message_id: continue if m["direction"] == "user_to_agent": - ctx.conversation.add(Message(role=Role.USER, content=m["content"])) + llm_messages.append(Message(role=Role.USER, content=m["content"])) elif m["direction"] == "agent_to_user": - ctx.conversation.add(Message(role=Role.ASSISTANT, content=m["content"])) + llm_messages.append(Message(role=Role.ASSISTANT, content=m["content"])) + + # Append the current user message + llm_messages.append(Message(role=Role.USER, content=user_content)) # Mark the user message as delivered manager.mark_message_delivered(message_id) chunk_id = f"chatcmpl-{uuid.uuid4().hex[:12]}" - async def generate(): - """Async generator yielding SSE-formatted chunks.""" - collected_content = "" + # Build extra kwargs for stream_full (e.g. tools from config) + stream_kwargs: Dict[str, Any] = {} + if config.get("tools"): + stream_kwargs["tools"] = config["tools"] - # Run agent.run() in a background thread + # Discover MCP tools and merge into stream_kwargs + mcp_adapters: Dict[str, Any] = {} + if app_state is not None: try: - result = await asyncio.to_thread(agent.run, user_content, context=ctx) + mcp_openai_tools, mcp_adapters = _get_mcp_tools(app_state) + if mcp_openai_tools: + existing_tools = stream_kwargs.get("tools", []) + stream_kwargs["tools"] = existing_tools + mcp_openai_tools + logger.info( + "Added %d MCP tools to streaming request", + len(mcp_openai_tools), + ) except Exception as exc: - logger.error("Managed agent stream error: %s", exc, exc_info=True) - error_data = { - "id": chunk_id, - "object": "chat.completion.chunk", - "model": model, - "choices": [{ - "index": 0, - "delta": {"content": f"Error: {exc}"}, - "finish_reason": "stop", - }], - } - yield f"data: {json.dumps(error_data)}\n\n" - yield "data: [DONE]\n\n" - return + logger.warning( + "Failed to get MCP tools for streaming: %s", exc, exc_info=True + ) - content = result.content or "" - collected_content = content + async def generate(): + """Async generator yielding SSE-formatted chunks with real token streaming.""" - # Emit tool results metadata if any - if result.tool_results: - tool_data = [] - for tr in result.tool_results: - tool_data.append({ - "tool_name": tr.tool_name, - "success": tr.success, - "output": tr.content, - "latency_ms": tr.latency_seconds * 1000, - }) - yield f"event: tool_results\ndata: {json.dumps({'results': tool_data})}\n\n" + collected_content = "" + messages_for_llm = list(llm_messages) + turns = 0 - # Stream content word-by-word for real-time feel - if content: - words = content.split(" ") - for i, word in enumerate(words): - token = word if i == 0 else " " + word - chunk_data = { + while turns < max_turns: + turns += 1 + turn_content = "" + tool_call_fragments: Dict[int, Dict[str, Any]] = {} + current_finish_reason = None + + try: + async for chunk in engine.stream_full( + messages_for_llm, + model=model, + temperature=temperature, + max_tokens=max_tokens, + **stream_kwargs, + ): + # Stream content tokens immediately to the client + if chunk.content: + turn_content += chunk.content + chunk_data = { + "id": chunk_id, + "object": "chat.completion.chunk", + "model": model, + "choices": [ + { + "index": 0, + "delta": {"content": chunk.content}, + "finish_reason": None, + } + ], + } + yield f"data: {json.dumps(chunk_data)}\n\n" + + # Accumulate tool_call fragments + if chunk.tool_calls: + _merge_tool_call_fragments( + tool_call_fragments, + chunk.tool_calls, + ) + + if chunk.finish_reason: + current_finish_reason = chunk.finish_reason + + except Exception as exc: + logger.error("Managed agent stream error: %s", exc, exc_info=True) + error_data = { "id": chunk_id, "object": "chat.completion.chunk", "model": model, - "choices": [{ - "index": 0, - "delta": {"content": token}, - "finish_reason": None, - }], + "choices": [ + { + "index": 0, + "delta": {"content": f"Error: {exc}"}, + "finish_reason": "stop", + } + ], } - yield f"data: {json.dumps(chunk_data)}\n\n" - await asyncio.sleep(0.012) + yield f"data: {json.dumps(error_data)}\n\n" + yield "data: [DONE]\n\n" + return + + # Handle tool calls: execute tools and loop for next turn + if tool_call_fragments and current_finish_reason == "tool_calls": + # Build the assistant message with tool_calls + sorted_tcs = [ + tool_call_fragments[i] for i in sorted(tool_call_fragments.keys()) + ] + + # Emit tool_calls metadata as SSE event + tool_meta = [] + for tc in sorted_tcs: + tool_meta.append( + { + "tool_name": tc["function"]["name"], + "arguments": tc["function"]["arguments"], + } + ) + yield ( + f"event: tool_calls\ndata: {json.dumps({'calls': tool_meta})}\n\n" + ) + + # Add assistant message with tool_calls to conversation + from openjarvis.core.types import ToolCall as MsgToolCall + + assistant_msg = Message( + role=Role.ASSISTANT, + content=turn_content or None, + tool_calls=[ + MsgToolCall( + id=tc["id"], + name=tc["function"]["name"], + arguments=tc["function"]["arguments"], + ) + for tc in sorted_tcs + ], + ) + messages_for_llm.append(assistant_msg) + + # Execute each tool call and append results + for tc in sorted_tcs: + tool_name = tc["function"]["name"] + tool_args = tc["function"]["arguments"] + tool_result_content = f"Tool '{tool_name}' not available" + + try: + # Try MCP adapter first (external tools) + mcp_adapter = mcp_adapters.get(tool_name) + if mcp_adapter is not None: + try: + parsed_args = json.loads(tool_args) if tool_args else {} + except (json.JSONDecodeError, TypeError): + parsed_args = {} + result = mcp_adapter.execute(**parsed_args) + tool_result_content = result.content + else: + # Try to use ToolExecutor if tools are configured + from openjarvis.core.registry import ToolRegistry + from openjarvis.tools._stubs import ( + ToolCall as StubToolCall, + ) + from openjarvis.tools._stubs import ( + ToolExecutor, + ) + + tool_cls = ToolRegistry.get(tool_name) + if tool_cls is not None: + tool_instance = tool_cls() + executor = ToolExecutor(tools=[tool_instance], bus=bus) + result = executor.execute( + StubToolCall( + id=tc["id"], + name=tool_name, + arguments=tool_args, + ), + ) + tool_result_content = result.content + else: + logger.warning( + "Tool '%s' not found in registry or MCP adapters", + tool_name, + ) + except Exception as tool_exc: + logger.error( + "Tool execution error for %s: %s", + tool_name, + tool_exc, + exc_info=True, + ) + tool_result_content = f"Error executing {tool_name}: {tool_exc}" + + # Emit tool result as SSE event + tool_event_data = json.dumps( + {"tool_name": tool_name, "output": tool_result_content} + ) + yield (f"event: tool_result\ndata: {tool_event_data}\n\n") + + # Add tool result message to conversation + messages_for_llm.append( + Message( + role=Role.TOOL, + content=tool_result_content, + tool_call_id=tc["id"], + name=tool_name, + ) + ) + + # Continue to next turn (loop back to stream_full) + collected_content += turn_content + continue + + # No tool calls — this is the final response + collected_content += turn_content + break # Final chunk with finish_reason final_data = { "id": chunk_id, "object": "chat.completion.chunk", "model": model, - "choices": [{ - "index": 0, - "delta": {}, - "finish_reason": "stop", - }], + "choices": [ + { + "index": 0, + "delta": {}, + "finish_reason": "stop", + } + ], } yield f"data: {json.dumps(final_data)}\n\n" yield "data: [DONE]\n\n" @@ -403,7 +669,9 @@ async def _stream_managed_agent( manager.store_agent_response(agent_id, collected_content) except Exception as store_exc: logger.error( - "Failed to store agent response: %s", store_exc, exc_info=True, + "Failed to store agent response: %s", + store_exc, + exc_info=True, ) return StreamingResponse( @@ -507,9 +775,7 @@ def create_agent_manager_router( try: manager.start_tick(agent_id) except ValueError: - raise HTTPException( - status_code=409, detail="Agent is already running" - ) + raise HTTPException(status_code=409, detail="Agent is already running") # Re-use the server's engine + model so we don't pick a # random model from Ollama's list. @@ -523,17 +789,22 @@ def create_agent_manager_router( from openjarvis.core.events import get_event_bus executor = AgentExecutor( - manager=manager, event_bus=get_event_bus(), + manager=manager, + event_bus=get_event_bus(), ) system = _make_lightweight_system( - server_engine, server_model, server_config, + server_engine, + server_model, + server_config, ) executor.set_system(system) executor.execute_tick(agent_id) except Exception as exc: logger.error( "Run-tick failed for agent %s: %s", - agent_id, exc, exc_info=True, + agent_id, + exc, + exc_info=True, ) try: manager.end_tick(agent_id) @@ -630,7 +901,8 @@ def create_agent_manager_router( # Auto-recover error-state agents on immediate messages if req.mode == "immediate" and agent_record["status"] in ( - "error", "needs_attention", + "error", + "needs_attention", ): manager.update_agent(agent_id, status="idle") @@ -657,33 +929,39 @@ def create_agent_manager_router( def _immediate_tick(): _start = _time.time() logger.info( - "Immediate tick starting for agent %s " - "(model=%s)", - agent_id, _srv_model, + "Immediate tick starting for agent %s (model=%s)", + agent_id, + _srv_model, ) try: executor = AgentExecutor( - manager=manager, event_bus=get_event_bus(), + manager=manager, + event_bus=get_event_bus(), ) system = _make_lightweight_system( - _srv_engine, _srv_model, _srv_config, + _srv_engine, + _srv_model, + _srv_config, ) executor.set_system(system) logger.info( "Immediate tick: system ready in %.1fs, " "executing tick for agent %s", - _time.time() - _start, agent_id, + _time.time() - _start, + agent_id, ) executor.execute_tick(agent_id) logger.info( - "Immediate tick completed for agent %s " - "in %.1fs", - agent_id, _time.time() - _start, + "Immediate tick completed for agent %s in %.1fs", + agent_id, + _time.time() - _start, ) except Exception as exc: logger.error( "Immediate tick failed for agent %s: %s", - agent_id, exc, exc_info=True, + agent_id, + exc, + exc_info=True, ) try: manager.end_tick(agent_id) @@ -691,11 +969,13 @@ def create_agent_manager_router( pass manager.update_agent(agent_id, status="error") manager.update_summary_memory( - agent_id, f"ERROR: {exc}", + agent_id, + f"ERROR: {exc}", ) threading.Thread( - target=_immediate_tick, daemon=True, + target=_immediate_tick, + daemon=True, ).start() return msg @@ -715,6 +995,7 @@ def create_agent_manager_router( message_id=msg["id"], engine=engine, bus=bus, + app_state=request.app.state, ) # ── State inspection ───────────────────────────────────── @@ -820,9 +1101,7 @@ def create_agent_manager_router( @templates_router.post("/{template_id}/instantiate") async def instantiate_template(template_id: str, req: CreateAgentRequest): - return manager.create_from_template( - template_id, req.name, overrides=req.config - ) + return manager.create_from_template(template_id, req.name, overrides=req.config) # ── Global agent endpoints ─────────────────────────────── @@ -854,8 +1133,26 @@ def create_agent_manager_router( tools_router = APIRouter(prefix="/v1/tools", tags=["tools"]) @tools_router.get("") - def list_tools(): - return {"tools": build_tools_list()} + def list_tools(request: Request): + items = build_tools_list() + try: + mcp_tools, _ = _get_mcp_tools(request.app.state) + for tool in mcp_tools: + fn = tool.get("function", {}) + items.append( + { + "name": fn.get("name", ""), + "description": fn.get("description", ""), + "category": "mcp", + "source": "mcp", + "requires_credentials": False, + "credential_keys": [], + "configured": True, + } + ) + except Exception: + pass + return {"tools": items} @tools_router.post("/{tool_name}/credentials") async def save_tool_credentials(tool_name: str, request: Request): @@ -871,6 +1168,7 @@ def create_agent_manager_router( @tools_router.get("/{tool_name}/credentials/status") def credential_status(tool_name: str): from openjarvis.core.credentials import get_credential_status + return get_credential_status(tool_name) return agents_router, templates_router, global_router, tools_router diff --git a/src/openjarvis/server/stream_bridge.py b/src/openjarvis/server/stream_bridge.py index 9a96577b..386a677f 100644 --- a/src/openjarvis/server/stream_bridge.py +++ b/src/openjarvis/server/stream_bridge.py @@ -119,17 +119,15 @@ class AgentStreamBridge: from openjarvis.core.types import Message, Role for m in self._request.messages[:-1]: - role = ( - Role(m.role) - if m.role in {r.value for r in Role} - else Role.USER + role = Role(m.role) if m.role in {r.value for r in Role} else Role.USER + ctx.conversation.add( + Message( + role=role, + content=m.content or "", + name=m.name, + tool_call_id=m.tool_call_id, + ) ) - ctx.conversation.add(Message( - role=role, - content=m.content or "", - name=m.name, - tool_call_id=m.tool_call_id, - )) input_text = ( self._request.messages[-1].content if self._request.messages else "" @@ -166,9 +164,11 @@ class AgentStreamBridge: first_chunk = ChatCompletionChunk( id=self._chunk_id, model=self._model, - choices=[StreamChoice( - delta=DeltaMessage(role="assistant"), - )], + choices=[ + StreamChoice( + delta=DeltaMessage(role="assistant"), + ) + ], ) yield f"data: {first_chunk.model_dump_json()}\n\n" @@ -202,18 +202,18 @@ class AgentStreamBridge: "Please try a shorter message." ) elif "400" in error_str: - error_content = ( - f"The model returned an error: {error_str}" - ) + error_content = f"The model returned an error: {error_str}" else: error_content = f"Sorry, an error occurred: {error_str}" error_chunk = ChatCompletionChunk( id=self._chunk_id, model=self._model, - choices=[StreamChoice( - delta=DeltaMessage(content=error_content), - finish_reason="stop", - )], + choices=[ + StreamChoice( + delta=DeltaMessage(content=error_content), + finish_reason="stop", + ) + ], ) yield f"data: {error_chunk.model_dump_json()}\n\n" yield "data: [DONE]\n\n" @@ -222,31 +222,88 @@ class AgentStreamBridge: # Emit tool results metadata if any tool_results_data = [] for tr in agent_result.tool_results: - tool_results_data.append({ - "tool_name": tr.tool_name, - "success": tr.success, - "output": tr.content, - "latency_ms": tr.latency_seconds * 1000, - }) + tool_results_data.append( + { + "tool_name": tr.tool_name, + "success": tr.success, + "output": tr.content, + "latency_ms": tr.latency_seconds * 1000, + } + ) if tool_results_data: yield self._format_named_event( - "tool_results", {"results": tool_results_data}, + "tool_results", + {"results": tool_results_data}, ) - # Stream content progressively (word-by-word) for a - # real-time feel, then send a final chunk with usage. + # Stream content using real LLM token streaming via + # engine.stream_full() when the engine is available. content = agent_result.content or "" - if content: + engine = getattr(self._agent, "_engine", None) + used_real_streaming = False + + if engine is not None and hasattr(engine, "stream_full") and content: + # Re-stream using the engine for real token delivery. + # Build the same messages the agent used for its final turn. + try: + from openjarvis.core.types import Message as MsgType + from openjarvis.core.types import Role as RoleType + + replay_messages = [] + for m in self._request.messages: + role = ( + RoleType(m.role) + if m.role in {r.value for r in RoleType} + else RoleType.USER + ) + replay_messages.append( + MsgType( + role=role, + content=m.content or "", + name=m.name, + tool_call_id=m.tool_call_id, + ) + ) + + async for sc in engine.stream_full( + replay_messages, + model=self._model, + ): + if sc.content: + chunk = ChatCompletionChunk( + id=self._chunk_id, + model=self._model, + choices=[ + StreamChoice( + delta=DeltaMessage(content=sc.content), + ) + ], + ) + yield f"data: {chunk.model_dump_json()}\n\n" + used_real_streaming = True + except Exception as stream_exc: + import logging as _logging + + _logger = _logging.getLogger("openjarvis.server") + _logger.warning( + "Real streaming failed, falling back to word replay: %s", + stream_exc, + ) + + # Fallback: word-by-word replay if real streaming was not used + if not used_real_streaming and content: words = content.split(" ") for i, word in enumerate(words): token = word if i == 0 else " " + word chunk = ChatCompletionChunk( id=self._chunk_id, model=self._model, - choices=[StreamChoice( - delta=DeltaMessage(content=token), - )], + choices=[ + StreamChoice( + delta=DeltaMessage(content=token), + ) + ], ) yield f"data: {chunk.model_dump_json()}\n\n" await asyncio.sleep(0.012) @@ -254,7 +311,8 @@ class AgentStreamBridge: # Final chunk: finish_reason + usage prompt_tokens = agent_result.metadata.get("prompt_tokens", 0) completion_tokens = agent_result.metadata.get( - "completion_tokens", 0, + "completion_tokens", + 0, ) total_tokens = agent_result.metadata.get("total_tokens", 0) if total_tokens == 0: @@ -266,10 +324,12 @@ class AgentStreamBridge: final_chunk = ChatCompletionChunk( id=self._chunk_id, model=self._model, - choices=[StreamChoice( - delta=DeltaMessage(), - finish_reason="stop", - )], + choices=[ + StreamChoice( + delta=DeltaMessage(), + finish_reason="stop", + ) + ], ) final_data = json.loads(final_chunk.model_dump_json()) final_data["usage"] = UsageInfo( diff --git a/src/openjarvis/system.py b/src/openjarvis/system.py index b2dc1619..d24421f2 100644 --- a/src/openjarvis/system.py +++ b/src/openjarvis/system.py @@ -49,6 +49,7 @@ class JarvisSystem: agent_executor: Optional[Any] = None # AgentExecutor speech_backend: Optional[Any] = None # SpeechBackend _learning_orchestrator: Optional[Any] = None # LearningOrchestrator + _mcp_clients: List = field(default_factory=list) def ask( self, @@ -371,6 +372,14 @@ class JarvisSystem: channel_bridge.on_message(_on_channel_message) + def _close_mcp_clients(self) -> None: + """Close all persistent MCP client connections.""" + for client in self._mcp_clients: + try: + client.close() + except Exception: + logger.debug("Error closing MCP client", exc_info=True) + def close(self) -> None: """Release resources.""" if self.scheduler and hasattr(self.scheduler, "stop"): @@ -393,6 +402,7 @@ class JarvisSystem: self.agent_manager.close() if self.agent_scheduler is not None: self.agent_scheduler.stop() + self._close_mcp_clients() def __enter__(self) -> JarvisSystem: return self @@ -431,6 +441,7 @@ class SystemBuilder: self._workflow: Optional[bool] = None self._sessions: Optional[bool] = None self._speech: Optional[bool] = None + self._mcp_clients: List = [] def engine(self, key: str) -> SystemBuilder: self._engine_key = key @@ -671,6 +682,8 @@ class SystemBuilder: speech_backend=speech_backend, ) system._learning_orchestrator = learning_orchestrator + # Transfer MCP clients so JarvisSystem.close() can shut them down + system._mcp_clients = list(getattr(self, "_mcp_clients", [])) # Wire system reference — must happen before scheduler.start() if system.agent_executor is not None: system.agent_executor.set_system(system) @@ -1069,24 +1082,60 @@ class SystemBuilder: logger.warning("Failed to set up learning orchestrator: %s", exc) return None - @staticmethod - def _discover_external_mcp(server_cfg) -> List[BaseTool]: - """Discover tools from an external MCP server configuration.""" + def _discover_external_mcp(self, server_cfg) -> List[BaseTool]: + """Discover tools from an external MCP server configuration. + + Supports both stdio (command + args) and Streamable HTTP (url) + transports. Persists MCP clients on ``self._mcp_clients`` so + that transports stay alive for runtime tool calls. + """ import json from openjarvis.mcp.client import MCPClient - from openjarvis.mcp.transport import StdioTransport + from openjarvis.mcp.transport import StdioTransport, StreamableHTTPTransport from openjarvis.tools.mcp_adapter import MCPToolProvider cfg = json.loads(server_cfg) if isinstance(server_cfg, str) else server_cfg + name = cfg.get("name", "") + url = cfg.get("url") command = cfg.get("command", "") args = cfg.get("args", []) - if not command: + + # Build transport based on config keys + if url: + transport = StreamableHTTPTransport(url=url) + elif command: + transport = StdioTransport(command=[command] + args) + else: + logger.warning( + "MCP server '%s' has neither 'url' nor 'command' — skipping", + name, + ) return [] - transport = StdioTransport(command=command, args=args) + client = MCPClient(transport) + client.initialize() + + # Persist client so the transport stays alive for tool calls + self._mcp_clients.append(client) + provider = MCPToolProvider(client) - return provider.discover() + discovered = provider.discover() + + # Per-server tool filtering + include_tools = set(cfg.get("include_tools", [])) + exclude_tools = set(cfg.get("exclude_tools", [])) + if include_tools: + discovered = [t for t in discovered if t.spec.name in include_tools] + if exclude_tools: + discovered = [t for t in discovered if t.spec.name not in exclude_tools] + + logger.info( + "Discovered %d tools from MCP server '%s'", + len(discovered), + name, + ) + return discovered __all__ = ["JarvisSystem", "SystemBuilder"] diff --git a/src/openjarvis/telemetry/instrumented_engine.py b/src/openjarvis/telemetry/instrumented_engine.py index 1ebe5d06..29b0cb89 100644 --- a/src/openjarvis/telemetry/instrumented_engine.py +++ b/src/openjarvis/telemetry/instrumented_engine.py @@ -4,11 +4,12 @@ from __future__ import annotations import statistics import time +from collections.abc import AsyncIterator from typing import Any, Dict, List, Optional, Sequence from openjarvis.core.events import EventBus, EventType from openjarvis.core.types import Message, TelemetryRecord -from openjarvis.engine._stubs import InferenceEngine +from openjarvis.engine._stubs import InferenceEngine, StreamChunk from openjarvis.telemetry.gpu_monitor import GpuSample # --------------------------------------------------------------------------- @@ -30,8 +31,14 @@ def _percentile(data: list[float], p: float) -> float: def _compute_itl_stats(itl_values_ms: list[float]) -> dict: """Compute ITL summary statistics from a list of inter-token latencies in ms.""" if not itl_values_ms: - return {"mean": 0.0, "median": 0.0, "p90": 0.0, - "p95": 0.0, "p99": 0.0, "std": 0.0} + return { + "mean": 0.0, + "median": 0.0, + "p90": 0.0, + "p95": 0.0, + "p99": 0.0, + "std": 0.0, + } return { "mean": statistics.mean(itl_values_ms), "median": statistics.median(itl_values_ms), @@ -78,9 +85,13 @@ class InstrumentedEngine(InferenceEngine): **kwargs: Any, ) -> Dict[str, Any]: """Generate with telemetry recording.""" - self._bus.publish(EventType.INFERENCE_START, { - "model": model, "message_count": len(messages), - }) + self._bus.publish( + EventType.INFERENCE_START, + { + "model": model, + "message_count": len(messages), + }, + ) gpu_sample: Optional[GpuSample] = None energy_sample: Optional[Any] = None @@ -90,19 +101,28 @@ class InstrumentedEngine(InferenceEngine): if self._energy_monitor is not None: with self._energy_monitor.sample() as energy_sample: result = self._inner.generate( - messages, model=model, temperature=temperature, - max_tokens=max_tokens, **kwargs, + messages, + model=model, + temperature=temperature, + max_tokens=max_tokens, + **kwargs, ) elif self._gpu_monitor is not None: with self._gpu_monitor.sample() as gpu_sample: result = self._inner.generate( - messages, model=model, temperature=temperature, - max_tokens=max_tokens, **kwargs, + messages, + model=model, + temperature=temperature, + max_tokens=max_tokens, + **kwargs, ) else: result = self._inner.generate( - messages, model=model, temperature=temperature, - max_tokens=max_tokens, **kwargs, + messages, + model=model, + temperature=temperature, + max_tokens=max_tokens, + **kwargs, ) latency = time.time() - t0 @@ -155,9 +175,7 @@ class InstrumentedEngine(InferenceEngine): energy_per_output_token = ( energy_joules / completion_tokens if completion_tokens > 0 else 0.0 ) - throughput_per_watt = ( - throughput / power_watts if power_watts > 0 else 0.0 - ) + throughput_per_watt = throughput / power_watts if power_watts > 0 else 0.0 # --- Tier 2.1: Phase energy split --- decode_latency = latency - prefill_latency if prefill_latency > 0 else 0.0 @@ -171,13 +189,15 @@ class InstrumentedEngine(InferenceEngine): # --- Tier 3: Non-streaming mean ITL approximation --- mean_itl_ms = ( (decode_latency / completion_tokens) * 1000 - if completion_tokens > 0 and decode_latency > 0 else 0.0 + if completion_tokens > 0 and decode_latency > 0 + else 0.0 ) # --- Tier 4: Per-inference efficiency --- tokens_per_joule = ( completion_tokens / energy_joules - if energy_joules > 0 and completion_tokens > 0 else 0.0 + if energy_joules > 0 and completion_tokens > 0 + else 0.0 ) engine_id = getattr(self._inner, "engine_id", "unknown") @@ -275,9 +295,13 @@ class InstrumentedEngine(InferenceEngine): **kwargs: Any, ) -> Any: """Stream with per-token timing and full telemetry recording.""" - self._bus.publish(EventType.INFERENCE_START, { - "model": model, "message_count": len(messages), - }) + self._bus.publish( + EventType.INFERENCE_START, + { + "model": model, + "message_count": len(messages), + }, + ) t0 = time.time() token_timestamps: list[float] = [] @@ -289,8 +313,11 @@ class InstrumentedEngine(InferenceEngine): if self._energy_monitor is not None: with self._energy_monitor.sample() as energy_sample: async for token in self._inner.stream( - messages, model=model, temperature=temperature, - max_tokens=max_tokens, **kwargs, + messages, + model=model, + temperature=temperature, + max_tokens=max_tokens, + **kwargs, ): token_timestamps.append(time.time()) token_count += 1 @@ -298,16 +325,22 @@ class InstrumentedEngine(InferenceEngine): elif self._gpu_monitor is not None: with self._gpu_monitor.sample() as gpu_sample: async for token in self._inner.stream( - messages, model=model, temperature=temperature, - max_tokens=max_tokens, **kwargs, + messages, + model=model, + temperature=temperature, + max_tokens=max_tokens, + **kwargs, ): token_timestamps.append(time.time()) token_count += 1 yield token else: async for token in self._inner.stream( - messages, model=model, temperature=temperature, - max_tokens=max_tokens, **kwargs, + messages, + model=model, + temperature=temperature, + max_tokens=max_tokens, + **kwargs, ): token_timestamps.append(time.time()) token_count += 1 @@ -362,9 +395,7 @@ class InstrumentedEngine(InferenceEngine): energy_per_output_token = ( energy_joules / token_count if token_count > 0 else 0.0 ) - throughput_per_watt = ( - throughput / power_watts if power_watts > 0 else 0.0 - ) + throughput_per_watt = throughput / power_watts if power_watts > 0 else 0.0 # Phase energy split decode_latency = latency - prefill_latency if prefill_latency > 0 else 0.0 @@ -378,7 +409,8 @@ class InstrumentedEngine(InferenceEngine): # Per-inference efficiency tokens_per_joule = ( token_count / energy_joules - if energy_joules > 0 and token_count > 0 else 0.0 + if energy_joules > 0 and token_count > 0 + else 0.0 ) engine_id = getattr(self._inner, "engine_id", "unknown") @@ -436,6 +468,25 @@ class InstrumentedEngine(InferenceEngine): self._bus.publish(EventType.INFERENCE_END, event_data) self._bus.publish(EventType.TELEMETRY_RECORD, {"record": record}) + async def stream_full( + self, + messages: Sequence[Message], + *, + model: str, + temperature: float = 0.7, + max_tokens: int = 1024, + **kwargs: Any, + ) -> AsyncIterator["StreamChunk"]: + """Delegate to inner engine's stream_full for tool-call support.""" + async for chunk in self._inner.stream_full( + messages, + model=model, + temperature=temperature, + max_tokens=max_tokens, + **kwargs, + ): + yield chunk + def list_models(self) -> List[str]: return self._inner.list_models() diff --git a/tests/engine/test_cloud_stream_full.py b/tests/engine/test_cloud_stream_full.py new file mode 100644 index 00000000..ce73d83a --- /dev/null +++ b/tests/engine/test_cloud_stream_full.py @@ -0,0 +1,466 @@ +"""Tests for CloudEngine.stream_full, _stream_full_openai, _stream_full_anthropic, +and _prepare_anthropic_messages.""" + +from __future__ import annotations + +from typing import Any, List +from unittest.mock import MagicMock + +import pytest + +from openjarvis.core.types import Message, Role, ToolCall +from openjarvis.engine._stubs import StreamChunk + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +def _make_cloud_engine(**overrides: Any) -> Any: + """Create a CloudEngine without calling __init__ (no env vars needed).""" + from openjarvis.engine.cloud import CloudEngine + + engine = CloudEngine.__new__(CloudEngine) + engine._openai_client = overrides.get("openai_client") + engine._anthropic_client = overrides.get("anthropic_client") + engine._google_client = overrides.get("google_client") + engine._openrouter_client = overrides.get("openrouter_client") + engine._minimax_client = overrides.get("minimax_client") + return engine + + +def _openai_chunk( + *, + content: str | None = None, + tool_calls: list | None = None, + finish_reason: str | None = None, +) -> MagicMock: + """Build a mock OpenAI streaming chunk.""" + delta = MagicMock() + delta.content = content + delta.tool_calls = tool_calls + choice = MagicMock() + choice.delta = delta + choice.finish_reason = finish_reason + chunk = MagicMock() + chunk.choices = [choice] + return chunk + + +def _openai_tool_call_delta( + *, + index: int = 0, + tc_id: str = "", + name: str = "", + arguments: str = "", +) -> MagicMock: + tc = MagicMock() + tc.index = index + tc.id = tc_id + tc.function = MagicMock() + tc.function.name = name + tc.function.arguments = arguments + return tc + + +# --------------------------------------------------------------------------- +# _stream_full_openai tests +# --------------------------------------------------------------------------- + + +@pytest.mark.asyncio +async def test_stream_full_openai_content(): + """Mock OpenAI streaming response with content chunks.""" + mock_client = MagicMock() + chunks = [ + _openai_chunk(content="Hello"), + _openai_chunk(content=" world"), + _openai_chunk(finish_reason="stop"), + ] + mock_client.chat.completions.create.return_value = iter(chunks) + + engine = _make_cloud_engine(openai_client=mock_client) + msgs = [Message(role=Role.USER, content="hi")] + + result: List[StreamChunk] = [] + async for sc in engine._stream_full_openai( + msgs, + model="gpt-4o", + temperature=0.7, + max_tokens=100, + ): + result.append(sc) + + assert len(result) == 3 + assert result[0].content == "Hello" + assert result[1].content == " world" + assert result[2].finish_reason == "stop" + + +@pytest.mark.asyncio +async def test_stream_full_openai_tool_calls(): + """Mock response with tool_call deltas, verify StreamChunk.tool_calls format.""" + mock_client = MagicMock() + tc1 = _openai_tool_call_delta(index=0, tc_id="call_1", name="calc", arguments="") + tc2 = _openai_tool_call_delta(index=0, tc_id="", name="", arguments='{"x": 1}') + chunks = [ + _openai_chunk(tool_calls=[tc1]), + _openai_chunk(tool_calls=[tc2]), + _openai_chunk(finish_reason="tool_calls"), + ] + mock_client.chat.completions.create.return_value = iter(chunks) + + engine = _make_cloud_engine(openai_client=mock_client) + msgs = [Message(role=Role.USER, content="calc")] + + result: List[StreamChunk] = [] + async for sc in engine._stream_full_openai( + msgs, + model="gpt-4o", + temperature=0.7, + max_tokens=100, + ): + result.append(sc) + + assert result[0].tool_calls is not None + assert result[0].tool_calls[0]["function"]["name"] == "calc" + assert result[0].tool_calls[0]["id"] == "call_1" + assert result[1].tool_calls[0]["function"]["arguments"] == '{"x": 1}' + assert result[2].finish_reason == "tool_calls" + + +@pytest.mark.asyncio +async def test_stream_full_openai_finish_reason(): + """Verify finish_reason='tool_calls' and 'stop' propagated correctly.""" + mock_client = MagicMock() + chunks_stop = [ + _openai_chunk(content="ok"), + _openai_chunk(finish_reason="stop"), + ] + mock_client.chat.completions.create.return_value = iter(chunks_stop) + + engine = _make_cloud_engine(openai_client=mock_client) + msgs = [Message(role=Role.USER, content="hi")] + + result = [] + async for sc in engine._stream_full_openai( + msgs, + model="gpt-4o", + temperature=0.7, + max_tokens=100, + ): + result.append(sc) + + assert result[-1].finish_reason == "stop" + + # Now test tool_calls finish + tc = _openai_tool_call_delta(index=0, tc_id="c1", name="fn", arguments="{}") + chunks_tc = [ + _openai_chunk(tool_calls=[tc]), + _openai_chunk(finish_reason="tool_calls"), + ] + mock_client.chat.completions.create.return_value = iter(chunks_tc) + + result2 = [] + async for sc in engine._stream_full_openai( + msgs, + model="gpt-4o", + temperature=0.7, + max_tokens=100, + ): + result2.append(sc) + + assert result2[-1].finish_reason == "tool_calls" + + +# --------------------------------------------------------------------------- +# _stream_full_anthropic tests +# --------------------------------------------------------------------------- + + +def _anthropic_event(event_type: str, **kwargs: Any) -> MagicMock: + """Build a mock Anthropic stream event.""" + event = MagicMock() + event.type = event_type + for k, v in kwargs.items(): + setattr(event, k, v) + return event + + +@pytest.mark.asyncio +async def test_stream_full_anthropic_content(): + """Mock Anthropic stream events with text content.""" + # Build content_block_start with text type + text_block = MagicMock() + text_block.type = "text" + + # Build text delta + text_delta = MagicMock() + text_delta.type = "text_delta" + text_delta.text = "Hello world" + + # Build message_delta with stop + msg_delta = MagicMock() + msg_delta.stop_reason = "end_turn" + + events = [ + _anthropic_event("content_block_start", content_block=text_block), + _anthropic_event("content_block_delta", delta=text_delta), + _anthropic_event("message_delta", delta=msg_delta), + ] + + mock_stream = MagicMock() + mock_stream.__enter__ = MagicMock(return_value=iter(events)) + mock_stream.__exit__ = MagicMock(return_value=False) + + mock_anthropic = MagicMock() + mock_anthropic.messages.stream.return_value = mock_stream + + engine = _make_cloud_engine(anthropic_client=mock_anthropic) + msgs = [Message(role=Role.USER, content="hi")] + + result: List[StreamChunk] = [] + async for sc in engine._stream_full_anthropic( + msgs, + model="claude-sonnet-4-20250514", + temperature=0.7, + max_tokens=100, + ): + result.append(sc) + + # Should have text content and a finish reason + content_chunks = [r for r in result if r.content is not None] + assert len(content_chunks) >= 1 + assert content_chunks[0].content == "Hello world" + + finish_chunks = [r for r in result if r.finish_reason is not None] + assert len(finish_chunks) == 1 + assert finish_chunks[0].finish_reason == "stop" + + +@pytest.mark.asyncio +async def test_stream_full_anthropic_tool_calls(): + """Mock Anthropic tool_use events, verify OpenAI-delta-format tool_calls.""" + # content_block_start with tool_use + tool_block = MagicMock() + tool_block.type = "tool_use" + tool_block.id = "toolu_123" + tool_block.name = "get_weather" + + # input_json_delta + json_delta = MagicMock() + json_delta.type = "input_json_delta" + json_delta.partial_json = '{"city": "Berlin"}' + + # message_delta with tool_use stop + msg_delta = MagicMock() + msg_delta.stop_reason = "tool_use" + + events = [ + _anthropic_event("content_block_start", content_block=tool_block), + _anthropic_event("content_block_delta", delta=json_delta), + _anthropic_event("message_delta", delta=msg_delta), + ] + + mock_stream = MagicMock() + mock_stream.__enter__ = MagicMock(return_value=iter(events)) + mock_stream.__exit__ = MagicMock(return_value=False) + + mock_anthropic = MagicMock() + mock_anthropic.messages.stream.return_value = mock_stream + + engine = _make_cloud_engine(anthropic_client=mock_anthropic) + msgs = [Message(role=Role.USER, content="weather?")] + + result: List[StreamChunk] = [] + async for sc in engine._stream_full_anthropic( + msgs, + model="claude-sonnet-4-20250514", + temperature=0.7, + max_tokens=100, + ): + result.append(sc) + + # First chunk: tool_use start with name + assert result[0].tool_calls is not None + assert result[0].tool_calls[0]["function"]["name"] == "get_weather" + assert result[0].tool_calls[0]["id"] == "toolu_123" + + # Second chunk: arguments fragment + assert result[1].tool_calls is not None + assert result[1].tool_calls[0]["function"]["arguments"] == '{"city": "Berlin"}' + + # Third chunk: finish with tool_calls + assert result[2].finish_reason == "tool_calls" + + +@pytest.mark.asyncio +async def test_stream_full_anthropic_finish_reason(): + """message_delta with stop_reason='tool_use' maps to finish_reason='tool_calls'.""" + msg_delta_tool = MagicMock() + msg_delta_tool.stop_reason = "tool_use" + + msg_delta_stop = MagicMock() + msg_delta_stop.stop_reason = "end_turn" + + # Test tool_use -> tool_calls + events_tool = [_anthropic_event("message_delta", delta=msg_delta_tool)] + mock_stream = MagicMock() + mock_stream.__enter__ = MagicMock(return_value=iter(events_tool)) + mock_stream.__exit__ = MagicMock(return_value=False) + + mock_anthropic = MagicMock() + mock_anthropic.messages.stream.return_value = mock_stream + + engine = _make_cloud_engine(anthropic_client=mock_anthropic) + msgs = [Message(role=Role.USER, content="test")] + + result = [] + async for sc in engine._stream_full_anthropic( + msgs, + model="claude-sonnet-4-20250514", + temperature=0.7, + max_tokens=100, + ): + result.append(sc) + assert result[0].finish_reason == "tool_calls" + + # Test end_turn -> stop + events_stop = [_anthropic_event("message_delta", delta=msg_delta_stop)] + mock_stream2 = MagicMock() + mock_stream2.__enter__ = MagicMock(return_value=iter(events_stop)) + mock_stream2.__exit__ = MagicMock(return_value=False) + mock_anthropic.messages.stream.return_value = mock_stream2 + + result2 = [] + async for sc in engine._stream_full_anthropic( + msgs, + model="claude-sonnet-4-20250514", + temperature=0.7, + max_tokens=100, + ): + result2.append(sc) + assert result2[0].finish_reason == "stop" + + +# --------------------------------------------------------------------------- +# _prepare_anthropic_messages tests +# --------------------------------------------------------------------------- + + +def test_prepare_anthropic_messages_system(): + """System message extracted separately from chat messages.""" + engine = _make_cloud_engine() + msgs = [ + Message(role=Role.SYSTEM, content="You are helpful"), + Message(role=Role.USER, content="Hello"), + ] + + system_text, chat_msgs = engine._prepare_anthropic_messages(msgs) + assert system_text == "You are helpful" + assert len(chat_msgs) == 1 + assert chat_msgs[0]["role"] == "user" + assert chat_msgs[0]["content"] == "Hello" + + +def test_prepare_anthropic_messages_tool_result(): + """Tool role converted to user + tool_result content block.""" + engine = _make_cloud_engine() + msgs = [ + Message(role=Role.USER, content="What's the weather?"), + Message( + role=Role.TOOL, + content='{"temp": 20}', + tool_call_id="call_abc", + ), + ] + + system_text, chat_msgs = engine._prepare_anthropic_messages(msgs) + assert system_text == "" + assert len(chat_msgs) == 2 + # Second message is the tool result wrapped as user + tool_msg = chat_msgs[1] + assert tool_msg["role"] == "user" + assert isinstance(tool_msg["content"], list) + assert tool_msg["content"][0]["type"] == "tool_result" + assert tool_msg["content"][0]["tool_use_id"] == "call_abc" + assert tool_msg["content"][0]["content"] == '{"temp": 20}' + + +def test_prepare_anthropic_messages_tool_calls(): + """Assistant with tool_calls converted to content blocks with tool_use.""" + engine = _make_cloud_engine() + msgs = [ + Message( + role=Role.ASSISTANT, + content="Let me check.", + tool_calls=[ + ToolCall( + id="call_1", name="get_weather", arguments='{"city": "Berlin"}' + ), + ], + ), + ] + + system_text, chat_msgs = engine._prepare_anthropic_messages(msgs) + assert len(chat_msgs) == 1 + blocks = chat_msgs[0]["content"] + assert blocks[0]["type"] == "text" + assert blocks[0]["text"] == "Let me check." + assert blocks[1]["type"] == "tool_use" + assert blocks[1]["id"] == "call_1" + assert blocks[1]["name"] == "get_weather" + assert blocks[1]["input"] == {"city": "Berlin"} + + +# --------------------------------------------------------------------------- +# stream_full routing tests +# --------------------------------------------------------------------------- + + +@pytest.mark.asyncio +async def test_stream_full_routes_to_anthropic(): + """model='claude-xxx' routes to _stream_full_anthropic.""" + msg_delta = MagicMock() + msg_delta.stop_reason = "end_turn" + events = [_anthropic_event("message_delta", delta=msg_delta)] + mock_stream = MagicMock() + mock_stream.__enter__ = MagicMock(return_value=iter(events)) + mock_stream.__exit__ = MagicMock(return_value=False) + + mock_anthropic = MagicMock() + mock_anthropic.messages.stream.return_value = mock_stream + + engine = _make_cloud_engine(anthropic_client=mock_anthropic) + msgs = [Message(role=Role.USER, content="test")] + + result = [] + async for sc in engine.stream_full(msgs, model="claude-sonnet-4-20250514"): + result.append(sc) + + # Verify Anthropic client was used + mock_anthropic.messages.stream.assert_called_once() + assert any(r.finish_reason is not None for r in result) + + +@pytest.mark.asyncio +async def test_stream_full_routes_to_openai(): + """model='gpt-xxx' routes to _stream_full_openai.""" + mock_client = MagicMock() + chunks = [ + _openai_chunk(content="hi"), + _openai_chunk(finish_reason="stop"), + ] + mock_client.chat.completions.create.return_value = iter(chunks) + + engine = _make_cloud_engine(openai_client=mock_client) + msgs = [Message(role=Role.USER, content="test")] + + result = [] + async for sc in engine.stream_full(msgs, model="gpt-4o"): + result.append(sc) + + # Verify OpenAI client was used + mock_client.chat.completions.create.assert_called_once() + assert result[0].content == "hi" + assert result[1].finish_reason == "stop" diff --git a/tests/engine/test_engine_wrappers.py b/tests/engine/test_engine_wrappers.py new file mode 100644 index 00000000..edfbcc4a --- /dev/null +++ b/tests/engine/test_engine_wrappers.py @@ -0,0 +1,142 @@ +"""Tests for InstrumentedEngine, GuardrailsEngine, and MultiEngine stream_full +delegation.""" + +from __future__ import annotations + +from collections.abc import AsyncIterator +from typing import Any, Dict, List + +import pytest + +from openjarvis.core.events import EventBus +from openjarvis.core.types import Message, Role +from openjarvis.engine._stubs import InferenceEngine, StreamChunk +from openjarvis.engine.multi import MultiEngine +from openjarvis.security.guardrails import GuardrailsEngine +from openjarvis.telemetry.instrumented_engine import InstrumentedEngine + +# --------------------------------------------------------------------------- +# Fake engine that yields predetermined StreamChunks via stream_full +# --------------------------------------------------------------------------- + + +class _FakeStreamFullEngine(InferenceEngine): + engine_id = "fake-sf" + + def __init__(self, chunks: list[StreamChunk]) -> None: + self._chunks = chunks + + def generate(self, messages, *, model, **kwargs) -> Dict[str, Any]: + return {"content": "ok", "usage": {}} + + async def stream(self, messages, *, model, **kwargs) -> AsyncIterator[str]: + yield "ok" + + async def stream_full( + self, messages, *, model, **kwargs + ) -> AsyncIterator[StreamChunk]: + for c in self._chunks: + yield c + + def list_models(self) -> List[str]: + return ["fake-model"] + + def health(self) -> bool: + return True + + +# --------------------------------------------------------------------------- +# InstrumentedEngine.stream_full delegation +# --------------------------------------------------------------------------- + + +@pytest.mark.asyncio +async def test_instrumented_delegates_stream_full(): + """InstrumentedEngine.stream_full delegates to inner engine.""" + expected = [ + StreamChunk(content="Hello"), + StreamChunk(content=" world"), + StreamChunk(finish_reason="stop"), + ] + inner = _FakeStreamFullEngine(expected) + bus = EventBus(record_history=True) + engine = InstrumentedEngine(inner, bus) + + result = [] + async for chunk in engine.stream_full( + [Message(role=Role.USER, content="test")], + model="fake-model", + ): + result.append(chunk) + + assert len(result) == 3 + assert result[0].content == "Hello" + assert result[1].content == " world" + assert result[2].finish_reason == "stop" + + +# --------------------------------------------------------------------------- +# GuardrailsEngine.stream_full delegation +# --------------------------------------------------------------------------- + + +@pytest.mark.asyncio +async def test_guardrails_delegates_stream_full(): + """GuardrailsEngine.stream_full delegates to wrapped engine.""" + expected = [ + StreamChunk(content="safe output"), + StreamChunk(finish_reason="stop"), + ] + inner = _FakeStreamFullEngine(expected) + engine = GuardrailsEngine(inner, scanners=[]) + + result = [] + async for chunk in engine.stream_full( + [Message(role=Role.USER, content="test")], + model="fake-model", + ): + result.append(chunk) + + assert len(result) == 2 + assert result[0].content == "safe output" + assert result[1].finish_reason == "stop" + + +# --------------------------------------------------------------------------- +# MultiEngine.stream_full routing +# --------------------------------------------------------------------------- + + +@pytest.mark.asyncio +async def test_multi_routes_stream_full_by_model(): + """MultiEngine routes stream_full to the correct engine by model name.""" + chunks_a = [StreamChunk(content="from A"), StreamChunk(finish_reason="stop")] + chunks_b = [StreamChunk(content="from B"), StreamChunk(finish_reason="stop")] + + engine_a = _FakeStreamFullEngine(chunks_a) + engine_a.list_models = lambda: ["model-a"] + + engine_b = _FakeStreamFullEngine(chunks_b) + engine_b.list_models = lambda: ["model-b"] + + multi = MultiEngine([("a", engine_a), ("b", engine_b)]) + + # Route to engine A + result_a = [] + async for chunk in multi.stream_full( + [Message(role=Role.USER, content="test")], + model="model-a", + ): + result_a.append(chunk) + + assert result_a[0].content == "from A" + + # Route to engine B + result_b = [] + async for chunk in multi.stream_full( + [Message(role=Role.USER, content="test")], + model="model-b", + ): + result_b.append(chunk) + + assert result_b[0].content == "from B" diff --git a/tests/engine/test_stream_full.py b/tests/engine/test_stream_full.py new file mode 100644 index 00000000..4003738c --- /dev/null +++ b/tests/engine/test_stream_full.py @@ -0,0 +1,250 @@ +"""Tests for StreamChunk dataclass and stream_full() engine method.""" + +from __future__ import annotations + +import json +from collections.abc import AsyncIterator +from typing import Any, Dict, List +from unittest.mock import MagicMock + +import pytest + +from openjarvis.core.types import Message, Role +from openjarvis.engine._stubs import InferenceEngine, StreamChunk + +# --------------------------------------------------------------------------- +# StreamChunk dataclass tests +# --------------------------------------------------------------------------- + + +class TestStreamChunk: + def test_defaults(self): + chunk = StreamChunk() + assert chunk.content is None + assert chunk.tool_calls is None + assert chunk.finish_reason is None + assert chunk.usage is None + + def test_content_only(self): + chunk = StreamChunk(content="hello") + assert chunk.content == "hello" + assert chunk.finish_reason is None + + def test_finish_reason(self): + chunk = StreamChunk(finish_reason="stop") + assert chunk.content is None + assert chunk.finish_reason == "stop" + + def test_tool_calls(self): + tc = [{"index": 0, "function": {"name": "calc", "arguments": "{}"}}] + chunk = StreamChunk(tool_calls=tc) + assert chunk.tool_calls == tc + + def test_usage(self): + usage = {"prompt_tokens": 10, "completion_tokens": 5, "total_tokens": 15} + chunk = StreamChunk(usage=usage) + assert chunk.usage == usage + + def test_all_fields(self): + chunk = StreamChunk( + content="hi", + tool_calls=[{"index": 0}], + finish_reason="tool_calls", + usage={"total_tokens": 1}, + ) + assert chunk.content == "hi" + assert chunk.tool_calls is not None + assert chunk.finish_reason == "tool_calls" + assert chunk.usage is not None + + +# --------------------------------------------------------------------------- +# Concrete engine stub for testing default stream_full() +# --------------------------------------------------------------------------- + + +class _FakeEngine(InferenceEngine): + """Minimal engine that yields predefined tokens via stream().""" + + engine_id = "fake" + + def __init__(self, tokens: list[str]) -> None: + self._tokens = tokens + + def generate(self, messages, *, model, **kwargs) -> Dict[str, Any]: + return {"content": "".join(self._tokens), "usage": {}} + + async def stream(self, messages, *, model, **kwargs) -> AsyncIterator[str]: + for t in self._tokens: + yield t + + def list_models(self) -> List[str]: + return ["fake-model"] + + def health(self) -> bool: + return True + + +class TestDefaultStreamFull: + """Test the default stream_full() implementation that wraps stream().""" + + @pytest.mark.asyncio + async def test_wraps_stream_tokens(self): + engine = _FakeEngine(["Hello", " world", "!"]) + chunks = [] + async for chunk in engine.stream_full( + [Message(role=Role.USER, content="test")], + model="fake-model", + ): + chunks.append(chunk) + + # Should have 3 content chunks + 1 finish chunk + assert len(chunks) == 4 + assert chunks[0].content == "Hello" + assert chunks[1].content == " world" + assert chunks[2].content == "!" + assert chunks[3].finish_reason == "stop" + assert chunks[3].content is None + + @pytest.mark.asyncio + async def test_empty_stream(self): + engine = _FakeEngine([]) + chunks = [] + async for chunk in engine.stream_full( + [Message(role=Role.USER, content="test")], + model="fake-model", + ): + chunks.append(chunk) + + # Should have just the finish chunk + assert len(chunks) == 1 + assert chunks[0].finish_reason == "stop" + + @pytest.mark.asyncio + async def test_kwargs_passed_through(self): + """Verify that temperature/max_tokens reach stream().""" + engine = _FakeEngine(["ok"]) + chunks = [] + async for chunk in engine.stream_full( + [Message(role=Role.USER, content="test")], + model="fake-model", + temperature=0.1, + max_tokens=50, + ): + chunks.append(chunk) + + assert len(chunks) == 2 + assert chunks[0].content == "ok" + + +# --------------------------------------------------------------------------- +# OpenAI-compatible stream_full() with mock HTTP response +# --------------------------------------------------------------------------- + + +class TestOpenAICompatStreamFull: + """Test _OpenAICompatibleEngine.stream_full() with mocked HTTP.""" + + @pytest.mark.asyncio + async def test_parses_sse_with_content_and_finish(self): + from openjarvis.engine._openai_compat import _OpenAICompatibleEngine + + # Build mock SSE lines + sse_lines = [] + for token in ["Hello", " world"]: + chunk = { + "choices": [{"delta": {"content": token}, "finish_reason": None}], + } + sse_lines.append(f"data: {json.dumps(chunk)}") + # Final chunk with finish_reason + final = { + "choices": [{"delta": {}, "finish_reason": "stop"}], + "usage": {"prompt_tokens": 5, "completion_tokens": 2, "total_tokens": 7}, + } + sse_lines.append(f"data: {json.dumps(final)}") + sse_lines.append("data: [DONE]") + + # Mock the httpx client stream context manager + mock_resp = MagicMock() + mock_resp.raise_for_status = MagicMock() + mock_resp.iter_lines.return_value = iter(sse_lines) + + engine = _OpenAICompatibleEngine.__new__(_OpenAICompatibleEngine) + engine.engine_id = "test" + engine._host = "http://localhost:8000" + engine._api_prefix = "/v1" + + mock_client = MagicMock() + mock_stream_ctx = MagicMock() + mock_stream_ctx.__enter__ = MagicMock(return_value=mock_resp) + mock_stream_ctx.__exit__ = MagicMock(return_value=False) + mock_client.stream.return_value = mock_stream_ctx + engine._client = mock_client + + chunks = [] + async for chunk in engine.stream_full( + [Message(role=Role.USER, content="test")], + model="test-model", + ): + chunks.append(chunk) + + # Should have: "Hello", " world", finish+usage + assert len(chunks) == 3 + assert chunks[0].content == "Hello" + assert chunks[1].content == " world" + assert chunks[2].finish_reason == "stop" + assert chunks[2].usage is not None + assert chunks[2].usage["total_tokens"] == 7 + + @pytest.mark.asyncio + async def test_parses_tool_call_fragments(self): + from openjarvis.engine._openai_compat import _OpenAICompatibleEngine + + # Simulate streamed tool_call fragments + _tc1 = ( + '{"choices": [{"delta": {"tool_calls": [{"index": 0, "id": "call_1",' + ' "function": {"name": "calc", "arguments": ""}}]},' + ' "finish_reason": null}]}' + ) + _tc2 = ( + '{"choices": [{"delta": {"tool_calls": [{"index": 0,' + ' "function": {"name": "", "arguments": "{\\"x\\": 1}"}}]},' + ' "finish_reason": null}]}' + ) + sse_lines = [ + f"data: {_tc1}", + f"data: {_tc2}", + 'data: {"choices": [{"delta": {}, "finish_reason": "tool_calls"}]}', + "data: [DONE]", + ] + + mock_resp = MagicMock() + mock_resp.raise_for_status = MagicMock() + mock_resp.iter_lines.return_value = iter(sse_lines) + + engine = _OpenAICompatibleEngine.__new__(_OpenAICompatibleEngine) + engine.engine_id = "test" + engine._host = "http://localhost:8000" + engine._api_prefix = "/v1" + + mock_client = MagicMock() + mock_stream_ctx = MagicMock() + mock_stream_ctx.__enter__ = MagicMock(return_value=mock_resp) + mock_stream_ctx.__exit__ = MagicMock(return_value=False) + mock_client.stream.return_value = mock_stream_ctx + engine._client = mock_client + + chunks = [] + async for chunk in engine.stream_full( + [Message(role=Role.USER, content="test")], + model="test-model", + ): + chunks.append(chunk) + + # First chunk has tool_calls with name + assert chunks[0].tool_calls is not None + assert chunks[0].tool_calls[0]["function"]["name"] == "calc" + # Second chunk has arguments fragment + assert chunks[1].tool_calls is not None + # Third chunk has finish_reason="tool_calls" + assert chunks[2].finish_reason == "tool_calls" diff --git a/tests/mcp/test_client_extended.py b/tests/mcp/test_client_extended.py new file mode 100644 index 00000000..5857a5e1 --- /dev/null +++ b/tests/mcp/test_client_extended.py @@ -0,0 +1,176 @@ +"""Extended tests for MCPClient — initialize params, notify, context manager.""" + +from __future__ import annotations + +from unittest.mock import MagicMock + +import pytest + +from openjarvis.mcp.client import MCPClient +from openjarvis.mcp.protocol import MCPRequest, MCPResponse +from openjarvis.tools._stubs import ToolSpec + + +@pytest.fixture +def mock_transport(): + """A mock transport that returns configurable responses.""" + transport = MagicMock() + transport.send.return_value = MCPResponse(result={}) + return transport + + +class TestInitialize: + def test_sends_correct_params(self, mock_transport): + """initialize() must send protocolVersion, capabilities, clientInfo.""" + mock_transport.send.return_value = MCPResponse( + result={"protocolVersion": "2025-03-26", "capabilities": {"tools": {}}} + ) + client = MCPClient(mock_transport) + client.initialize() + + # First call is the initialize request + init_call = mock_transport.send.call_args_list[0] + req = init_call[0][0] + assert isinstance(req, MCPRequest) + assert req.method == "initialize" + assert req.params["protocolVersion"] == "2025-03-26" + assert req.params["capabilities"] == {} + assert req.params["clientInfo"]["name"] == "openjarvis" + assert req.params["clientInfo"]["version"] == "0.1.0" + + def test_sends_initialized_notification(self, mock_transport): + """After initialize, notifications/initialized must be sent via + send_notification.""" + mock_transport.send.return_value = MCPResponse( + result={"protocolVersion": "2025-03-26", "capabilities": {}} + ) + client = MCPClient(mock_transport) + client.initialize() + + # initialize request goes via send(), notification via send_notification() + assert mock_transport.send.call_count == 1 + mock_transport.send_notification.assert_called_once() + req = mock_transport.send_notification.call_args[0][0] + assert req.method == "notifications/initialized" + assert req.id is None # notifications must not have an id + + def test_stores_capabilities(self, mock_transport): + """initialize() should store server capabilities.""" + mock_transport.send.return_value = MCPResponse( + result={"capabilities": {"tools": {"listChanged": True}}} + ) + client = MCPClient(mock_transport) + client.initialize() + assert client._capabilities == {"tools": {"listChanged": True}} + + +class TestNotify: + def test_notify_sends_request(self, mock_transport): + """notify() should send a notification with the given method and params.""" + client = MCPClient(mock_transport) + client.notify("notifications/cancelled", {"requestId": 42}) + + mock_transport.send_notification.assert_called_once() + req = mock_transport.send_notification.call_args[0][0] + assert req.method == "notifications/cancelled" + assert req.params == {"requestId": 42} + assert req.id is None # notifications must omit id + + def test_notify_defaults_empty_params(self, mock_transport): + """notify() with no params should send empty dict.""" + client = MCPClient(mock_transport) + client.notify("notifications/initialized") + + req = mock_transport.send_notification.call_args[0][0] + assert req.params == {} + + def test_notify_json_has_no_id(self, mock_transport): + """The serialized notification JSON must not contain an 'id' field.""" + import json + + client = MCPClient(mock_transport) + client.notify("notifications/initialized") + + req = mock_transport.send_notification.call_args[0][0] + payload = json.loads(req.to_json()) + assert "id" not in payload + + +class TestContextManager: + def test_context_manager_calls_close(self, mock_transport): + """Using MCPClient as context manager should call close on exit.""" + with MCPClient(mock_transport) as client: + assert client is not None + mock_transport.close.assert_called_once() + + def test_context_manager_closes_on_exception(self, mock_transport): + """close() should be called even if an exception occurs.""" + with pytest.raises(ValueError): + with MCPClient(mock_transport): + raise ValueError("test error") + mock_transport.close.assert_called_once() + + +class TestListTools: + def test_parses_tool_specs(self, mock_transport): + """list_tools() should return ToolSpec objects from server response.""" + mock_transport.send.return_value = MCPResponse( + result={ + "tools": [ + { + "name": "get_entities", + "description": "Get HA entities", + "inputSchema": { + "type": "object", + "properties": {"domain": {"type": "string"}}, + }, + }, + { + "name": "call_service", + "description": "Call HA service", + "inputSchema": {"type": "object", "properties": {}}, + }, + ] + } + ) + client = MCPClient(mock_transport) + tools = client.list_tools() + + assert len(tools) == 2 + assert all(isinstance(t, ToolSpec) for t in tools) + assert tools[0].name == "get_entities" + assert tools[0].description == "Get HA entities" + assert "properties" in tools[0].parameters + assert tools[1].name == "call_service" + + def test_empty_tools_list(self, mock_transport): + """list_tools() with no tools should return empty list.""" + mock_transport.send.return_value = MCPResponse(result={"tools": []}) + client = MCPClient(mock_transport) + assert client.list_tools() == [] + + +class TestCallTool: + def test_call_tool_sends_correct_params(self, mock_transport): + """call_tool() should send method=tools/call with name and arguments.""" + mock_transport.send.return_value = MCPResponse( + result={"content": [{"type": "text", "text": "ok"}], "isError": False} + ) + client = MCPClient(mock_transport) + result = client.call_tool("get_entities", {"domain": "light"}) + + req = mock_transport.send.call_args[0][0] + assert req.method == "tools/call" + assert req.params == {"name": "get_entities", "arguments": {"domain": "light"}} + assert result["isError"] is False + + def test_call_tool_no_arguments(self, mock_transport): + """call_tool() with no arguments passes empty dict.""" + mock_transport.send.return_value = MCPResponse( + result={"content": [{"type": "text", "text": "done"}], "isError": False} + ) + client = MCPClient(mock_transport) + client.call_tool("ping") + + req = mock_transport.send.call_args[0][0] + assert req.params == {"name": "ping", "arguments": {}} diff --git a/tests/mcp/test_discovery.py b/tests/mcp/test_discovery.py new file mode 100644 index 00000000..ac9c061e --- /dev/null +++ b/tests/mcp/test_discovery.py @@ -0,0 +1,195 @@ +"""Tests for _discover_external_mcp in SystemBuilder.""" + +from __future__ import annotations + +from unittest.mock import MagicMock, patch + +import pytest + +from openjarvis.tools._stubs import ToolSpec + + +def _make_mock_tool(name: str) -> MagicMock: + """Create a mock BaseTool with the given name.""" + tool = MagicMock() + tool.spec = ToolSpec(name=name, description=f"Mock {name}") + return tool + + +@pytest.fixture +def builder(): + """Create a minimal SystemBuilder instance for testing _discover_external_mcp.""" + from openjarvis.system import SystemBuilder + + def _minimal_init(self): + self._mcp_clients = [] + + with patch.object(SystemBuilder, "__init__", _minimal_init): + instance = SystemBuilder.__new__(SystemBuilder) + instance.__init__() + return instance + + +# Patch targets: the method uses local imports, so we patch the actual classes +# in their source modules (which is where the local from-imports resolve to). +_PATCH_HTTP = "openjarvis.mcp.transport.StreamableHTTPTransport" +_PATCH_STDIO = "openjarvis.mcp.transport.StdioTransport" +_PATCH_CLIENT = "openjarvis.mcp.client.MCPClient" +_PATCH_PROVIDER = "openjarvis.tools.mcp_adapter.MCPToolProvider" +_PATCH_LOGGER = "openjarvis.system.logger" + + +class TestDiscoverHTTPServer: + @patch(_PATCH_PROVIDER) + @patch(_PATCH_CLIENT) + @patch(_PATCH_HTTP) + def test_url_config_uses_http_transport( + self, mock_transport_cls, mock_client_cls, mock_provider_cls, builder + ): + """Config with 'url' should create StreamableHTTPTransport.""" + mock_tools = [_make_mock_tool("get_entities"), _make_mock_tool("call_service")] + mock_provider_cls.return_value.discover.return_value = mock_tools + + cfg = {"name": "ha-mcp", "url": "http://172.16.3.1:9583/mcp"} + result = builder._discover_external_mcp(cfg) + + mock_transport_cls.assert_called_once_with(url="http://172.16.3.1:9583/mcp") + mock_client_cls.return_value.initialize.assert_called_once() + assert len(result) == 2 + assert result[0].spec.name == "get_entities" + + +class TestDiscoverStdioServer: + @patch(_PATCH_PROVIDER) + @patch(_PATCH_CLIENT) + @patch(_PATCH_STDIO) + def test_command_config_uses_stdio_transport( + self, mock_transport_cls, mock_client_cls, mock_provider_cls, builder + ): + """Config with 'command' + 'args' should create StdioTransport.""" + mock_tools = [_make_mock_tool("read_file")] + mock_provider_cls.return_value.discover.return_value = mock_tools + + cfg = {"name": "fs-server", "command": "node", "args": ["server.js", "--stdio"]} + result = builder._discover_external_mcp(cfg) + + mock_transport_cls.assert_called_once_with( + command=["node", "server.js", "--stdio"] + ) + assert len(result) == 1 + assert result[0].spec.name == "read_file" + + +class TestDiscoverInvalidConfig: + @patch(_PATCH_LOGGER) + def test_no_url_no_command_returns_empty(self, mock_logger, builder): + """Config with neither 'url' nor 'command' should return [] and log warning.""" + cfg = {"name": "broken-server"} + result = builder._discover_external_mcp(cfg) + + assert result == [] + mock_logger.warning.assert_called_once() + assert "neither" in mock_logger.warning.call_args[0][0].lower() + + +class TestToolFiltering: + @patch(_PATCH_PROVIDER) + @patch(_PATCH_CLIENT) + @patch(_PATCH_HTTP) + def test_include_tools_filter( + self, mock_transport_cls, mock_client_cls, mock_provider_cls, builder + ): + """include_tools should keep only the listed tools.""" + mock_tools = [ + _make_mock_tool("tool1"), + _make_mock_tool("tool2"), + _make_mock_tool("tool3"), + ] + mock_provider_cls.return_value.discover.return_value = mock_tools + + cfg = { + "name": "filtered", + "url": "http://localhost:8080/mcp", + "include_tools": ["tool1"], + } + result = builder._discover_external_mcp(cfg) + + assert len(result) == 1 + assert result[0].spec.name == "tool1" + + @patch(_PATCH_PROVIDER) + @patch(_PATCH_CLIENT) + @patch(_PATCH_HTTP) + def test_exclude_tools_filter( + self, mock_transport_cls, mock_client_cls, mock_provider_cls, builder + ): + """exclude_tools should remove the listed tools.""" + mock_tools = [ + _make_mock_tool("tool1"), + _make_mock_tool("tool2"), + _make_mock_tool("tool3"), + ] + mock_provider_cls.return_value.discover.return_value = mock_tools + + cfg = { + "name": "filtered", + "url": "http://localhost:8080/mcp", + "exclude_tools": ["tool2"], + } + result = builder._discover_external_mcp(cfg) + + names = [t.spec.name for t in result] + assert "tool2" not in names + assert "tool1" in names + assert "tool3" in names + + +class TestClientPersistence: + @patch(_PATCH_PROVIDER) + @patch(_PATCH_CLIENT) + @patch(_PATCH_HTTP) + def test_client_stored_in_mcp_clients( + self, mock_transport_cls, mock_client_cls, mock_provider_cls, builder + ): + """After discovery, the MCPClient should be persisted on _mcp_clients.""" + mock_provider_cls.return_value.discover.return_value = [] + + cfg = {"name": "test", "url": "http://localhost:8080/mcp"} + builder._discover_external_mcp(cfg) + + assert hasattr(builder, "_mcp_clients") + assert len(builder._mcp_clients) == 1 + assert builder._mcp_clients[0] is mock_client_cls.return_value + + @patch(_PATCH_PROVIDER) + @patch(_PATCH_CLIENT) + @patch(_PATCH_HTTP) + def test_multiple_servers_accumulate_clients( + self, mock_transport_cls, mock_client_cls, mock_provider_cls, builder + ): + """Multiple discover calls should accumulate clients.""" + mock_provider_cls.return_value.discover.return_value = [] + + for i in range(3): + cfg = {"name": f"server-{i}", "url": f"http://localhost:{8080 + i}/mcp"} + builder._discover_external_mcp(cfg) + + assert len(builder._mcp_clients) == 3 + + +class TestStringConfig: + @patch(_PATCH_PROVIDER) + @patch(_PATCH_CLIENT) + @patch(_PATCH_HTTP) + def test_json_string_config_parsed( + self, mock_transport_cls, mock_client_cls, mock_provider_cls, builder + ): + """Config passed as JSON string should be parsed correctly.""" + import json + + mock_provider_cls.return_value.discover.return_value = [] + + cfg_str = json.dumps({"name": "test", "url": "http://localhost:8080/mcp"}) + builder._discover_external_mcp(cfg_str) + + mock_transport_cls.assert_called_once_with(url="http://localhost:8080/mcp") diff --git a/tests/mcp/test_streamable_http_transport.py b/tests/mcp/test_streamable_http_transport.py new file mode 100644 index 00000000..0bb9b979 --- /dev/null +++ b/tests/mcp/test_streamable_http_transport.py @@ -0,0 +1,142 @@ +"""Tests for the StreamableHTTPTransport class.""" + +from __future__ import annotations + +import json +from unittest.mock import MagicMock, patch + +import pytest + +from openjarvis.mcp.protocol import MCPRequest + + +@pytest.fixture +def _mock_httpx_client(): + """Patch httpx.Client so no real HTTP connections are made.""" + with patch("httpx.Client") as mock_cls, patch("httpx.Timeout"): + mock_instance = MagicMock() + mock_cls.return_value = mock_instance + yield mock_instance + + +def _make_http_response(result, *, session_id=None): + """Build a mock httpx.Response with the given JSON-RPC result.""" + resp = MagicMock() + resp.text = json.dumps({"jsonrpc": "2.0", "id": 1, "result": result}) + resp.raise_for_status = MagicMock() + headers = {} + if session_id is not None: + headers["mcp-session-id"] = session_id + resp.headers = headers + return resp + + +class TestStreamableHTTPTransport: + def test_send_request(self, _mock_httpx_client): + """Verify correct URL, headers, JSON body, and MCPResponse parsing.""" + from openjarvis.mcp.transport import StreamableHTTPTransport + + mock_client = _mock_httpx_client + mock_client.post.return_value = _make_http_response({"tools": []}) + + transport = StreamableHTTPTransport("http://localhost:9583/mcp") + req = MCPRequest(method="tools/list", id=1) + resp = transport.send(req) + + # Verify the POST call + mock_client.post.assert_called_once() + call_kwargs = mock_client.post.call_args + assert call_kwargs[0][0] == "http://localhost:9583/mcp" + headers = call_kwargs[1]["headers"] + assert headers["Content-Type"] == "application/json" + assert "application/json" in headers["Accept"] + assert "text/event-stream" in headers["Accept"] + + # Verify JSON body matches the request + sent_json = call_kwargs[1]["json"] + assert sent_json["method"] == "tools/list" + assert sent_json["id"] == 1 + assert sent_json["jsonrpc"] == "2.0" + + # Verify response parsing + assert resp.error is None + assert resp.result == {"tools": []} + + def test_session_id_tracking(self, _mock_httpx_client): + """First response sets Mcp-Session-Id, subsequent requests include it.""" + from openjarvis.mcp.transport import StreamableHTTPTransport + + mock_client = _mock_httpx_client + # First response sets the session id + mock_client.post.return_value = _make_http_response( + {"capabilities": {}}, session_id="sess-abc-123" + ) + + transport = StreamableHTTPTransport("http://localhost:9583/mcp") + req1 = MCPRequest(method="initialize", id=1) + transport.send(req1) + + assert transport._session_id == "sess-abc-123" + + # Second request — prepare new response without session header + mock_client.post.return_value = _make_http_response({"tools": []}) + req2 = MCPRequest(method="tools/list", id=2) + transport.send(req2) + + # Verify the second call included the session id header + second_call_headers = mock_client.post.call_args[1]["headers"] + assert second_call_headers["Mcp-Session-Id"] == "sess-abc-123" + + def test_first_request_has_no_session_id(self, _mock_httpx_client): + """First request should not include Mcp-Session-Id header.""" + from openjarvis.mcp.transport import StreamableHTTPTransport + + mock_client = _mock_httpx_client + mock_client.post.return_value = _make_http_response({}) + + transport = StreamableHTTPTransport("http://localhost:9583/mcp") + transport.send(MCPRequest(method="initialize", id=1)) + + first_call_headers = mock_client.post.call_args[1]["headers"] + assert "Mcp-Session-Id" not in first_call_headers + + def test_connect_error_handling(self, _mock_httpx_client): + """httpx.ConnectError should be wrapped in RuntimeError.""" + import httpx + + from openjarvis.mcp.transport import StreamableHTTPTransport + + mock_client = _mock_httpx_client + mock_client.post.side_effect = httpx.ConnectError("Connection refused") + + transport = StreamableHTTPTransport("http://localhost:9583/mcp") + with pytest.raises(RuntimeError, match="Failed to connect"): + transport.send(MCPRequest(method="initialize", id=1)) + + def test_timeout_error_handling(self, _mock_httpx_client): + """httpx.TimeoutException should be wrapped in RuntimeError.""" + import httpx + + from openjarvis.mcp.transport import StreamableHTTPTransport + + mock_client = _mock_httpx_client + mock_client.post.side_effect = httpx.TimeoutException("Read timed out") + + transport = StreamableHTTPTransport("http://localhost:9583/mcp") + with pytest.raises(RuntimeError, match="Timeout communicating"): + transport.send(MCPRequest(method="initialize", id=1)) + + def test_close(self, _mock_httpx_client): + """close() should close the underlying httpx client.""" + from openjarvis.mcp.transport import StreamableHTTPTransport + + mock_client = _mock_httpx_client + transport = StreamableHTTPTransport("http://localhost:9583/mcp") + transport.close() + mock_client.close.assert_called_once() + + def test_backward_compat_alias(self): + """SSETransport should be the same class as StreamableHTTPTransport.""" + from openjarvis.mcp.transport import SSETransport, StreamableHTTPTransport + + assert SSETransport is StreamableHTTPTransport diff --git a/tests/mcp/test_transport.py b/tests/mcp/test_transport.py index f1cb545e..4cd10e86 100644 --- a/tests/mcp/test_transport.py +++ b/tests/mcp/test_transport.py @@ -5,13 +5,18 @@ from __future__ import annotations import json import sys import textwrap -from unittest.mock import MagicMock +from unittest.mock import MagicMock, patch import pytest from openjarvis.mcp.protocol import MCPRequest from openjarvis.mcp.server import MCPServer -from openjarvis.mcp.transport import InProcessTransport, SSETransport, StdioTransport +from openjarvis.mcp.transport import ( + InProcessTransport, + SSETransport, + StdioTransport, + StreamableHTTPTransport, +) from openjarvis.tools.calculator import CalculatorTool from openjarvis.tools.think import ThinkTool @@ -156,65 +161,108 @@ class TestStdioTransport: transport.close() # Should not raise -class TestSSETransport: - def test_send_receive(self, monkeypatch): - """Mock httpx to simulate HTTP response.""" +class TestStreamableHTTPTransport: + """Tests for StreamableHTTPTransport (also aliased as SSETransport).""" + + def _make_mock_response(self, body: dict) -> MagicMock: + """Create a mock httpx.Response with the given JSON body.""" mock_response = MagicMock() - mock_response.text = json.dumps( + mock_response.text = json.dumps(body) + mock_response.headers = {} + mock_response.raise_for_status = MagicMock() + return mock_response + + @patch("httpx.Client") + def test_send_receive(self, mock_client_cls): + """Mock httpx.Client to simulate HTTP response.""" + mock_client = MagicMock() + mock_client_cls.return_value = mock_client + mock_client.post.return_value = self._make_mock_response( {"jsonrpc": "2.0", "id": 1, "result": {"tools": []}} ) - mock_response.raise_for_status = MagicMock() - mock_httpx = MagicMock() - mock_httpx.post.return_value = mock_response - monkeypatch.setitem(sys.modules, "httpx", mock_httpx) - - transport = SSETransport("http://localhost:8080/mcp") + transport = StreamableHTTPTransport("http://localhost:8080/mcp") req = MCPRequest(method="tools/list", id=1) resp = transport.send(req) assert resp.error is None assert resp.result == {"tools": []} - def test_send_posts_json(self, monkeypatch): + @patch("httpx.Client") + def test_send_posts_json(self, mock_client_cls): """Verify the HTTP POST includes correct headers and body.""" - mock_response = MagicMock() - mock_response.text = json.dumps({"jsonrpc": "2.0", "id": 1, "result": {}}) - mock_response.raise_for_status = MagicMock() + mock_client = MagicMock() + mock_client_cls.return_value = mock_client + mock_client.post.return_value = self._make_mock_response( + {"jsonrpc": "2.0", "id": 1, "result": {}} + ) - mock_httpx = MagicMock() - mock_httpx.post.return_value = mock_response - monkeypatch.setitem(sys.modules, "httpx", mock_httpx) - - transport = SSETransport("http://localhost:8080/mcp") + transport = StreamableHTTPTransport("http://localhost:8080/mcp") req = MCPRequest(method="initialize", id=1) transport.send(req) - call_args = mock_httpx.post.call_args + call_args = mock_client.post.call_args assert call_args[0][0] == "http://localhost:8080/mcp" assert call_args[1]["headers"]["Content-Type"] == "application/json" - def test_close_is_noop(self): - transport = SSETransport("http://localhost:8080/mcp") - transport.close() # Should not raise + @patch("httpx.Client") + def test_close_closes_client(self, mock_client_cls): + """close() should close the underlying httpx.Client.""" + mock_client = MagicMock() + mock_client_cls.return_value = mock_client - def test_error_response(self, monkeypatch): + transport = StreamableHTTPTransport("http://localhost:8080/mcp") + transport.close() + mock_client.close.assert_called_once() + + @patch("httpx.Client") + def test_error_response(self, mock_client_cls): """Simulate server returning an error response.""" - mock_response = MagicMock() - mock_response.text = json.dumps( + mock_client = MagicMock() + mock_client_cls.return_value = mock_client + mock_client.post.return_value = self._make_mock_response( { "jsonrpc": "2.0", "id": 1, "error": {"code": -32601, "message": "Not found"}, } ) - mock_response.raise_for_status = MagicMock() - mock_httpx = MagicMock() - mock_httpx.post.return_value = mock_response - monkeypatch.setitem(sys.modules, "httpx", mock_httpx) - - transport = SSETransport("http://localhost:8080/mcp") + transport = StreamableHTTPTransport("http://localhost:8080/mcp") req = MCPRequest(method="unknown", id=1) resp = transport.send(req) assert resp.error is not None assert resp.error["code"] == -32601 + + @patch("httpx.Client") + def test_session_id_tracking(self, mock_client_cls): + """Verify Mcp-Session-Id is tracked from response and sent on subsequent + requests.""" + mock_client = MagicMock() + mock_client_cls.return_value = mock_client + + # First response sets a session id + first_response = self._make_mock_response( + {"jsonrpc": "2.0", "id": 1, "result": {}} + ) + first_response.headers = {"mcp-session-id": "sess-abc-123"} + + # Second response + second_response = self._make_mock_response( + {"jsonrpc": "2.0", "id": 2, "result": {}} + ) + second_response.headers = {} + + mock_client.post.side_effect = [first_response, second_response] + + transport = StreamableHTTPTransport("http://localhost:8080/mcp") + transport.send(MCPRequest(method="initialize", id=1)) + assert transport._session_id == "sess-abc-123" + + transport.send(MCPRequest(method="tools/list", id=2)) + # Second call should include session id in headers + second_call_headers = mock_client.post.call_args_list[1][1]["headers"] + assert second_call_headers["Mcp-Session-Id"] == "sess-abc-123" + + def test_sse_transport_alias(self): + """SSETransport should be an alias for StreamableHTTPTransport.""" + assert SSETransport is StreamableHTTPTransport diff --git a/tests/server/test_agent_manager_routes.py b/tests/server/test_agent_manager_routes.py index b8dbd57c..dd908d67 100644 --- a/tests/server/test_agent_manager_routes.py +++ b/tests/server/test_agent_manager_routes.py @@ -244,39 +244,40 @@ def test_run_agent_concurrent_returns_409(tmp_path): @pytest.mark.skipif(not HAS_FASTAPI, reason="fastapi not installed") class TestAgentManagerStreaming: - """Tests for the SSE streaming mode of the managed-agent messages endpoint.""" + """Tests for the SSE streaming mode of the managed-agent messages endpoint. + + The new implementation uses engine.stream_full() for real token streaming + instead of agent.run() + word-by-word replay. + """ @pytest.fixture def _mock_engine(self): + """Create a mock engine with a working stream_full() method.""" + from openjarvis.engine._stubs import StreamChunk + engine = MagicMock() engine.engine_id = "mock" engine._model = "test-model" engine.health.return_value = True + + # Default stream_full: echo the last user message token-by-token + async def _stream_full(messages, *, model, **kwargs): + # Find the last user message content + last_content = "" + for m in reversed(messages): + if hasattr(m, "role") and m.role.value == "user": + last_content = m.content + break + response = f"Echo: {last_content}" + for token in response.split(" "): + yield StreamChunk(content=token + " ") + yield StreamChunk(finish_reason="stop") + + engine.stream_full = _stream_full return engine @pytest.fixture - def _mock_agent_cls(self): - """Register a mock agent class in the AgentRegistry for testing.""" - from openjarvis.agents._stubs import AgentResult - from openjarvis.core.registry import AgentRegistry - - class _MockStreamAgent: - agent_id = "mock_stream" - - def __init__(self, engine, model, **kwargs): - self._engine = engine - self._model = model - - def run(self, input_text, context=None, **kwargs): - return AgentResult(content=f"Echo: {input_text}", turns=1) - - # Register under a unique key for test isolation - AgentRegistry._entries()["_test_stream"] = _MockStreamAgent - yield _MockStreamAgent - AgentRegistry._entries().pop("_test_stream", None) - - @pytest.fixture - def stream_client(self, manager, _mock_engine, _mock_agent_cls): + def stream_client(self, manager, _mock_engine): from fastapi import FastAPI from openjarvis.server.agent_manager_routes import create_agent_manager_router @@ -293,10 +294,11 @@ class TestAgentManagerStreaming: app.include_router(tools_router) return TestClient(app) - def test_send_message_stream(self, manager, stream_client, _mock_agent_cls): + def test_send_message_stream(self, manager, stream_client): """Test streaming mode returns SSE response with [DONE] sentinel.""" agent = manager.create_agent( - name="streamer", agent_type="_test_stream", + name="streamer", + agent_type="simple", ) resp = stream_client.post( f"/v1/managed-agents/{agent['id']}/messages", @@ -312,10 +314,11 @@ class TestAgentManagerStreaming: # Last data line must be [DONE] assert data_lines[-1].strip() == "data: [DONE]" - def test_send_message_stream_content(self, manager, stream_client, _mock_agent_cls): - """Test streaming returns the correct agent response content.""" + def test_send_message_stream_real_tokens(self, manager, stream_client): + """Content arrives as real tokens, not word-burst after completion.""" agent = manager.create_agent( - name="streamer2", agent_type="_test_stream", + name="streamer_tokens", + agent_type="simple", ) resp = stream_client.post( f"/v1/managed-agents/{agent['id']}/messages", @@ -324,7 +327,7 @@ class TestAgentManagerStreaming: assert resp.status_code == 200 # Collect content tokens from stream - content = "" + content_chunks = [] for line in resp.text.strip().split("\n"): if line.startswith("data:") and "[DONE]" not in line: raw = line[5:].strip() @@ -335,16 +338,19 @@ class TestAgentManagerStreaming: choices = data.get("choices", [{}]) delta_content = choices[0].get("delta", {}).get("content") if delta_content: - content += delta_content + content_chunks.append(delta_content) - assert content == "Echo: Hello world" + # Should have multiple token chunks (real streaming, not single burst) + assert len(content_chunks) > 1 + full_content = "".join(content_chunks) + assert "Echo:" in full_content + assert "Hello world" in full_content - def test_send_message_stream_stores_response( - self, manager, stream_client, _mock_agent_cls, - ): + def test_send_message_stream_stores_response(self, manager, stream_client): """After streaming, agent response is persisted in the DB.""" agent = manager.create_agent( - name="streamer3", agent_type="_test_stream", + name="streamer3", + agent_type="simple", ) resp = stream_client.post( f"/v1/managed-agents/{agent['id']}/messages", @@ -362,12 +368,11 @@ class TestAgentManagerStreaming: agent_msg = next(m for m in messages if m["direction"] == "agent_to_user") assert "persist me" in agent_msg["content"] - def test_send_message_stream_finish_reason( - self, manager, stream_client, _mock_agent_cls, - ): + def test_send_message_stream_finish_reason(self, manager, stream_client): """The final chunk before [DONE] has finish_reason='stop'.""" agent = manager.create_agent( - name="streamer4", agent_type="_test_stream", + name="streamer4", + agent_type="simple", ) resp = stream_client.post( f"/v1/managed-agents/{agent['id']}/messages", @@ -385,3 +390,38 @@ class TestAgentManagerStreaming: # Last chunk should have finish_reason="stop" assert chunks[-1]["choices"][0]["finish_reason"] == "stop" + + def test_send_message_stream_error_handling(self, manager): + """Engine errors are reported gracefully via SSE.""" + + error_engine = MagicMock() + error_engine.engine_id = "error" + error_engine._model = "test-model" + + async def _stream_full_error(messages, *, model, **kwargs): + raise RuntimeError("LLM connection failed") + yield # make it a generator # noqa: E501 + + error_engine.stream_full = _stream_full_error + + from fastapi import FastAPI + from fastapi.testclient import TestClient as TC + + from openjarvis.server.agent_manager_routes import create_agent_manager_router + + app = FastAPI() + app.state.engine = error_engine + app.state.bus = None + routers = create_agent_manager_router(manager) + for r in routers: + app.include_router(r) + client = TC(app) + + agent = manager.create_agent(name="err_agent", agent_type="simple") + resp = client.post( + f"/v1/managed-agents/{agent['id']}/messages", + json={"content": "fail", "stream": True}, + ) + assert resp.status_code == 200 + assert "Error:" in resp.text or "error" in resp.text.lower() + assert "data: [DONE]" in resp.text diff --git a/tests/server/test_mcp_tools_cache.py b/tests/server/test_mcp_tools_cache.py new file mode 100644 index 00000000..d3bcdfa4 --- /dev/null +++ b/tests/server/test_mcp_tools_cache.py @@ -0,0 +1,154 @@ +"""Tests for _get_mcp_tools() caching in agent_manager_routes.""" + +from __future__ import annotations + +import json +from unittest.mock import MagicMock, patch + +import pytest + +pytest.importorskip("fastapi", reason="fastapi required for server route tests") + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +class _FakeAppState: + """Minimal app_state substitute with dynamic attributes.""" + + pass + + +def _make_config(*, enabled: bool = True, servers_json: str = "[]") -> MagicMock: + """Build a mock config with tools.mcp.enabled and tools.mcp.servers.""" + config = MagicMock() + config.tools.mcp.enabled = enabled + config.tools.mcp.servers = servers_json + return config + + +def _make_tool_spec(name: str, description: str = "") -> MagicMock: + spec = MagicMock() + spec.name = name + spec.description = description + spec.parameters = {"type": "object", "properties": {}} + return spec + + +def _make_adapter(name: str) -> MagicMock: + adapter = MagicMock() + adapter.spec = _make_tool_spec(name) + return adapter + + +# --------------------------------------------------------------------------- +# Tests +# --------------------------------------------------------------------------- + + +@patch("openjarvis.core.config.load_config") +def test_returns_tools_from_mcp_server(mock_load_config: MagicMock): + """With a mocked MCP server, discovered tools are returned.""" + from openjarvis.server.agent_manager_routes import _get_mcp_tools + + server_cfg = [{"name": "test-server", "url": "http://localhost:9999"}] + mock_load_config.return_value = _make_config( + servers_json=json.dumps(server_cfg), + ) + + mock_adapter = _make_adapter("get_weather") + + with ( + patch("openjarvis.mcp.transport.StreamableHTTPTransport"), + patch("openjarvis.mcp.client.MCPClient"), + patch("openjarvis.tools.mcp_adapter.MCPToolProvider") as MockProvider, + ): + MockProvider.return_value.discover.return_value = [mock_adapter] + + app_state = _FakeAppState() + tools, adapters = _get_mcp_tools(app_state) + + assert len(tools) == 1 + assert tools[0]["function"]["name"] == "get_weather" + assert "get_weather" in adapters + + +@patch("openjarvis.core.config.load_config") +def test_caches_successful_discovery(mock_load_config: MagicMock): + """Second call returns cached result without re-discovering.""" + from openjarvis.server.agent_manager_routes import _get_mcp_tools + + server_cfg = [{"name": "test-server", "url": "http://localhost:9999"}] + mock_load_config.return_value = _make_config( + servers_json=json.dumps(server_cfg), + ) + + mock_adapter = _make_adapter("cached_tool") + + with ( + patch("openjarvis.mcp.transport.StreamableHTTPTransport"), + patch("openjarvis.mcp.client.MCPClient"), + patch("openjarvis.tools.mcp_adapter.MCPToolProvider") as MockProvider, + ): + MockProvider.return_value.discover.return_value = [mock_adapter] + + app_state = _FakeAppState() + + # First call discovers + tools1, _ = _get_mcp_tools(app_state) + assert len(tools1) == 1 + + # Second call should use cache (discover not called again) + discover_call_count = MockProvider.return_value.discover.call_count + tools2, _ = _get_mcp_tools(app_state) + assert len(tools2) == 1 + assert MockProvider.return_value.discover.call_count == discover_call_count + + +@patch("openjarvis.core.config.load_config") +def test_does_not_cache_empty_results(mock_load_config: MagicMock): + """Failed/empty discovery is not cached so it can be retried.""" + from openjarvis.server.agent_manager_routes import _get_mcp_tools + + server_cfg = [{"name": "failing-server", "url": "http://localhost:9999"}] + mock_load_config.return_value = _make_config( + servers_json=json.dumps(server_cfg), + ) + + with ( + patch("openjarvis.mcp.transport.StreamableHTTPTransport"), + patch("openjarvis.mcp.client.MCPClient"), + patch("openjarvis.tools.mcp_adapter.MCPToolProvider") as MockProvider, + ): + # First call: discovery returns empty + MockProvider.return_value.discover.return_value = [] + app_state = _FakeAppState() + + tools1, _ = _get_mcp_tools(app_state) + assert len(tools1) == 0 + + # Verify no cache was set (empty result) + assert getattr(app_state, "_mcp_tools_cache", None) is None + + # Second call: discovery now returns something + mock_adapter = _make_adapter("retry_tool") + MockProvider.return_value.discover.return_value = [mock_adapter] + + tools2, _ = _get_mcp_tools(app_state) + assert len(tools2) == 1 + assert tools2[0]["function"]["name"] == "retry_tool" + + +@patch("openjarvis.core.config.load_config") +def test_handles_config_load_failure(mock_load_config: MagicMock): + """Config load failure returns empty, no crash.""" + from openjarvis.server.agent_manager_routes import _get_mcp_tools + + mock_load_config.side_effect = RuntimeError("config broken") + + app_state = _FakeAppState() + tools, adapters = _get_mcp_tools(app_state) + + assert tools == [] + assert adapters == {} From 728896ae867d8c103f823918a930ee492eb96ef5 Mon Sep 17 00:00:00 2001 From: Jon Saad-Falcon <41205309+jonsaadfalcon@users.noreply.github.com> Date: Fri, 27 Mar 2026 11:15:37 -0700 Subject: [PATCH 34/44] docs: add agent wizard simplification design spec Sub-project A spec covering: 2-step wizard, smart template defaults, rich system prompt templates, tool wiring fix, recommended model endpoint, and Advanced settings collapse. Co-Authored-By: Claude Opus 4.6 (1M context) --- ...3-27-agent-wizard-simplification-design.md | 433 ++++++++++++++++++ 1 file changed, 433 insertions(+) create mode 100644 docs/superpowers/specs/2026-03-27-agent-wizard-simplification-design.md diff --git a/docs/superpowers/specs/2026-03-27-agent-wizard-simplification-design.md b/docs/superpowers/specs/2026-03-27-agent-wizard-simplification-design.md new file mode 100644 index 00000000..8c594e60 --- /dev/null +++ b/docs/superpowers/specs/2026-03-27-agent-wizard-simplification-design.md @@ -0,0 +1,433 @@ +# Agent Wizard Simplification + Smart Defaults + Tool Wiring + +**Date:** 2026-03-27 +**Status:** Approved +**Scope:** Sub-project A — wizard UX, template defaults, system prompts, tool wiring, recommended model endpoint + +## Problem + +The current agent creation wizard has 12+ fields across 3 steps, requires users to understand memory extraction strategies and retrieval backends, and doesn't wire the selected tools to the executor (`tools=[]`). The result: agents are hard to create and can't use the tools users selected. + +## Goals + +1. Simplify wizard to 2 steps (template → name + instruction) +2. Smart defaults from templates (tools, model, schedule, retrieval) +3. Collapse Advanced settings with sensible defaults +4. Wire tools from config to executor (fix `tools=[]` gap) +5. Rich system prompt templates with tool documentation +6. Backend endpoint for hardware-appropriate model recommendation + +## Non-Goals + +- OAuth credential flows (Sub-project B) +- Tab population — Tasks, Memory, Logs (Sub-project C) +- New agent types or templates beyond the existing 3 + Custom + +--- + +## Architecture + +### 1. Wizard Flow + +**Step 1: Template Selection** + +A 2x2 grid of template cards: +- Research Monitor (icon: 🔬) +- Inbox Triager (icon: 📥) +- Code Reviewer (icon: 🔍) +- Custom Agent (icon: ⚙️) + +Each card shows: name, 1-line description, tool tags. Clicking a card advances to Step 2. + +**Step 2: Configuration** + +Two required fields at top: +- **Agent Name** — text input +- **What should this agent do?** — textarea for user instruction + +Three pre-filled fields below (editable): +- **Intelligence** — dropdown, pre-selected to backend-recommended model, shows "(recommended)" tag +- **Schedule** — pre-filled from template (e.g. "Every day at 9:00 AM" for Research Monitor), editable dropdown for manual/cron/interval +- **Tools** — shown as purple tags, auto-selected from template, clickable to add/remove + +Collapsed `
` section for Advanced Settings: +- Memory Extraction +- Observation Compression +- Retrieval Strategy +- Task Decomposition +- Max Turns +- Temperature +- Budget +- Learning Policy + +**Launch button** at the bottom — no Step 3 review page. + +**Custom Agent variation:** Same layout but model defaults to recommended, schedule defaults to manual, tools default to empty (user adds from Advanced or from the tools tag area), and all Advanced settings use universal defaults. + +### 2. Template Defaults + +Each template TOML defines all fields. Universal defaults (for Custom Agent and any unset template field) are: + +| Field | Universal Default | +|-------|-------------------| +| Memory Extraction | `structured_json` | +| Observation Compression | `summarize` | +| Retrieval Strategy | `sqlite` (BM25/FTS5) | +| Task Decomposition | `hierarchical` | +| Max Turns | `25` | +| Temperature | `0.3` | +| Budget | Unlimited | +| Learning Policy | None | + +Template-specific overrides: + +**research_monitor.toml:** +| Field | Value | +|-------|-------| +| schedule_type | `cron` | +| schedule_value | `0 9 * * *` | +| tools | `web_search`, `http_request`, `file_read`, `file_write`, `memory_store`, `memory_retrieve`, `think` | +| temperature | `0.3` | +| task_decomposition | `phased` | + +**inbox_triager.toml:** +| Field | Value | +|-------|-------| +| schedule_type | `interval` | +| schedule_value | `1800` | +| tools | `channel_send`, `channel_list`, `memory_store`, `memory_retrieve`, `think`, `web_search`, `file_write` | +| max_turns | `20` | +| retrieval_strategy | `sqlite` | +| task_decomposition | `phased` | + +**code_reviewer.toml:** +| Field | Value | +|-------|-------| +| schedule_type | `interval` | +| schedule_value | `3600` | +| tools | `file_read`, `file_write`, `shell_exec`, `git_status`, `git_diff`, `git_commit`, `git_log`, `apply_patch`, `code_interpreter`, `memory_store`, `memory_retrieve`, `think` | +| temperature | `0.1` | +| memory_extraction | `scratchpad` | +| retrieval_strategy | `sqlite` | +| task_decomposition | `monolithic` | + +### 3. System Prompt Templates + +Each template includes a rich system prompt that documents available tools, workflow steps, and quality standards. The user's instruction is inserted at `{instruction}`. + +**Research Monitor system prompt:** + +``` +You are a Research Monitor agent. Your job is to systematically search for new papers, articles, and developments on your assigned topics, store important findings in memory, and produce concise summaries. + +## Your Assigned Topic +{instruction} + +## Available Tools +You have access to these tools. Use them proactively: + +- **web_search(query)** — Search the web for recent articles, papers, and news. Use specific, targeted queries. Try multiple search angles to get comprehensive coverage. +- **http_request(url, method)** — Fetch a specific URL to read full article content. Use after finding promising URLs via web_search. +- **file_read(path)** / **file_write(path, content)** — Read and write local files. Use to save detailed reports or read reference material. +- **memory_store(key, content)** — Store findings for future reference across sessions. Use structured keys like "finding:2026-03-27:topic-name". +- **memory_retrieve(query)** — Recall previously stored findings. Always check what you already know before searching again. +- **think(thought)** — Reason through complex decisions before acting. Use when planning search strategy or evaluating source quality. + +## How to Work +1. Start by checking memory (memory_retrieve) for what you already know about this topic. +2. Search the web with 2-3 different query angles using web_search. +3. For promising results, fetch the full content via http_request. +4. Extract key findings: title, authors/source, date, main contribution, relevance to your assigned topic. +5. Store each significant finding in memory with a structured key. +6. Produce a concise summary of new developments found. + +## Quality Standards +- Be thorough: try multiple search queries, not just one. +- Be concise: summaries should be 2-4 paragraphs, not essays. +- Be structured: use headers, bullet points, and dates. +- Be honest: if you cannot find recent information, say so clearly. Never fabricate or hallucinate sources. +- Prioritize recency: newer findings are more valuable than old ones. +- Cite sources: include URLs or paper titles for every claim. +``` + +**Code Reviewer system prompt:** + +``` +You are a Code Reviewer agent. Your job is to monitor a repository for recent changes, review code quality, identify potential bugs, suggest improvements, and store your findings. + +## Your Review Focus +{instruction} + +## Available Tools +You have access to these tools. Use them systematically: + +- **git_status()** — Check the current state of the repository. Start here. +- **git_diff()** — View uncommitted changes or diffs between commits. +- **git_log()** — View recent commit history to understand what changed. +- **file_read(path)** — Read source files for detailed review. +- **file_write(path, content)** — Write review notes or suggested patches. +- **shell_exec(command)** — Run linters, tests, or other analysis commands. +- **code_interpreter(code)** — Execute code snippets to verify behavior. +- **apply_patch(patch)** — Apply a suggested fix as a patch. +- **memory_store(key, content)** / **memory_retrieve(query)** — Track review history across sessions. +- **think(thought)** — Reason through complex code logic before commenting. + +## How to Work +1. Run git_status and git_log to understand recent changes. +2. For each significant change, read the affected files with file_read. +3. Analyze for: bugs, security issues, performance problems, readability. +4. Run relevant tests or linters via shell_exec if available. +5. Store findings in memory for tracking across sessions. +6. Produce a summary: what changed, what's good, what needs attention. + +## Quality Standards +- Focus on substance: real bugs and security issues over style nitpicks. +- Be specific: reference exact file paths and line numbers. +- Be constructive: suggest fixes, not just problems. +- Prioritize severity: critical bugs > performance > readability. +- Don't comment on formatting if a linter handles it. +``` + +**Inbox Triager system prompt:** + +``` +You are an Inbox Triager agent. Your job is to monitor incoming messages across email and messaging channels, categorize them by priority and topic, summarize key information, and flag items that need immediate attention. + +## Your Triage Instructions +{instruction} + +## Available Tools +You have access to these tools. Use them to process incoming messages: + +- **channel_list()** — List available messaging channels and their recent messages. +- **channel_send(channel, message)** — Send a message to a channel (for forwarding urgent items or sending status updates). +- **web_search(query)** — Search for context on unfamiliar senders or topics mentioned in messages. +- **file_write(path, content)** — Save triage reports or summaries to local files. +- **memory_store(key, content)** / **memory_retrieve(query)** — Track message history, sender patterns, and priority rules across sessions. +- **think(thought)** — Reason through priority decisions before categorizing. + +## How to Work +1. Check memory for your existing triage rules and sender patterns. +2. List channels to see new incoming messages. +3. For each message, categorize by priority: urgent, important, informational, low. +4. For urgent items, forward via channel_send with a brief summary. +5. Store triage decisions in memory for pattern learning. +6. Produce a summary: counts by priority, key action items, anything unusual. + +## Quality Standards +- Never miss urgent items: err on the side of flagging too much. +- Be concise: triage summaries should be scannable in 30 seconds. +- Learn patterns: remember which senders/topics are usually important. +- Respect context: a message from your boss is higher priority than a newsletter. +- Group related messages: thread continuations should be triaged together. +``` + +**Custom Agent system prompt (generic):** + +``` +You are a personal AI agent. Follow the user's instructions carefully and use your available tools to accomplish the task. + +## Your Instructions +{instruction} + +## Available Tools +{tool_descriptions} + +## How to Work +1. Understand the user's request fully before acting. +2. Use the think tool to plan your approach for complex tasks. +3. Execute steps one at a time, checking results before proceeding. +4. Store important findings in memory for future reference. +5. Provide clear, structured responses. + +## Quality Standards +- Be thorough but concise. +- Use tools proactively — don't just generate text when you can take action. +- Be honest about limitations. +- Cite sources when making factual claims. +``` + +For Custom Agent, `{tool_descriptions}` is dynamically generated from the user's selected tools at creation time, listing each tool's name and description from the ToolRegistry. + +### 4. Tool Wiring in Executor + +**File:** `src/openjarvis/agents/executor.py`, method `_invoke_agent()` + +Current code (broken): +```python +agent_instance = agent_cls(engine, model, system_prompt=..., tools=[]) +``` + +Fixed code: +```python +# Read tool names from agent config +tool_names = config.get("tools", []) +if isinstance(tool_names, str): + tool_names = [t.strip() for t in tool_names.split(",") if t.strip()] + +# Resolve tool instances from ToolRegistry +tool_instances = [] +if tool_names: + from openjarvis.server.agent_manager_routes import _ensure_registries_populated + _ensure_registries_populated() + from openjarvis.core.registry import ToolRegistry + + for name in tool_names: + tool_cls = ToolRegistry.get(name) + if tool_cls is not None: + try: + tool = tool_cls() + # Inject runtime deps (engine, memory) using existing helper + self._inject_tool_deps(tool) + tool_instances.append(tool) + except Exception: + logger.warning("Failed to instantiate tool %s", name) + +agent_instance = agent_cls( + engine, model, + system_prompt=config.get("system_prompt"), + tools=tool_instances, +) +``` + +A new `_inject_tool_deps` instance method on the executor injects runtime dependencies into tools that need them. It mirrors the logic in `SystemBuilder._inject_tool_deps` (system.py:920-945): for `llm` tools inject engine+model, for `memory_*` tools inject memory_backend, for `channel_*` tools inject channel_backend. References are taken from `self._system` (the lightweight system). + +### 5. Recommended Model Endpoint + +**File:** `src/openjarvis/server/agent_manager_routes.py` (or a new lightweight route) + +``` +GET /v1/recommended-model + +Response: +{ + "model": "qwen3.5:9b", + "reason": "Second-largest model that fits in 64.0 GB RAM" +} +``` + +Logic: +1. Get available models from engine via `engine.list_models()` +2. Filter to local models (exclude cloud models like gpt-4o) +3. Parse parameter count from model name (e.g. "qwen3.5:9b" → 9.0) +4. Sort by parameter count descending +5. Pick the second-largest (leaves headroom for OS/apps) +6. If only 1 model available, pick it +7. Return model ID and human-readable reason + +The frontend wizard calls this on mount and pre-selects it in the Intelligence dropdown with a "(recommended)" badge. + +### 6. Frontend Changes + +**File:** `frontend/src/pages/AgentsPage.tsx` + +**Wizard state simplification:** +Remove the 3-step state machine. Replace with: +- `wizardStep: 1 | 2` +- `selectedTemplate: string | null` +- `name: string` +- `instruction: string` +- `model: string` (pre-filled from `/v1/recommended-model`) +- `scheduleType: string` (pre-filled from template) +- `scheduleValue: string` (pre-filled from template) +- `selectedTools: string[]` (pre-filled from template) +- Advanced settings: all pre-filled from template, editable in collapsed section + +**handleLaunch():** +1. Build the system prompt by inserting `instruction` into the template's prompt template +2. Build config object with all fields +3. POST to `/v1/managed-agents` +4. No review step — launches directly + +**File:** `frontend/src/lib/api.ts` + +Add: +```typescript +export async function fetchRecommendedModel(): Promise<{ model: string; reason: string }> { + const res = await fetch(`${getBase()}/v1/recommended-model`); + if (!res.ok) throw new Error(`Failed: ${res.status}`); + return res.json(); +} +``` + +### 7. Template TOML Updates + +Each template TOML gets new fields: + +```toml +[template] +id = "research_monitor" +name = "Research Monitor" +description = "Searches papers, news, blogs on your topic. Stores findings in memory." +icon = "🔬" +agent_type = "monitor_operative" + +# Schedule defaults +schedule_type = "cron" +schedule_value = "0 9 * * *" + +# Tools (wired to executor) +tools = ["web_search", "http_request", "file_read", "file_write", "memory_store", "memory_retrieve", "think"] + +# Agent behavior +max_turns = 25 +temperature = 0.3 +memory_extraction = "structured_json" +observation_compression = "summarize" +retrieval_strategy = "sqlite" +task_decomposition = "phased" + +# System prompt template — full text in Section 3 of this spec +# Uses {instruction} placeholder, replaced at creation time +system_prompt_template = "..." # See Section 3: Research Monitor system prompt +``` + +The `system_prompt_template` field replaces the current `system_prompt` field. At creation time, `{instruction}` is replaced with the user's instruction. For Custom Agent, `{tool_descriptions}` is also replaced with dynamically generated tool docs. + +--- + +## Data Flow Summary + +``` +User picks template → template TOML loaded +User types name + instruction +Backend recommends model → pre-selected + ↓ +Frontend builds config: + config.tools = template.tools + config.model = recommended_model + config.schedule_type = template.schedule_type + config.system_prompt = template.system_prompt_template.format(instruction=...) + config.memory_extraction = template.memory_extraction (or universal default) + ... (all other fields) + ↓ +POST /v1/managed-agents { name, config } + ↓ +Manager stores in SQLite (config as JSON) + ↓ +On tick: executor reads config + → resolves tools from ToolRegistry + → constructs agent with tools + system_prompt + → agent.run() can now call web_search, file_read, etc. +``` + +## Files Changed + +| File | Changes | +|------|---------| +| `frontend/src/pages/AgentsPage.tsx` | Rewrite wizard: 2-step, smart defaults, collapsed Advanced | +| `frontend/src/lib/api.ts` | Add `fetchRecommendedModel()` | +| `src/openjarvis/server/agent_manager_routes.py` | Add `/v1/recommended-model` endpoint | +| `src/openjarvis/agents/executor.py` | Wire tools from config, add `_inject_tool_deps` | +| `src/openjarvis/agents/templates/research_monitor.toml` | Add `system_prompt_template`, `icon`, update defaults | +| `src/openjarvis/agents/templates/code_reviewer.toml` | Add `system_prompt_template`, `icon`, update defaults | +| `src/openjarvis/agents/templates/inbox_triager.toml` | Add `system_prompt_template`, `icon`, update defaults | +| `src/openjarvis/agents/manager.py` | Handle `system_prompt_template` → `system_prompt` expansion in `create_from_template()` | + +## Testing + +- Lint: `uv run ruff check src/ tests/` +- Unit: `uv run pytest tests/server/test_agent_manager_routes.py tests/core/test_config.py -v` +- Manual: create each template agent in browser, verify tools are wired, verify system prompt contains instruction +- Manual: create Custom Agent, verify recommended model pre-selected, verify tools can be added +- Manual: verify Advanced settings collapse/expand and persist values From 264998b36ce3677d5a2393cc1b0743b5dba390e0 Mon Sep 17 00:00:00 2001 From: Jon Saad-Falcon <41205309+jonsaadfalcon@users.noreply.github.com> Date: Fri, 27 Mar 2026 11:17:31 -0700 Subject: [PATCH 35/44] fix: filter outlier entries from leaderboard instead of recomputing MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replace the recompute approach from #146 with per-token outlier detection. Entries whose energy, FLOPs, or dollar savings exceed generous per-token thresholds (~1000x legitimate values) are hidden from the leaderboard entirely. All displayed values come directly from the database — no rewriting of submitted data. Co-Authored-By: Claude Opus 4.6 (1M context) --- docs/javascripts/leaderboard.js | 62 ++++++++++++++------------------- frontend/src/App.tsx | 5 +-- 2 files changed, 28 insertions(+), 39 deletions(-) diff --git a/docs/javascripts/leaderboard.js b/docs/javascripts/leaderboard.js index 093a3022..ce22868c 100644 --- a/docs/javascripts/leaderboard.js +++ b/docs/javascripts/leaderboard.js @@ -9,27 +9,24 @@ var allRows = []; var currentPage = 0; - // Claude Opus 4.6 constants — recompute energy and FLOPs from - // total_tokens so submitted values cannot be gamed. - var CLAUDE_PARAMS_B = 137; - var CLAUDE_FLOPS_PER_TOKEN = 4.0e12; - var CLAUDE_WH_PER_FLOP = 0.5 / (1000 * CLAUDE_FLOPS_PER_TOKEN); - // Max possible $/token: all tokens are output at $25/1M - var MAX_DOLLAR_PER_TOKEN = 25.0 / 1e6; + // Outlier detection — hide entries with values that are physically + // implausible relative to their token count. Thresholds are ~1000x + // above legitimate per-token values to avoid false positives. + var MAX_ENERGY_WH_PER_TOKEN = 10; // legit ≈ 0.001 Wh/tok + var MAX_FLOPS_PER_TOKEN = 1e17; // legit ≈ 1e12 /tok + var MAX_DOLLAR_PER_TOKEN = 25.0 / 1e6; // hard ceiling: $25/1M output - function recomputeFlops(totalTokens) { - var n = Number(totalTokens) || 0; - if (n <= 0) return 0; - return 2.0 * CLAUDE_PARAMS_B * 1e9 * n; - } - - function recomputeEnergy(totalTokens) { - return recomputeFlops(totalTokens) * CLAUDE_WH_PER_FLOP; - } - - function clampDollars(dollarSavings, totalTokens) { - var max = (Number(totalTokens) || 0) * MAX_DOLLAR_PER_TOKEN; - return Math.min(Number(dollarSavings) || 0, max); + function isOutlier(row) { + var tokens = Number(row.total_tokens) || 0; + if (tokens <= 0) return false; + var energy = Number(row.energy_wh_saved) || 0; + var flops = Number(row.flops_saved) || 0; + var dollars = Number(row.dollar_savings) || 0; + return ( + energy / tokens > MAX_ENERGY_WH_PER_TOKEN || + flops / tokens > MAX_FLOPS_PER_TOKEN || + dollars / tokens > MAX_DOLLAR_PER_TOKEN + ); } function escapeHtml(s) { @@ -65,19 +62,15 @@ var medal = rank === 1 ? "\uD83E\uDD47" : rank === 2 ? "\uD83E\uDD48" : rank === 3 ? "\uD83E\uDD49" : ""; var row = pageRows[j]; - var tokens = Number(row.total_tokens || 0); - var dollars = clampDollars(row.dollar_savings, tokens); - var flops = recomputeFlops(tokens); - var energyWh = recomputeEnergy(tokens); html += "" + '' + (medal || rank) + "" + '' + escapeHtml(row.display_name) + "" + - '$' + dollars.toFixed(4) + "" + - '' + energyWh.toFixed(2) + "" + - '' + fmtLarge(flops) + "" + + '$' + Number(row.dollar_savings || 0).toFixed(4) + "" + + '' + Number(row.energy_wh_saved || 0).toFixed(2) + "" + + '' + fmtLarge(Number(row.flops_saved || 0)) + "" + '' + Number(row.total_calls || 0).toLocaleString() + "" + - '' + tokens.toLocaleString() + "" + + '' + Number(row.total_tokens || 0).toLocaleString() + "" + ""; } tbody.innerHTML = html; @@ -144,18 +137,17 @@ return; } - allRows = rows; + allRows = rows.filter(function (r) { return !isOutlier(r); }); currentPage = 0; - var totalMembers = rows.length; + var totalMembers = allRows.length; var totalDollars = 0; var totalRequests = 0; var totalTokens = 0; - for (var i = 0; i < rows.length; i++) { - var t = Number(rows[i].total_tokens || 0); - totalDollars += clampDollars(rows[i].dollar_savings, t); - totalRequests += Number(rows[i].total_calls || 0); - totalTokens += t; + for (var i = 0; i < allRows.length; i++) { + totalDollars += Number(allRows[i].dollar_savings || 0); + totalRequests += Number(allRows[i].total_calls || 0); + totalTokens += Number(allRows[i].total_tokens || 0); } var elMembers = document.getElementById("stat-members"); diff --git a/frontend/src/App.tsx b/frontend/src/App.tsx index c6ccf928..e627931f 100644 --- a/frontend/src/App.tsx +++ b/frontend/src/App.tsx @@ -71,9 +71,6 @@ export default function App() { (p) => p.provider === 'claude-opus-4.6', ); const dollarSavings = claudeEntry ? claudeEntry.total_cost : 0; - // Cap at theoretical max ($25/1M output tokens) to prevent spoofed values - const maxDollars = (data.total_tokens * 25) / 1_000_000; - const clampedDollars = Math.min(dollarSavings, maxDollars); const energySaved = data.per_provider.reduce( (sum, p) => sum + (p.energy_wh || 0), 0, @@ -88,7 +85,7 @@ export default function App() { email: optInEmail, total_calls: data.total_calls, total_tokens: data.total_tokens, - dollar_savings: clampedDollars, + dollar_savings: dollarSavings, energy_wh_saved: energySaved, flops_saved: flopsSaved, token_counting_version: data.token_counting_version ?? 1, From 8899662fc5ab0ca959195eb915b02201111f19fc Mon Sep 17 00:00:00 2001 From: krypticmouse Date: Fri, 27 Mar 2026 18:19:09 +0000 Subject: [PATCH 36/44] fix: line-too-long lint error in agent_manager_routes.py Co-Authored-By: Claude Opus 4.6 (1M context) --- src/openjarvis/server/agent_manager_routes.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/openjarvis/server/agent_manager_routes.py b/src/openjarvis/server/agent_manager_routes.py index 11c5404b..05a8991f 100644 --- a/src/openjarvis/server/agent_manager_routes.py +++ b/src/openjarvis/server/agent_manager_routes.py @@ -240,7 +240,9 @@ def build_tools_list() -> List[Dict[str, Any]]: items.append( { "name": name, - "description": f"{name.replace('_', ' ').title()} messaging channel", + "description": ( + f"{name.replace('_', ' ').title()} messaging channel" + ), "category": "communication", "source": "channel", "requires_credentials": len(cred_keys) > 0, From 049d6ff543b477eda1750e6cf95ca14bb241c77d Mon Sep 17 00:00:00 2001 From: Jon Saad-Falcon <41205309+jonsaadfalcon@users.noreply.github.com> Date: Fri, 27 Mar 2026 11:21:01 -0700 Subject: [PATCH 37/44] docs: add agent wizard simplification implementation plan 7-task plan covering template TOMLs, manager expansion, executor tool wiring, recommended-model endpoint, frontend API, wizard rewrite, and integration testing. Co-Authored-By: Claude Opus 4.6 (1M context) --- .../2026-03-27-agent-wizard-simplification.md | 1164 +++++++++++++++++ 1 file changed, 1164 insertions(+) create mode 100644 docs/superpowers/plans/2026-03-27-agent-wizard-simplification.md diff --git a/docs/superpowers/plans/2026-03-27-agent-wizard-simplification.md b/docs/superpowers/plans/2026-03-27-agent-wizard-simplification.md new file mode 100644 index 00000000..29a12856 --- /dev/null +++ b/docs/superpowers/plans/2026-03-27-agent-wizard-simplification.md @@ -0,0 +1,1164 @@ +# Agent Wizard Simplification Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Simplify the agent creation wizard from 12+ fields / 3 steps down to 2 required fields / 2 steps, wire tools from config to executor, and add rich system prompt templates. + +**Architecture:** Backend-first approach. Update template TOMLs with system_prompt_template and new defaults, add tool wiring to executor, add /v1/recommended-model endpoint, then rewrite the frontend wizard. Each task is independently testable. + +**Tech Stack:** Python (FastAPI, SQLite), TypeScript (React), TOML templates + +--- + +## File Map + +| File | Action | Responsibility | +|------|--------|----------------| +| `src/openjarvis/agents/templates/research_monitor.toml` | Modify | Add system_prompt_template, icon, update defaults | +| `src/openjarvis/agents/templates/code_reviewer.toml` | Modify | Add system_prompt_template, icon, update defaults | +| `src/openjarvis/agents/templates/inbox_triager.toml` | Modify | Add system_prompt_template, icon, update defaults | +| `src/openjarvis/agents/manager.py` | Modify | Expand system_prompt_template in create_from_template() | +| `src/openjarvis/agents/executor.py` | Modify | Wire tools from config, add _inject_tool_deps | +| `src/openjarvis/server/agent_manager_routes.py` | Modify | Add /v1/recommended-model endpoint | +| `frontend/src/lib/api.ts` | Modify | Add fetchRecommendedModel() | +| `frontend/src/pages/AgentsPage.tsx` | Modify | Rewrite LaunchWizard to 2-step flow | +| `tests/agents/test_executor_tools.py` | Create | Test tool wiring in executor | +| `tests/agents/test_template_prompts.py` | Create | Test system_prompt_template expansion | +| `tests/server/test_recommended_model.py` | Create | Test /v1/recommended-model endpoint | + +--- + +### Task 1: Update Template TOMLs with system_prompt_template and new defaults + +**Files:** +- Modify: `src/openjarvis/agents/templates/research_monitor.toml` +- Modify: `src/openjarvis/agents/templates/code_reviewer.toml` +- Modify: `src/openjarvis/agents/templates/inbox_triager.toml` + +- [ ] **Step 1: Update research_monitor.toml** + +Replace the entire file with: + +```toml +[template] +id = "research_monitor" +name = "Research Monitor" +description = "Searches papers, news, blogs on your topic. Stores findings in memory." +icon = "🔬" +agent_type = "monitor_operative" +schedule_type = "cron" +schedule_value = "0 9 * * *" +tools = ["web_search", "http_request", "file_read", "file_write", "memory_store", "memory_retrieve", "think"] +max_turns = 25 +temperature = 0.3 +memory_extraction = "structured_json" +observation_compression = "summarize" +retrieval_strategy = "sqlite" +task_decomposition = "phased" +system_prompt_template = """You are a Research Monitor agent. Your job is to systematically search for new papers, articles, and developments on your assigned topics, store important findings in memory, and produce concise summaries. + +## Your Assigned Topic +{instruction} + +## Available Tools +You have access to these tools. Use them proactively: + +- **web_search(query)** — Search the web for recent articles, papers, and news. Use specific, targeted queries. Try multiple search angles to get comprehensive coverage. +- **http_request(url, method)** — Fetch a specific URL to read full article content. Use after finding promising URLs via web_search. +- **file_read(path)** / **file_write(path, content)** — Read and write local files. Use to save detailed reports or read reference material. +- **memory_store(key, content)** — Store findings for future reference across sessions. Use structured keys like "finding:YYYY-MM-DD:topic-name". +- **memory_retrieve(query)** — Recall previously stored findings. Always check what you already know before searching again. +- **think(thought)** — Reason through complex decisions before acting. Use when planning search strategy or evaluating source quality. + +## How to Work +1. Start by checking memory (memory_retrieve) for what you already know about this topic. +2. Search the web with 2-3 different query angles using web_search. +3. For promising results, fetch the full content via http_request. +4. Extract key findings: title, authors/source, date, main contribution, relevance to your assigned topic. +5. Store each significant finding in memory with a structured key. +6. Produce a concise summary of new developments found. + +## Quality Standards +- Be thorough: try multiple search queries, not just one. +- Be concise: summaries should be 2-4 paragraphs, not essays. +- Be structured: use headers, bullet points, and dates. +- Be honest: if you cannot find recent information, say so clearly. Never fabricate or hallucinate sources. +- Prioritize recency: newer findings are more valuable than old ones. +- Cite sources: include URLs or paper titles for every claim.""" +``` + +- [ ] **Step 2: Update code_reviewer.toml** + +Replace the entire file with: + +```toml +[template] +id = "code_reviewer" +name = "Code Reviewer" +description = "Monitors a repository for changes, reviews code quality, and identifies bugs." +icon = "🔍" +agent_type = "monitor_operative" +schedule_type = "interval" +schedule_value = "3600" +tools = ["file_read", "file_write", "shell_exec", "git_status", "git_diff", "git_commit", "git_log", "apply_patch", "code_interpreter", "memory_store", "memory_retrieve", "think"] +max_turns = 15 +temperature = 0.1 +memory_extraction = "scratchpad" +observation_compression = "summarize" +retrieval_strategy = "sqlite" +task_decomposition = "monolithic" +system_prompt_template = """You are a Code Reviewer agent. Your job is to monitor a repository for recent changes, review code quality, identify potential bugs, suggest improvements, and store your findings. + +## Your Review Focus +{instruction} + +## Available Tools +You have access to these tools. Use them systematically: + +- **git_status()** — Check the current state of the repository. Start here. +- **git_diff()** — View uncommitted changes or diffs between commits. +- **git_log()** — View recent commit history to understand what changed. +- **file_read(path)** — Read source files for detailed review. +- **file_write(path, content)** — Write review notes or suggested patches. +- **shell_exec(command)** — Run linters, tests, or other analysis commands. +- **code_interpreter(code)** — Execute code snippets to verify behavior. +- **apply_patch(patch)** — Apply a suggested fix as a patch. +- **memory_store(key, content)** / **memory_retrieve(query)** — Track review history across sessions. +- **think(thought)** — Reason through complex code logic before commenting. + +## How to Work +1. Run git_status and git_log to understand recent changes. +2. For each significant change, read the affected files with file_read. +3. Analyze for: bugs, security issues, performance problems, readability. +4. Run relevant tests or linters via shell_exec if available. +5. Store findings in memory for tracking across sessions. +6. Produce a summary: what changed, what's good, what needs attention. + +## Quality Standards +- Focus on substance: real bugs and security issues over style nitpicks. +- Be specific: reference exact file paths and line numbers. +- Be constructive: suggest fixes, not just problems. +- Prioritize severity: critical bugs > performance > readability. +- Don't comment on formatting if a linter handles it.""" +``` + +- [ ] **Step 3: Update inbox_triager.toml** + +Replace the entire file with: + +```toml +[template] +id = "inbox_triager" +name = "Inbox Triager" +description = "Monitors email and messaging channels, categorizes and summarizes by priority." +icon = "📥" +agent_type = "monitor_operative" +schedule_type = "interval" +schedule_value = "1800" +tools = ["channel_send", "channel_list", "memory_store", "memory_retrieve", "think", "web_search", "file_write"] +max_turns = 20 +temperature = 0.3 +memory_extraction = "structured_json" +observation_compression = "summarize" +retrieval_strategy = "sqlite" +task_decomposition = "phased" +system_prompt_template = """You are an Inbox Triager agent. Your job is to monitor incoming messages across email and messaging channels, categorize them by priority and topic, summarize key information, and flag items that need immediate attention. + +## Your Triage Instructions +{instruction} + +## Available Tools +You have access to these tools. Use them to process incoming messages: + +- **channel_list()** — List available messaging channels and their recent messages. +- **channel_send(channel, message)** — Send a message to a channel (for forwarding urgent items or sending status updates). +- **web_search(query)** — Search for context on unfamiliar senders or topics mentioned in messages. +- **file_write(path, content)** — Save triage reports or summaries to local files. +- **memory_store(key, content)** / **memory_retrieve(query)** — Track message history, sender patterns, and priority rules across sessions. +- **think(thought)** — Reason through priority decisions before categorizing. + +## How to Work +1. Check memory for your existing triage rules and sender patterns. +2. List channels to see new incoming messages. +3. For each message, categorize by priority: urgent, important, informational, low. +4. For urgent items, forward via channel_send with a brief summary. +5. Store triage decisions in memory for pattern learning. +6. Produce a summary: counts by priority, key action items, anything unusual. + +## Quality Standards +- Never miss urgent items: err on the side of flagging too much. +- Be concise: triage summaries should be scannable in 30 seconds. +- Learn patterns: remember which senders/topics are usually important. +- Respect context: a message from your boss is higher priority than a newsletter. +- Group related messages: thread continuations should be triaged together.""" +``` + +- [ ] **Step 4: Commit template updates** + +```bash +git add src/openjarvis/agents/templates/research_monitor.toml src/openjarvis/agents/templates/code_reviewer.toml src/openjarvis/agents/templates/inbox_triager.toml +git commit -m "feat: add system_prompt_template, icon, update defaults in agent templates" +``` + +--- + +### Task 2: Manager — expand system_prompt_template on creation + +**Files:** +- Modify: `src/openjarvis/agents/manager.py:478-491` +- Create: `tests/agents/test_template_prompts.py` + +- [ ] **Step 1: Write failing test** + +Create `tests/agents/test_template_prompts.py`: + +```python +"""Tests for system_prompt_template expansion in agent creation.""" + +from __future__ import annotations + +import tempfile +from pathlib import Path + +from openjarvis.agents.manager import AgentManager + + +def test_create_from_template_expands_system_prompt(tmp_path): + """system_prompt_template should be expanded with the instruction.""" + mgr = AgentManager(db_path=str(tmp_path / "test.db")) + agent = mgr.create_from_template( + "research_monitor", + "Test Agent", + overrides={"instruction": "Monitor AI safety papers"}, + ) + config = agent["config"] + # system_prompt should contain the expanded instruction + assert "Monitor AI safety papers" in config.get("system_prompt", "") + # system_prompt_template should NOT be in the stored config + assert "system_prompt_template" not in config + mgr.close() + + +def test_create_from_template_without_instruction(tmp_path): + """Template with no instruction should still have a system_prompt.""" + mgr = AgentManager(db_path=str(tmp_path / "test.db")) + agent = mgr.create_from_template("research_monitor", "Test Agent") + config = agent["config"] + assert "system_prompt" in config + assert len(config["system_prompt"]) > 100 # non-trivial prompt + mgr.close() + + +def test_create_from_template_preserves_icon(tmp_path): + """Template icon field should be preserved in config.""" + mgr = AgentManager(db_path=str(tmp_path / "test.db")) + agent = mgr.create_from_template("research_monitor", "Test Agent") + config = agent["config"] + assert config.get("icon") == "🔬" + mgr.close() +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `uv run pytest tests/agents/test_template_prompts.py -v` +Expected: FAIL — `system_prompt_template` is currently stored raw, not expanded. + +- [ ] **Step 3: Implement system_prompt_template expansion** + +In `src/openjarvis/agents/manager.py`, modify `create_from_template()` (around line 478): + +```python +def create_from_template( + self, template_id: str, name: str, overrides: Optional[Dict[str, Any]] = None +) -> Dict[str, Any]: + """Create an agent from a template with optional overrides.""" + templates = self.list_templates() + tpl = next((t for t in templates if t.get("id") == template_id), None) + if not tpl: + raise ValueError(f"Template not found: {template_id}") + skip = {"id", "name", "description", "source"} + config = {k: v for k, v in tpl.items() if k not in skip} + if overrides: + config.update(overrides) + agent_type = config.pop("agent_type", "monitor_operative") + + # Expand system_prompt_template with instruction + prompt_tpl = config.pop("system_prompt_template", "") + if prompt_tpl: + instruction = config.get("instruction", "") + config["system_prompt"] = prompt_tpl.format( + instruction=instruction or "(No specific instruction provided)", + ) + + return self.create_agent(name=name, agent_type=agent_type, config=config) +``` + +- [ ] **Step 4: Run test to verify it passes** + +Run: `uv run pytest tests/agents/test_template_prompts.py -v` +Expected: PASS + +- [ ] **Step 5: Commit** + +```bash +git add src/openjarvis/agents/manager.py tests/agents/test_template_prompts.py +git commit -m "feat: expand system_prompt_template with instruction on agent creation" +``` + +--- + +### Task 3: Executor — wire tools from config + +**Files:** +- Modify: `src/openjarvis/agents/executor.py:242-248` +- Create: `tests/agents/test_executor_tools.py` + +- [ ] **Step 1: Write failing test** + +Create `tests/agents/test_executor_tools.py`: + +```python +"""Tests for tool wiring in AgentExecutor.""" + +from __future__ import annotations + +import tempfile +import time +from unittest.mock import MagicMock, patch + +import pytest + +from openjarvis.agents._stubs import AgentResult +from openjarvis.agents.executor import AgentExecutor +from openjarvis.agents.manager import AgentManager +from openjarvis.core.events import EventBus + + +class FakeSystem: + def __init__(self): + self.engine = MagicMock() + self.engine.generate.return_value = { + "content": "test response", + "usage": {"prompt_tokens": 10, "completion_tokens": 5, "total_tokens": 15}, + "finish_reason": "stop", + } + self.model = "test-model" + self.config = None + self.memory_backend = None + + +def test_executor_resolves_tools_from_config(tmp_path): + """Executor should resolve tool names from config into tool instances.""" + mgr = AgentManager(db_path=str(tmp_path / "test.db")) + agent = mgr.create_agent( + "test", + agent_type="monitor_operative", + config={ + "system_prompt": "You are a test agent.", + "tools": ["think"], + "instruction": "test", + }, + ) + mgr.send_message(agent["id"], "hello", mode="immediate") + + executor = AgentExecutor(manager=mgr, event_bus=EventBus()) + executor.set_system(FakeSystem()) + + # Patch _invoke_agent to capture the tools passed to the agent + captured_tools = [] + original_invoke = executor._invoke_agent + + def capture_invoke(agent_dict): + # We can't easily intercept the agent constructor, + # so just verify the method doesn't crash with tools + return original_invoke(agent_dict) + + # Just verify it doesn't crash — tool resolution happens inside + executor.execute_tick(agent["id"]) + result_agent = mgr.get_agent(agent["id"]) + assert result_agent["status"] == "idle" + assert result_agent["total_runs"] == 1 + mgr.close() +``` + +- [ ] **Step 2: Run test to verify baseline** + +Run: `uv run pytest tests/agents/test_executor_tools.py -v` +Expected: May pass (tools=[] doesn't crash) but tools aren't actually wired. + +- [ ] **Step 3: Add _inject_tool_deps to executor and wire tools** + +In `src/openjarvis/agents/executor.py`, add the `_inject_tool_deps` method after `_set_activity` (around line 55): + +```python +def _inject_tool_deps(self, tool: Any) -> None: + """Inject runtime dependencies into a tool instance. + + Mirrors SystemBuilder._inject_tool_deps but uses the + lightweight system's references. + """ + if self._system is None: + return + name = getattr(getattr(tool, "spec", None), "name", "") + if name == "llm": + if hasattr(tool, "_engine"): + tool._engine = self._system.engine + if hasattr(tool, "_model"): + tool._model = self._system.model + elif name == "retrieval" or name.startswith("memory_"): + if hasattr(tool, "_backend"): + tool._backend = getattr(self._system, "memory_backend", None) + elif name.startswith("channel_"): + if hasattr(tool, "_channel"): + tool._channel = getattr(self._system, "channel_backend", None) +``` + +Then modify the agent construction in `_invoke_agent()` (around line 242). Replace: + +```python + # Construct agent instance (BaseAgent requires engine, model as positional args) + agent_instance = agent_cls( + engine, + model, + system_prompt=config.get("system_prompt"), + tools=[], + ) +``` + +With: + +```python + # Resolve tools from config via ToolRegistry + tool_names = config.get("tools", []) + if isinstance(tool_names, str): + tool_names = [t.strip() for t in tool_names.split(",") if t.strip()] + + tool_instances: list[Any] = [] + if tool_names: + try: + from openjarvis.server.agent_manager_routes import ( + _ensure_registries_populated, + ) + + _ensure_registries_populated() + except ImportError: + pass + from openjarvis.core.registry import ToolRegistry + + for tname in tool_names: + if ToolRegistry.contains(tname): + try: + tool_cls = ToolRegistry.get(tname) + tool = tool_cls() + self._inject_tool_deps(tool) + tool_instances.append(tool) + except Exception: + logger.warning("Failed to instantiate tool %s", tname) + if tool_instances: + logger.info( + "Agent %s: resolved %d/%d tools", + agent["name"], len(tool_instances), len(tool_names), + ) + + # Construct agent instance + agent_instance = agent_cls( + engine, + model, + system_prompt=config.get("system_prompt"), + tools=tool_instances, + ) +``` + +- [ ] **Step 4: Run test to verify it passes** + +Run: `uv run pytest tests/agents/test_executor_tools.py -v` +Expected: PASS + +- [ ] **Step 5: Run existing tests for regression** + +Run: `uv run pytest tests/server/test_agent_manager_routes.py tests/core/test_config.py -v` +Expected: All PASS + +- [ ] **Step 6: Lint** + +Run: `uv run ruff check src/openjarvis/agents/executor.py` +Expected: All checks passed + +- [ ] **Step 7: Commit** + +```bash +git add src/openjarvis/agents/executor.py tests/agents/test_executor_tools.py +git commit -m "feat: wire tools from agent config to executor via ToolRegistry" +``` + +--- + +### Task 4: Backend — /v1/recommended-model endpoint + +**Files:** +- Modify: `src/openjarvis/server/agent_manager_routes.py` +- Create: `tests/server/test_recommended_model.py` + +- [ ] **Step 1: Write failing test** + +Create `tests/server/test_recommended_model.py`: + +```python +"""Tests for /v1/recommended-model endpoint.""" + +from __future__ import annotations + +import re +from unittest.mock import MagicMock + +import pytest + +try: + from fastapi.testclient import TestClient + + HAS_FASTAPI = True +except ImportError: + HAS_FASTAPI = False + + +@pytest.mark.skipif(not HAS_FASTAPI, reason="fastapi not installed") +def test_recommended_model_picks_second_largest(): + """Should pick the second-largest local model.""" + from fastapi import FastAPI + + from openjarvis.server.agent_manager_routes import ( + _parse_param_count, + _pick_recommended_model, + ) + + models = ["qwen3.5:0.8b", "qwen3.5:4b", "qwen3.5:9b", "qwen3.5:35b"] + result = _pick_recommended_model(models) + assert result["model"] == "qwen3.5:9b" + + +@pytest.mark.skipif(not HAS_FASTAPI, reason="fastapi not installed") +def test_recommended_model_single_model(): + """With only one model, pick it.""" + from openjarvis.server.agent_manager_routes import _pick_recommended_model + + result = _pick_recommended_model(["qwen3.5:4b"]) + assert result["model"] == "qwen3.5:4b" + + +@pytest.mark.skipif(not HAS_FASTAPI, reason="fastapi not installed") +def test_recommended_model_filters_cloud(): + """Cloud models should be excluded.""" + from openjarvis.server.agent_manager_routes import _pick_recommended_model + + models = ["qwen3.5:4b", "gpt-4o", "claude-3.5-sonnet", "qwen3.5:9b"] + result = _pick_recommended_model(models) + # Should only consider local models + assert result["model"] in ("qwen3.5:4b", "qwen3.5:9b") + + +@pytest.mark.skipif(not HAS_FASTAPI, reason="fastapi not installed") +def test_parse_param_count(): + """Parse parameter counts from model names.""" + from openjarvis.server.agent_manager_routes import _parse_param_count + + assert _parse_param_count("qwen3.5:9b") == 9.0 + assert _parse_param_count("qwen3.5:0.8b") == 0.8 + assert _parse_param_count("qwen3.5:35b") == 35.0 + assert _parse_param_count("gpt-4o") == 0.0 # unparseable +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `uv run pytest tests/server/test_recommended_model.py -v` +Expected: FAIL — `_parse_param_count` and `_pick_recommended_model` don't exist. + +- [ ] **Step 3: Implement helper functions and endpoint** + +In `src/openjarvis/server/agent_manager_routes.py`, add before the `_ensure_registries_populated` function (around line 105): + +```python +import re as _re + + +def _parse_param_count(model_name: str) -> float: + """Extract parameter count in billions from model name. + + Examples: 'qwen3.5:9b' → 9.0, 'qwen3.5:0.8b' → 0.8 + """ + m = _re.search(r":(\d+(?:\.\d+)?)b", model_name.lower()) + return float(m.group(1)) if m else 0.0 + + +_CLOUD_PREFIXES = ("gpt-", "claude-", "gemini-", "o1-", "o3-", "o4-") + + +def _pick_recommended_model( + model_ids: list[str], +) -> dict[str, str]: + """Pick the second-largest local model from a list.""" + local = [ + m for m in model_ids + if not any(m.startswith(p) for p in _CLOUD_PREFIXES) + ] + if not local: + return {"model": model_ids[0] if model_ids else "", "reason": "Only model available"} + sized = sorted(local, key=_parse_param_count, reverse=True) + if len(sized) == 1: + return {"model": sized[0], "reason": "Only local model available"} + pick = sized[1] # second-largest + params = _parse_param_count(pick) + return { + "model": pick, + "reason": f"Second-largest local model ({params}B parameters)", + } +``` + +Then inside `create_agent_manager_router()`, add the endpoint on the `tools_router` (since it's already a utility router). Add after the `credential_status` endpoint (around line 785): + +```python + @tools_router.get("/recommended-model") + def recommended_model(request: Request): + engine = getattr(request.app.state, "engine", None) + if engine is None: + return {"model": "", "reason": "No engine available"} + try: + models = engine.list_models() + except Exception: + models = [] + return _pick_recommended_model(models) +``` + +Note: The endpoint path will be `/v1/tools/recommended-model` since it's on the tools_router with prefix `/v1/tools`. Alternatively, put it on the `global_router` to get `/v1/recommended-model`. Let's use `global_router`: + +```python + @global_router.get("/v1/recommended-model") + def recommended_model(request: Request): + engine = getattr(request.app.state, "engine", None) + if engine is None: + return {"model": "", "reason": "No engine available"} + try: + models = engine.list_models() + except Exception: + models = [] + return _pick_recommended_model(models) +``` + +- [ ] **Step 4: Run test to verify it passes** + +Run: `uv run pytest tests/server/test_recommended_model.py -v` +Expected: PASS + +- [ ] **Step 5: Lint** + +Run: `uv run ruff check src/openjarvis/server/agent_manager_routes.py` +Expected: All checks passed + +- [ ] **Step 6: Commit** + +```bash +git add src/openjarvis/server/agent_manager_routes.py tests/server/test_recommended_model.py +git commit -m "feat: add /v1/recommended-model endpoint" +``` + +--- + +### Task 5: Frontend API — add fetchRecommendedModel + +**Files:** +- Modify: `frontend/src/lib/api.ts` + +- [ ] **Step 1: Add fetchRecommendedModel function** + +In `frontend/src/lib/api.ts`, add after the `fetchModels` function (around line 101): + +```typescript +export async function fetchRecommendedModel(): Promise<{ model: string; reason: string }> { + const res = await fetch(`${getBase()}/v1/recommended-model`); + if (!res.ok) return { model: '', reason: 'Failed to fetch' }; + return res.json(); +} +``` + +- [ ] **Step 2: Commit** + +```bash +git add frontend/src/lib/api.ts +git commit -m "feat: add fetchRecommendedModel API function" +``` + +--- + +### Task 6: Frontend — rewrite LaunchWizard to 2-step flow + +**Files:** +- Modify: `frontend/src/pages/AgentsPage.tsx:207-932` (the WizardState interface and LaunchWizard component) + +This is the largest task. The wizard is ~725 lines. We replace it with a simpler 2-step flow. + +- [ ] **Step 1: Add fetchRecommendedModel to imports** + +At the top of `frontend/src/pages/AgentsPage.tsx`, add `fetchRecommendedModel` to the api imports: + +```typescript +import { + fetchManagedAgents, + fetchAgentTasks, + fetchAgentChannels, + fetchAgentMessages, + fetchTemplates, + createManagedAgent, + pauseManagedAgent, + resumeManagedAgent, + deleteManagedAgent, + runManagedAgent, + recoverManagedAgent, + sendAgentMessage, + fetchLearningLog, + triggerLearning, + fetchAgentTraces, + fetchManagedAgent, + fetchAvailableTools, + saveToolCredentials, + fetchModels, + updateManagedAgent, + fetchRecommendedModel, +} from '../lib/api'; +``` + +- [ ] **Step 2: Replace WizardState interface** + +Replace the WizardState interface (lines 207-222) with: + +```typescript +interface WizardState { + step: 1 | 2; + templateId: string; + templateData: AgentTemplate | null; + name: string; + instruction: string; + model: string; + scheduleType: string; + scheduleValue: string; + selectedTools: string[]; + budget: string; + routerPolicy: string; + memoryExtraction: string; + observationCompression: string; + retrievalStrategy: string; + taskDecomposition: string; + maxTurns: number; + temperature: number; +} +``` + +- [ ] **Step 3: Rewrite LaunchWizard component** + +Replace the entire `LaunchWizard` function (from `function LaunchWizard` through its closing `}`, approximately lines 224-932) with the new 2-step wizard. This is the full replacement code: + +```typescript +function LaunchWizard({ + templates, + onClose, + onLaunched, +}: { + templates: AgentTemplate[]; + onClose: () => void; + onLaunched: () => void; +}) { + const UNIVERSAL_DEFAULTS = { + memoryExtraction: 'structured_json', + observationCompression: 'summarize', + retrievalStrategy: 'sqlite', + taskDecomposition: 'hierarchical', + maxTurns: 25, + temperature: 0.3, + }; + + const [wizard, setWizard] = useState({ + step: 1, + templateId: '', + templateData: null, + name: '', + instruction: '', + model: '', + scheduleType: 'manual', + scheduleValue: '', + selectedTools: [], + budget: '', + routerPolicy: '', + ...UNIVERSAL_DEFAULTS, + }); + const [launching, setLaunching] = useState(false); + const [recommendedModel, setRecommendedModel] = useState(''); + const models = useAppStore((s) => s.models); + + // Fetch recommended model on mount + useEffect(() => { + fetchRecommendedModel().then((r) => { + setRecommendedModel(r.model); + if (!wizard.model) { + setWizard((w) => ({ ...w, model: r.model })); + } + }).catch(() => {}); + }, []); + + function selectTemplate(tpl: AgentTemplate | null) { + if (tpl) { + setWizard((w) => ({ + ...w, + step: 2, + templateId: tpl.id, + templateData: tpl, + name: '', + instruction: '', + model: recommendedModel || w.model, + scheduleType: (tpl as any).schedule_type || 'manual', + scheduleValue: (tpl as any).schedule_value || '', + selectedTools: (tpl as any).tools || [], + memoryExtraction: (tpl as any).memory_extraction || UNIVERSAL_DEFAULTS.memoryExtraction, + observationCompression: (tpl as any).observation_compression || UNIVERSAL_DEFAULTS.observationCompression, + retrievalStrategy: (tpl as any).retrieval_strategy || UNIVERSAL_DEFAULTS.retrievalStrategy, + taskDecomposition: (tpl as any).task_decomposition || UNIVERSAL_DEFAULTS.taskDecomposition, + maxTurns: (tpl as any).max_turns || UNIVERSAL_DEFAULTS.maxTurns, + temperature: (tpl as any).temperature ?? UNIVERSAL_DEFAULTS.temperature, + })); + } else { + // Custom Agent + setWizard((w) => ({ + ...w, + step: 2, + templateId: '', + templateData: null, + name: '', + instruction: '', + model: recommendedModel || w.model, + scheduleType: 'manual', + scheduleValue: '', + selectedTools: [], + ...UNIVERSAL_DEFAULTS, + })); + } + } + + async function handleLaunch() { + if (!wizard.name.trim()) { toast.error('Name is required'); return; } + setLaunching(true); + try { + const config: Record = { + schedule_type: wizard.scheduleType, + schedule_value: wizard.scheduleValue || undefined, + tools: wizard.selectedTools, + learning_enabled: !!wizard.routerPolicy, + memory_extraction: wizard.memoryExtraction, + observation_compression: wizard.observationCompression, + retrieval_strategy: wizard.retrievalStrategy, + task_decomposition: wizard.taskDecomposition, + max_turns: wizard.maxTurns, + temperature: wizard.temperature, + }; + if (wizard.budget) config.budget = parseFloat(wizard.budget); + if (wizard.instruction.trim()) config.instruction = wizard.instruction.trim(); + if (wizard.model) config.model = wizard.model; + if (wizard.routerPolicy) config.router_policy = wizard.routerPolicy; + + await createManagedAgent({ + name: wizard.name.trim(), + template_id: wizard.templateId || undefined, + config, + }); + toast.success(`Agent "${wizard.name}" created`); + onLaunched(); + } catch (err: any) { + toast.error(err.message || 'Failed to create agent'); + } finally { + setLaunching(false); + } + } + + const formatScheduleLabel = (type: string, value: string) => { + if (type === 'manual') return 'Manual (run on demand)'; + if (type === 'cron') return `Cron: ${value}`; + if (type === 'interval') { + const secs = parseInt(value, 10); + if (secs >= 3600) return `Every ${secs / 3600}h`; + if (secs >= 60) return `Every ${secs / 60}m`; + return `Every ${secs}s`; + } + return type; + }; + + // ── Step 1: Template Selection ── + if (wizard.step === 1) { + return ( +
+
+
+

New Agent — Choose Template

+ +
+
+ {templates.map((tpl) => ( + + ))} + +
+
+
+ ); + } + + // ── Step 2: Configuration ── + return ( +
+
+
+
+ +

+ {wizard.templateData ? `New ${wizard.templateData.name}` : 'New Custom Agent'} +

+
+ +
+ +
+ {/* Name */} +
+ + setWizard((w) => ({ ...w, name: e.target.value }))} + placeholder="e.g. AI Research Tracker" + className="w-full px-3 py-2 rounded-lg text-sm bg-transparent" + style={{ border: '1px solid var(--color-border)', color: 'var(--color-text)' }} + /> +
+ + {/* Instruction */} +
+ +