mirror of
https://github.com/open-jarvis/OpenJarvis.git
synced 2026-07-28 14:07:55 +00:00
Prathap PandGitHub
c2756964a7
* fix(channels): wire channel→agent handler and fix Telegram send pipeline * format code * add supported tests
555 lines
18 KiB
Python
555 lines
18 KiB
Python
"""Extended integration tests for new components (Phase 6+)."""
|
|
|
|
from __future__ import annotations
|
|
|
|
from unittest.mock import MagicMock
|
|
|
|
import pytest
|
|
|
|
from openjarvis.agents._stubs import AgentContext, AgentResult
|
|
from openjarvis.core.events import EventBus, EventType
|
|
from openjarvis.core.registry import AgentRegistry, ToolRegistry
|
|
from openjarvis.core.types import (
|
|
Conversation,
|
|
Message,
|
|
Role,
|
|
)
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Helpers
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def _register_all():
|
|
"""Ensure agents and tools are registered."""
|
|
from openjarvis.agents.native_openhands import NativeOpenHandsAgent
|
|
from openjarvis.agents.native_react import NativeReActAgent
|
|
from openjarvis.tools.calculator import CalculatorTool
|
|
from openjarvis.tools.think import ThinkTool
|
|
|
|
for key, cls in [
|
|
("native_react", NativeReActAgent),
|
|
("native_openhands", NativeOpenHandsAgent),
|
|
]:
|
|
if not AgentRegistry.contains(key):
|
|
AgentRegistry.register_value(key, cls)
|
|
|
|
for key, cls in [
|
|
("calculator", CalculatorTool),
|
|
("think", ThinkTool),
|
|
]:
|
|
if not ToolRegistry.contains(key):
|
|
ToolRegistry.register_value(key, cls)
|
|
|
|
|
|
def _make_engine(responses):
|
|
"""Create a mock engine returning a sequence of responses."""
|
|
engine = MagicMock()
|
|
engine.engine_id = "mock"
|
|
engine.health.return_value = True
|
|
engine.list_models.return_value = ["test-model"]
|
|
if isinstance(responses, list):
|
|
engine.generate.side_effect = responses
|
|
else:
|
|
engine.generate.return_value = responses
|
|
return engine
|
|
|
|
|
|
def _simple_response(content, model="test-model"):
|
|
return {
|
|
"content": content,
|
|
"usage": {
|
|
"prompt_tokens": 10,
|
|
"completion_tokens": 5,
|
|
"total_tokens": 15,
|
|
},
|
|
"model": model,
|
|
"finish_reason": "stop",
|
|
}
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# ReAct pipeline integration
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
class TestReActPipeline:
|
|
"""End-to-end: ReAct agent with calculator tool."""
|
|
|
|
def test_react_with_calculator_e2e(self):
|
|
_register_all()
|
|
from openjarvis.agents.native_react import NativeReActAgent
|
|
from openjarvis.tools.calculator import CalculatorTool
|
|
|
|
responses = [
|
|
_simple_response(
|
|
"Thought: I need to calculate 2+2.\n"
|
|
"Action: calculator\n"
|
|
'Action Input: {"expression":"2+2"}'
|
|
),
|
|
_simple_response("Thought: The result is 4.\nFinal Answer: 2+2 equals 4."),
|
|
]
|
|
engine = _make_engine(responses)
|
|
bus = EventBus(record_history=True)
|
|
agent = NativeReActAgent(
|
|
engine,
|
|
"test-model",
|
|
tools=[CalculatorTool()],
|
|
bus=bus,
|
|
)
|
|
result = agent.run("What is 2+2?")
|
|
|
|
assert isinstance(result, AgentResult)
|
|
assert "4" in result.content
|
|
assert result.turns == 2
|
|
assert len(result.tool_results) == 1
|
|
assert result.tool_results[0].content == "4.0"
|
|
|
|
def test_react_with_think_tool(self):
|
|
_register_all()
|
|
from openjarvis.agents.native_react import NativeReActAgent
|
|
from openjarvis.tools.think import ThinkTool
|
|
|
|
responses = [
|
|
_simple_response(
|
|
"Thought: Let me reason about this.\n"
|
|
"Action: think\n"
|
|
'Action Input: {"thought":"Step 1: analyze"}'
|
|
),
|
|
_simple_response(
|
|
"Thought: I have my analysis.\nFinal Answer: The answer is clear."
|
|
),
|
|
]
|
|
engine = _make_engine(responses)
|
|
agent = NativeReActAgent(
|
|
engine,
|
|
"test-model",
|
|
tools=[ThinkTool()],
|
|
)
|
|
result = agent.run("Analyze this.")
|
|
assert result.turns == 2
|
|
assert result.tool_results[0].success is True
|
|
|
|
def test_react_direct_answer(self):
|
|
"""ReAct returns immediately when no tool use is needed."""
|
|
_register_all()
|
|
from openjarvis.agents.native_react import NativeReActAgent
|
|
|
|
engine = _make_engine(
|
|
_simple_response("Thought: This is simple.\nFinal Answer: Hello!")
|
|
)
|
|
agent = NativeReActAgent(engine, "test-model")
|
|
result = agent.run("Say hello")
|
|
assert result.content == "Hello!"
|
|
assert result.turns == 1
|
|
|
|
def test_react_event_chain(self):
|
|
"""Verify agent-level event chain through ReAct run.
|
|
|
|
INFERENCE_START/END are now published by InstrumentedEngine,
|
|
not by agents directly.
|
|
"""
|
|
_register_all()
|
|
from openjarvis.agents.native_react import NativeReActAgent
|
|
|
|
engine = _make_engine(_simple_response("Thought: done.\nFinal Answer: ok"))
|
|
bus = EventBus(record_history=True)
|
|
agent = NativeReActAgent(
|
|
engine,
|
|
"test-model",
|
|
bus=bus,
|
|
)
|
|
agent.run("Test")
|
|
|
|
types = [e.event_type for e in bus.history]
|
|
assert EventType.AGENT_TURN_START in types
|
|
assert EventType.AGENT_TURN_END in types
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# OpenHands pipeline integration
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
class TestOpenHandsPipeline:
|
|
"""End-to-end: OpenHands agent with code execution."""
|
|
|
|
def test_openhands_code_execution_e2e(self):
|
|
_register_all()
|
|
from openjarvis.agents.native_openhands import NativeOpenHandsAgent
|
|
from openjarvis.tools.code_interpreter import (
|
|
CodeInterpreterTool,
|
|
)
|
|
|
|
if not ToolRegistry.contains("code_interpreter"):
|
|
ToolRegistry.register_value(
|
|
"code_interpreter",
|
|
CodeInterpreterTool,
|
|
)
|
|
|
|
responses = [
|
|
_simple_response("I'll calculate this:\n```python\nprint(2 + 2)\n```"),
|
|
_simple_response("The result is 4."),
|
|
]
|
|
engine = _make_engine(responses)
|
|
agent = NativeOpenHandsAgent(
|
|
engine,
|
|
"test-model",
|
|
tools=[CodeInterpreterTool()],
|
|
)
|
|
result = agent.run("What is 2+2?")
|
|
|
|
assert isinstance(result, AgentResult)
|
|
assert result.turns == 2
|
|
assert len(result.tool_results) == 1
|
|
# The code_interpreter actually runs print(2+2)
|
|
assert "4" in result.tool_results[0].content
|
|
|
|
def test_openhands_direct_answer(self):
|
|
"""OpenHands returns directly when no code is needed."""
|
|
_register_all()
|
|
from openjarvis.agents.native_openhands import NativeOpenHandsAgent
|
|
|
|
engine = _make_engine(_simple_response("Hello! How can I help?"))
|
|
agent = NativeOpenHandsAgent(engine, "test-model")
|
|
result = agent.run("Say hello")
|
|
assert result.content == "Hello! How can I help?"
|
|
assert result.turns == 1
|
|
|
|
def test_openhands_event_chain(self):
|
|
"""Verify event chain through OpenHands run."""
|
|
_register_all()
|
|
from openjarvis.agents.native_openhands import NativeOpenHandsAgent
|
|
|
|
engine = _make_engine(_simple_response("Direct answer."))
|
|
bus = EventBus(record_history=True)
|
|
agent = NativeOpenHandsAgent(
|
|
engine,
|
|
"test-model",
|
|
bus=bus,
|
|
)
|
|
agent.run("Test")
|
|
|
|
types = [e.event_type for e in bus.history]
|
|
assert EventType.AGENT_TURN_START in types
|
|
assert EventType.AGENT_TURN_END in types
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# MCP integration
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
class TestMCPIntegration:
|
|
"""MCP server + client with real tools."""
|
|
|
|
def test_mcp_server_with_all_tools(self):
|
|
from openjarvis.mcp.client import MCPClient
|
|
from openjarvis.mcp.server import MCPServer
|
|
from openjarvis.mcp.transport import InProcessTransport
|
|
from openjarvis.tools.calculator import CalculatorTool
|
|
from openjarvis.tools.think import ThinkTool
|
|
|
|
tools = [CalculatorTool(), ThinkTool()]
|
|
server = MCPServer(tools)
|
|
transport = InProcessTransport(server)
|
|
client = MCPClient(transport)
|
|
|
|
# Initialize
|
|
caps = client.initialize()
|
|
assert "serverInfo" in caps
|
|
|
|
# List tools
|
|
specs = client.list_tools()
|
|
names = [s.name for s in specs]
|
|
assert "calculator" in names
|
|
assert "think" in names
|
|
|
|
# Call calculator
|
|
result = client.call_tool(
|
|
"calculator",
|
|
{"expression": "10*5"},
|
|
)
|
|
assert result["content"][0]["text"] == "50.0"
|
|
assert result["isError"] is False
|
|
|
|
# Call think
|
|
result = client.call_tool(
|
|
"think",
|
|
{"thought": "reasoning step"},
|
|
)
|
|
assert result["content"][0]["text"] == "reasoning step"
|
|
|
|
client.close()
|
|
|
|
def test_mcp_unknown_tool_error(self):
|
|
from openjarvis.mcp.client import MCPClient
|
|
from openjarvis.mcp.protocol import MCPError
|
|
from openjarvis.mcp.server import MCPServer
|
|
from openjarvis.mcp.transport import InProcessTransport
|
|
from openjarvis.tools.calculator import CalculatorTool
|
|
|
|
server = MCPServer([CalculatorTool()])
|
|
client = MCPClient(InProcessTransport(server))
|
|
client.initialize()
|
|
|
|
with pytest.raises(MCPError):
|
|
client.call_tool("nonexistent", {})
|
|
|
|
client.close()
|
|
|
|
def test_mcp_roundtrip_lifecycle(self):
|
|
"""Full lifecycle: init -> list -> call -> result."""
|
|
from openjarvis.mcp.client import MCPClient
|
|
from openjarvis.mcp.server import MCPServer
|
|
from openjarvis.mcp.transport import InProcessTransport
|
|
from openjarvis.tools.calculator import CalculatorTool
|
|
|
|
server = MCPServer([CalculatorTool()])
|
|
client = MCPClient(InProcessTransport(server))
|
|
|
|
# 1. Initialize
|
|
caps = client.initialize()
|
|
assert caps["protocolVersion"] == "2025-11-25"
|
|
|
|
# 2. Discover tools
|
|
tools = client.list_tools()
|
|
assert len(tools) == 1
|
|
assert tools[0].name == "calculator"
|
|
|
|
# 3. Call tool
|
|
result = client.call_tool(
|
|
"calculator",
|
|
{"expression": "7+3"},
|
|
)
|
|
assert result["content"][0]["text"] == "10.0"
|
|
|
|
# 4. Close
|
|
client.close()
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Cross-engine consistency
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
class TestCrossEngineConsistency:
|
|
"""Same query through different mock engine configs."""
|
|
|
|
def test_same_query_same_format(self):
|
|
"""All engines return the same result dict shape."""
|
|
_register_all()
|
|
from openjarvis.agents.native_react import NativeReActAgent
|
|
|
|
for engine_name in ["vllm", "ollama", "mock"]:
|
|
engine = _make_engine(
|
|
_simple_response(
|
|
"Thought: ok.\nFinal Answer: Result",
|
|
model="test-model",
|
|
)
|
|
)
|
|
engine.engine_id = engine_name
|
|
agent = NativeReActAgent(engine, "test-model")
|
|
result = agent.run("Test query")
|
|
assert isinstance(result, AgentResult)
|
|
assert result.content == "Result"
|
|
|
|
def test_tool_calls_across_engines(self):
|
|
"""Tool calling works regardless of engine mock."""
|
|
_register_all()
|
|
from openjarvis.agents.native_react import NativeReActAgent
|
|
from openjarvis.tools.calculator import CalculatorTool
|
|
|
|
for engine_name in ["vllm", "ollama"]:
|
|
responses = [
|
|
_simple_response(
|
|
"Thought: calc.\n"
|
|
"Action: calculator\n"
|
|
'Action Input: {"expression":"3*3"}'
|
|
),
|
|
_simple_response("Thought: got 9.\nFinal Answer: 9"),
|
|
]
|
|
engine = _make_engine(responses)
|
|
engine.engine_id = engine_name
|
|
agent = NativeReActAgent(
|
|
engine,
|
|
"test-model",
|
|
tools=[CalculatorTool()],
|
|
)
|
|
result = agent.run("What is 3*3?")
|
|
assert result.content == "9"
|
|
tr = result.tool_results[0]
|
|
assert tr.content == "9.0"
|
|
assert tr.success is True
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Memory pipeline integration
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
class TestMemoryPipeline:
|
|
"""Index and retrieve across available backends."""
|
|
|
|
def test_sqlite_index_and_retrieve(self, tmp_path):
|
|
from openjarvis.tools.storage.sqlite import SQLiteMemory
|
|
|
|
backend = SQLiteMemory(db_path=str(tmp_path / "mem.db"))
|
|
backend.store(
|
|
"Machine learning uses data to learn patterns",
|
|
source="ml.md",
|
|
)
|
|
backend.store(
|
|
"Python is a versatile programming language",
|
|
source="py.md",
|
|
)
|
|
results = backend.retrieve("machine learning")
|
|
assert len(results) >= 1
|
|
assert "machine" in results[0].content.lower()
|
|
|
|
def test_bm25_index_and_retrieve(self, tmp_path):
|
|
try:
|
|
from openjarvis.tools.storage.bm25 import BM25Memory
|
|
except ImportError:
|
|
pytest.skip("rank_bm25 not installed")
|
|
|
|
backend = BM25Memory()
|
|
backend.store("Neural networks for NLP", source="a.md")
|
|
backend.store("Database indexing strategies", source="b.md")
|
|
results = backend.retrieve("neural NLP")
|
|
assert len(results) >= 1
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Model catalog integration
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
class TestModelCatalogIntegration:
|
|
"""All registered models have valid metadata."""
|
|
|
|
def test_all_models_routable(self):
|
|
"""Every model in catalog has required fields."""
|
|
from openjarvis.intelligence.model_catalog import (
|
|
BUILTIN_MODELS,
|
|
)
|
|
|
|
for spec in BUILTIN_MODELS:
|
|
assert spec.model_id, "model_id required"
|
|
assert spec.context_length > 0
|
|
if spec.requires_api_key:
|
|
assert spec.provider
|
|
|
|
def test_local_models_have_engine_compat(self):
|
|
"""Every local model has at least one engine."""
|
|
from openjarvis.intelligence.model_catalog import (
|
|
BUILTIN_MODELS,
|
|
)
|
|
|
|
local = [s for s in BUILTIN_MODELS if not s.requires_api_key]
|
|
for spec in local:
|
|
assert len(spec.supported_engines) >= 1, f"{spec.model_id} has no engines"
|
|
|
|
def test_cloud_models_require_api_key(self):
|
|
"""All cloud models require an API key."""
|
|
from openjarvis.intelligence.model_catalog import (
|
|
BUILTIN_MODELS,
|
|
)
|
|
|
|
cloud_ids = [
|
|
"gpt-5-mini",
|
|
"claude-opus-4-6",
|
|
"claude-sonnet-4-6",
|
|
"claude-haiku-4-5",
|
|
"gemini-2.5-pro",
|
|
"gemini-2.5-flash",
|
|
"gemini-3-pro",
|
|
"gemini-3-flash",
|
|
]
|
|
for mid in cloud_ids:
|
|
matches = [s for s in BUILTIN_MODELS if s.model_id == mid]
|
|
assert len(matches) == 1, f"Missing {mid}"
|
|
assert matches[0].requires_api_key is True
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Agent routing matrix (lightweight integration)
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
class TestAgentRoutingMatrix:
|
|
"""Agents run consistently across different configurations."""
|
|
|
|
@pytest.mark.parametrize(
|
|
"agent_key",
|
|
["native_react", "native_openhands"],
|
|
)
|
|
def test_agent_returns_valid_result(self, agent_key):
|
|
_register_all()
|
|
|
|
engine = _make_engine(
|
|
_simple_response(
|
|
"Thought: done.\nFinal Answer: ok"
|
|
if agent_key == "native_react"
|
|
else "The answer is ok"
|
|
)
|
|
)
|
|
agent_cls = AgentRegistry.get(agent_key)
|
|
agent = agent_cls(engine, "test-model")
|
|
result = agent.run("Test")
|
|
|
|
assert isinstance(result, AgentResult)
|
|
assert result.turns >= 1
|
|
assert len(result.content) > 0
|
|
|
|
@pytest.mark.parametrize(
|
|
"agent_key",
|
|
["native_react", "native_openhands"],
|
|
)
|
|
def test_agent_emits_events(self, agent_key):
|
|
_register_all()
|
|
|
|
engine = _make_engine(
|
|
_simple_response(
|
|
"Thought: done.\nFinal Answer: ok"
|
|
if agent_key == "native_react"
|
|
else "Direct answer"
|
|
)
|
|
)
|
|
bus = EventBus(record_history=True)
|
|
agent_cls = AgentRegistry.get(agent_key)
|
|
agent = agent_cls(engine, "test-model", bus=bus)
|
|
agent.run("Test")
|
|
|
|
types = [e.event_type for e in bus.history]
|
|
assert EventType.AGENT_TURN_START in types
|
|
assert EventType.AGENT_TURN_END in types
|
|
|
|
def test_context_passing(self):
|
|
"""Agents accept and use AgentContext."""
|
|
_register_all()
|
|
from openjarvis.agents.native_react import NativeReActAgent
|
|
|
|
engine = _make_engine(
|
|
_simple_response(
|
|
"Thought: I see the system message.\nFinal Answer: Got context."
|
|
)
|
|
)
|
|
conv = Conversation()
|
|
conv.add(
|
|
Message(
|
|
role=Role.SYSTEM,
|
|
content="You are helpful.",
|
|
)
|
|
)
|
|
ctx = AgentContext(conversation=conv)
|
|
agent = NativeReActAgent(engine, "test-model")
|
|
result = agent.run("Hello", context=ctx)
|
|
assert result.content == "Got context."
|
|
|
|
# Verify engine received system message
|
|
call_args = engine.generate.call_args
|
|
msgs = call_args[0][0]
|
|
# First message is ReAct system prompt, then context
|
|
assert any(m.content == "You are helpful." for m in msgs)
|