diff --git a/MagicMock/load_config().security.audit_log_path/4610918928 b/MagicMock/load_config().security.audit_log_path/4610918928 new file mode 100644 index 00000000..93e421ea Binary files /dev/null and b/MagicMock/load_config().security.audit_log_path/4610918928 differ diff --git a/MagicMock/load_config().security.audit_log_path/4612274128 b/MagicMock/load_config().security.audit_log_path/4612274128 new file mode 100644 index 00000000..93e421ea Binary files /dev/null and b/MagicMock/load_config().security.audit_log_path/4612274128 differ diff --git a/MagicMock/load_config().security.audit_log_path/4660834048 b/MagicMock/load_config().security.audit_log_path/4660834048 new file mode 100644 index 00000000..93e421ea Binary files /dev/null and b/MagicMock/load_config().security.audit_log_path/4660834048 differ diff --git a/MagicMock/load_config().security.audit_log_path/4661125504 b/MagicMock/load_config().security.audit_log_path/4661125504 new file mode 100644 index 00000000..93e421ea Binary files /dev/null and b/MagicMock/load_config().security.audit_log_path/4661125504 differ diff --git a/src/openjarvis/a2a/__init__.py b/src/openjarvis/a2a/__init__.py index 7c8b4702..f57af7c2 100644 --- a/src/openjarvis/a2a/__init__.py +++ b/src/openjarvis/a2a/__init__.py @@ -1,10 +1,16 @@ """Agent-to-Agent protocol — Google A2A spec implementation.""" + from openjarvis.a2a.client import A2AClient from openjarvis.a2a.protocol import A2ARequest, A2AResponse, A2ATask, AgentCard from openjarvis.a2a.server import A2AServer from openjarvis.a2a.tool import A2AAgentTool __all__ = [ - "A2AAgentTool", "A2AClient", "A2ARequest", "A2AResponse", - "A2AServer", "A2ATask", "AgentCard", + "A2AAgentTool", + "A2AClient", + "A2ARequest", + "A2AResponse", + "A2AServer", + "A2ATask", + "AgentCard", ] diff --git a/src/openjarvis/a2a/client.py b/src/openjarvis/a2a/client.py index cf40092e..a8ecea77 100644 --- a/src/openjarvis/a2a/client.py +++ b/src/openjarvis/a2a/client.py @@ -22,6 +22,7 @@ class A2AClient: def discover(self) -> AgentCard: """Fetch the agent card from /.well-known/agent.json.""" import httpx + resp = httpx.get( f"{self._base_url}/.well-known/agent.json", timeout=self._timeout, @@ -41,6 +42,7 @@ class A2AClient: def send_task(self, input_text: str, **kwargs: Any) -> A2ATask: """Send a task to the remote agent and return the result.""" import httpx + request = A2ARequest( method="tasks/send", params={ @@ -69,6 +71,7 @@ class A2AClient: def get_task(self, task_id: str) -> A2ATask: """Get the status of a previously submitted task.""" import httpx + request = A2ARequest( method="tasks/get", params={"id": task_id}, @@ -90,6 +93,7 @@ class A2AClient: def cancel_task(self, task_id: str) -> A2ATask: """Cancel a running task.""" import httpx + request = A2ARequest( method="tasks/cancel", params={"id": task_id}, diff --git a/src/openjarvis/a2a/protocol.py b/src/openjarvis/a2a/protocol.py index ce90fd45..d3ee6b72 100644 --- a/src/openjarvis/a2a/protocol.py +++ b/src/openjarvis/a2a/protocol.py @@ -21,6 +21,7 @@ class TaskState(str, Enum): @dataclass(slots=True) class AgentCard: """Agent discovery card served at /.well-known/agent.json.""" + name: str description: str = "" url: str = "" @@ -44,6 +45,7 @@ class AgentCard: @dataclass class A2ATask: """An A2A task with state machine.""" + task_id: str = field(default_factory=lambda: uuid.uuid4().hex[:16]) state: TaskState = TaskState.SUBMITTED input_text: str = "" @@ -65,6 +67,7 @@ class A2ATask: @dataclass(slots=True) class A2ARequest: """JSON-RPC 2.0 request for A2A.""" + method: str params: Dict[str, Any] = field(default_factory=dict) request_id: str = field(default_factory=lambda: uuid.uuid4().hex[:8]) @@ -84,6 +87,7 @@ class A2ARequest: @dataclass(slots=True) class A2AResponse: """JSON-RPC 2.0 response for A2A.""" + result: Any = None error: Optional[Dict[str, Any]] = None request_id: str = "" diff --git a/src/openjarvis/a2a/server.py b/src/openjarvis/a2a/server.py index 56c0ccdd..29ca5eba 100644 --- a/src/openjarvis/a2a/server.py +++ b/src/openjarvis/a2a/server.py @@ -102,7 +102,9 @@ class A2AServer: return A2AResponse(result=task.to_dict(), request_id=req_id).to_dict() def _handle_task_cancel( - self, params: Dict[str, Any], req_id: str, + self, + params: Dict[str, Any], + req_id: str, ) -> Dict[str, Any]: """Handle tasks/cancel — cancel a running task.""" task_id = params.get("id", "") diff --git a/src/openjarvis/agents/channel_agent.py b/src/openjarvis/agents/channel_agent.py index bf721169..205b607b 100644 --- a/src/openjarvis/agents/channel_agent.py +++ b/src/openjarvis/agents/channel_agent.py @@ -55,9 +55,7 @@ def classify_query(text: str) -> str: # ChannelAgent # --------------------------------------------------------------------------- -_ESCALATION_TEMPLATE = ( - "{preview}...\n\n---\nFull report ready — open in OpenJarvis:\nopenjarvis://research/{session_id}" -) +_ESCALATION_TEMPLATE = "{preview}...\n\n---\nFull report ready — open in OpenJarvis:\nopenjarvis://research/{session_id}" _LONG_RESPONSE_THRESHOLD = 500 diff --git a/src/openjarvis/agents/claude_code.py b/src/openjarvis/agents/claude_code.py index 8fb77473..8fb3c470 100644 --- a/src/openjarvis/agents/claude_code.py +++ b/src/openjarvis/agents/claude_code.py @@ -35,9 +35,7 @@ _OUTPUT_END = "---OPENJARVIS_OUTPUT_END---" _RUNNER_SRC = Path(__file__).resolve().parent / "claude_code_runner" if not _RUNNER_SRC.exists(): _RUNNER_SRC = ( - Path(__file__).resolve().parents[2] - / "_node_modules" - / "claude_code_runner" + Path(__file__).resolve().parents[2] / "_node_modules" / "claude_code_runner" ) diff --git a/src/openjarvis/agents/deep_research.py b/src/openjarvis/agents/deep_research.py index 04a6c57b..9724e575 100644 --- a/src/openjarvis/agents/deep_research.py +++ b/src/openjarvis/agents/deep_research.py @@ -30,6 +30,7 @@ def _tc_args(tc: dict) -> str: return tc["function"]["arguments"] return tc["arguments"] + DEEP_RESEARCH_SYSTEM_PROMPT = """\ /no_think You are a deep research agent with access to a personal knowledge base \ @@ -201,9 +202,7 @@ class DeepResearchAgent(ToolUsingAgent): # If content is empty but we have prior tool results, # force one more generation without tools to synthesize if not content.strip() and all_tool_results: - messages.append( - Message(role=Role.ASSISTANT, content=content) - ) + messages.append(Message(role=Role.ASSISTANT, content=content)) messages.append( Message( role=Role.USER, diff --git a/src/openjarvis/agents/executor.py b/src/openjarvis/agents/executor.py index 713d1a58..44b48883 100644 --- a/src/openjarvis/agents/executor.py +++ b/src/openjarvis/agents/executor.py @@ -87,7 +87,7 @@ class AgentExecutor: agent_cls = AgentRegistry.get(agent_type) agent = agent_cls( - engine=getattr(self._manager, '_engine', None), + engine=getattr(self._manager, "_engine", None), system_prompt=system_prompt, bus=self._bus, ) @@ -113,10 +113,13 @@ class AgentExecutor: logger.error("Agent %s not found", agent_id) return - self._bus.publish(EventType.AGENT_TICK_START, { - "agent_id": agent_id, - "agent_name": agent["name"], - }) + self._bus.publish( + EventType.AGENT_TICK_START, + { + "agent_id": agent_id, + "agent_name": agent["name"], + }, + ) # Activity tracking: subscribe to tool/inference events def _on_activity(event: Any) -> None: @@ -131,14 +134,16 @@ class AgentExecutor: def _on_tool_start(event: Any) -> None: if event.data.get("agent") == agent_id: - trace_steps.append({ - "type": "tool_call", - "input": { - "tool": event.data.get("tool"), - "args": event.data.get("args"), - }, - "start_time": event.timestamp, - }) + trace_steps.append( + { + "type": "tool_call", + "input": { + "tool": event.data.get("tool"), + "args": event.data.get("args"), + }, + "start_time": event.timestamp, + } + ) def _on_tool_end(event: Any) -> None: if event.data.get("agent") == agent_id and trace_steps: @@ -175,8 +180,13 @@ class AgentExecutor: if self._trace_store: self._save_trace( - agent_id, agent, result, error_info, - tick_start, tick_duration, trace_steps, + agent_id, + agent, + result, + error_info, + tick_start, + tick_duration, + trace_steps, ) def _run_with_retries(self, agent: dict) -> AgentResult: @@ -193,7 +203,11 @@ class AgentExecutor: delay = retry_delay(attempt) logger.info( "Agent %s tick retry %d/%d in %ds: %s", - agent["id"], attempt + 1, _MAX_RETRIES, delay, e, + agent["id"], + attempt + 1, + _MAX_RETRIES, + delay, + e, ) time.sleep(delay) except Exception as e: @@ -203,7 +217,11 @@ class AgentExecutor: delay = retry_delay(attempt) logger.info( "Agent %s tick retry %d/%d in %ds: %s", - agent["id"], attempt + 1, _MAX_RETRIES, delay, e, + agent["id"], + attempt + 1, + _MAX_RETRIES, + delay, + e, ) time.sleep(delay) @@ -225,17 +243,16 @@ class AgentExecutor: engine = self._system.engine if self._system else None if engine is None: raise FatalError("No engine available in JarvisSystem") - model = config.get("model") or ( - self._system.model - if self._system else "" - ) + model = config.get("model") or (self._system.model if self._system else "") if not model: raise FatalError("No model configured for agent") logger.info( "Agent %s [%s]: using model=%s, engine=%s", - agent["name"], agent["id"], - model, type(engine).__name__, + agent["name"], + agent["id"], + model, + type(engine).__name__, ) self._set_activity(agent["id"], f"Loading model {model}...") @@ -289,7 +306,9 @@ class AgentExecutor: if tool_instances: logger.info( "Agent %s: resolved %d/%d tools", - agent["name"], len(tool_instances), len(tool_names), + agent["name"], + len(tool_instances), + len(tool_names), ) # Construct agent instance @@ -317,7 +336,8 @@ class AgentExecutor: self._manager.mark_message_delivered(m["id"]) logger.info( "Agent %s: delivering %d pending message(s)", - agent["name"], len(pending), + agent["name"], + len(pending), ) self._set_activity( agent["id"], @@ -325,8 +345,7 @@ class AgentExecutor: ) else: logger.info( - "Agent %s: no pending messages, running with " - "instruction only", + "Agent %s: no pending messages, running with instruction only", agent["name"], ) @@ -363,7 +382,8 @@ class AgentExecutor: if query: results = self._system.memory_backend.retrieve( - query, top_k=ctx_cfg.top_k, + query, + top_k=ctx_cfg.top_k, ) memory_results = [ r for r in results if r.score >= ctx_cfg.min_score @@ -383,7 +403,8 @@ class AgentExecutor: self._set_activity(agent["id"], "Generating response...") logger.info( "Agent %s: calling agent.run() with %d chars input", - agent["name"], len(input_text), + agent["name"], + len(input_text), ) _t0 = time.time() result = agent_instance.run(input_text, context=agent_ctx) @@ -392,7 +413,8 @@ class AgentExecutor: # Qwen3.5 thinking mode consuming all tokens). if not (result.content or "").strip(): self._set_activity( - agent["id"], "Retrying (empty response)...", + agent["id"], + "Retrying (empty response)...", ) logger.warning( "Agent %s: empty content, retrying once", @@ -404,7 +426,8 @@ class AgentExecutor: logger.info( "Agent %s: agent.run() completed in %.1fs, " "content_len=%d, turns=%d, tokens=%s", - agent["name"], _elapsed, + agent["name"], + _elapsed, len(result.content or ""), result.turns, result.metadata.get("total_tokens", "?"), @@ -434,7 +457,9 @@ class AgentExecutor: "suggested_action": suggest_action(error), "stack_trace_summary": "".join( traceback.format_exception(type(error), error, error.__traceback__)[-3:] - )[:1000] if error.__traceback__ else "", + )[:1000] + if error.__traceback__ + else "", } def _finalize_tick( @@ -449,9 +474,9 @@ class AgentExecutor: if error is None: # Success logger.info( - "Tick succeeded for agent %s in %.1fs, " - "response_len=%d", - agent_id, duration, + "Tick succeeded for agent %s in %.1fs, response_len=%d", + agent_id, + duration, len(result.content or "") if result else 0, ) self._manager.end_tick(agent_id) @@ -466,7 +491,8 @@ class AgentExecutor: ) in_tokens = result.metadata.get("prompt_tokens", 0) out_tokens = result.metadata.get( - "completion_tokens", 0, + "completion_tokens", + 0, ) cost = result.metadata.get("cost", 0.0) budget_kwargs: dict[str, Any] = {"stall_retries": 0} @@ -481,7 +507,8 @@ class AgentExecutor: self._manager.update_agent(agent_id, **budget_kwargs) self._manager.update_summary_memory( - agent_id, result.content[:2000], + agent_id, + result.content[:2000], ) self._manager.store_agent_response(agent_id, result.content[:2000]) @@ -498,49 +525,68 @@ class AgentExecutor: exceeded = True if exceeded: self._manager.update_agent(agent_id, status="budget_exceeded") - self._bus.publish(EventType.AGENT_BUDGET_EXCEEDED, { - "agent_id": agent_id, - "total_cost": agent_data["total_cost"], - "total_tokens": agent_data["total_tokens"], - "max_cost": max_cost, - "max_tokens": max_tokens, - }) - self._bus.publish(EventType.AGENT_TICK_END, { - "agent_id": agent_id, - "duration": duration, - "status": "ok", - }) + self._bus.publish( + EventType.AGENT_BUDGET_EXCEEDED, + { + "agent_id": agent_id, + "total_cost": agent_data["total_cost"], + "total_tokens": agent_data["total_tokens"], + "max_cost": max_cost, + "max_tokens": max_tokens, + }, + ) + self._bus.publish( + EventType.AGENT_TICK_END, + { + "agent_id": agent_id, + "duration": duration, + "status": "ok", + }, + ) elif isinstance(error, EscalateError): logger.warning( "Tick escalated for agent %s after %.1fs: %s", - agent_id, duration, error, + agent_id, + duration, + error, ) self._manager.end_tick(agent_id) self._manager.update_agent(agent_id, status="needs_attention") - self._bus.publish(EventType.AGENT_TICK_ERROR, { - "agent_id": agent_id, - "error": str(error), - "error_type": "escalate", - "duration": duration, - }) + self._bus.publish( + EventType.AGENT_TICK_ERROR, + { + "agent_id": agent_id, + "error": str(error), + "error_type": "escalate", + "duration": duration, + }, + ) else: logger.error( "Tick failed for agent %s after %.1fs: %s", - agent_id, duration, error, exc_info=error, + agent_id, + duration, + error, + exc_info=error, ) self._manager.end_tick(agent_id) self._manager.update_agent(agent_id, status="error") # Write error detail to summary_memory so frontend can display it error_msg = str(error)[:2000] self._manager.update_summary_memory(agent_id, f"ERROR: {error_msg}") - self._bus.publish(EventType.AGENT_TICK_ERROR, { - "agent_id": agent_id, - "error": str(error), - "error_type": ( - "fatal" if isinstance(error, FatalError) else "retryable_exhausted" - ), - "duration": duration, - }) + self._bus.publish( + EventType.AGENT_TICK_ERROR, + { + "agent_id": agent_id, + "error": str(error), + "error_type": ( + "fatal" + if isinstance(error, FatalError) + else "retryable_exhausted" + ), + "duration": duration, + }, + ) def _save_trace( self, @@ -557,17 +603,19 @@ class AgentExecutor: steps = [] for s in trace_steps: - steps.append(TraceStep( - step_type=( - StepType.TOOL_CALL - if s["type"] == "tool_call" - else StepType.GENERATE - ), - input=s.get("input", {}), - output=s.get("output", {}), - duration_seconds=s.get("duration", 0), - timestamp=s.get("start_time", tick_start), - )) + steps.append( + TraceStep( + step_type=( + StepType.TOOL_CALL + if s["type"] == "tool_call" + else StepType.GENERATE + ), + input=s.get("input", {}), + output=s.get("output", {}), + duration_seconds=s.get("duration", 0), + timestamp=s.get("start_time", tick_start), + ) + ) metadata: dict[str, Any] = {} if error is not None: @@ -590,5 +638,7 @@ class AgentExecutor: self._trace_store.save(trace) except Exception: logger.warning( - "Failed to save trace for agent %s", agent_id, exc_info=True, + "Failed to save trace for agent %s", + agent_id, + exc_info=True, ) diff --git a/src/openjarvis/agents/loop_guard.py b/src/openjarvis/agents/loop_guard.py index cd19983d..f7e92cfe 100644 --- a/src/openjarvis/agents/loop_guard.py +++ b/src/openjarvis/agents/loop_guard.py @@ -13,17 +13,19 @@ from openjarvis.core.events import EventBus, EventType @dataclass(slots=True) class LoopGuardConfig: """Configuration for the loop guard.""" + enabled: bool = True - max_identical_calls: int = 3 # SHA-256 of (tool_name, arguments) - ping_pong_window: int = 6 # detect A-B-A-B cycling - poll_tool_budget: int = 5 # max calls to same polling tool - max_context_messages: int = 100 # context overflow threshold - warn_before_block: bool = True # warn on first cycle, block on second + max_identical_calls: int = 3 # SHA-256 of (tool_name, arguments) + ping_pong_window: int = 6 # detect A-B-A-B cycling + poll_tool_budget: int = 5 # max calls to same polling tool + max_context_messages: int = 100 # context overflow threshold + warn_before_block: bool = True # warn on first cycle, block on second @dataclass(slots=True) class LoopVerdict: """Result of a loop guard check.""" + blocked: bool = False reason: str = "" warned: bool = False @@ -54,13 +56,12 @@ class LoopGuard: try: from openjarvis._rust_bridge import get_rust_module + _rust = get_rust_module() self._rust_impl = _rust.LoopGuard( max_identical=config.max_identical_calls, max_ping_pong=( - config.ping_pong_window // 2 - if config.ping_pong_window > 1 - else 2 + config.ping_pong_window // 2 if config.ping_pong_window > 1 else 2 ), poll_budget=config.poll_tool_budget, ) @@ -93,9 +94,7 @@ class LoopGuard: def _python_check(self, tool_name: str, arguments: str) -> LoopVerdict: """Pure-Python fallback when Rust backend is not available.""" # 1. Hash tracking — identical calls - call_hash = hashlib.sha256( - f"{tool_name}:{arguments}".encode() - ).hexdigest()[:16] + call_hash = hashlib.sha256(f"{tool_name}:{arguments}".encode()).hexdigest()[:16] self._call_counts[call_hash] = self._call_counts.get(call_hash, 0) + 1 if self._call_counts[call_hash] > self._config.max_identical_calls: self._emit_triggered("identical_call", tool_name) @@ -139,12 +138,12 @@ class LoopGuard: @staticmethod def _is_system(msg: object) -> bool: """Check if a message has role == system.""" - return getattr(msg, 'role', None) == 'system' + return getattr(msg, "role", None) == "system" @staticmethod def _is_tool(msg: object) -> bool: """Check if a message has role == tool.""" - return getattr(msg, 'role', None) == 'tool' + return getattr(msg, "role", None) == "tool" def compress_context(self, messages: list) -> list: """Apply 4-stage context overflow recovery to message list. @@ -164,14 +163,19 @@ class LoopGuard: for i, msg in enumerate(messages): if i < threshold and self._is_tool(msg): from openjarvis.core.types import Message, Role - compressed.append(Message( - role=Role.TOOL, - content="[Tool result truncated]", - tool_call_id=getattr( - msg, 'tool_call_id', None, - ), - name=getattr(msg, 'name', None), - )) + + compressed.append( + Message( + role=Role.TOOL, + content="[Tool result truncated]", + tool_call_id=getattr( + msg, + "tool_call_id", + None, + ), + name=getattr(msg, "name", None), + ) + ) else: compressed.append(msg) @@ -179,16 +183,9 @@ class LoopGuard: return compressed # Stage 2: Sliding window — keep system + recent - system_msgs = [ - m for m in compressed if self._is_system(m) - ] - non_system = [ - m for m in compressed - if not self._is_system(m) - ] - window_size = ( - self._config.max_context_messages - len(system_msgs) - ) + system_msgs = [m for m in compressed if self._is_system(m)] + non_system = [m for m in compressed if not self._is_system(m)] + window_size = self._config.max_context_messages - len(system_msgs) if len(non_system) > window_size: non_system = non_system[-window_size:] compressed = system_msgs + non_system @@ -198,24 +195,18 @@ class LoopGuard: # Stage 3: Drop tool call/result pairs from middle keep_start = max( - len(system_msgs), len(compressed) // 10, + len(system_msgs), + len(compressed) // 10, ) keep_end = len(compressed) // 2 - compressed = ( - compressed[:keep_start] + compressed[-keep_end:] - ) + compressed = compressed[:keep_start] + compressed[-keep_end:] if len(compressed) <= self._config.max_context_messages: return compressed # Stage 4: Extreme — system + last 2 exchanges - sys_final = [ - m for m in compressed if self._is_system(m) - ] - tail = [ - m for m in compressed - if not self._is_system(m) - ] + sys_final = [m for m in compressed if self._is_system(m)] + tail = [m for m in compressed if not self._is_system(m)] return sys_final + tail[-4:] def reset(self) -> None: @@ -234,7 +225,7 @@ class LoopGuard: # Check for period-2 pattern (A-B-A-B) for period in (2, 3): if n >= period * 2: - tail = seq[-period * 2:] + tail = seq[-period * 2 :] pattern = tail[:period] if all(tail[i] == pattern[i % period] for i in range(len(tail))): return True diff --git a/src/openjarvis/agents/native_openhands.py b/src/openjarvis/agents/native_openhands.py index 811c3047..e0eae061 100644 --- a/src/openjarvis/agents/native_openhands.py +++ b/src/openjarvis/agents/native_openhands.py @@ -284,9 +284,7 @@ class NativeOpenHandsAgent(ToolUsingAgent): total_usage = {"prompt_tokens": 0, "completion_tokens": 0, "total_tokens": 0} # Build OpenAI-format tool schemas for native function calling - openai_tools = ( - self._executor.get_openai_tools() if self._tools else [] - ) + openai_tools = self._executor.get_openai_tools() if self._tools else [] # Side dict for Gemini thought_signatures (ToolCall uses slots) _thought_sigs: dict[str, bytes] = {} diff --git a/src/openjarvis/agents/scheduler.py b/src/openjarvis/agents/scheduler.py index 42cac3ef..157bcf40 100644 --- a/src/openjarvis/agents/scheduler.py +++ b/src/openjarvis/agents/scheduler.py @@ -162,7 +162,11 @@ class AgentScheduler: for agent_id, info in due: agent = self._manager.get_agent(agent_id) if agent is None or agent["status"] in ( - "paused", "archived", "running", "budget_exceeded", "stalled", + "paused", + "archived", + "running", + "budget_exceeded", + "stalled", ): continue @@ -177,11 +181,12 @@ class AgentScheduler: if agent_id in self._agents: if info["schedule_type"] == "cron": self._agents[agent_id]["next_fire"] = _next_cron_fire( - str(info["schedule_value"]), now, + str(info["schedule_value"]), + now, ) elif info["schedule_type"] == "interval": - self._agents[agent_id]["next_fire"] = ( - now + float(info["schedule_value"]) + self._agents[agent_id]["next_fire"] = now + float( + info["schedule_value"] ) # Manual: stays at inf @@ -214,22 +219,30 @@ class AgentScheduler: self._manager.update_agent(agent["id"], status="error") logger.warning( "Agent %s stall retries exhausted (%d/%d), setting error", - agent["id"], current_retries, max_retries, + agent["id"], + current_retries, + max_retries, ) else: self._manager.end_tick(agent["id"]) # Release concurrency guard self._manager.update_agent( - agent["id"], stall_retries=current_retries + 1, + agent["id"], + stall_retries=current_retries + 1, ) if self._bus: - self._bus.publish(EventType.AGENT_STALL_DETECTED, { - "agent_id": agent["id"], - "last_activity_at": last_activity, - "stall_retries": current_retries + 1, - }) + self._bus.publish( + EventType.AGENT_STALL_DETECTED, + { + "agent_id": agent["id"], + "last_activity_at": last_activity, + "stall_retries": current_retries + 1, + }, + ) logger.warning( "Agent %s stalled (retry %d/%d)", - agent["id"], current_retries + 1, max_retries, + agent["id"], + current_retries + 1, + max_retries, ) # -- Learning tick counting ------------------------------------------------ @@ -258,9 +271,12 @@ class AgentScheduler: if self._tick_counts[agent_id] >= threshold: self._tick_counts[agent_id] = 0 if self._bus: - self._bus.publish(EventType.AGENT_LEARNING_STARTED, { - "agent_id": agent_id, - }) + self._bus.publish( + EventType.AGENT_LEARNING_STARTED, + { + "agent_id": agent_id, + }, + ) logger.info( "Learning triggered for agent %s after %d ticks", agent_id, diff --git a/src/openjarvis/bench/energy.py b/src/openjarvis/bench/energy.py index ab823fff..abc35e1f 100644 --- a/src/openjarvis/bench/energy.py +++ b/src/openjarvis/bench/energy.py @@ -64,9 +64,7 @@ class EnergyBenchmark(BaseBenchmark): from openjarvis.telemetry.steady_state import SteadyStateDetector detector = SteadyStateDetector() - energy_method = getattr( - energy_monitor, "energy_method", lambda: "" - )() + energy_method = getattr(energy_monitor, "energy_method", lambda: "")() for _ in range(num_samples): t0 = time.time() @@ -78,9 +76,7 @@ class EnergyBenchmark(BaseBenchmark): usage = result.get("usage", {}) tokens = usage.get("completion_tokens", 0) energy_j = sample.energy_joules - power_w = ( - energy_j / elapsed if elapsed > 0 else 0.0 - ) + power_w = energy_j / elapsed if elapsed > 0 else 0.0 tps = tokens / elapsed if elapsed > 0 else 0.0 ept = energy_j / tokens if tokens > 0 else 0.0 @@ -145,9 +141,7 @@ class EnergyBenchmark(BaseBenchmark): samples=num_samples, errors=errors, warmup_samples=warmup_samples, - steady_state_samples=( - ss_result.steady_state_samples if ss_result else 0 - ), + steady_state_samples=(ss_result.steady_state_samples if ss_result else 0), steady_state_reached=( ss_result.steady_state_reached if ss_result else False ), diff --git a/src/openjarvis/channels/bluebubbles.py b/src/openjarvis/channels/bluebubbles.py index a3d527be..1dca4f36 100644 --- a/src/openjarvis/channels/bluebubbles.py +++ b/src/openjarvis/channels/bluebubbles.py @@ -96,7 +96,8 @@ class BlueBubblesChannel(BaseChannel): self._publish_sent(channel, content, conversation_id) return True logger.warning( - "BlueBubbles API returned status %d", resp.status_code, + "BlueBubbles API returned status %d", + resp.status_code, ) return False except Exception: diff --git a/src/openjarvis/channels/discord_channel.py b/src/openjarvis/channels/discord_channel.py index 202b2577..b4a44a44 100644 --- a/src/openjarvis/channels/discord_channel.py +++ b/src/openjarvis/channels/discord_channel.py @@ -62,7 +62,8 @@ class DiscordChannel(BaseChannel): import discord # noqa: F401 self._listener_thread = threading.Thread( - target=self._gateway_loop, daemon=True, + target=self._gateway_loop, + daemon=True, ) self._listener_thread.start() self._status = ChannelStatus.CONNECTED @@ -107,7 +108,10 @@ class DiscordChannel(BaseChannel): payload["message_reference"] = {"message_id": conversation_id} resp = httpx.post( - url, json=payload, headers=headers, timeout=10.0, + url, + json=payload, + headers=headers, + timeout=10.0, ) if resp.status_code < 300: self._publish_sent(channel, content, conversation_id) diff --git a/src/openjarvis/channels/email_channel.py b/src/openjarvis/channels/email_channel.py index bf58f6bb..312ad262 100644 --- a/src/openjarvis/channels/email_channel.py +++ b/src/openjarvis/channels/email_channel.py @@ -86,7 +86,8 @@ class EmailChannel(BaseChannel): if self._imap_host: self._listener_thread = threading.Thread( - target=self._imap_poll_loop, daemon=True, + target=self._imap_poll_loop, + daemon=True, ) self._listener_thread.start() @@ -121,7 +122,8 @@ class EmailChannel(BaseChannel): msg["From"] = self._username msg["To"] = channel msg["Subject"] = (metadata or {}).get( - "subject", "Message from OpenJarvis", + "subject", + "Message from OpenJarvis", ) if conversation_id: msg["In-Reply-To"] = conversation_id @@ -166,7 +168,8 @@ class EmailChannel(BaseChannel): try: if self._use_tls: imap = imaplib.IMAP4_SSL( - self._imap_host, self._imap_port, + self._imap_host, + self._imap_port, ) else: imap = imaplib.IMAP4(self._imap_host, self._imap_port) diff --git a/src/openjarvis/channels/feishu.py b/src/openjarvis/channels/feishu.py index 7d68fb91..6c73e735 100644 --- a/src/openjarvis/channels/feishu.py +++ b/src/openjarvis/channels/feishu.py @@ -81,8 +81,7 @@ class FeishuChannel(BaseChannel): # Obtain tenant_access_token token_url = ( - "https://open.feishu.cn/open-apis/auth/v3/" - "tenant_access_token/internal" + "https://open.feishu.cn/open-apis/auth/v3/tenant_access_token/internal" ) token_resp = httpx.post( token_url, @@ -119,13 +118,17 @@ class FeishuChannel(BaseChannel): } resp = httpx.post( - msg_url, json=payload, headers=headers, timeout=10.0, + msg_url, + json=payload, + headers=headers, + timeout=10.0, ) if resp.status_code < 300: self._publish_sent(channel, content, conversation_id) return True logger.warning( - "Feishu API returned status %d", resp.status_code, + "Feishu API returned status %d", + resp.status_code, ) return False except Exception: diff --git a/src/openjarvis/channels/gmail.py b/src/openjarvis/channels/gmail.py index 95f8c729..10e4a9e2 100644 --- a/src/openjarvis/channels/gmail.py +++ b/src/openjarvis/channels/gmail.py @@ -53,10 +53,12 @@ class GmailChannel(BaseChannel): bus: Optional[EventBus] = None, ) -> None: self._credentials_path = credentials_path or os.environ.get( - "GMAIL_CREDENTIALS_PATH", "", + "GMAIL_CREDENTIALS_PATH", + "", ) self._token_path = token_path or os.environ.get( - "GMAIL_TOKEN_PATH", "", + "GMAIL_TOKEN_PATH", + "", ) self._user_id = user_id self._poll_interval = poll_interval @@ -89,7 +91,8 @@ class GmailChannel(BaseChannel): creds: Any = None if self._token_path and os.path.exists(self._token_path): creds = Credentials.from_authorized_user_file( - self._token_path, SCOPES, + self._token_path, + SCOPES, ) if not creds or not creds.valid: @@ -101,7 +104,8 @@ class GmailChannel(BaseChannel): self._credentials_path, ): flow = InstalledAppFlow.from_client_secrets_file( - self._credentials_path, SCOPES, + self._credentials_path, + SCOPES, ) creds = flow.run_local_server(port=0) else: @@ -120,7 +124,8 @@ class GmailChannel(BaseChannel): # Start polling thread self._listener_thread = threading.Thread( - target=self._poll_loop, daemon=True, + target=self._poll_loop, + daemon=True, ) self._listener_thread.start() except ImportError: @@ -165,7 +170,8 @@ class GmailChannel(BaseChannel): msg["To"] = channel msg["From"] = self._user_id msg["Subject"] = (metadata or {}).get( - "subject", "Message from OpenJarvis", + "subject", + "Message from OpenJarvis", ) raw = base64.urlsafe_b64encode(msg.as_bytes()).decode("utf-8") @@ -174,7 +180,8 @@ class GmailChannel(BaseChannel): body["threadId"] = conversation_id self._service.users().messages().send( - userId=self._user_id, body=body, + userId=self._user_id, + body=body, ).execute() self._publish_sent(channel, content, conversation_id) diff --git a/src/openjarvis/channels/google_chat.py b/src/openjarvis/channels/google_chat.py index ee0f1454..a61e0232 100644 --- a/src/openjarvis/channels/google_chat.py +++ b/src/openjarvis/channels/google_chat.py @@ -39,7 +39,8 @@ class GoogleChatChannel(BaseChannel): bus: Optional[EventBus] = None, ) -> None: self._webhook_url = webhook_url or os.environ.get( - "GOOGLE_CHAT_WEBHOOK_URL", "", + "GOOGLE_CHAT_WEBHOOK_URL", + "", ) self._bus = bus self._handlers: List[ChannelHandler] = [] @@ -80,13 +81,16 @@ class GoogleChatChannel(BaseChannel): payload: Dict[str, Any] = {"text": content} resp = httpx.post( - self._webhook_url, json=payload, timeout=10.0, + self._webhook_url, + json=payload, + timeout=10.0, ) if resp.status_code < 300: self._publish_sent(channel, content, conversation_id) return True logger.warning( - "Google Chat API returned status %d", resp.status_code, + "Google Chat API returned status %d", + resp.status_code, ) return False except Exception: diff --git a/src/openjarvis/channels/imessage_daemon.py b/src/openjarvis/channels/imessage_daemon.py index 2a4ec569..9ef72d95 100644 --- a/src/openjarvis/channels/imessage_daemon.py +++ b/src/openjarvis/channels/imessage_daemon.py @@ -20,13 +20,9 @@ from typing import Any, Dict, List logger = logging.getLogger(__name__) -_DEFAULT_DB_PATH = str( - Path.home() / "Library" / "Messages" / "chat.db" -) +_DEFAULT_DB_PATH = str(Path.home() / "Library" / "Messages" / "chat.db") _POLL_INTERVAL = 5 -_PID_FILE = str( - Path.home() / ".openjarvis" / "imessage-agent.pid" -) +_PID_FILE = str(Path.home() / ".openjarvis" / "imessage-agent.pid") def poll_new_messages( @@ -37,9 +33,7 @@ def poll_new_messages( ) -> List[Dict[str, Any]]: """Return new incoming messages since last_rowid.""" try: - conn = sqlite3.connect( - f"file:{db_path}?mode=ro", uri=True - ) + conn = sqlite3.connect(f"file:{db_path}?mode=ro", uri=True) conn.row_factory = sqlite3.Row except sqlite3.OperationalError: return [] @@ -68,7 +62,7 @@ def send_imessage(chat_identifier: str, message: str) -> bool: escaped = message.replace("\\", "\\\\").replace('"', '\\"') script = ( f'tell application "Messages"\n' - f' set targetChat to a reference to ' + f" set targetChat to a reference to " f'chat id "{chat_identifier}"\n' f' send "{escaped}" to targetChat\n' f"end tell" @@ -152,12 +146,8 @@ def run_daemon( def _get_max_rowid(db_path: str) -> int: """Get the current max ROWID from chat.db.""" try: - conn = sqlite3.connect( - f"file:{db_path}?mode=ro", uri=True - ) - row = conn.execute( - "SELECT MAX(ROWID) FROM message" - ).fetchone() + conn = sqlite3.connect(f"file:{db_path}?mode=ro", uri=True) + row = conn.execute("SELECT MAX(ROWID) FROM message").fetchone() conn.close() return row[0] or 0 except sqlite3.OperationalError: diff --git a/src/openjarvis/channels/line_channel.py b/src/openjarvis/channels/line_channel.py index 330a57c8..c55ccaec 100644 --- a/src/openjarvis/channels/line_channel.py +++ b/src/openjarvis/channels/line_channel.py @@ -65,8 +65,7 @@ class LineChannel(BaseChannel): import linebot # noqa: F401 except ImportError: raise ImportError( - "line-bot-sdk not installed. Install with: " - "uv sync --extra channel-line" + "line-bot-sdk not installed. Install with: uv sync --extra channel-line" ) self._status = ChannelStatus.CONNECTED diff --git a/src/openjarvis/channels/mastodon_channel.py b/src/openjarvis/channels/mastodon_channel.py index fdb11109..26230baa 100644 --- a/src/openjarvis/channels/mastodon_channel.py +++ b/src/openjarvis/channels/mastodon_channel.py @@ -43,12 +43,8 @@ class MastodonChannel(BaseChannel): access_token: str = "", bus: Optional[EventBus] = None, ) -> None: - self._api_base_url = api_base_url or os.environ.get( - "MASTODON_API_BASE_URL", "" - ) - self._access_token = access_token or os.environ.get( - "MASTODON_ACCESS_TOKEN", "" - ) + self._api_base_url = api_base_url or os.environ.get("MASTODON_API_BASE_URL", "") + self._access_token = access_token or os.environ.get("MASTODON_ACCESS_TOKEN", "") self._bus = bus self._handlers: List[ChannelHandler] = [] self._status = ChannelStatus.DISCONNECTED diff --git a/src/openjarvis/channels/matrix_channel.py b/src/openjarvis/channels/matrix_channel.py index a1e65ebf..5ddc6538 100644 --- a/src/openjarvis/channels/matrix_channel.py +++ b/src/openjarvis/channels/matrix_channel.py @@ -94,13 +94,17 @@ class MatrixChannel(BaseChannel): } resp = httpx.put( - url, json=payload, headers=headers, timeout=10.0, + url, + json=payload, + headers=headers, + timeout=10.0, ) if resp.status_code < 300: self._publish_sent(channel, content, conversation_id) return True logger.warning( - "Matrix API returned status %d", resp.status_code, + "Matrix API returned status %d", + resp.status_code, ) return False except Exception: diff --git a/src/openjarvis/channels/mattermost.py b/src/openjarvis/channels/mattermost.py index 96875a45..8fc6060a 100644 --- a/src/openjarvis/channels/mattermost.py +++ b/src/openjarvis/channels/mattermost.py @@ -91,13 +91,17 @@ class MattermostChannel(BaseChannel): payload["root_id"] = conversation_id resp = httpx.post( - url, json=payload, headers=headers, timeout=10.0, + url, + json=payload, + headers=headers, + timeout=10.0, ) if resp.status_code < 300: self._publish_sent(channel, content, conversation_id) return True logger.warning( - "Mattermost API returned status %d", resp.status_code, + "Mattermost API returned status %d", + resp.status_code, ) return False except Exception: diff --git a/src/openjarvis/channels/nostr_channel.py b/src/openjarvis/channels/nostr_channel.py index 9bd0e685..8ec47276 100644 --- a/src/openjarvis/channels/nostr_channel.py +++ b/src/openjarvis/channels/nostr_channel.py @@ -45,9 +45,7 @@ class NostrChannel(BaseChannel): bus: Optional[EventBus] = None, ) -> None: self._private_key = private_key or os.environ.get("NOSTR_PRIVATE_KEY", "") - relays_str = relays or os.environ.get( - "NOSTR_RELAYS", "wss://relay.damus.io" - ) + relays_str = relays or os.environ.get("NOSTR_RELAYS", "wss://relay.damus.io") self._relays = [r.strip() for r in relays_str.split(",") if r.strip()] self._bus = bus self._handlers: List[ChannelHandler] = [] @@ -65,8 +63,7 @@ class NostrChannel(BaseChannel): import pynostr # noqa: F401 except ImportError: raise ImportError( - "pynostr not installed. Install with: " - "uv sync --extra channel-nostr" + "pynostr not installed. Install with: uv sync --extra channel-nostr" ) self._status = ChannelStatus.CONNECTED diff --git a/src/openjarvis/channels/reddit_channel.py b/src/openjarvis/channels/reddit_channel.py index 9c2afb53..c390697f 100644 --- a/src/openjarvis/channels/reddit_channel.py +++ b/src/openjarvis/channels/reddit_channel.py @@ -78,8 +78,7 @@ class RedditChannel(BaseChannel): import praw # noqa: F401 except ImportError: raise ImportError( - "praw not installed. Install with: " - "uv sync --extra channel-reddit" + "praw not installed. Install with: uv sync --extra channel-reddit" ) self._status = ChannelStatus.CONNECTED diff --git a/src/openjarvis/channels/signal_channel.py b/src/openjarvis/channels/signal_channel.py index 97279c90..558149ed 100644 --- a/src/openjarvis/channels/signal_channel.py +++ b/src/openjarvis/channels/signal_channel.py @@ -44,7 +44,8 @@ class SignalChannel(BaseChannel): ) -> None: self._api_url = api_url or os.environ.get("SIGNAL_API_URL", "") self._phone_number = phone_number or os.environ.get( - "SIGNAL_PHONE_NUMBER", "", + "SIGNAL_PHONE_NUMBER", + "", ) self._bus = bus self._handlers: List[ChannelHandler] = [] @@ -94,7 +95,8 @@ class SignalChannel(BaseChannel): self._publish_sent(channel, content, conversation_id) return True logger.warning( - "Signal API returned status %d", resp.status_code, + "Signal API returned status %d", + resp.status_code, ) return False except Exception: diff --git a/src/openjarvis/channels/slack.py b/src/openjarvis/channels/slack.py index 6728da30..fea74067 100644 --- a/src/openjarvis/channels/slack.py +++ b/src/openjarvis/channels/slack.py @@ -72,7 +72,8 @@ class SlackChannel(BaseChannel): return self._listener_thread = threading.Thread( - target=self._socket_mode_loop, daemon=True, + target=self._socket_mode_loop, + daemon=True, ) self._listener_thread.start() self._status = ChannelStatus.CONNECTED @@ -120,7 +121,10 @@ class SlackChannel(BaseChannel): payload["thread_ts"] = conversation_id resp = httpx.post( - url, json=payload, headers=headers, timeout=10.0, + url, + json=payload, + headers=headers, + timeout=10.0, ) if resp.status_code < 300: data = resp.json() @@ -130,7 +134,8 @@ class SlackChannel(BaseChannel): logger.warning("Slack API error: %s", data.get("error")) return False logger.warning( - "Slack API returned status %d", resp.status_code, + "Slack API returned status %d", + resp.status_code, ) return False except Exception: diff --git a/src/openjarvis/channels/teams.py b/src/openjarvis/channels/teams.py index 4fbd50e1..20059570 100644 --- a/src/openjarvis/channels/teams.py +++ b/src/openjarvis/channels/teams.py @@ -46,9 +46,8 @@ class TeamsChannel(BaseChannel): ) -> None: self._app_id = app_id or os.environ.get("TEAMS_APP_ID", "") self._app_password = app_password or os.environ.get("TEAMS_APP_PASSWORD", "") - self._service_url = ( - service_url - or os.environ.get("TEAMS_SERVICE_URL", "https://smba.trafficmanager.net/teams") + self._service_url = service_url or os.environ.get( + "TEAMS_SERVICE_URL", "https://smba.trafficmanager.net/teams" ) self._bus = bus self._handlers: List[ChannelHandler] = [] @@ -99,13 +98,17 @@ class TeamsChannel(BaseChannel): payload["replyToId"] = conversation_id resp = httpx.post( - url, json=payload, headers=headers, timeout=10.0, + url, + json=payload, + headers=headers, + timeout=10.0, ) if resp.status_code < 300: self._publish_sent(channel, content, conversation_id) return True logger.warning( - "Teams API returned status %d", resp.status_code, + "Teams API returned status %d", + resp.status_code, ) return False except Exception: diff --git a/src/openjarvis/channels/telegram.py b/src/openjarvis/channels/telegram.py index 7cfa2146..1063e4d0 100644 --- a/src/openjarvis/channels/telegram.py +++ b/src/openjarvis/channels/telegram.py @@ -70,7 +70,8 @@ class TelegramChannel(BaseChannel): from telegram.ext import ApplicationBuilder # noqa: F401 self._listener_thread = threading.Thread( - target=self._poll_loop, daemon=True, + target=self._poll_loop, + daemon=True, ) self._listener_thread.start() self._status = ChannelStatus.CONNECTED diff --git a/src/openjarvis/channels/twilio_sms.py b/src/openjarvis/channels/twilio_sms.py index 97281573..332292e1 100644 --- a/src/openjarvis/channels/twilio_sms.py +++ b/src/openjarvis/channels/twilio_sms.py @@ -45,15 +45,9 @@ class TwilioSMSChannel(BaseChannel): phone_number: str = "", bus: Optional[EventBus] = None, ) -> None: - self._account_sid = account_sid or os.environ.get( - "TWILIO_ACCOUNT_SID", "" - ) - self._auth_token = auth_token or os.environ.get( - "TWILIO_AUTH_TOKEN", "" - ) - self._phone_number = phone_number or os.environ.get( - "TWILIO_PHONE_NUMBER", "" - ) + self._account_sid = account_sid or os.environ.get("TWILIO_ACCOUNT_SID", "") + self._auth_token = auth_token or os.environ.get("TWILIO_AUTH_TOKEN", "") + self._phone_number = phone_number or os.environ.get("TWILIO_PHONE_NUMBER", "") self._bus = bus self._client: Any = None self._handlers: List[ChannelHandler] = [] @@ -61,9 +55,7 @@ class TwilioSMSChannel(BaseChannel): def connect(self) -> None: try: - self._client = _create_twilio_client( - self._account_sid, self._auth_token - ) + self._client = _create_twilio_client(self._account_sid, self._auth_token) self._status = ChannelStatus.CONNECTED except Exception: logger.exception("Failed to create Twilio client") @@ -93,9 +85,7 @@ class TwilioSMSChannel(BaseChannel): self._publish_sent(channel, content, conversation_id) return True except Exception: - logger.exception( - "Failed to send SMS to %s", channel - ) + logger.exception("Failed to send SMS to %s", channel) return False def status(self) -> ChannelStatus: diff --git a/src/openjarvis/channels/twitch_channel.py b/src/openjarvis/channels/twitch_channel.py index 1ee2a674..d50232b2 100644 --- a/src/openjarvis/channels/twitch_channel.py +++ b/src/openjarvis/channels/twitch_channel.py @@ -51,9 +51,7 @@ class TwitchChannel(BaseChannel): initial_channels: str = "", bus: Optional[EventBus] = None, ) -> None: - self._access_token = access_token or os.environ.get( - "TWITCH_ACCESS_TOKEN", "" - ) + self._access_token = access_token or os.environ.get("TWITCH_ACCESS_TOKEN", "") self._client_id = client_id or os.environ.get("TWITCH_CLIENT_ID", "") self._nick = nick or os.environ.get("TWITCH_NICK", "") self._initial_channels = initial_channels or os.environ.get( @@ -75,8 +73,7 @@ class TwitchChannel(BaseChannel): import twitchio # noqa: F401 except ImportError: raise ImportError( - "twitchio not installed. Install with: " - "uv sync --extra channel-twitch" + "twitchio not installed. Install with: uv sync --extra channel-twitch" ) self._status = ChannelStatus.CONNECTED diff --git a/src/openjarvis/channels/twitter.py b/src/openjarvis/channels/twitter.py index 5a14dfd2..0ba60d6e 100644 --- a/src/openjarvis/channels/twitter.py +++ b/src/openjarvis/channels/twitter.py @@ -55,15 +55,18 @@ class TwitterChannel(BaseChannel): bus: Optional[EventBus] = None, ) -> None: self._bearer_token = bearer_token or os.environ.get( - "TWITTER_BEARER_TOKEN", "", + "TWITTER_BEARER_TOKEN", + "", ) self._api_key = api_key or os.environ.get("TWITTER_API_KEY", "") self._api_secret = api_secret or os.environ.get("TWITTER_API_SECRET", "") self._access_token = access_token or os.environ.get( - "TWITTER_ACCESS_TOKEN", "", + "TWITTER_ACCESS_TOKEN", + "", ) self._access_secret = access_secret or os.environ.get( - "TWITTER_ACCESS_SECRET", "", + "TWITTER_ACCESS_SECRET", + "", ) self._poll_interval = poll_interval self._bus = bus @@ -99,7 +102,8 @@ class TwitterChannel(BaseChannel): if self._access_token: self._poll_thread = threading.Thread( - target=self._poll_loop, daemon=True, + target=self._poll_loop, + daemon=True, ) self._poll_thread.start() except ImportError: @@ -189,7 +193,8 @@ class TwitterChannel(BaseChannel): continue response = self._client.get_users_mentions( - id=user_id, **kwargs, + id=user_id, + **kwargs, ) if response and response.data: @@ -225,7 +230,10 @@ class TwitterChannel(BaseChannel): self._stop_event.wait(self._poll_interval) def _publish_sent( - self, channel: str, content: str, conversation_id: str, + self, + channel: str, + content: str, + conversation_id: str, ) -> None: """Publish a CHANNEL_MESSAGE_SENT event on the bus.""" if self._bus is not None: diff --git a/src/openjarvis/channels/viber_channel.py b/src/openjarvis/channels/viber_channel.py index bae63bb3..509e3334 100644 --- a/src/openjarvis/channels/viber_channel.py +++ b/src/openjarvis/channels/viber_channel.py @@ -64,8 +64,7 @@ class ViberChannel(BaseChannel): import viberbot # noqa: F401 except ImportError: raise ImportError( - "viberbot not installed. Install with: " - "uv sync --extra channel-viber" + "viberbot not installed. Install with: uv sync --extra channel-viber" ) self._status = ChannelStatus.CONNECTED diff --git a/src/openjarvis/channels/webhook.py b/src/openjarvis/channels/webhook.py index c221ffe7..5d6ca64d 100644 --- a/src/openjarvis/channels/webhook.py +++ b/src/openjarvis/channels/webhook.py @@ -95,14 +95,18 @@ class WebhookChannel(BaseChannel): headers["X-Webhook-Secret"] = self._secret resp = httpx.request( - self._method, self._url, json=payload, - headers=headers, timeout=10.0, + self._method, + self._url, + json=payload, + headers=headers, + timeout=10.0, ) if resp.status_code < 300: self._publish_sent(channel, content, conversation_id) return True logger.warning( - "Webhook returned status %d", resp.status_code, + "Webhook returned status %d", + resp.status_code, ) return False except Exception: diff --git a/src/openjarvis/channels/whatsapp.py b/src/openjarvis/channels/whatsapp.py index eacc4f8b..75bc9eef 100644 --- a/src/openjarvis/channels/whatsapp.py +++ b/src/openjarvis/channels/whatsapp.py @@ -44,7 +44,8 @@ class WhatsAppChannel(BaseChannel): ) -> None: self._token = access_token or os.environ.get("WHATSAPP_ACCESS_TOKEN", "") self._phone_number_id = phone_number_id or os.environ.get( - "WHATSAPP_PHONE_NUMBER_ID", "", + "WHATSAPP_PHONE_NUMBER_ID", + "", ) self._bus = bus self._handlers: List[ChannelHandler] = [] @@ -82,10 +83,7 @@ class WhatsAppChannel(BaseChannel): try: import httpx - url = ( - f"https://graph.facebook.com/v21.0/" - f"{self._phone_number_id}/messages" - ) + url = f"https://graph.facebook.com/v21.0/{self._phone_number_id}/messages" headers = { "Authorization": f"Bearer {self._token}", "Content-Type": "application/json", @@ -98,13 +96,17 @@ class WhatsAppChannel(BaseChannel): } resp = httpx.post( - url, json=payload, headers=headers, timeout=10.0, + url, + json=payload, + headers=headers, + timeout=10.0, ) if resp.status_code < 300: self._publish_sent(channel, content, conversation_id) return True logger.warning( - "WhatsApp API returned status %d", resp.status_code, + "WhatsApp API returned status %d", + resp.status_code, ) return False except Exception: diff --git a/src/openjarvis/channels/whatsapp_baileys.py b/src/openjarvis/channels/whatsapp_baileys.py index bfbdd704..42c48886 100644 --- a/src/openjarvis/channels/whatsapp_baileys.py +++ b/src/openjarvis/channels/whatsapp_baileys.py @@ -109,8 +109,7 @@ class WhatsAppBaileysChannel(BaseChannel): pkg_dst = runtime / "package.json" pkg_src = _BRIDGE_SRC / "package.json" if pkg_src.exists() and ( - not pkg_dst.exists() - or pkg_src.stat().st_mtime > pkg_dst.stat().st_mtime + not pkg_dst.exists() or pkg_src.stat().st_mtime > pkg_dst.stat().st_mtime ): shutil.copy2(pkg_src, pkg_dst) @@ -169,7 +168,8 @@ class WhatsAppBaileysChannel(BaseChannel): bufsize=1, ) self._reader_thread = threading.Thread( - target=self._reader_loop, daemon=True, + target=self._reader_loop, + daemon=True, ) self._reader_thread.start() logger.info( @@ -218,11 +218,13 @@ class WhatsAppBaileysChannel(BaseChannel): return False try: - self._write_command({ - "type": "send", - "jid": channel, - "text": content, - }) + self._write_command( + { + "type": "send", + "jid": channel, + "text": content, + } + ) self._publish_sent(channel, content, conversation_id) return True except Exception: diff --git a/src/openjarvis/channels/xmpp_channel.py b/src/openjarvis/channels/xmpp_channel.py index 465b77dc..f36e030b 100644 --- a/src/openjarvis/channels/xmpp_channel.py +++ b/src/openjarvis/channels/xmpp_channel.py @@ -70,8 +70,7 @@ class XMPPChannel(BaseChannel): import slixmpp # noqa: F401 except ImportError: raise ImportError( - "slixmpp not installed. Install with: " - "uv sync --extra channel-xmpp" + "slixmpp not installed. Install with: uv sync --extra channel-xmpp" ) self._status = ChannelStatus.CONNECTED diff --git a/src/openjarvis/channels/zulip_channel.py b/src/openjarvis/channels/zulip_channel.py index 90dc4ffd..8d91e706 100644 --- a/src/openjarvis/channels/zulip_channel.py +++ b/src/openjarvis/channels/zulip_channel.py @@ -71,8 +71,7 @@ class ZulipChannel(BaseChannel): import zulip # noqa: F401 except ImportError: raise ImportError( - "zulip not installed. Install with: " - "uv sync --extra channel-zulip" + "zulip not installed. Install with: uv sync --extra channel-zulip" ) self._status = ChannelStatus.CONNECTED diff --git a/src/openjarvis/cli/__init__.py b/src/openjarvis/cli/__init__.py index f6125746..1589944f 100644 --- a/src/openjarvis/cli/__init__.py +++ b/src/openjarvis/cli/__init__.py @@ -15,8 +15,8 @@ from openjarvis.cli.chat_cmd import chat from openjarvis.cli.compose_cmd import compose from openjarvis.cli.config_cmd import config from openjarvis.cli.connect_cmd import connect -from openjarvis.cli.deep_research_setup_cmd import deep_research_setup from openjarvis.cli.daemon_cmd import restart, start, status, stop +from openjarvis.cli.deep_research_setup_cmd import deep_research_setup from openjarvis.cli.doctor_cmd import doctor from openjarvis.cli.eval_cmd import eval_group from openjarvis.cli.feedback_cmd import feedback_group diff --git a/src/openjarvis/cli/_version_check.py b/src/openjarvis/cli/_version_check.py index ef13f3f2..d95f4933 100644 --- a/src/openjarvis/cli/_version_check.py +++ b/src/openjarvis/cli/_version_check.py @@ -74,11 +74,15 @@ def _get_latest_version(current: str) -> str | None: try: _CACHE_PATH.parent.mkdir(parents=True, exist_ok=True) - _CACHE_PATH.write_text(json.dumps({ - "last_check": time.time(), - "latest_version": latest, - "current_version": current, - })) + _CACHE_PATH.write_text( + json.dumps( + { + "last_check": time.time(), + "latest_version": latest, + "current_version": current, + } + ) + ) except Exception: pass diff --git a/src/openjarvis/cli/add_cmd.py b/src/openjarvis/cli/add_cmd.py index 6ada1883..7bde76c9 100644 --- a/src/openjarvis/cli/add_cmd.py +++ b/src/openjarvis/cli/add_cmd.py @@ -70,7 +70,9 @@ _MCP_TEMPLATES = { @click.argument("server_name") @click.option("--key", default=None, help="API key or token for the server.") @click.option( - "--args", "extra_args", default=None, + "--args", + "extra_args", + default=None, help="Additional arguments (comma-separated).", ) def add(server_name: str, key: str | None, extra_args: str | None) -> None: diff --git a/src/openjarvis/cli/agent_cmd.py b/src/openjarvis/cli/agent_cmd.py index 0958ca4e..db24144c 100644 --- a/src/openjarvis/cli/agent_cmd.py +++ b/src/openjarvis/cli/agent_cmd.py @@ -72,13 +72,19 @@ def list_agents() -> None: tasks = mgr.list_tasks(a["id"]) bindings = mgr.list_channel_bindings(a["id"]) status_style = { - "idle": "dim", "running": "green", "paused": "yellow", - "error": "red", "archived": "dim strike", + "idle": "dim", + "running": "green", + "paused": "yellow", + "error": "red", + "archived": "dim strike", }.get(a["status"], "") table.add_row( - a["id"], a["name"], a["agent_type"], + a["id"], + a["name"], + a["agent_type"], f"[{status_style}]{a['status']}[/{status_style}]", - str(len(tasks)), str(len(bindings)), + str(len(tasks)), + str(len(bindings)), ) console.print(table) except Exception as exc: @@ -303,8 +309,10 @@ def templates() -> None: table.add_column("Description", style="white") for t in tpls: table.add_row( - t.get("id", ""), t.get("name", ""), - t.get("source", ""), t.get("description", "")[:60], + t.get("id", ""), + t.get("name", ""), + t.get("source", ""), + t.get("description", "")[:60], ) console.print(table) except Exception as exc: @@ -314,6 +322,7 @@ def templates() -> None: def _get_system(): """Build a JarvisSystem for CLI commands that need scheduler/executor.""" from openjarvis.system import SystemBuilder + try: return SystemBuilder().build() except RuntimeError as exc: @@ -332,6 +341,7 @@ def _get_scheduler_and_executor(system=None): def launch(): """Interactive agent launcher.""" from openjarvis.agents.manager import AgentManager as _AM + templates = _AM.list_templates() click.echo("Available templates:") for i, t in enumerate(templates, 1): @@ -363,7 +373,8 @@ def launch(): manager = _get_manager() if template: agent_data = manager.create_from_template( - template["id"], name=name, + template["id"], + name=name, ) agent_config = agent_data.get("config", {}) agent_config.update(config) @@ -374,7 +385,7 @@ def launch(): agent_data = manager.create_agent( name=name, agent_type=agent_type, config=config ) - click.echo(f"\nAgent \"{agent_data['name']}\" created (ID: {agent_data['id']})") + click.echo(f'\nAgent "{agent_data["name"]}" created (ID: {agent_data["id"]})') @agent.command("start") @@ -391,7 +402,7 @@ def start_agent(agent_id): click.echo("Scheduler not available", err=True) raise SystemExit(1) scheduler.register_agent(agent_id) - click.echo(f"Agent \"{agent_data['name']}\" registered with scheduler") + click.echo(f'Agent "{agent_data["name"]}" registered with scheduler') @agent.command("stop") @@ -408,7 +419,7 @@ def stop_agent(agent_id): click.echo("Scheduler not available", err=True) raise SystemExit(1) scheduler.deregister_agent(agent_id) - click.echo(f"Agent \"{agent_data['name']}\" deregistered from scheduler") + click.echo(f'Agent "{agent_data["name"]}" deregistered from scheduler') @agent.command("run") @@ -423,7 +434,7 @@ def run_agent(agent_id): if agent_data["status"] == "archived": click.echo(f"Agent {agent_id} is archived and cannot be run", err=True) raise SystemExit(1) - click.echo(f"Running tick for \"{agent_data['name']}\"...") + click.echo(f'Running tick for "{agent_data["name"]}"...') _, executor, _ = _get_scheduler_and_executor() if executor is None: click.echo("Executor not available", err=True) @@ -465,9 +476,7 @@ def status(): max_cost = cfg.get("max_cost", 0) if max_cost > 0: pct = min(100, int(a.get("total_cost", 0) / max_cost * 100)) - budget = ( - f"${a.get('total_cost', 0):.2f}/{max_cost:.0f} ({pct}%)" - ) + budget = f"${a.get('total_cost', 0):.2f}/{max_cost:.0f} ({pct}%)" else: budget = f"${a.get('total_cost', 0):.2f}" # Last seen column @@ -491,9 +500,7 @@ def status(): @agent.command("learning") @click.argument("agent_id") -@click.option( - "--run", "trigger_run", is_flag=True, help="Trigger manual learning run" -) +@click.option("--run", "trigger_run", is_flag=True, help="Trigger manual learning run") def learning(agent_id, trigger_run): """Show learning history or trigger a manual run.""" import datetime @@ -505,35 +512,35 @@ def learning(agent_id, trigger_run): raise SystemExit(1) if trigger_run: - click.echo(f"Triggering learning for \"{agent_data['name']}\"...") + click.echo(f'Triggering learning for "{agent_data["name"]}"...') from openjarvis.core.events import EventType, get_event_bus bus = get_event_bus() - bus.publish( - EventType.AGENT_LEARNING_STARTED, {"agent_id": agent_id} - ) + bus.publish(EventType.AGENT_LEARNING_STARTED, {"agent_id": agent_id}) click.echo("Learning triggered.") return logs = manager.list_learning_log(agent_id) if not logs: - click.echo(f"No learning history for \"{agent_data['name']}\"") + click.echo(f'No learning history for "{agent_data["name"]}"') return - click.echo(f"Learning history for \"{agent_data['name']}\":") + click.echo(f'Learning history for "{agent_data["name"]}":') for entry in logs: - ts = datetime.datetime.fromtimestamp( - entry["created_at"] - ).strftime("%Y-%m-%d %H:%M") + ts = datetime.datetime.fromtimestamp(entry["created_at"]).strftime( + "%Y-%m-%d %H:%M" + ) click.echo( - f" {ts} [{entry['event_type']}] " - f"{entry.get('description', '')[:60]}" + f" {ts} [{entry['event_type']}] {entry.get('description', '')[:60]}" ) @agent.command("trace") @click.argument("agent_id") @click.option( - "--run", "run_number", default=None, type=int, + "--run", + "run_number", + default=None, + type=int, help="Specific run number", ) @click.option("--limit", "-n", default=10, help="Number of traces to show") @@ -551,36 +558,32 @@ def trace(agent_id, run_number, limit): raise SystemExit(1) config = load_config() - store = TraceStore( - config.traces.db_path or "~/.openjarvis/traces.db" - ) + store = TraceStore(config.traces.db_path or "~/.openjarvis/traces.db") traces = store.list_traces(agent=agent_id, limit=limit) if not traces: - click.echo(f"No traces for \"{agent_data['name']}\"") + click.echo(f'No traces for "{agent_data["name"]}"') return if run_number is not None and 1 <= run_number <= len(traces): t = traces[run_number - 1] click.echo( - f"Trace #{run_number} — {t.outcome} " - f"({t.total_latency_seconds:.1f}s)" + f"Trace #{run_number} — {t.outcome} ({t.total_latency_seconds:.1f}s)" ) for i, step in enumerate(t.steps, 1): click.echo( - f" Step {i}: [{step.step_type.value}] " - f"{step.duration_seconds:.2f}s" + f" Step {i}: [{step.step_type.value}] {step.duration_seconds:.2f}s" ) inp = str(step.input)[:80] out = str(step.output)[:80] click.echo(f" In: {inp}") click.echo(f" Out: {out}") else: - click.echo(f"Traces for \"{agent_data['name']}\":") + click.echo(f'Traces for "{agent_data["name"]}":') for i, t in enumerate(traces, 1): - ts = datetime.datetime.fromtimestamp( - t.started_at - ).strftime("%Y-%m-%d %H:%M") + ts = datetime.datetime.fromtimestamp(t.started_at).strftime( + "%Y-%m-%d %H:%M" + ) click.echo( f" #{i} {ts} {t.outcome} " f"{t.total_latency_seconds:.1f}s " @@ -602,9 +605,9 @@ def logs(agent_id, limit): raise SystemExit(1) checkpoints = manager.list_checkpoints(agent_id) if not checkpoints: - click.echo(f"No execution history for \"{agent_data['name']}\"") + click.echo(f'No execution history for "{agent_data["name"]}"') return - click.echo(f"Recent runs for \"{agent_data['name']}\":") + click.echo(f'Recent runs for "{agent_data["name"]}":') for cp in checkpoints[:limit]: ts = datetime.datetime.fromtimestamp(cp["created_at"]).strftime( "%Y-%m-%d %H:%M" @@ -651,8 +654,10 @@ def watch(agent_id): click.echo("Watching agent events... (press Ctrl+C to stop)") bus = get_event_bus() agent_events = [ - EventType.AGENT_TICK_START, EventType.AGENT_TICK_END, - EventType.AGENT_TICK_ERROR, EventType.AGENT_BUDGET_EXCEEDED, + EventType.AGENT_TICK_START, + EventType.AGENT_TICK_END, + EventType.AGENT_TICK_ERROR, + EventType.AGENT_BUDGET_EXCEEDED, ] def _on_event(event): diff --git a/src/openjarvis/cli/auth_cmd.py b/src/openjarvis/cli/auth_cmd.py index c4a5e5ea..2d6409c4 100644 --- a/src/openjarvis/cli/auth_cmd.py +++ b/src/openjarvis/cli/auth_cmd.py @@ -46,16 +46,11 @@ def create_key() -> None: content += f'\n[server.auth]\napi_key = "{key}"\n' config_path.write_text(content) - os.chmod( - config_path, stat.S_IRUSR | stat.S_IWUSR - ) # 0600 + os.chmod(config_path, stat.S_IRUSR | stat.S_IWUSR) # 0600 click.echo(f"API key generated: {key}") click.echo(f"Stored in: {config_path}") - click.echo( - "File permissions set to 0600 " - "(user-only read/write)." - ) + click.echo("File permissions set to 0600 (user-only read/write).") @auth.command("revoke-key") @@ -71,8 +66,6 @@ def revoke_key() -> None: click.echo("No API key found in config.") return - content = re.sub( - r'api_key\s*=\s*"[^"]*"', 'api_key = ""', content - ) + content = re.sub(r'api_key\s*=\s*"[^"]*"', 'api_key = ""', content) config_path.write_text(content) click.echo("API key revoked.") diff --git a/src/openjarvis/cli/bench_cmd.py b/src/openjarvis/cli/bench_cmd.py index 7f2dbc37..ab6f431d 100644 --- a/src/openjarvis/cli/bench_cmd.py +++ b/src/openjarvis/cli/bench_cmd.py @@ -61,7 +61,7 @@ def _detect_stat_groups(metrics: dict[str, float]) -> dict[str, dict[str, float] for key, val in metrics.items(): for pfx in _STATS_PREFIXES: if key.startswith(pfx): - base = key[len(pfx):] + base = key[len(pfx) :] groups.setdefault(base, {})[pfx.rstrip("_")] = val break return groups @@ -152,27 +152,46 @@ def bench() -> None: @click.option("-m", "--model", "model_name", default=None, help="Model to benchmark.") @click.option("-e", "--engine", "engine_key", default=None, help="Engine backend.") @click.option( - "-n", "--samples", "num_samples", default=10, type=int, + "-n", + "--samples", + "num_samples", + default=10, + type=int, help="Number of samples per benchmark.", ) @click.option( - "-b", "--benchmark", "benchmark_name", default=None, + "-b", + "--benchmark", + "benchmark_name", + default=None, help="Specific benchmark to run (default: all).", ) @click.option( - "-o", "--output", "output_path", default=None, type=click.Path(), + "-o", + "--output", + "output_path", + default=None, + type=click.Path(), help="Write JSONL results to file.", ) @click.option( - "--json", "output_json", is_flag=True, + "--json", + "output_json", + is_flag=True, help="Output JSON summary to stdout.", ) @click.option( - "-w", "--warmup", "warmup", default=0, type=int, + "-w", + "--warmup", + "warmup", + default=0, + type=int, help="Number of warmup iterations before measurement.", ) @click.option( - "--setup-energy", "setup_energy", is_flag=True, + "--setup-energy", + "setup_energy", + is_flag=True, help="Run energy monitor setup script when missing (for energy benchmark).", ) def run( @@ -250,16 +269,12 @@ def run( import platform setup_script = ( - Path(__file__).resolve().parents[3] - / "scripts" - / "setup-energy-monitor.sh" - ) - is_darwin_arm = ( - platform.system() == "Darwin" - and platform.machine() == "arm64" + Path(__file__).resolve().parents[3] / "scripts" / "setup-energy-monitor.sh" ) + is_darwin_arm = platform.system() == "Darwin" and platform.machine() == "arm64" extra_hint = ( - "openjarvis[energy-apple]" if is_darwin_arm + "openjarvis[energy-apple]" + if is_darwin_arm else "openjarvis[gpu-metrics]" if platform.system() == "Linux" else "openjarvis[energy-all]" @@ -314,8 +329,10 @@ def run( f"[bold cyan]Running {len(benchmarks)} benchmark(s)...[/bold cyan]", ): results = suite.run_all( - engine, model_name, - num_samples=num_samples, warmup_samples=warmup, + engine, + model_name, + num_samples=num_samples, + warmup_samples=warmup, energy_monitor=energy_monitor, ) diff --git a/src/openjarvis/cli/channel_cmd.py b/src/openjarvis/cli/channel_cmd.py index 25f6c649..5ba8342a 100644 --- a/src/openjarvis/cli/channel_cmd.py +++ b/src/openjarvis/cli/channel_cmd.py @@ -148,7 +148,9 @@ def channel() -> None: @channel.command("list") @click.option( - "--channel-type", default=None, help=_CHANNEL_TYPE_HELP, + "--channel-type", + default=None, + help=_CHANNEL_TYPE_HELP, ) def channel_list( channel_type: Optional[str], @@ -186,7 +188,9 @@ def channel_list( @click.argument("target") @click.argument("message") @click.option( - "--channel-type", default=None, help=_CHANNEL_TYPE_HELP, + "--channel-type", + default=None, + help=_CHANNEL_TYPE_HELP, ) def channel_send( target: str, @@ -216,7 +220,9 @@ def channel_send( @channel.command("status") @click.option( - "--channel-type", default=None, help=_CHANNEL_TYPE_HELP, + "--channel-type", + default=None, + help=_CHANNEL_TYPE_HELP, ) def channel_status( channel_type: Optional[str], diff --git a/src/openjarvis/cli/channels_cmd.py b/src/openjarvis/cli/channels_cmd.py index e4a9ecac..49abb33b 100644 --- a/src/openjarvis/cli/channels_cmd.py +++ b/src/openjarvis/cli/channels_cmd.py @@ -27,12 +27,14 @@ def channels_status() -> None: if is_running(): table.add_row( - "iMessage", "[green]running[/green]", + "iMessage", + "[green]running[/green]", "Polling chat.db", ) else: table.add_row( - "iMessage", "[dim]stopped[/dim]", + "iMessage", + "[dim]stopped[/dim]", "jarvis channels imessage-start ", ) @@ -47,7 +49,8 @@ def channels_status() -> None: help="Run in background.", ) def imessage_start( - chat_identifier: str, background: bool, + chat_identifier: str, + background: bool, ) -> None: """Start the iMessage daemon for CHAT_IDENTIFIER. @@ -61,9 +64,7 @@ def imessage_start( console = Console() if is_running(): - console.print( - "[yellow]iMessage daemon already running.[/yellow]" - ) + console.print("[yellow]iMessage daemon already running.[/yellow]") return if background: @@ -71,9 +72,11 @@ def imessage_start( proc = subprocess.Popen( [ - sys.executable, "-m", + sys.executable, + "-m", "openjarvis.channels.imessage_daemon", - "--chat", chat_identifier, + "--chat", + chat_identifier, ], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, @@ -88,8 +91,7 @@ def imessage_start( ) else: console.print( - f"[green]Starting iMessage daemon[/green] " - f"— monitoring {chat_identifier}" + f"[green]Starting iMessage daemon[/green] — monitoring {chat_identifier}" ) console.print("Press Ctrl+C to stop.\n") @@ -117,13 +119,16 @@ def imessage_start( KnowledgeSearchTool(retriever=retriever), KnowledgeSQLTool(store=store), ScanChunksTool( - store=store, engine=engine, + store=store, + engine=engine, model="qwen3.5:4b", ), ThinkTool(), ] agent = DeepResearchAgent( - engine=engine, model="qwen3.5:4b", tools=tools, + engine=engine, + model="qwen3.5:4b", + tools=tools, ) def handler(text: str) -> str: @@ -145,6 +150,4 @@ def imessage_stop() -> None: if stop_daemon(): console.print("[green]iMessage daemon stopped.[/green]") else: - console.print( - "[dim]iMessage daemon is not running.[/dim]" - ) + console.print("[dim]iMessage daemon is not running.[/dim]") diff --git a/src/openjarvis/cli/chat_cmd.py b/src/openjarvis/cli/chat_cmd.py index f78f19f2..d1b7731c 100644 --- a/src/openjarvis/cli/chat_cmd.py +++ b/src/openjarvis/cli/chat_cmd.py @@ -162,8 +162,7 @@ def chat( continue elif cmd == "/model": console.print( - f"Model: [cyan]{model}[/cyan] " - f"Engine: [cyan]{engine_name}[/cyan]" + f"Model: [cyan]{model}[/cyan] Engine: [cyan]{engine_name}[/cyan]" ) continue elif cmd == "/help": @@ -183,9 +182,7 @@ def chat( for msg in history: role_str = msg.role if isinstance(msg.role, str) else msg.role.value role = role_str.upper() - console.print( - f"[bold]{role}:[/bold] {msg.content[:200]}" - ) + console.print(f"[bold]{role}:[/bold] {msg.content[:200]}") continue # Add user message @@ -196,9 +193,7 @@ def chat( if agent is not None: response = agent.run(user_input) content = ( - response.content - if hasattr(response, "content") - else str(response) + response.content if hasattr(response, "content") else str(response) ) else: result = engine.generate(history, model=model) diff --git a/src/openjarvis/cli/compose_cmd.py b/src/openjarvis/cli/compose_cmd.py index 89d82d6e..c3509c5a 100644 --- a/src/openjarvis/cli/compose_cmd.py +++ b/src/openjarvis/cli/compose_cmd.py @@ -31,7 +31,10 @@ def compose() -> None: @compose.command("list") @click.option( - "-k", "--kind", "kind", default=None, + "-k", + "--kind", + "kind", + default=None, type=click.Choice(["discrete", "operator"]), help="Filter by recipe kind.", ) @@ -203,13 +206,19 @@ def compose_run(name: str, query: tuple[str, ...], output_json: bool) -> None: if output_json: import json as json_mod + if isinstance(result, str): click.echo(json_mod.dumps({"content": result}, indent=2)) else: - click.echo(json_mod.dumps({ - "content": result.content, - "turns": getattr(result, "turns", 1), - }, indent=2)) + click.echo( + json_mod.dumps( + { + "content": result.content, + "turns": getattr(result, "turns", 1), + }, + indent=2, + ) + ) else: content = result if isinstance(result, str) else result.content click.echo(content) @@ -228,19 +237,33 @@ def compose_run(name: str, query: tuple[str, ...], output_json: bool) -> None: @compose.command("bench") @click.argument("name") @click.option( - "-b", "--benchmark", "benchmark", default=None, multiple=True, + "-b", + "--benchmark", + "benchmark", + default=None, + multiple=True, help="Override benchmarks (can specify multiple).", ) @click.option( - "-n", "--max-samples", "max_samples", type=int, default=None, + "-n", + "--max-samples", + "max_samples", + type=int, + default=None, help="Maximum samples per benchmark.", ) @click.option( - "--judge", "judge_model", default=None, + "--judge", + "judge_model", + default=None, help="LLM judge model override.", ) @click.option( - "-v", "--verbose", "verbose", is_flag=True, default=False, + "-v", + "--verbose", + "verbose", + is_flag=True, + default=False, help="Verbose logging.", ) def compose_bench( @@ -302,9 +325,7 @@ def compose_bench( results_table.add_column("Errors", justify="right", style="red") for i, rc in enumerate(run_configs, 1): - console.print( - f"\n[bold]Run {i}/{len(run_configs)}:[/bold] {rc.benchmark}" - ) + console.print(f"\n[bold]Run {i}/{len(run_configs)}:[/bold] {rc.benchmark}") try: summary = _run_single(rc, console=console) results_table.add_row( diff --git a/src/openjarvis/cli/dashboard.py b/src/openjarvis/cli/dashboard.py index cc962afc..8476c847 100644 --- a/src/openjarvis/cli/dashboard.py +++ b/src/openjarvis/cli/dashboard.py @@ -30,6 +30,7 @@ class DashboardApp: """Check if textual is available.""" try: import textual # noqa: F401 + return True except ImportError: return False @@ -86,21 +87,16 @@ class DashboardApp: classes="panel", ) yield Log( - id="events-panel", classes="panel", + id="events-panel", + classes="panel", ) yield Static( - "Telemetry\n" - "─────────\n" - "Throughput: --\n" - "Latency: --\n" - "Energy: --", + "Telemetry\n─────────\nThroughput: --\nLatency: --\nEnergy: --", id="telemetry-panel", classes="panel", ) yield Static( - "Agent Activity\n" - "──────────────\n" - "No active agents.", + "Agent Activity\n──────────────\nNo active agents.", id="agent-panel", classes="panel", ) @@ -113,6 +109,7 @@ class DashboardApp: # Try to connect to event bus try: from openjarvis.core.events import get_event_bus + bus = get_event_bus() def _on_event(event: Any) -> None: @@ -124,6 +121,7 @@ class DashboardApp: logger.debug("Event serialization failed: %s", exc) from openjarvis.core.events import EventType + for et in EventType: bus.subscribe(et, _on_event) except Exception: @@ -137,30 +135,16 @@ class DashboardApp: lines = ["System Status", "─────────────"] try: from openjarvis.core.config import load_config + config = load_config() - lines.append( - f"Engine: {config.engine.default}" - ) - model = ( - config.intelligence.default_model - or 'auto' - ) + lines.append(f"Engine: {config.engine.default}") + model = config.intelligence.default_model or "auto" lines.append(f"Model: {model}") - backend = ( - config.tools.storage.default_backend - ) + backend = config.tools.storage.default_backend lines.append(f"Memory: {backend}") - sec = ( - 'enabled' - if config.security.enabled - else 'disabled' - ) + sec = "enabled" if config.security.enabled else "disabled" lines.append(f"Security: {sec}") - tel = ( - 'enabled' - if config.telemetry.enabled - else 'disabled' - ) + tel = "enabled" if config.telemetry.enabled else "disabled" lines.append(f"Telemetry: {tel}") except Exception: lines.append("Config: not loaded") @@ -175,8 +159,7 @@ def launch_dashboard(config: Optional[Any] = None) -> None: app = DashboardApp(config=config) if not app.available(): raise ImportError( - "TUI dashboard requires 'textual'. " - "Install with: uv sync --extra dashboard" + "TUI dashboard requires 'textual'. Install with: uv sync --extra dashboard" ) app.run() diff --git a/src/openjarvis/cli/deep_research_setup_cmd.py b/src/openjarvis/cli/deep_research_setup_cmd.py index 5e9af601..e9ae7165 100644 --- a/src/openjarvis/cli/deep_research_setup_cmd.py +++ b/src/openjarvis/cli/deep_research_setup_cmd.py @@ -57,26 +57,32 @@ def detect_local_sources( notes_path = notes_db_path or _DEFAULT_NOTES_DB if notes_path.exists(): - sources.append({ - "connector_id": "apple_notes", - "display_name": "Apple Notes", - "config": {"db_path": str(notes_path)}, - }) + sources.append( + { + "connector_id": "apple_notes", + "display_name": "Apple Notes", + "config": {"db_path": str(notes_path)}, + } + ) imessage_path = imessage_db_path or _DEFAULT_IMESSAGE_DB if imessage_path.exists(): - sources.append({ - "connector_id": "imessage", - "display_name": "iMessage", - "config": {"db_path": str(imessage_path)}, - }) + sources.append( + { + "connector_id": "imessage", + "display_name": "iMessage", + "config": {"db_path": str(imessage_path)}, + } + ) if obsidian_vault_path and obsidian_vault_path.is_dir(): - sources.append({ - "connector_id": "obsidian", - "display_name": "Obsidian / Markdown", - "config": {"vault_path": str(obsidian_vault_path)}, - }) + sources.append( + { + "connector_id": "obsidian", + "display_name": "Obsidian / Markdown", + "config": {"vault_path": str(obsidian_vault_path)}, + } + ) return sources @@ -137,11 +143,13 @@ def detect_token_sources( continue if not data or not any(v for v in data.values() if v): continue - sources.append({ - "connector_id": ts["connector_id"], - "display_name": ts["display_name"], - "config": {}, - }) + sources.append( + { + "connector_id": ts["connector_id"], + "display_name": ts["display_name"], + "config": {}, + } + ) return sources @@ -154,8 +162,7 @@ def _prompt_connect_sources(console: Console) -> List[Dict[str, Any]]: while True: unconnected = [ - ts for ts in _TOKEN_SOURCES - if not (cdir / ts["creds_file"]).exists() + ts for ts in _TOKEN_SOURCES if not (cdir / ts["creds_file"]).exists() ] if not unconnected: console.print("[dim]All token sources already connected.[/dim]") @@ -165,10 +172,7 @@ def _prompt_connect_sources(console: Console) -> List[Dict[str, Any]]: break names = [ts["connector_id"] for ts in unconnected] - labels = [ - f"{ts['display_name']} ({ts['connector_id']})" - for ts in unconnected - ] + labels = [f"{ts['display_name']} ({ts['connector_id']})" for ts in unconnected] console.print("Available:") for label in labels: console.print(f" {label}") @@ -185,11 +189,13 @@ def _prompt_connect_sources(console: Console) -> List[Dict[str, Any]]: connector.handle_callback(token.strip()) console.print(f" [green]{ts['display_name']}: connected![/green]") - connected.append({ - "connector_id": choice, - "display_name": ts["display_name"], - "config": {}, - }) + connected.append( + { + "connector_id": choice, + "display_name": ts["display_name"], + "config": {}, + } + ) return connected @@ -203,27 +209,35 @@ def _instantiate_connector(connector_id: str, config: Dict[str, Any]) -> Any: """Lazily import and instantiate a connector by ID.""" if connector_id == "apple_notes": from openjarvis.connectors.apple_notes import AppleNotesConnector + return AppleNotesConnector(db_path=config.get("db_path", "")) elif connector_id == "imessage": from openjarvis.connectors.imessage import IMessageConnector + return IMessageConnector(db_path=config.get("db_path", "")) elif connector_id == "obsidian": from openjarvis.connectors.obsidian import ObsidianConnector + return ObsidianConnector(vault_path=config.get("vault_path", "")) elif connector_id == "gmail_imap": from openjarvis.connectors.gmail_imap import GmailIMAPConnector + return GmailIMAPConnector() elif connector_id == "outlook": from openjarvis.connectors.outlook import OutlookConnector + return OutlookConnector() elif connector_id == "slack": from openjarvis.connectors.slack_connector import SlackConnector + return SlackConnector() elif connector_id == "notion": from openjarvis.connectors.notion import NotionConnector + return NotionConnector() elif connector_id == "granola": from openjarvis.connectors.granola import GranolaConnector + return GranolaConnector() else: msg = f"Unknown connector: {connector_id}" @@ -277,8 +291,7 @@ def _launch_chat(store: KnowledgeStore, console: Console) -> None: engine = OllamaEngine() if not engine.health(): console.print( - "[red]Ollama is not running.[/red] Start it with: " - "[bold]ollama serve[/bold]" + "[red]Ollama is not running.[/red] Start it with: [bold]ollama serve[/bold]" ) return @@ -404,18 +417,15 @@ def deep_research_setup(obsidian_vault: Optional[str], skip_chat: bool) -> None: for src in all_sources: try: connector = _instantiate_connector( - src["connector_id"], src["config"], + src["connector_id"], + src["config"], ) pipeline = IngestionPipeline(store) engine = SyncEngine(pipeline) chunks = engine.sync(connector) - console.print( - f" {src['display_name']}: [green]{chunks} chunks[/green]" - ) + console.print(f" {src['display_name']}: [green]{chunks} chunks[/green]") except Exception as exc: # noqa: BLE001 - console.print( - f" {src['display_name']}: [red]error: {exc}[/red]" - ) + console.print(f" {src['display_name']}: [red]error: {exc}[/red]") total = store.count() console.print( diff --git a/src/openjarvis/cli/eval_cmd.py b/src/openjarvis/cli/eval_cmd.py index 3f03fb30..740f24aa 100644 --- a/src/openjarvis/cli/eval_cmd.py +++ b/src/openjarvis/cli/eval_cmd.py @@ -28,7 +28,8 @@ KNOWN_BENCHMARKS = { "swebench": {"category": "agentic", "description": "SWE-bench code patches"}, "swefficiency": {"category": "agentic", "description": "SWEfficiency optimization"}, "terminalbench": { - "category": "agentic", "description": "TerminalBench terminal tasks", + "category": "agentic", + "description": "TerminalBench terminal tasks", }, "terminalbench-native": { "category": "agentic", @@ -98,95 +99,155 @@ def eval_list() -> None: @eval_group.command("run") @click.option( - "-c", "--config", "config_path", default=None, type=click.Path(), + "-c", + "--config", + "config_path", + default=None, + type=click.Path(), help="TOML config file for suite runs.", ) @click.option( - "-b", "--benchmark", "benchmark", default=None, + "-b", + "--benchmark", + "benchmark", + default=None, help="Benchmark to run (e.g. supergpqa, gaia, frames, wildchat).", ) @click.option( - "-m", "--model", "model", default=None, + "-m", + "--model", + "model", + default=None, help="Model identifier.", ) @click.option( - "-n", "--max-samples", "max_samples", type=int, default=None, + "-n", + "--max-samples", + "max_samples", + type=int, + default=None, help="Maximum samples to evaluate.", ) @click.option( - "--backend", "backend", default="jarvis-direct", + "--backend", + "backend", + default="jarvis-direct", help="Inference backend (jarvis-direct or jarvis-agent).", ) @click.option( - "--agent", "agent_name", default=None, + "--agent", + "agent_name", + default=None, help="Agent name for jarvis-agent backend.", ) @click.option( - "-e", "--engine", "engine_key", default=None, + "-e", + "--engine", + "engine_key", + default=None, help="Engine key (ollama, vllm, cloud, ...).", ) @click.option( - "--tools", "tools", default="", + "--tools", + "tools", + default="", help="Comma-separated tool names.", ) @click.option( - "--telemetry/--no-telemetry", "telemetry", default=False, + "--telemetry/--no-telemetry", + "telemetry", + default=False, help="Enable telemetry collection during eval.", ) @click.option( - "--gpu-metrics/--no-gpu-metrics", "gpu_metrics", default=False, + "--gpu-metrics/--no-gpu-metrics", + "gpu_metrics", + default=False, help="Enable GPU metrics collection.", ) @click.option( - "--seed", "seed", type=int, default=42, + "--seed", + "seed", + type=int, + default=42, help="Random seed.", ) @click.option( - "--temperature", "temperature", type=float, default=0.0, + "--temperature", + "temperature", + type=float, + default=0.0, help="Generation temperature.", ) @click.option( - "--max-tokens", "max_tokens", type=int, default=2048, + "--max-tokens", + "max_tokens", + type=int, + default=2048, help="Max output tokens.", ) @click.option( - "--model-filter", "model_filter", default=None, + "--model-filter", + "model_filter", + default=None, help="Filter models by name substring (for multi-model configs).", ) @click.option( - "-o", "--output", "output_path", default=None, type=click.Path(), + "-o", + "--output", + "output_path", + default=None, + type=click.Path(), help="Output JSONL path.", ) @click.option( - "--wandb-project", "wandb_project", default="", + "--wandb-project", + "wandb_project", + default="", help="W&B project name (enables W&B tracking).", ) @click.option( - "--wandb-entity", "wandb_entity", default="", + "--wandb-entity", + "wandb_entity", + default="", help="W&B entity (team or user).", ) @click.option( - "--wandb-tags", "wandb_tags", default="", + "--wandb-tags", + "wandb_tags", + default="", help="Comma-separated W&B tags.", ) @click.option( - "--wandb-group", "wandb_group", default="", + "--wandb-group", + "wandb_group", + default="", help="W&B run group.", ) @click.option( - "--sheets-id", "sheets_spreadsheet_id", default="", + "--sheets-id", + "sheets_spreadsheet_id", + default="", help="Google Sheets spreadsheet ID.", ) @click.option( - "--sheets-worksheet", "sheets_worksheet", default="Results", + "--sheets-worksheet", + "sheets_worksheet", + default="Results", help="Google Sheets worksheet name.", ) @click.option( - "--sheets-creds", "sheets_credentials_path", default="", + "--sheets-creds", + "sheets_credentials_path", + default="", help="Path to Google service account JSON.", ) @click.option( - "-v", "--verbose", "verbose", is_flag=True, default=False, + "-v", + "--verbose", + "verbose", + is_flag=True, + default=False, help="Verbose logging.", ) def eval_run( @@ -237,13 +298,9 @@ def eval_run( # Filter by model name substring if requested if model_filter: - run_configs = [ - rc for rc in run_configs if model_filter in rc.model - ] + run_configs = [rc for rc in run_configs if model_filter in rc.model] if not run_configs: - console.print( - f"[red]No models match filter '{model_filter}'[/red]" - ) + console.print(f"[red]No models match filter '{model_filter}'[/red]") sys.exit(1) console.print( @@ -257,9 +314,7 @@ def eval_run( try: from openjarvis.evals.cli import _run_single except ImportError: - console.print( - "[red]Eval CLI module not available.[/red]" - ) + console.print("[red]Eval CLI module not available.[/red]") sys.exit(1) for i, rc in enumerate(run_configs, 1): @@ -286,9 +341,7 @@ def eval_run( ) if benchmark not in KNOWN_BENCHMARKS: - console.print( - f"[yellow]Warning: unknown benchmark '{benchmark}'[/yellow]" - ) + console.print(f"[yellow]Warning: unknown benchmark '{benchmark}'[/yellow]") try: from openjarvis.evals.core.types import RunConfig @@ -299,9 +352,7 @@ def eval_run( ) sys.exit(1) - tool_list = ( - [t.strip() for t in tools.split(",") if t.strip()] if tools else [] - ) + tool_list = [t.strip() for t in tools.split(",") if t.strip()] if tools else [] config = RunConfig( benchmark=benchmark, @@ -340,9 +391,7 @@ def eval_run( f"({summary.correct}/{summary.scored_samples})" ) except ImportError: - console.print( - "[red]Eval CLI module not available.[/red]" - ) + console.print("[red]Eval CLI module not available.[/red]") sys.exit(1) except Exception as exc: console.print(f"[red]Error: {exc}[/red]") @@ -352,7 +401,9 @@ def eval_run( @eval_group.command("compare") @click.argument("result_files", nargs=-1, required=True) @click.option( - "--metric", "metric", default="accuracy", + "--metric", + "metric", + default="accuracy", help="Metric to compare across runs (default: accuracy).", ) def eval_compare(result_files: tuple[str, ...], metric: str) -> None: @@ -367,13 +418,15 @@ def eval_compare(result_files: tuple[str, ...], metric: str) -> None: if summary_path.exists(): with open(summary_path) as f: data = json.load(f) - rows.append({ - "file": path.name, - "benchmark": data.get("benchmark", "?"), - "model": data.get("model", "?"), - "value": data.get(metric, "N/A"), - "samples": data.get("total_samples", 0), - }) + rows.append( + { + "file": path.name, + "benchmark": data.get("benchmark", "?"), + "model": data.get("model", "?"), + "value": data.get(metric, "N/A"), + "samples": data.get("total_samples", 0), + } + ) elif path.exists() and path.suffix == ".jsonl": # Fall back to reading JSONL and computing metric on-the-fly records = [] @@ -384,29 +437,25 @@ def eval_compare(result_files: tuple[str, ...], metric: str) -> None: records.append(json.loads(line)) if records: if metric == "accuracy": - scored = [ - r for r in records - if r.get("is_correct") is not None - ] + scored = [r for r in records if r.get("is_correct") is not None] correct = [r for r in scored if r["is_correct"]] - value = ( - len(correct) / len(scored) if scored else 0.0 - ) + value = len(correct) / len(scored) if scored else 0.0 else: values = [ - r[metric] for r in records + r[metric] + for r in records if metric in r and r[metric] is not None ] - value = ( - sum(values) / len(values) if values else "N/A" - ) - rows.append({ - "file": path.name, - "benchmark": records[0].get("benchmark", "?"), - "model": records[0].get("model", "?"), - "value": value, - "samples": len(records), - }) + value = sum(values) / len(values) if values else "N/A" + rows.append( + { + "file": path.name, + "benchmark": records[0].get("benchmark", "?"), + "model": records[0].get("model", "?"), + "value": value, + "samples": len(records), + } + ) else: console.print(f"[yellow]Skipping missing file: {path_str}[/yellow]") @@ -429,8 +478,11 @@ def eval_compare(result_files: tuple[str, ...], metric: str) -> None: val = row["value"] val_str = f"{val:.4f}" if isinstance(val, float) else str(val) table.add_row( - row["file"], row["benchmark"], row["model"], - val_str, str(row["samples"]), + row["file"], + row["benchmark"], + row["model"], + val_str, + str(row["samples"]), ) console.print(table) @@ -454,8 +506,7 @@ def eval_report(result_file: str) -> None: console.print(f" [cyan]Model:[/cyan] {data.get('model', '?')}") console.print(f" [cyan]Backend:[/cyan] {data.get('backend', '?')}") console.print( - f" [cyan]Accuracy:[/cyan] " - f"[bold]{data.get('accuracy', 0.0):.4f}[/bold]" + f" [cyan]Accuracy:[/cyan] [bold]{data.get('accuracy', 0.0):.4f}[/bold]" ) console.print(f" [cyan]Samples:[/cyan] {data.get('total_samples', 0)}") console.print(f" [cyan]Scored:[/cyan] {data.get('scored_samples', 0)}") @@ -466,8 +517,7 @@ def eval_report(result_file: str) -> None: f"{data.get('mean_latency_seconds', 0.0):.4f}s (mean)" ) console.print( - f" [cyan]Cost:[/cyan] " - f"${data.get('total_cost_usd', 0.0):.6f}" + f" [cyan]Cost:[/cyan] ${data.get('total_cost_usd', 0.0):.6f}" ) # Per-subject breakdown @@ -521,9 +571,7 @@ def eval_report(result_file: str) -> None: console.print(f" [cyan]Total:[/cyan] {len(records)}") console.print(f" [cyan]Scored:[/cyan] {len(scored)}") console.print(f" [cyan]Correct:[/cyan] {len(correct)}") - console.print( - f" [cyan]Accuracy:[/cyan] [bold]{accuracy:.4f}[/bold]" - ) + console.print(f" [cyan]Accuracy:[/cyan] [bold]{accuracy:.4f}[/bold]") console.print(f" [cyan]Errors:[/cyan] {len(errors)}") diff --git a/src/openjarvis/cli/feedback_cmd.py b/src/openjarvis/cli/feedback_cmd.py index 20b7bb9d..3dfd24e6 100644 --- a/src/openjarvis/cli/feedback_cmd.py +++ b/src/openjarvis/cli/feedback_cmd.py @@ -18,7 +18,10 @@ def feedback_group() -> None: @feedback_group.command("score") @click.argument("trace_id") @click.option( - "-s", "--score", type=float, required=True, + "-s", + "--score", + type=float, + required=True, help="Feedback score (0.0-1.0).", ) def feedback_score(trace_id: str, score: float) -> None: @@ -43,13 +46,9 @@ def feedback_score(trace_id: str, score: float) -> None: store.close() if updated: - console.print( - f"[green]Recorded score {score} for trace {trace_id}[/green]" - ) + console.print(f"[green]Recorded score {score} for trace {trace_id}[/green]") else: - console.print( - f"[yellow]Trace '{trace_id}' not found.[/yellow]" - ) + console.print(f"[yellow]Trace '{trace_id}' not found.[/yellow]") except Exception as exc: console.print(f"[red]Error: {exc}[/red]") sys.exit(1) @@ -57,16 +56,22 @@ def feedback_score(trace_id: str, score: float) -> None: @feedback_group.command("thumbs") @click.option( - "--last", is_flag=True, default=False, + "--last", + is_flag=True, + default=False, help="Rate the last interaction.", ) @click.option( - "--up/--down", "thumbs_up", default=True, + "--up/--down", + "thumbs_up", + default=True, help="Thumbs up or down.", ) @click.argument("trace_id", required=False) def feedback_thumbs( - last: bool, thumbs_up: bool, trace_id: Optional[str], + last: bool, + thumbs_up: bool, + trace_id: Optional[str], ) -> None: """Rate a trace with thumbs up/down.""" console = Console() @@ -97,13 +102,9 @@ def feedback_thumbs( label = "thumbs up" if thumbs_up else "thumbs down" if updated: - console.print( - f"[green]Recorded {label} for trace {trace_id}[/green]" - ) + console.print(f"[green]Recorded {label} for trace {trace_id}[/green]") else: - console.print( - f"[yellow]Trace '{trace_id}' not found.[/yellow]" - ) + console.print(f"[yellow]Trace '{trace_id}' not found.[/yellow]") except Exception as exc: console.print(f"[red]Error: {exc}[/red]") sys.exit(1) @@ -111,7 +112,9 @@ def feedback_thumbs( @feedback_group.command("evaluate") @click.option( - "--since", type=str, default="7d", + "--since", + type=str, + default="7d", help="Evaluate traces since (e.g. 7d, 24h).", ) def feedback_evaluate(since: str) -> None: @@ -125,15 +128,11 @@ def feedback_evaluate(since: str) -> None: value = float(since[:-1]) seconds = value * multipliers.get(unit, 86400) except (ValueError, IndexError): - console.print( - f"[red]Invalid duration '{since}'. Use e.g. 7d, 24h, 30m.[/red]" - ) + console.print(f"[red]Invalid duration '{since}'. Use e.g. 7d, 24h, 30m.[/red]") sys.exit(1) since_ts = time.time() - seconds - console.print( - f"[cyan]Evaluating traces from the last {since}...[/cyan]" - ) + console.print(f"[cyan]Evaluating traces from the last {since}...[/cyan]") try: from openjarvis.core.config import DEFAULT_CONFIG_DIR @@ -153,8 +152,7 @@ def feedback_evaluate(since: str) -> None: return console.print( - "[yellow]LLM judge evaluation is not yet " - "fully implemented.[/yellow]" + "[yellow]LLM judge evaluation is not yet fully implemented.[/yellow]" ) except Exception as exc: console.print(f"[red]Error: {exc}[/red]") @@ -183,9 +181,7 @@ def feedback_stats() -> None: total = len(all_traces) scored_count = len(scored) mean_score = ( - sum(t.feedback for t in scored) / scored_count - if scored_count > 0 - else 0.0 + sum(t.feedback for t in scored) / scored_count if scored_count > 0 else 0.0 ) positive = sum(1 for t in scored if t.feedback >= 0.5) diff --git a/src/openjarvis/cli/gateway_cmd.py b/src/openjarvis/cli/gateway_cmd.py index 2137a164..f74e5eb2 100644 --- a/src/openjarvis/cli/gateway_cmd.py +++ b/src/openjarvis/cli/gateway_cmd.py @@ -31,27 +31,26 @@ def start(install: bool) -> None: if plat.system() == "Darwin": plist_path = ( - Path.home() - / "Library/LaunchAgents/com.openjarvis.gateway.plist" + Path.home() / "Library/LaunchAgents/com.openjarvis.gateway.plist" ) generate_launchd_plist(plist_path) click.echo(f"Wrote {plist_path}") subprocess.run( - ["launchctl", "load", str(plist_path)], check=False, + ["launchctl", "load", str(plist_path)], + check=False, ) else: service_path = ( - Path.home() - / ".config/systemd/user/openjarvis-gateway.service" + Path.home() / ".config/systemd/user/openjarvis-gateway.service" ) generate_systemd_service(service_path) click.echo(f"Wrote {service_path}") subprocess.run( - ["systemctl", "--user", "daemon-reload"], check=False, + ["systemctl", "--user", "daemon-reload"], + check=False, ) subprocess.run( - ["systemctl", "--user", "enable", "--now", - "openjarvis-gateway"], + ["systemctl", "--user", "enable", "--now", "openjarvis-gateway"], check=False, ) else: diff --git a/src/openjarvis/cli/host_cmd.py b/src/openjarvis/cli/host_cmd.py index f8dd33e7..8013dfb4 100644 --- a/src/openjarvis/cli/host_cmd.py +++ b/src/openjarvis/cli/host_cmd.py @@ -136,9 +136,7 @@ def _install_backend(backend: str, console: Console) -> bool: display = info["display"] console.print() - console.print( - f"[yellow]Backend [bold]{display}[/bold] is not installed.[/yellow]" - ) + console.print(f"[yellow]Backend [bold]{display}[/bold] is not installed.[/yellow]") uv_available = shutil.which("uv") is not None in_uv_project = os.path.exists("pyproject.toml") and os.path.exists("uv.lock") @@ -168,9 +166,7 @@ def _install_backend(backend: str, console: Console) -> bool: console.print(f"[bold]Running:[/bold] {install_label}") result = subprocess.run(install_cmd) if result.returncode != 0: - console.print( - f"[red]Installation failed (exit {result.returncode}).[/red]" - ) + console.print(f"[red]Installation failed (exit {result.returncode}).[/red]") return False console.print(f"[green]{display} installed successfully.[/green]\n") @@ -208,11 +204,7 @@ def _install_binary_backend(backend: str, console: Console) -> bool: " Or from source:\n" " https://github.com/exo-explore/exo" ), - "uzu": ( - "Install Uzu:\n" - "\n" - " See https://uzu.ai for installation instructions." - ), + "uzu": ("Install Uzu:\n\n See https://uzu.ai for installation instructions."), } console.print() @@ -241,8 +233,7 @@ def _install_binary_backend(backend: str, console: Console) -> bool: console.print("[green]Ollama installed successfully.[/green]\n") return True console.print( - "[red]Installation may have failed. " - "Check above for errors.[/red]" + "[red]Installation may have failed. Check above for errors.[/red]" ) return False @@ -340,9 +331,7 @@ def host( raise SystemExit(1) backend = detected name = _BACKENDS[backend]["display"] - console.print( - f"Auto-detected backend: [bold cyan]{name}[/bold cyan]" - ) + console.print(f"Auto-detected backend: [bold cyan]{name}[/bold cyan]") info = _BACKENDS[backend] @@ -381,12 +370,9 @@ def host( console.print() if backend != "ollama": + console.print(f"[dim]The model server will be available at {host_url}[/dim]") console.print( - f"[dim]The model server will be available at {host_url}[/dim]" - ) - console.print( - "[dim]OpenJarvis will auto-discover it. " - "Press Ctrl+C to stop.[/dim]\n" + "[dim]OpenJarvis will auto-discover it. Press Ctrl+C to stop.[/dim]\n" ) try: diff --git a/src/openjarvis/cli/log_config.py b/src/openjarvis/cli/log_config.py index 6cf6fa15..751a58fd 100644 --- a/src/openjarvis/cli/log_config.py +++ b/src/openjarvis/cli/log_config.py @@ -58,12 +58,12 @@ def setup_logging( log_dir.mkdir(parents=True, exist_ok=True) log_file = log_dir / "cli.log" file_handler = RotatingFileHandler( - str(log_file), maxBytes=5 * 1024 * 1024, backupCount=3, + str(log_file), + maxBytes=5 * 1024 * 1024, + backupCount=3, ) file_handler.setLevel(logging.DEBUG) - file_fmt = logging.Formatter( - "%(asctime)s %(levelname)s %(name)s: %(message)s" - ) + file_fmt = logging.Formatter("%(asctime)s %(levelname)s %(name)s: %(message)s") file_handler.setFormatter(file_fmt) logger.addHandler(file_handler) diff --git a/src/openjarvis/cli/memory_cmd.py b/src/openjarvis/cli/memory_cmd.py index 43f4e5b7..8042f982 100644 --- a/src/openjarvis/cli/memory_cmd.py +++ b/src/openjarvis/cli/memory_cmd.py @@ -43,15 +43,21 @@ def memory() -> None: @memory.command() @click.argument("path") @click.option( - "--backend", "-b", default=None, + "--backend", + "-b", + default=None, help="Override the default memory backend.", ) @click.option( - "--chunk-size", default=512, type=int, + "--chunk-size", + default=512, + type=int, help="Chunk size in tokens.", ) @click.option( - "--chunk-overlap", default=64, type=int, + "--chunk-overlap", + default=64, + type=int, help="Overlap between chunks in tokens.", ) def index( @@ -108,11 +114,16 @@ def index( @memory.command() @click.argument("query", nargs=-1, required=True) @click.option( - "--top-k", "-k", default=5, type=int, + "--top-k", + "-k", + default=5, + type=int, help="Number of results to return.", ) @click.option( - "--backend", "-b", default=None, + "--backend", + "-b", + default=None, help="Override the default memory backend.", ) def search( @@ -158,7 +169,9 @@ def search( @memory.command() @click.option( - "--backend", "-b", default=None, + "--backend", + "-b", + default=None, help="Override the default memory backend.", ) def stats(backend: str | None) -> None: diff --git a/src/openjarvis/cli/model.py b/src/openjarvis/cli/model.py index 453bcca2..9e5f5617 100644 --- a/src/openjarvis/cli/model.py +++ b/src/openjarvis/cli/model.py @@ -97,15 +97,11 @@ def info(model_name: str) -> None: spec = ModelRegistry.get(model_name) params = f"{spec.parameter_count_b}B" if spec.parameter_count_b else "unknown" active = ( - f"{spec.active_parameter_count_b}B" - if spec.active_parameter_count_b - else "-" + f"{spec.active_parameter_count_b}B" if spec.active_parameter_count_b else "-" ) ctx_len = f"{spec.context_length:,}" if spec.context_length else "unknown" vram = f"{spec.min_vram_gb}GB" if spec.min_vram_gb else "-" - engines_str = ( - ", ".join(spec.supported_engines) if spec.supported_engines else "-" - ) + engines_str = ", ".join(spec.supported_engines) if spec.supported_engines else "-" provider = spec.provider or "-" api_key = "required" if spec.requires_api_key else "not required" lines = [ @@ -163,6 +159,7 @@ def ollama_pull(host: str, model_name: str, console: Console) -> bool: ) as resp: resp.raise_for_status() import json + for line in resp.iter_lines(): if not line.strip(): continue @@ -219,9 +216,7 @@ def hf_download(repo: str, filename: str | None, console: Console) -> bool: @model.command() @click.argument("model_name") -@click.option( - "--engine", default=None, help="Engine to download for." -) +@click.option("--engine", default=None, help="Engine to download for.") def pull(model_name: str, engine: str | None) -> None: """Download a model.""" console = Console() diff --git a/src/openjarvis/cli/operators_cmd.py b/src/openjarvis/cli/operators_cmd.py index 6a6b33d1..c0881a79 100644 --- a/src/openjarvis/cli/operators_cmd.py +++ b/src/openjarvis/cli/operators_cmd.py @@ -29,10 +29,10 @@ def list_operators() -> None: from openjarvis.operators.loader import load_operator config = load_config() - manifests_dir = Path( - config.operators.manifests_dir - ).expanduser() if hasattr(config, "operators") else ( - DEFAULT_CONFIG_DIR / "operators" + manifests_dir = ( + Path(config.operators.manifests_dir).expanduser() + if hasattr(config, "operators") + else (DEFAULT_CONFIG_DIR / "operators") ) # Also check project-local operators/ directory @@ -55,8 +55,7 @@ def list_operators() -> None: if not found: console.print("[dim]No operators discovered.[/dim]") console.print( - f"[dim]Place TOML manifests in {manifests_dir} " - f"or ./operators/[/dim]" + f"[dim]Place TOML manifests in {manifests_dir} or ./operators/[/dim]" ) return @@ -189,6 +188,7 @@ def logs(operator_id: str, lines: int) -> None: db_path = config.scheduler.db_path if not db_path: from openjarvis.core.config import DEFAULT_CONFIG_DIR + db_path = str(DEFAULT_CONFIG_DIR / "scheduler.db") store = SchedulerStore(db_path=db_path) diff --git a/src/openjarvis/cli/optimize_cmd.py b/src/openjarvis/cli/optimize_cmd.py index 0fa75ecd..3f586b2a 100644 --- a/src/openjarvis/cli/optimize_cmd.py +++ b/src/openjarvis/cli/optimize_cmd.py @@ -18,27 +18,43 @@ def optimize_group() -> None: @optimize_group.command("run") @click.option( - "-c", "--config", "config_path", type=str, default=None, + "-c", + "--config", + "config_path", + type=str, + default=None, help="TOML config file for the optimization run.", ) @click.option( - "-b", "--benchmark", type=str, default=None, + "-b", + "--benchmark", + type=str, + default=None, help="Benchmark name (e.g. supergpqa, mmlu-pro).", ) @click.option( - "-t", "--trials", type=int, default=20, + "-t", + "--trials", + type=int, + default=20, help="Maximum number of trials.", ) @click.option( - "--optimizer-model", type=str, default="claude-sonnet-4-6", + "--optimizer-model", + type=str, + default="claude-sonnet-4-6", help="Model used by the LLM optimizer.", ) @click.option( - "--max-samples", type=int, default=50, + "--max-samples", + type=int, + default=50, help="Maximum samples per trial evaluation.", ) @click.option( - "--output-dir", type=str, default="results/optimize/", + "--output-dir", + type=str, + default="results/optimize/", help="Directory for trial output files.", ) def optimize_run( @@ -58,9 +74,7 @@ def optimize_run( try: from openjarvis.learning.optimize.config import load_optimize_config except ImportError: - console.print( - "[red]Optimization framework not available.[/red]" - ) + console.print("[red]Optimization framework not available.[/red]") sys.exit(1) try: @@ -167,7 +181,8 @@ def optimize_run( early_stop = 5 if data is not None: early_stop = data.get("optimize", {}).get( - "early_stop_patience", early_stop, + "early_stop_patience", + early_stop, ) engine = OptimizationEngine( @@ -200,9 +215,7 @@ def optimize_run( f"(accuracy={run.best_trial.accuracy:.4f})" ) except ImportError as exc: - console.print( - f"[red]Missing dependency for optimization: {exc}[/red]" - ) + console.print(f"[red]Missing dependency for optimization: {exc}[/red]") sys.exit(1) except Exception as exc: console.print(f"[red]Optimization failed: {exc}[/red]") @@ -321,7 +334,10 @@ def optimize_results(run_id: str) -> None: @optimize_group.command("best") @click.argument("run_id") @click.option( - "-o", "--output", type=str, default=None, + "-o", + "--output", + type=str, + default=None, help="Output recipe path (TOML).", ) def optimize_best(run_id: str, output: Optional[str]) -> None: @@ -350,15 +366,11 @@ def optimize_best(run_id: str, output: Optional[str]) -> None: console.print("[yellow]No best trial found in this run.[/yellow]") return - output_path = Path( - output or f"results/optimize/best_{run_id}.toml" - ) + output_path = Path(output or f"results/optimize/best_{run_id}.toml") engine = OptimizationEngine.__new__(OptimizationEngine) engine.export_best_recipe(run, output_path) - console.print( - f"[green]Best recipe exported to:[/green] {output_path}" - ) + console.print(f"[green]Best recipe exported to:[/green] {output_path}") console.print( f" Trial: {run.best_trial.trial_id} " f"(accuracy={run.best_trial.accuracy:.4f})" @@ -370,11 +382,16 @@ def optimize_best(run_id: str, output: Optional[str]) -> None: @optimize_group.command("personal") @click.argument("action", type=click.Choice(["synthesize", "run"])) @click.option( - "--workflow", type=str, default="default", + "--workflow", + type=str, + default="default", help="Workflow ID for personal optimization.", ) @click.option( - "-t", "--trials", type=int, default=10, + "-t", + "--trials", + type=int, + default=10, help="Maximum trials for personal optimization.", ) def optimize_personal(action: str, workflow: str, trials: int) -> None: @@ -396,8 +413,7 @@ def optimize_personal(action: str, workflow: str, trials: int) -> None: f"workflow '{workflow}' (max {trials} trials)...[/cyan]" ) console.print( - "[yellow]Personal optimization is not yet " - "fully implemented.[/yellow]" + "[yellow]Personal optimization is not yet fully implemented.[/yellow]" ) diff --git a/src/openjarvis/cli/scheduler_cmd.py b/src/openjarvis/cli/scheduler_cmd.py index cc368622..094af437 100644 --- a/src/openjarvis/cli/scheduler_cmd.py +++ b/src/openjarvis/cli/scheduler_cmd.py @@ -17,9 +17,9 @@ def _get_store() -> "SchedulerStore": # noqa: F821 from openjarvis.scheduler.store import SchedulerStore config = load_config() - db_path = getattr( - getattr(config, "scheduler", None), "db_path", None - ) or str(DEFAULT_CONFIG_DIR / "scheduler.db") + db_path = getattr(getattr(config, "scheduler", None), "db_path", None) or str( + DEFAULT_CONFIG_DIR / "scheduler.db" + ) return SchedulerStore(db_path) @@ -38,13 +38,15 @@ def scheduler() -> None: @scheduler.command("create") @click.argument("prompt") @click.option( - "--type", "schedule_type", + "--type", + "schedule_type", required=True, type=click.Choice(["cron", "interval", "once"]), help="Schedule type.", ) @click.option( - "--value", "schedule_value", + "--value", + "schedule_value", required=True, help="Schedule value (cron expr, seconds, or ISO datetime).", ) @@ -82,7 +84,8 @@ def scheduler_create( @scheduler.command("list") @click.option( - "--status", default=None, + "--status", + default=None, type=click.Choice(["active", "paused", "completed", "cancelled"]), help="Filter by status.", ) @@ -197,14 +200,10 @@ def scheduler_logs(task_id: str, limit: int) -> None: table.add_column("Error", max_width=30) for log in logs: - success_str = ( - "[green]yes[/green]" if log["success"] else "[red]no[/red]" - ) + success_str = "[green]yes[/green]" if log["success"] else "[red]no[/red]" result_text = log["result"] result_short = ( - (result_text[:37] + "...") - if len(result_text) > 40 - else result_text + (result_text[:37] + "...") if len(result_text) > 40 else result_text ) table.add_row( str(log["id"]), @@ -221,7 +220,9 @@ def scheduler_logs(task_id: str, limit: int) -> None: @scheduler.command("start") @click.option( - "--poll-interval", default=60, type=int, + "--poll-interval", + default=60, + type=int, help="Seconds between poll cycles.", ) def scheduler_start(poll_interval: int) -> None: diff --git a/src/openjarvis/cli/serve.py b/src/openjarvis/cli/serve.py index a00b7811..28d5e70a 100644 --- a/src/openjarvis/cli/serve.py +++ b/src/openjarvis/cli/serve.py @@ -26,13 +26,18 @@ logger = logging.getLogger(__name__) @click.command() @click.option("--host", default=None, help="Bind address (default: config).") @click.option( - "--port", default=None, type=int, + "--port", + default=None, + type=int, help="Port number (default: config).", ) @click.option("-e", "--engine", "engine_key", default=None, help="Engine backend.") @click.option("-m", "--model", "model_name", default=None, help="Default model.") @click.option( - "-a", "--agent", "agent_name", default=None, + "-a", + "--agent", + "agent_name", + default=None, help="Agent for non-streaming requests (simple, orchestrator, react, openhands).", ) def serve( @@ -94,12 +99,14 @@ def serve( # Apply security guardrails from openjarvis.security import setup_security + sec = setup_security(config, engine, bus) engine = sec.engine # If cloud API keys are set, wrap with MultiEngine so both local # and cloud models appear in the model list and can be used. import os + _has_cloud = ( os.environ.get("OPENAI_API_KEY") or os.environ.get("ANTHROPIC_API_KEY") @@ -183,13 +190,13 @@ def serve( if configured: if isinstance(configured, list): allowed = { - t.strip() for t in configured + t.strip() + for t in configured if isinstance(t, str) and t.strip() } else: allowed = { - t.strip() for t in configured.split(",") - if t.strip() + t.strip() for t in configured.split(",") if t.strip() } else: allowed = _DEFAULT_TOOLS @@ -214,6 +221,7 @@ def serve( agent = agent_cls(engine, model_name, **agent_kwargs) except Exception as exc: import traceback + console.print(f"[yellow]Agent '{agent_key}' failed to load: {exc}[/yellow]") traceback.print_exc() @@ -254,6 +262,7 @@ def serve( speech_backend = None try: from openjarvis.speech._discovery import get_speech_backend + speech_backend = get_speech_backend(config) if speech_backend: console.print(f" Speech: [cyan]{speech_backend.backend_id}[/cyan]") @@ -287,6 +296,7 @@ def serve( executor = AgentExecutor(manager=agent_manager, event_bus=bus) from openjarvis.system import SystemBuilder + system = SystemBuilder(config).build() executor.set_system(system) @@ -298,7 +308,8 @@ def serve( for ag in agent_manager.list_agents(): sched_type = ag.get("config", {}).get("schedule_type", "manual") if sched_type in ("cron", "interval") and ag["status"] not in ( - "archived", "error", + "archived", + "error", ): agent_scheduler.register_agent(ag["id"]) agent_scheduler.start() @@ -316,7 +327,8 @@ def serve( mem_key = config.memory.default_backend if MemoryRegistry.contains(mem_key): memory_backend = MemoryRegistry.create( - mem_key, db_path=config.memory.db_path, + mem_key, + db_path=config.memory.db_path, ) console.print(" Memory: [cyan]active[/cyan]") except Exception as exc: @@ -329,33 +341,21 @@ def serve( if not api_key: try: import tomllib + _cfg_path = str( - __import__("pathlib").Path.home() - / ".openjarvis" / "config.toml" + __import__("pathlib").Path.home() / ".openjarvis" / "config.toml" ) with open(_cfg_path, "rb") as _f: _raw = tomllib.load(_f) - api_key = ( - _raw.get("server", {}) - .get("auth", {}) - .get("api_key", "") - ) + api_key = _raw.get("server", {}).get("auth", {}).get("api_key", "") except (FileNotFoundError, ImportError): pass webhook_config = { - "twilio_auth_token": _os.environ.get( - "TWILIO_AUTH_TOKEN", "" - ), - "bluebubbles_password": _os.environ.get( - "BLUEBUBBLES_PASSWORD", "" - ), - "whatsapp_verify_token": _os.environ.get( - "WHATSAPP_VERIFY_TOKEN", "" - ), - "whatsapp_app_secret": _os.environ.get( - "WHATSAPP_APP_SECRET", "" - ), + "twilio_auth_token": _os.environ.get("TWILIO_AUTH_TOKEN", ""), + "bluebubbles_password": _os.environ.get("BLUEBUBBLES_PASSWORD", ""), + "whatsapp_verify_token": _os.environ.get("WHATSAPP_VERIFY_TOKEN", ""), + "whatsapp_app_secret": _os.environ.get("WHATSAPP_APP_SECRET", ""), } # Wrap existing channel in ChannelBridge orchestrator @@ -369,9 +369,7 @@ def serve( ) session_store = SessionStore() - channels = { - channel_bridge.channel_id: channel_bridge - } + channels = {channel_bridge.channel_id: channel_bridge} channel_bridge = ChannelBridge( channels=channels, session_store=session_store, @@ -380,14 +378,17 @@ def serve( agent_manager=agent_manager, ) except Exception as exc: - logger.debug( - "ChannelBridge init skipped: %s", exc - ) + logger.debug("ChannelBridge init skipped: %s", exc) app = create_app( - engine, model_name, agent=agent, bus=bus, - engine_name=engine_name, agent_name=agent_key or "", - channel_bridge=channel_bridge, config=config, + engine, + model_name, + agent=agent, + bus=bus, + engine_name=engine_name, + agent_name=agent_key or "", + channel_bridge=channel_bridge, + config=config, memory_backend=memory_backend, speech_backend=speech_backend, agent_manager=agent_manager, @@ -405,4 +406,5 @@ def serve( ) import uvicorn + uvicorn.run(app, host=bind_host, port=bind_port, log_level="info") diff --git a/src/openjarvis/cli/telemetry_cmd.py b/src/openjarvis/cli/telemetry_cmd.py index 32130f22..ddd7772f 100644 --- a/src/openjarvis/cli/telemetry_cmd.py +++ b/src/openjarvis/cli/telemetry_cmd.py @@ -27,7 +27,11 @@ def telemetry() -> None: @telemetry.command() @click.option( - "-n", "--top", "top_n", default=10, type=int, + "-n", + "--top", + "top_n", + default=10, + type=int, help="Number of top models to show.", ) def stats(top_n: int) -> None: @@ -76,13 +80,9 @@ def stats(top_n: int) -> None: # Per-model table if summary.per_model: has_energy = any( - ms.total_energy_joules > 0 - for ms in summary.per_model[:top_n] - ) - has_itl = any( - ms.avg_mean_itl_ms > 0 - for ms in summary.per_model[:top_n] + ms.total_energy_joules > 0 for ms in summary.per_model[:top_n] ) + has_itl = any(ms.avg_mean_itl_ms > 0 for ms in summary.per_model[:top_n]) model_table = Table(title=f"Top {top_n} Models") model_table.add_column("Model", style="cyan") model_table.add_column("Calls", justify="right") @@ -121,13 +121,9 @@ def stats(top_n: int) -> None: # Per-engine table if summary.per_engine: has_engine_energy = any( - es.total_energy_joules > 0 - for es in summary.per_engine - ) - has_engine_itl = any( - es.avg_mean_itl_ms > 0 - for es in summary.per_engine + es.total_energy_joules > 0 for es in summary.per_engine ) + has_engine_itl = any(es.avg_mean_itl_ms > 0 for es in summary.per_engine) engine_table = Table(title="Engines") engine_table.add_column("Engine", style="cyan") engine_table.add_column("Calls", justify="right") @@ -171,11 +167,19 @@ def stats(top_n: int) -> None: @telemetry.command() @click.option( - "-f", "--format", "fmt", default="json", type=click.Choice(["json", "csv"]), + "-f", + "--format", + "fmt", + default="json", + type=click.Choice(["json", "csv"]), help="Output format.", ) @click.option( - "-o", "--output", "output_path", default=None, type=click.Path(), + "-o", + "--output", + "output_path", + default=None, + type=click.Path(), help="Output file path (default: stdout).", ) def export(fmt: str, output_path: str | None) -> None: @@ -207,7 +211,10 @@ def export(fmt: str, output_path: str | None) -> None: @telemetry.command() @click.option( - "-y", "--yes", "confirmed", is_flag=True, + "-y", + "--yes", + "confirmed", + is_flag=True, help="Skip confirmation prompt.", ) def clear(confirmed: bool) -> None: diff --git a/src/openjarvis/cli/tunnel_cmd.py b/src/openjarvis/cli/tunnel_cmd.py index a3047283..b5e82217 100644 --- a/src/openjarvis/cli/tunnel_cmd.py +++ b/src/openjarvis/cli/tunnel_cmd.py @@ -12,9 +12,7 @@ from openjarvis.core.config import DEFAULT_CONFIG_PATH @click.group("tunnel", invoke_without_command=True) -@click.option( - "--port", default=8000, help="Local port to tunnel." -) +@click.option("--port", default=8000, help="Local port to tunnel.") @click.pass_context def tunnel(ctx: click.Context, port: int) -> None: """Start a Cloudflare tunnel or manage config.""" @@ -31,10 +29,7 @@ def tunnel(ctx: click.Context, port: int) -> None: ) sys.exit(1) - click.echo( - f"Starting Cloudflare tunnel " - f"to localhost:{port}..." - ) + click.echo(f"Starting Cloudflare tunnel to localhost:{port}...") click.echo("Press Ctrl+C to stop.\n") try: @@ -66,10 +61,7 @@ def status() -> None: if "public_url" in content: for line in content.splitlines(): if "public_url" in line: - click.echo( - "Configured tunnel URL: " - f"{line.split('=', 1)[1].strip()}" - ) + click.echo(f"Configured tunnel URL: {line.split('=', 1)[1].strip()}") return click.echo("No tunnel URL configured.") diff --git a/src/openjarvis/cli/vault_cmd.py b/src/openjarvis/cli/vault_cmd.py index 6636942f..5aa6a58d 100644 --- a/src/openjarvis/cli/vault_cmd.py +++ b/src/openjarvis/cli/vault_cmd.py @@ -21,8 +21,7 @@ def _get_or_create_key() -> bytes: from cryptography.fernet import Fernet except ImportError: raise ImportError( - "cryptography not installed. Install with: " - "uv sync --extra security-signing" + "cryptography not installed. Install with: uv sync --extra security-signing" ) if _VAULT_KEY_FILE.exists(): @@ -41,6 +40,7 @@ def _load_vault() -> dict: return {} try: from cryptography.fernet import Fernet + key = _get_or_create_key() f = Fernet(key) encrypted = _VAULT_FILE.read_bytes() @@ -53,6 +53,7 @@ def _load_vault() -> dict: def _save_vault(data: dict) -> None: """Encrypt and save vault contents.""" from cryptography.fernet import Fernet + key = _get_or_create_key() f = Fernet(key) plaintext = json.dumps(data).encode() diff --git a/src/openjarvis/cli/workflow_cmd.py b/src/openjarvis/cli/workflow_cmd.py index f1f3bc0a..f7665120 100644 --- a/src/openjarvis/cli/workflow_cmd.py +++ b/src/openjarvis/cli/workflow_cmd.py @@ -52,8 +52,7 @@ def run(workflow_name: str, input_text: str | None) -> None: console.print(f"[green]Workflow '{workflow_name}' started.[/green]") # Full execution would need a JarvisSystem — just report for now console.print( - "[dim]Note: Full workflow execution" - " requires a running system.[/dim]" + "[dim]Note: Full workflow execution requires a running system.[/dim]" ) except ImportError: console.print("[red]Workflow system not available.[/red]") diff --git a/src/openjarvis/connectors/dropbox.py b/src/openjarvis/connectors/dropbox.py index d7372db1..61331c50 100644 --- a/src/openjarvis/connectors/dropbox.py +++ b/src/openjarvis/connectors/dropbox.py @@ -26,9 +26,7 @@ _DROPBOX_API_BASE = "https://api.dropboxapi.com/2" _DROPBOX_CONTENT_BASE = "https://content.dropboxapi.com/2" _DROPBOX_AUTH_URL = "https://www.dropbox.com/oauth2/authorize" _DROPBOX_SCOPES = ["files.metadata.read", "files.content.read"] -_DEFAULT_CREDENTIALS_PATH = str( - DEFAULT_CONFIG_DIR / "connectors" / "dropbox.json" -) +_DEFAULT_CREDENTIALS_PATH = str(DEFAULT_CONFIG_DIR / "connectors" / "dropbox.json") # File extensions whose content can be downloaded and stored as text _TEXT_EXTENSIONS = { @@ -231,9 +229,7 @@ class DropboxConnector(BaseConnector): entries_seen: int = 0 while True: - list_resp = _dropbox_api_list_folder( - token, path="", cursor=list_cursor - ) + list_resp = _dropbox_api_list_folder(token, path="", cursor=list_cursor) entries: List[Dict[str, Any]] = list_resp.get("entries", []) entries_seen += len(entries) diff --git a/src/openjarvis/connectors/granola.py b/src/openjarvis/connectors/granola.py index a9335ba9..9426f704 100644 --- a/src/openjarvis/connectors/granola.py +++ b/src/openjarvis/connectors/granola.py @@ -26,9 +26,7 @@ from openjarvis.tools._stubs import ToolSpec # --------------------------------------------------------------------------- _GRANOLA_API_BASE = "https://public-api.granola.ai" -_DEFAULT_CREDENTIALS_PATH = str( - DEFAULT_CONFIG_DIR / "connectors" / "granola.json" -) +_DEFAULT_CREDENTIALS_PATH = str(DEFAULT_CONFIG_DIR / "connectors" / "granola.json") # --------------------------------------------------------------------------- # Module-level API functions (easy to patch in tests) @@ -122,9 +120,7 @@ def _format_note_content(note: Dict[str, Any]) -> str: parts: List[str] = [] # Summary section - summary_markdown: str = ( - note.get("summary", {}) or {} - ).get("markdown", "") + summary_markdown: str = (note.get("summary", {}) or {}).get("markdown", "") parts.append("## Summary") parts.append(summary_markdown) diff --git a/src/openjarvis/connectors/notion.py b/src/openjarvis/connectors/notion.py index e8104ef1..60fd6857 100644 --- a/src/openjarvis/connectors/notion.py +++ b/src/openjarvis/connectors/notion.py @@ -27,9 +27,7 @@ from openjarvis.tools._stubs import ToolSpec _NOTION_API_BASE = "https://api.notion.com/v1" _NOTION_VERSION = "2022-06-28" -_DEFAULT_CREDENTIALS_PATH = str( - DEFAULT_CONFIG_DIR / "connectors" / "notion.json" -) +_DEFAULT_CREDENTIALS_PATH = str(DEFAULT_CONFIG_DIR / "connectors" / "notion.json") # --------------------------------------------------------------------------- # Module-level API functions (easy to patch in tests) diff --git a/src/openjarvis/connectors/outlook.py b/src/openjarvis/connectors/outlook.py index 9302c11d..e23e4fad 100644 --- a/src/openjarvis/connectors/outlook.py +++ b/src/openjarvis/connectors/outlook.py @@ -17,9 +17,7 @@ from openjarvis.connectors.gmail_imap import GmailIMAPConnector from openjarvis.core.config import DEFAULT_CONFIG_DIR from openjarvis.core.registry import ConnectorRegistry -_DEFAULT_CREDENTIALS_PATH = str( - DEFAULT_CONFIG_DIR / "connectors" / "outlook.json" -) +_DEFAULT_CREDENTIALS_PATH = str(DEFAULT_CONFIG_DIR / "connectors" / "outlook.json") @ConnectorRegistry.register("outlook") diff --git a/src/openjarvis/connectors/slack_connector.py b/src/openjarvis/connectors/slack_connector.py index 7a318665..2049318b 100644 --- a/src/openjarvis/connectors/slack_connector.py +++ b/src/openjarvis/connectors/slack_connector.py @@ -26,9 +26,7 @@ from openjarvis.tools._stubs import ToolSpec _SLACK_API_BASE = "https://slack.com/api" _SLACK_AUTH_ENDPOINT = "https://slack.com/oauth/v2/authorize" _SLACK_SCOPES = "channels:history,channels:read,users:read" -_DEFAULT_CREDENTIALS_PATH = str( - DEFAULT_CONFIG_DIR / "connectors" / "slack.json" -) +_DEFAULT_CREDENTIALS_PATH = str(DEFAULT_CONFIG_DIR / "connectors" / "slack.json") # --------------------------------------------------------------------------- # Module-level API functions (easy to patch in tests) @@ -263,9 +261,7 @@ class SlackConnector(BaseConnector): # Step 2: paginate through channels while True: - channels_resp = _slack_api_conversations_list( - token, cursor=channels_cursor - ) + channels_resp = _slack_api_conversations_list(token, cursor=channels_cursor) channels: List[Dict[str, Any]] = channels_resp.get("channels", []) for channel in channels: @@ -280,9 +276,7 @@ class SlackConnector(BaseConnector): history_resp = _slack_api_conversations_history( token, chan_id, cursor=history_cursor ) - messages: List[Dict[str, Any]] = history_resp.get( - "messages", [] - ) + messages: List[Dict[str, Any]] = history_resp.get("messages", []) for msg in messages: # Skip bot messages and non-content subtypes @@ -327,9 +321,7 @@ class SlackConnector(BaseConnector): yield doc next_history_cursor: str = ( - history_resp.get("response_metadata", {}).get( - "next_cursor", "" - ) + history_resp.get("response_metadata", {}).get("next_cursor", "") or "" ) if not history_resp.get("has_more") or not next_history_cursor: @@ -337,8 +329,7 @@ class SlackConnector(BaseConnector): history_cursor = next_history_cursor next_channels_cursor: str = ( - channels_resp.get("response_metadata", {}).get("next_cursor", "") - or "" + channels_resp.get("response_metadata", {}).get("next_cursor", "") or "" ) if not next_channels_cursor: self._last_cursor = None diff --git a/src/openjarvis/core/credentials.py b/src/openjarvis/core/credentials.py index cee6a019..04f95d70 100644 --- a/src/openjarvis/core/credentials.py +++ b/src/openjarvis/core/credentials.py @@ -3,6 +3,7 @@ Stores credentials in ~/.openjarvis/credentials.toml with 0o600 permissions. Thread-safe writes via lock. Sets os.environ on save for immediate effect. """ + from __future__ import annotations import os @@ -33,8 +34,10 @@ TOOL_CREDENTIALS: dict[str, list[str]] = { "viber": ["VIBER_AUTH_TOKEN"], "messenger": ["MESSENGER_PAGE_ACCESS_TOKEN", "MESSENGER_VERIFY_TOKEN"], "reddit": [ - "REDDIT_CLIENT_ID", "REDDIT_CLIENT_SECRET", - "REDDIT_USERNAME", "REDDIT_PASSWORD", + "REDDIT_CLIENT_ID", + "REDDIT_CLIENT_SECRET", + "REDDIT_USERNAME", + "REDDIT_PASSWORD", ], "mastodon": ["MASTODON_ACCESS_TOKEN", "MASTODON_API_BASE_URL"], "twitch": ["TWITCH_TOKEN", "TWITCH_CHANNEL"], diff --git a/src/openjarvis/daemon/session_expiry.py b/src/openjarvis/daemon/session_expiry.py index 87614ae9..bd518438 100644 --- a/src/openjarvis/daemon/session_expiry.py +++ b/src/openjarvis/daemon/session_expiry.py @@ -26,8 +26,7 @@ class SessionExpiryHook: self._executor.run_ephemeral( agent_type="simple", system_prompt=( - "You are a memory management agent. " - "Save important information." + "You are a memory management agent. Save important information." ), input_text=input_text, tools=["memory_manage", "skill_manage"], diff --git a/src/openjarvis/engine/apple_fm_shim.py b/src/openjarvis/engine/apple_fm_shim.py index 659e4d3f..215735e4 100644 --- a/src/openjarvis/engine/apple_fm_shim.py +++ b/src/openjarvis/engine/apple_fm_shim.py @@ -80,12 +80,14 @@ def health() -> JSONResponse: @app.get("/v1/models") def list_models() -> JSONResponse: - return JSONResponse({ - "object": "list", - "data": [ - {"id": MODEL_ID, "object": "model", "owned_by": "apple"}, - ], - }) + return JSONResponse( + { + "object": "list", + "data": [ + {"id": MODEL_ID, "object": "model", "owned_by": "apple"}, + ], + } + ) @app.post("/v1/chat/completions") @@ -96,21 +98,25 @@ async def chat_completions( session = apple_fm.LanguageModelSession() if req.stream: + async def generate(): cid = f"chatcmpl-{uuid.uuid4().hex[:12]}" async for token in session.stream_response( - prompt, max_tokens=req.max_tokens, + prompt, + max_tokens=req.max_tokens, ): chunk = { "id": cid, "object": "chat.completion.chunk", "created": int(time.time()), "model": MODEL_ID, - "choices": [{ - "index": 0, - "delta": {"content": token}, - "finish_reason": None, - }], + "choices": [ + { + "index": 0, + "delta": {"content": token}, + "finish_reason": None, + } + ], } yield f"data: {json.dumps(chunk)}\n\n" final = { @@ -118,34 +124,41 @@ async def chat_completions( "object": "chat.completion.chunk", "created": int(time.time()), "model": MODEL_ID, - "choices": [{ - "index": 0, - "delta": {}, - "finish_reason": "stop", - }], + "choices": [ + { + "index": 0, + "delta": {}, + "finish_reason": "stop", + } + ], } yield f"data: {json.dumps(final)}\n\n" yield "data: [DONE]\n\n" return StreamingResponse( - generate(), media_type="text/event-stream", + generate(), + media_type="text/event-stream", ) text = await session.respond(prompt, max_tokens=req.max_tokens) cid = f"chatcmpl-{uuid.uuid4().hex[:12]}" - return JSONResponse({ - "id": cid, - "object": "chat.completion", - "created": int(time.time()), - "model": MODEL_ID, - "choices": [{ - "index": 0, - "message": {"role": "assistant", "content": text}, - "finish_reason": "stop", - }], - "usage": { - "prompt_tokens": 0, - "completion_tokens": 0, - "total_tokens": 0, - }, - }) + return JSONResponse( + { + "id": cid, + "object": "chat.completion", + "created": int(time.time()), + "model": MODEL_ID, + "choices": [ + { + "index": 0, + "message": {"role": "assistant", "content": text}, + "finish_reason": "stop", + } + ], + "usage": { + "prompt_tokens": 0, + "completion_tokens": 0, + "total_tokens": 0, + }, + } + ) diff --git a/src/openjarvis/engine/nexa_shim.py b/src/openjarvis/engine/nexa_shim.py index 9c8bc160..9572ace5 100644 --- a/src/openjarvis/engine/nexa_shim.py +++ b/src/openjarvis/engine/nexa_shim.py @@ -81,12 +81,14 @@ def health() -> JSONResponse: @app.get("/v1/models") def list_models() -> JSONResponse: - return JSONResponse({ - "object": "list", - "data": [ - {"id": MODEL_ID, "object": "model", "owned_by": "nexa"}, - ], - }) + return JSONResponse( + { + "object": "list", + "data": [ + {"id": MODEL_ID, "object": "model", "owned_by": "nexa"}, + ], + } + ) @app.post("/v1/chat/completions") @@ -97,6 +99,7 @@ async def chat_completions( llm = _get_llm() if req.stream: + async def generate(): cid = f"chatcmpl-{uuid.uuid4().hex[:12]}" for token in llm.generate(prompt, max_tokens=req.max_tokens): @@ -105,11 +108,13 @@ async def chat_completions( "object": "chat.completion.chunk", "created": int(time.time()), "model": MODEL_ID, - "choices": [{ - "index": 0, - "delta": {"content": token}, - "finish_reason": None, - }], + "choices": [ + { + "index": 0, + "delta": {"content": token}, + "finish_reason": None, + } + ], } yield f"data: {json.dumps(chunk)}\n\n" final = { @@ -117,36 +122,43 @@ async def chat_completions( "object": "chat.completion.chunk", "created": int(time.time()), "model": MODEL_ID, - "choices": [{ - "index": 0, - "delta": {}, - "finish_reason": "stop", - }], + "choices": [ + { + "index": 0, + "delta": {}, + "finish_reason": "stop", + } + ], } yield f"data: {json.dumps(final)}\n\n" yield "data: [DONE]\n\n" return StreamingResponse( - generate(), media_type="text/event-stream", + generate(), + media_type="text/event-stream", ) text = llm.generate(prompt, max_tokens=req.max_tokens) if isinstance(text, list): text = "".join(text) cid = f"chatcmpl-{uuid.uuid4().hex[:12]}" - return JSONResponse({ - "id": cid, - "object": "chat.completion", - "created": int(time.time()), - "model": MODEL_ID, - "choices": [{ - "index": 0, - "message": {"role": "assistant", "content": text}, - "finish_reason": "stop", - }], - "usage": { - "prompt_tokens": 0, - "completion_tokens": 0, - "total_tokens": 0, - }, - }) + return JSONResponse( + { + "id": cid, + "object": "chat.completion", + "created": int(time.time()), + "model": MODEL_ID, + "choices": [ + { + "index": 0, + "message": {"role": "assistant", "content": text}, + "finish_reason": "stop", + } + ], + "usage": { + "prompt_tokens": 0, + "completion_tokens": 0, + "total_tokens": 0, + }, + } + ) diff --git a/src/openjarvis/engine/ollama.py b/src/openjarvis/engine/ollama.py index c6fcf489..c167bfbe 100644 --- a/src/openjarvis/engine/ollama.py +++ b/src/openjarvis/engine/ollama.py @@ -139,26 +139,36 @@ class OllamaEngine(InferenceEngine): } # Extract timing from Ollama response (nanoseconds → seconds) result["ttft"] = data.get("prompt_eval_duration", 0) / 1e9 - result["engine_timing"] = {k: data[k] for k in - ("total_duration", "load_duration", "prompt_eval_duration", "eval_duration") - if k in data} + result["engine_timing"] = { + k: data[k] + for k in ( + "total_duration", + "load_duration", + "prompt_eval_duration", + "eval_duration", + ) + if k in data + } # Extract tool calls if present raw_tool_calls = data.get("message", {}).get("tool_calls", []) if raw_tool_calls: tool_calls = [] for i, tc in enumerate(raw_tool_calls): raw_args = tc.get("function", {}).get( - "arguments", "{}", + "arguments", + "{}", + ) + tool_calls.append( + { + "id": tc.get("id", f"call_{i}"), + "name": tc.get("function", {}).get("name", ""), + "arguments": ( + json.dumps(raw_args) + if isinstance(raw_args, dict) + else raw_args + ), + } ) - tool_calls.append({ - "id": tc.get("id", f"call_{i}"), - "name": tc.get("function", {}).get("name", ""), - "arguments": ( - json.dumps(raw_args) - if isinstance(raw_args, dict) - else raw_args - ), - }) result["tool_calls"] = tool_calls return result @@ -199,8 +209,7 @@ class OllamaEngine(InferenceEngine): est_prompt = estimate_prompt_tokens(messages) full_prompt = max(reported_prompt, est_prompt) evaluated = ( - reported_prompt if reported_prompt > 0 - else full_prompt + reported_prompt if reported_prompt > 0 else full_prompt ) comp = chunk.get("eval_count", 0) self._last_stream_usage = { @@ -220,11 +229,14 @@ class OllamaEngine(InferenceEngine): resp = self._client.get("/api/tags") resp.raise_for_status() except ( - httpx.ConnectError, httpx.TimeoutException, httpx.HTTPStatusError, + httpx.ConnectError, + httpx.TimeoutException, + httpx.HTTPStatusError, ) as exc: logger.warning( "Failed to list models from Ollama at %s: %s", - self._host, exc, + self._host, + exc, ) return [] data = resp.json() diff --git a/src/openjarvis/evals/backends/jarvis_agent.py b/src/openjarvis/evals/backends/jarvis_agent.py index fc917e7e..2cc6b03b 100644 --- a/src/openjarvis/evals/backends/jarvis_agent.py +++ b/src/openjarvis/evals/backends/jarvis_agent.py @@ -57,8 +57,11 @@ class JarvisAgentBackend(InferenceBackend): max_tokens: int = 2048, ) -> str: result = self.generate_full( - prompt, model=model, system=system, - temperature=temperature, max_tokens=max_tokens, + prompt, + model=model, + system=system, + temperature=temperature, + max_tokens=max_tokens, ) return result["content"] diff --git a/src/openjarvis/evals/backends/jarvis_direct.py b/src/openjarvis/evals/backends/jarvis_direct.py index 107e4153..ba12d55e 100644 --- a/src/openjarvis/evals/backends/jarvis_direct.py +++ b/src/openjarvis/evals/backends/jarvis_direct.py @@ -43,8 +43,11 @@ class JarvisDirectBackend(InferenceBackend): max_tokens: int = 2048, ) -> str: result = self.generate_full( - prompt, model=model, system=system, - temperature=temperature, max_tokens=max_tokens, + prompt, + model=model, + system=system, + temperature=temperature, + max_tokens=max_tokens, ) return result["content"] @@ -66,8 +69,10 @@ class JarvisDirectBackend(InferenceBackend): t0 = time.monotonic() result = self._system.engine.generate( - messages, model=model, - temperature=temperature, max_tokens=max_tokens, + messages, + model=model, + temperature=temperature, + max_tokens=max_tokens, ) elapsed = time.monotonic() - t0 diff --git a/src/openjarvis/evals/backends/terminalbench_native.py b/src/openjarvis/evals/backends/terminalbench_native.py index b35d006a..41adc0c4 100644 --- a/src/openjarvis/evals/backends/terminalbench_native.py +++ b/src/openjarvis/evals/backends/terminalbench_native.py @@ -79,6 +79,7 @@ class TerminalBenchNativeBackend(InferenceBackend): # Use built-in agent (naive uses LiteLLM) from terminal_bench.agents.agent_name import AgentName + harness_kwargs["agent_name"] = AgentName(self._agent_name) harness_kwargs["agent_kwargs"] = { "llm": LiteLLM( @@ -95,13 +96,25 @@ class TerminalBenchNativeBackend(InferenceBackend): self._results = harness.run() return self._results - def generate(self, prompt: str, *, model: str, system: str = "", - temperature: float = 0.0, max_tokens: int = 2048) -> str: + def generate( + self, + prompt: str, + *, + model: str, + system: str = "", + temperature: float = 0.0, + max_tokens: int = 2048, + ) -> str: return "" def generate_full( - self, prompt: str, *, model: str, system: str = "", - temperature: float = 0.0, max_tokens: int = 2048, + self, + prompt: str, + *, + model: str, + system: str = "", + temperature: float = 0.0, + max_tokens: int = 2048, ) -> Dict[str, Any]: return {"content": "", "usage": {}, "model": model, "latency_seconds": 0.0} diff --git a/src/openjarvis/evals/cli.py b/src/openjarvis/evals/cli.py index 25ed1b5f..4781863a 100644 --- a/src/openjarvis/evals/cli.py +++ b/src/openjarvis/evals/cli.py @@ -45,7 +45,8 @@ BENCHMARKS = { "swebench": {"category": "agentic", "description": "SWE-bench code patches"}, "swefficiency": {"category": "agentic", "description": "SWEfficiency optimization"}, "terminalbench": { - "category": "agentic", "description": "TerminalBench terminal tasks", + "category": "agentic", + "description": "TerminalBench terminal tasks", }, "terminalbench-native": { "category": "agentic", @@ -140,13 +141,19 @@ def _setup_logging(verbose: bool) -> None: ) -def _build_backend(backend_name: str, engine_key: Optional[str], - agent_name: str, tools: list[str], - telemetry: bool = False, gpu_metrics: bool = False, - model: Optional[str] = None): +def _build_backend( + backend_name: str, + engine_key: Optional[str], + agent_name: str, + tools: list[str], + telemetry: bool = False, + gpu_metrics: bool = False, + model: Optional[str] = None, +): """Construct the appropriate backend.""" if backend_name == "jarvis-agent": from openjarvis.evals.backends.jarvis_agent import JarvisAgentBackend + return JarvisAgentBackend( engine_key=engine_key, agent_name=agent_name, @@ -157,6 +164,7 @@ def _build_backend(backend_name: str, engine_key: Optional[str], ) else: from openjarvis.evals.backends.jarvis_direct import JarvisDirectBackend + return JarvisDirectBackend( engine_key=engine_key, telemetry=telemetry, @@ -168,104 +176,137 @@ def _build_dataset(benchmark: str, subset: str | None = None): """Construct the dataset provider for a benchmark.""" if benchmark == "supergpqa": from openjarvis.evals.datasets.supergpqa import SuperGPQADataset + return SuperGPQADataset() elif benchmark == "gpqa": from openjarvis.evals.datasets.gpqa import GPQADataset + return GPQADataset() elif benchmark == "mmlu-pro": from openjarvis.evals.datasets.mmlu_pro import MMLUProDataset + return MMLUProDataset() elif benchmark == "math500": from openjarvis.evals.datasets.math500 import MATH500Dataset + return MATH500Dataset() elif benchmark == "natural-reasoning": from openjarvis.evals.datasets.natural_reasoning import NaturalReasoningDataset + return NaturalReasoningDataset() elif benchmark == "hle": from openjarvis.evals.datasets.hle import HLEDataset + return HLEDataset() elif benchmark == "simpleqa": from openjarvis.evals.datasets.simpleqa import SimpleQADataset + return SimpleQADataset() elif benchmark == "wildchat": from openjarvis.evals.datasets.wildchat import WildChatDataset + return WildChatDataset() elif benchmark == "ipw": from openjarvis.evals.datasets.ipw_mixed import IPWDataset + return IPWDataset() elif benchmark == "gaia": from openjarvis.evals.datasets.gaia import GAIADataset + return GAIADataset() elif benchmark == "frames": from openjarvis.evals.datasets.frames import FRAMESDataset + return FRAMESDataset() elif benchmark == "swebench": from openjarvis.evals.datasets.swebench import SWEBenchDataset + return SWEBenchDataset() elif benchmark == "swefficiency": from openjarvis.evals.datasets.swefficiency import SWEfficiencyDataset + return SWEfficiencyDataset() elif benchmark == "terminalbench": from openjarvis.evals.datasets.terminalbench import TerminalBenchDataset + return TerminalBenchDataset() elif benchmark == "terminalbench-native": from openjarvis.evals.datasets.terminalbench_native import ( TerminalBenchNativeDataset, ) + return TerminalBenchNativeDataset() elif benchmark == "email_triage": from openjarvis.evals.datasets.email_triage import EmailTriageDataset + return EmailTriageDataset() elif benchmark == "morning_brief": from openjarvis.evals.datasets.morning_brief import MorningBriefDataset + return MorningBriefDataset() elif benchmark == "research_mining": from openjarvis.evals.datasets.research_mining import ResearchMiningDataset + return ResearchMiningDataset() elif benchmark == "knowledge_base": from openjarvis.evals.datasets.knowledge_base import KnowledgeBaseDataset + return KnowledgeBaseDataset() elif benchmark == "coding_task": from openjarvis.evals.datasets.coding_task import CodingTaskDataset + return CodingTaskDataset() elif benchmark == "loghub": from openjarvis.evals.datasets.loghub import LogHubDataset + return LogHubDataset() elif benchmark == "ama-bench": from openjarvis.evals.datasets.ama_bench import AMABenchDataset + return AMABenchDataset() elif benchmark == "lifelong-agent": from openjarvis.evals.datasets.lifelong_agent import LifelongAgentDataset + return LifelongAgentDataset(subset=subset or "db_bench") elif benchmark == "deepplanning": from openjarvis.evals.datasets.deepplanning import DeepPlanningDataset + return DeepPlanningDataset() elif benchmark == "paperarena": from openjarvis.evals.datasets.paperarena import PaperArenaDataset + return PaperArenaDataset() elif benchmark == "webchorearena": from openjarvis.evals.datasets.webchorearena import WebChoreArenaDataset + return WebChoreArenaDataset() elif benchmark == "workarena": from openjarvis.evals.datasets.workarena import WorkArenaDataset + return WorkArenaDataset() elif benchmark == "coding_assistant": from openjarvis.evals.datasets.coding_assistant import CodingAssistantDataset + return CodingAssistantDataset() elif benchmark == "security_scanner": from openjarvis.evals.datasets.security_scanner import SecurityScannerDataset + return SecurityScannerDataset() elif benchmark == "daily_digest": from openjarvis.evals.datasets.daily_digest import DailyDigestDataset + return DailyDigestDataset() elif benchmark == "doc_qa": from openjarvis.evals.datasets.doc_qa import DocQADataset + return DocQADataset() elif benchmark == "browser_assistant": from openjarvis.evals.datasets.browser_assistant import BrowserAssistantDataset + return BrowserAssistantDataset() elif benchmark == "pinchbench": from openjarvis.evals.datasets.pinchbench import PinchBenchDataset + return PinchBenchDataset(path=subset) else: raise click.ClickException(f"Unknown benchmark: {benchmark}") @@ -275,101 +316,133 @@ def _build_scorer(benchmark: str, judge_backend, judge_model: str): """Construct the scorer for a benchmark.""" if benchmark == "supergpqa": from openjarvis.evals.scorers.supergpqa_mcq import SuperGPQAScorer + return SuperGPQAScorer(judge_backend, judge_model) elif benchmark == "gpqa": from openjarvis.evals.scorers.gpqa_mcq import GPQAScorer + return GPQAScorer(judge_backend, judge_model) elif benchmark == "mmlu-pro": from openjarvis.evals.scorers.mmlu_pro_mcq import MMLUProScorer + return MMLUProScorer(judge_backend, judge_model) elif benchmark == "math500" or benchmark == "natural-reasoning": from openjarvis.evals.scorers.reasoning_judge import ReasoningJudgeScorer + return ReasoningJudgeScorer(judge_backend, judge_model) elif benchmark == "hle": from openjarvis.evals.scorers.hle_judge import HLEScorer + return HLEScorer(judge_backend, judge_model) elif benchmark == "simpleqa": from openjarvis.evals.scorers.simpleqa_judge import SimpleQAScorer + return SimpleQAScorer(judge_backend, judge_model) elif benchmark == "wildchat": from openjarvis.evals.scorers.wildchat_judge import WildChatScorer + return WildChatScorer(judge_backend, judge_model) elif benchmark == "ipw": from openjarvis.evals.scorers.ipw_mixed import IPWMixedScorer + return IPWMixedScorer(judge_backend, judge_model) elif benchmark == "gaia": from openjarvis.evals.scorers.gaia_exact import GAIAScorer + return GAIAScorer(judge_backend, judge_model) elif benchmark == "frames": from openjarvis.evals.scorers.frames_judge import FRAMESScorer + return FRAMESScorer(judge_backend, judge_model) elif benchmark == "swebench": from openjarvis.evals.scorers.swebench_structural import SWEBenchScorer + return SWEBenchScorer(judge_backend, judge_model) elif benchmark == "swefficiency": from openjarvis.evals.scorers.swefficiency_structural import SWEfficiencyScorer + return SWEfficiencyScorer(judge_backend, judge_model) elif benchmark == "terminalbench": from openjarvis.evals.scorers.terminalbench_judge import TerminalBenchScorer + return TerminalBenchScorer(judge_backend, judge_model) elif benchmark == "terminalbench-native": from openjarvis.evals.scorers.terminalbench_native_structural import ( TerminalBenchNativeScorer, ) + return TerminalBenchNativeScorer(judge_backend, judge_model) elif benchmark == "email_triage": from openjarvis.evals.scorers.email_triage import EmailTriageScorer + return EmailTriageScorer(judge_backend, judge_model) elif benchmark == "morning_brief": from openjarvis.evals.scorers.morning_brief import MorningBriefScorer + return MorningBriefScorer(judge_backend, judge_model) elif benchmark == "research_mining": from openjarvis.evals.scorers.research_mining import ResearchMiningScorer + return ResearchMiningScorer(judge_backend, judge_model) elif benchmark == "knowledge_base": from openjarvis.evals.scorers.knowledge_base import KnowledgeBaseScorer + return KnowledgeBaseScorer(judge_backend, judge_model) elif benchmark == "coding_task": from openjarvis.evals.scorers.coding_task import CodingTaskScorer + return CodingTaskScorer(judge_backend, judge_model) elif benchmark == "loghub": from openjarvis.evals.scorers.loghub_scorer import LogHubScorer + return LogHubScorer(judge_backend, judge_model) elif benchmark == "ama-bench": from openjarvis.evals.scorers.ama_bench_judge import AMABenchScorer + return AMABenchScorer(judge_backend, judge_model) elif benchmark == "lifelong-agent": from openjarvis.evals.scorers.lifelong_agent_scorer import LifelongAgentScorer + return LifelongAgentScorer(judge_backend, judge_model) elif benchmark == "deepplanning": from openjarvis.evals.scorers.deepplanning_scorer import DeepPlanningScorer + return DeepPlanningScorer(judge_backend, judge_model) elif benchmark == "paperarena": from openjarvis.evals.scorers.paperarena_judge import PaperArenaScorer + return PaperArenaScorer(judge_backend, judge_model) elif benchmark == "webchorearena": from openjarvis.evals.scorers.webchorearena_scorer import WebChoreArenaScorer + return WebChoreArenaScorer(judge_backend, judge_model) elif benchmark == "workarena": from openjarvis.evals.scorers.workarena_scorer import WorkArenaScorer + return WorkArenaScorer(judge_backend, judge_model) elif benchmark == "coding_assistant": from openjarvis.evals.scorers.coding_assistant import CodingAssistantScorer + return CodingAssistantScorer(judge_backend, judge_model) elif benchmark == "security_scanner": from openjarvis.evals.scorers.security_scanner import SecurityScannerScorer + return SecurityScannerScorer(judge_backend, judge_model) elif benchmark == "daily_digest": from openjarvis.evals.scorers.daily_digest import DailyDigestScorer + return DailyDigestScorer(judge_backend, judge_model) elif benchmark == "doc_qa": from openjarvis.evals.scorers.doc_qa import DocQAScorer + return DocQAScorer(judge_backend, judge_model) elif benchmark == "browser_assistant": from openjarvis.evals.scorers.browser_assistant import BrowserAssistantScorer + return BrowserAssistantScorer(judge_backend, judge_model) elif benchmark == "pinchbench": from openjarvis.evals.scorers.pinchbench import PinchBenchScorer + return PinchBenchScorer(judge_backend, judge_model) else: raise click.ClickException(f"Unknown benchmark: {benchmark}") @@ -384,6 +457,7 @@ def _build_judge_backend(judge_model: str, engine_key: str = "cloud"): to use the backend rather than failing at startup. """ from openjarvis.evals.backends.jarvis_direct import JarvisDirectBackend + try: return JarvisDirectBackend(engine_key=engine_key) except RuntimeError as exc: @@ -391,7 +465,8 @@ def _build_judge_backend(judge_model: str, engine_key: str = "cloud"): "Judge backend (%s) unavailable: %s — " "deterministic scorers will still work; " "LLM-judge scorers will fail when scoring.", - engine_key, exc, + engine_key, + exc, ) return None @@ -421,25 +496,30 @@ def _build_trackers(config) -> list: if getattr(config, "wandb_project", ""): try: from openjarvis.evals.trackers.wandb_tracker import WandbTracker - trackers.append(WandbTracker( - project=config.wandb_project, - entity=getattr(config, "wandb_entity", ""), - tags=getattr(config, "wandb_tags", ""), - group=getattr(config, "wandb_group", ""), - )) + + trackers.append( + WandbTracker( + project=config.wandb_project, + entity=getattr(config, "wandb_entity", ""), + tags=getattr(config, "wandb_tags", ""), + group=getattr(config, "wandb_group", ""), + ) + ) except ImportError as exc: raise click.ClickException( - f"wandb not installed: {exc}\n" - "Install with: uv sync --extra eval-wandb" + f"wandb not installed: {exc}\nInstall with: uv sync --extra eval-wandb" ) from exc if getattr(config, "sheets_spreadsheet_id", ""): try: from openjarvis.evals.trackers.sheets_tracker import SheetsTracker - trackers.append(SheetsTracker( - spreadsheet_id=config.sheets_spreadsheet_id, - worksheet=getattr(config, "sheets_worksheet", "Results"), - credentials_path=getattr(config, "sheets_credentials_path", ""), - )) + + trackers.append( + SheetsTracker( + spreadsheet_id=config.sheets_spreadsheet_id, + worksheet=getattr(config, "sheets_worksheet", "Results"), + credentials_path=getattr(config, "sheets_credentials_path", ""), + ) + ) except ImportError as exc: raise click.ClickException( f"gspread not installed: {exc}\n" @@ -486,7 +566,8 @@ def _run_single(config, console: Optional[Console] = None) -> object: task = progress.add_task("Evaluating samples...", total=num_samples) summary = runner.run( progress_callback=lambda done, total: progress.update( - task, completed=done, + task, + completed=done, ), ) else: @@ -533,8 +614,7 @@ def _run_agentic( if config.benchmark == "pinchbench" and hasattr(dataset, "set_judge"): judge_engine = getattr(config, "judge_engine", "cloud") or "cloud" judge_model = ( - getattr(config, "judge_model", None) - or "anthropic/claude-opus-4-5" + getattr(config, "judge_model", None) or "anthropic/claude-opus-4-5" ) judge_backend = _build_judge_backend(judge_model, engine_key=judge_engine) dataset.set_judge(judge_backend, judge_model) @@ -576,7 +656,8 @@ def _run_agentic( monitor = create_energy_monitor() if monitor is not None: telemetry_session = TelemetrySession( - monitor=monitor, interval_ms=100, + monitor=monitor, + interval_ms=100, ) except ImportError: pass @@ -650,6 +731,7 @@ def _run_agentic( # Try HF dataset export (optional) try: from openjarvis.evals.core.export import export_hf_dataset + hf_path = run_dir / "hf_dataset" export_hf_dataset(traces, hf_path) console.print(f" [green]HF Arrow:[/green] {hf_path}") @@ -664,6 +746,7 @@ def _run_agentic( def _nullctx(): """Return a no-op context manager.""" from contextlib import nullcontext + return nullcontext() @@ -681,7 +764,8 @@ def _print_agentic_summary(console: Console, traces, config) -> None: total_wall = sum(t.total_wall_clock_s for t in traces) gpu_energies = [ - t.total_gpu_energy_joules for t in traces + t.total_gpu_energy_joules + for t in traces if t.total_gpu_energy_joules is not None ] total_gpu_energy = sum(gpu_energies) if gpu_energies else None @@ -710,7 +794,7 @@ def _print_agentic_summary(console: Console, traces, config) -> None: table.add_row("Wall clock", f"{total_wall:.1f}s") table.add_row( "Avg query time", - f"{total_wall/len(traces):.1f}s" if traces else "0s", + f"{total_wall / len(traces):.1f}s" if traces else "0s", ) if total_gpu_energy is not None: @@ -722,7 +806,7 @@ def _print_agentic_summary(console: Console, traces, config) -> None: if total_out_tok > 0 and total_wall > 0: table.add_row( "Throughput", - f"{total_out_tok/total_wall:.1f} tok/s", + f"{total_out_tok / total_wall:.1f} tok/s", ) console.print(table) @@ -746,18 +830,14 @@ def _run_from_config( if model_filter: run_configs = [rc for rc in run_configs if model_filter in rc.model] if not run_configs: - raise click.ClickException( - f"No models match filter '{model_filter}'" - ) + raise click.ClickException(f"No models match filter '{model_filter}'") suite_name = suite.meta.name or Path(config_path).stem # Banner + configuration print_banner(console) print_section(console, "Suite Configuration") - console.print( - f" [cyan]Suite:[/cyan] {suite_name}" - ) + console.print(f" [cyan]Suite:[/cyan] {suite_name}") if suite.meta.description: console.print(f" [cyan]Description:[/cyan] {suite.meta.description}") console.print( @@ -802,86 +882,166 @@ def main(): @main.command() -@click.option("-c", "--config", "config_path", default=None, - type=click.Path(), help="TOML config file for suite runs") -@click.option("-b", "--benchmark", default=None, - type=click.Choice(list(BENCHMARKS.keys())), - help="Benchmark to run") -@click.option("--backend", default="jarvis-direct", - type=click.Choice(list(BACKENDS.keys())), - help="Inference backend") -@click.option("-m", "--model", default=None, help="Model identifier") -@click.option("-e", "--engine", "engine_key", default=None, - help="Engine key (ollama, vllm, cloud, ...)") -@click.option("--agent", "agent_name", default="orchestrator", - help="Agent name for jarvis-agent backend") -@click.option("--tools", default="", help="Comma-separated tool names") -@click.option("-n", "--max-samples", type=int, default=None, - help="Maximum samples to evaluate") -@click.option("-w", "--max-workers", type=int, default=4, - help="Parallel workers") -@click.option("--judge-model", default="gpt-5-mini-2025-08-07", - help="LLM judge model") -@click.option("-o", "--output", "output_path", default=None, - help="Output JSONL path") -@click.option("--seed", type=int, default=42, help="Random seed") -@click.option("--split", "dataset_split", default=None, - help="Dataset split override") -@click.option("--temperature", type=float, default=0.0, - help="Generation temperature") -@click.option("--max-tokens", type=int, default=2048, - help="Max output tokens") -@click.option("--telemetry/--no-telemetry", default=False, - help="Enable telemetry collection during eval") -@click.option("--gpu-metrics/--no-gpu-metrics", default=False, - help="Enable GPU metrics collection") @click.option( - "--compact", is_flag=True, default=False, + "-c", + "--config", + "config_path", + default=None, + type=click.Path(), + help="TOML config file for suite runs", +) +@click.option( + "-b", + "--benchmark", + default=None, + type=click.Choice(list(BENCHMARKS.keys())), + help="Benchmark to run", +) +@click.option( + "--backend", + default="jarvis-direct", + type=click.Choice(list(BACKENDS.keys())), + help="Inference backend", +) +@click.option("-m", "--model", default=None, help="Model identifier") +@click.option( + "-e", + "--engine", + "engine_key", + default=None, + help="Engine key (ollama, vllm, cloud, ...)", +) +@click.option( + "--agent", + "agent_name", + default="orchestrator", + help="Agent name for jarvis-agent backend", +) +@click.option("--tools", default="", help="Comma-separated tool names") +@click.option( + "-n", "--max-samples", type=int, default=None, help="Maximum samples to evaluate" +) +@click.option("-w", "--max-workers", type=int, default=4, help="Parallel workers") +@click.option("--judge-model", default="gpt-5-mini-2025-08-07", help="LLM judge model") +@click.option("-o", "--output", "output_path", default=None, help="Output JSONL path") +@click.option("--seed", type=int, default=42, help="Random seed") +@click.option("--split", "dataset_split", default=None, help="Dataset split override") +@click.option("--temperature", type=float, default=0.0, help="Generation temperature") +@click.option("--max-tokens", type=int, default=2048, help="Max output tokens") +@click.option( + "--telemetry/--no-telemetry", + default=False, + help="Enable telemetry collection during eval", +) +@click.option( + "--gpu-metrics/--no-gpu-metrics", + default=False, + help="Enable GPU metrics collection", +) +@click.option( + "--compact", + is_flag=True, + default=False, help="Dense single-table output", ) @click.option( - "--trace-detail", is_flag=True, default=False, + "--trace-detail", + is_flag=True, + default=False, help="Full per-step trace listing", ) -@click.option("--wandb-project", default="", - help="W&B project name (enables tracking)") -@click.option("--wandb-entity", default="", - help="W&B entity (team or user)") -@click.option("--wandb-tags", default="", - help="Comma-separated W&B tags") -@click.option("--wandb-group", default="", - help="W&B run group") -@click.option("--sheets-id", "sheets_spreadsheet_id", default="", - help="Google Sheets spreadsheet ID") -@click.option("--sheets-worksheet", default="Results", - help="Google Sheets worksheet name") -@click.option("--sheets-creds", "sheets_credentials_path", - default="", - help="Service account JSON path") -@click.option("--model-filter", default=None, - help="Filter models by name substring (for multi-model configs)") -@click.option("--judge-engine", default="cloud", - help="Engine key for LLM judge (default: cloud). " - "Use 'vllm' to judge locally.") -@click.option("--agentic", is_flag=True, default=False, - help="Use AgenticRunner for multi-turn agent execution") -@click.option("--episode-mode", is_flag=True, default=False, - help="Sequential episode processing with lifelong learning " - "(required for lifelong-agent and similar benchmarks)") -@click.option("--concurrency", type=int, default=1, - help="Parallel query execution (AgenticRunner only)") -@click.option("--query-timeout", type=float, default=None, - help="Per-query wall-clock timeout in seconds (AgenticRunner only)") +@click.option("--wandb-project", default="", help="W&B project name (enables tracking)") +@click.option("--wandb-entity", default="", help="W&B entity (team or user)") +@click.option("--wandb-tags", default="", help="Comma-separated W&B tags") +@click.option("--wandb-group", default="", help="W&B run group") +@click.option( + "--sheets-id", + "sheets_spreadsheet_id", + default="", + help="Google Sheets spreadsheet ID", +) +@click.option( + "--sheets-worksheet", default="Results", help="Google Sheets worksheet name" +) +@click.option( + "--sheets-creds", + "sheets_credentials_path", + default="", + help="Service account JSON path", +) +@click.option( + "--model-filter", + default=None, + help="Filter models by name substring (for multi-model configs)", +) +@click.option( + "--judge-engine", + default="cloud", + help="Engine key for LLM judge (default: cloud). Use 'vllm' to judge locally.", +) +@click.option( + "--agentic", + is_flag=True, + default=False, + help="Use AgenticRunner for multi-turn agent execution", +) +@click.option( + "--episode-mode", + is_flag=True, + default=False, + help="Sequential episode processing with lifelong learning " + "(required for lifelong-agent and similar benchmarks)", +) +@click.option( + "--concurrency", + type=int, + default=1, + help="Parallel query execution (AgenticRunner only)", +) +@click.option( + "--query-timeout", + type=float, + default=None, + help="Per-query wall-clock timeout in seconds (AgenticRunner only)", +) @click.option("-v", "--verbose", is_flag=True, help="Verbose logging") @click.pass_context -def run(ctx, config_path, benchmark, backend, model, engine_key, agent_name, - tools, max_samples, max_workers, judge_model, output_path, seed, - dataset_split, temperature, max_tokens, telemetry, gpu_metrics, - compact, trace_detail, - wandb_project, wandb_entity, wandb_tags, wandb_group, - sheets_spreadsheet_id, sheets_worksheet, sheets_credentials_path, - model_filter, judge_engine, agentic, episode_mode, - concurrency, query_timeout, verbose): +def run( + ctx, + config_path, + benchmark, + backend, + model, + engine_key, + agent_name, + tools, + max_samples, + max_workers, + judge_model, + output_path, + seed, + dataset_split, + temperature, + max_tokens, + telemetry, + gpu_metrics, + compact, + trace_detail, + wandb_project, + wandb_entity, + wandb_tags, + wandb_group, + sheets_spreadsheet_id, + sheets_worksheet, + sheets_credentials_path, + model_filter, + judge_engine, + agentic, + episode_mode, + concurrency, + query_timeout, + verbose, +): """Run a single benchmark evaluation, or a full suite from a TOML config.""" _setup_logging(verbose) @@ -900,8 +1060,7 @@ def run(ctx, config_path, benchmark, backend, model, engine_key, agent_name, ) if model is None: raise click.UsageError( - "Missing option '-m' / '--model' " - "(required when --config is not provided)" + "Missing option '-m' / '--model' (required when --config is not provided)" ) from openjarvis.evals.core.types import RunConfig @@ -960,22 +1119,18 @@ def run(ctx, config_path, benchmark, backend, model, engine_key, agent_name, ) if episode_mode: console.print( - " [cyan]Mode:[/cyan] episode " - "(sequential + lifelong learning)" + " [cyan]Mode:[/cyan] episode (sequential + lifelong learning)" ) if agentic: # --- Agentic runner path --- print_section(console, "Agentic Evaluation") - console.print( - f" [cyan]Concurrency:[/cyan] {concurrency}" - ) + console.print(f" [cyan]Concurrency:[/cyan] {concurrency}") if query_timeout: - console.print( - f" [cyan]Timeout:[/cyan] {query_timeout}s per query" - ) + console.print(f" [cyan]Timeout:[/cyan] {query_timeout}s per query") _run_agentic( - config, console=console, + config, + console=console, concurrency=concurrency, query_timeout=query_timeout, ) @@ -1000,19 +1155,18 @@ def run(ctx, config_path, benchmark, backend, model, engine_key, agent_name, @main.command("run-all") @click.option("-m", "--model", required=True, help="Model identifier") -@click.option("-e", "--engine", "engine_key", default=None, - help="Engine key") -@click.option("-n", "--max-samples", type=int, default=None, - help="Max samples per benchmark") -@click.option("-w", "--max-workers", type=int, default=4, - help="Parallel workers") +@click.option("-e", "--engine", "engine_key", default=None, help="Engine key") +@click.option( + "-n", "--max-samples", type=int, default=None, help="Max samples per benchmark" +) +@click.option("-w", "--max-workers", type=int, default=4, help="Parallel workers") @click.option("--judge-model", default="gpt-5-mini-2025-08-07", help="LLM judge model") -@click.option("--output-dir", default="results/", - help="Output directory for results") +@click.option("--output-dir", default="results/", help="Output directory for results") @click.option("--seed", type=int, default=42, help="Random seed") @click.option("-v", "--verbose", is_flag=True, help="Verbose logging") -def run_all(model, engine_key, max_samples, max_workers, judge_model, - output_dir, seed, verbose): +def run_all( + model, engine_key, max_samples, max_workers, judge_model, output_dir, seed, verbose +): """Run all benchmarks.""" _setup_logging(verbose) @@ -1069,11 +1223,13 @@ def run_all(model, engine_key, max_samples, max_workers, judge_model, console=console, ) as progress: task = progress.add_task( - f"Evaluating {bench_name}...", total=max_samples, + f"Evaluating {bench_name}...", + total=max_samples, ) summary = runner.run( progress_callback=lambda done, total: progress.update( - task, completed=done, + task, + completed=done, ), ) else: diff --git a/src/openjarvis/evals/core/agentic_runner.py b/src/openjarvis/evals/core/agentic_runner.py index ec98da97..3e9f8ad9 100644 --- a/src/openjarvis/evals/core/agentic_runner.py +++ b/src/openjarvis/evals/core/agentic_runner.py @@ -36,10 +36,7 @@ def _compute_energy_delta( gpu_field: str = "gpu_energy_j", ) -> Optional[float]: """Compute energy delta from first to last reading for a field.""" - values = [ - getattr(s, gpu_field, None) - for s in readings - ] + values = [getattr(s, gpu_field, None) for s in readings] values = [v for v in values if v is not None and math.isfinite(v)] if len(values) >= 2: delta = values[-1] - values[0] @@ -52,10 +49,7 @@ def _compute_power_avg( power_field: str = "gpu_power_w", ) -> Optional[float]: """Compute average power across readings for a field.""" - values = [ - getattr(s, power_field, None) - for s in readings - ] + values = [getattr(s, power_field, None) for s in readings] values = [v for v in values if v is not None and math.isfinite(v)] return statistics.mean(values) if values else None @@ -64,9 +58,7 @@ def _compute_power_avg( # Patch extraction helpers # --------------------------------------------------------------------------- -_FENCED_DIFF_RE = re.compile( - r"```(?:diff|patch)\s*\n(.*?)```", re.DOTALL -) +_FENCED_DIFF_RE = re.compile(r"```(?:diff|patch)\s*\n(.*?)```", re.DOTALL) _UNIFIED_DIFF_MARKERS = ("diff --git", "--- a/", "+++ b/", "@@ ") @@ -160,16 +152,16 @@ class AgenticRunner: index, record, model, self._agent, self._event_recorder ) if self._query_timeout: - trace = await asyncio.wait_for( - fut, timeout=self._query_timeout - ) + trace = await asyncio.wait_for(fut, timeout=self._query_timeout) else: trace = await fut except asyncio.TimeoutError: elapsed = time.time() - start_time LOGGER.warning( "Query %s timed out after %.0fs (limit=%ss)", - query_id, elapsed, self._query_timeout, + query_id, + elapsed, + self._query_timeout, ) workload_type = getattr(record, "category", "agentic") trace = QueryTrace( @@ -185,12 +177,13 @@ class AgenticRunner: self._traces.append(trace) status = ( - "TIMEOUT" if trace.timed_out - else ("OK" if trace.completed else "FAIL") + "TIMEOUT" if trace.timed_out else ("OK" if trace.completed else "FAIL") ) LOGGER.info( "Task %s: %s in %.1fs", - query_id, status, trace.total_wall_clock_s, + query_id, + status, + trace.total_wall_clock_s, ) if self._run_dir: @@ -199,7 +192,8 @@ class AgenticRunner: if len(self._traces) % self._FLUSH_INTERVAL == 0: LOGGER.debug( "Processed %d/%d queries", - len(self._traces), len(work_items), + len(self._traces), + len(work_items), ) return self._traces @@ -213,7 +207,8 @@ class AgenticRunner: total = len(work_items) LOGGER.info( "Running %d queries with concurrency=%d", - total, self._concurrency, + total, + self._concurrency, ) result_slots: list[Optional[QueryTrace]] = [None] * total @@ -235,19 +230,23 @@ class AgenticRunner: fut = loop.run_in_executor( None, self._run_single_query_sync, - index, record, model, agent, recorder, + index, + record, + model, + agent, + recorder, ) if self._query_timeout: - trace = await asyncio.wait_for( - fut, timeout=self._query_timeout - ) + trace = await asyncio.wait_for(fut, timeout=self._query_timeout) else: trace = await fut except asyncio.TimeoutError: elapsed = time.time() - start_time LOGGER.warning( "Query %s timed out after %.0fs (limit=%ss)", - query_id, elapsed, self._query_timeout, + query_id, + elapsed, + self._query_timeout, ) workload_type = getattr(record, "category", "agentic") trace = QueryTrace( @@ -262,12 +261,15 @@ class AgenticRunner: ) status = ( - "TIMEOUT" if trace.timed_out + "TIMEOUT" + if trace.timed_out else ("OK" if trace.completed else "FAIL") ) LOGGER.info( "Task %s: %s in %.1fs", - query_id, status, trace.total_wall_clock_s, + query_id, + status, + trace.total_wall_clock_s, ) if self._run_dir: @@ -328,9 +330,11 @@ class AgenticRunner: agent_bus = getattr(agent, "bus", None) if agent_bus is not None: for etype in (EventType.TOOL_CALL_START, EventType.TOOL_CALL_END): + def _relay(event, _etype=etype): data = event.data if hasattr(event, "data") else {} event_recorder.record(_etype, **data) + agent_bus.subscribe(etype, _relay) _bus_unsubs.append((etype, _relay)) @@ -364,6 +368,7 @@ class AgenticRunner: # Envs with reset/step/evaluate use run_agent_loop # for multi-turn interaction with conversation history. if task_env is not None and hasattr(task_env, "run_agent_loop"): + def _generate(prompt: str) -> str: if hasattr(agent, "ask"): r = agent.ask(prompt) @@ -402,10 +407,8 @@ class AgenticRunner: result = agent.run(record.problem) response_text = getattr(result, "content", str(result)) result_tokens = { - "input_tokens": getattr(result, "input_tokens", 0) - or 0, - "output_tokens": getattr(result, "output_tokens", 0) - or 0, + "input_tokens": getattr(result, "input_tokens", 0) or 0, + "output_tokens": getattr(result, "output_tokens", 0) or 0, "cost_usd": getattr(result, "cost_usd", 0.0) or 0.0, } else: @@ -499,13 +502,15 @@ class AgenticRunner: out_tok = result_tokens.get("output_tokens", 0) cost = result_tokens.get("cost_usd", 0.0) if not turns and (in_tok > 0 or out_tok > 0): - turns = [TurnTrace( - turn_index=0, - input_tokens=in_tok, - output_tokens=out_tok, - wall_clock_s=end_time - start_time, - cost_usd=cost if cost else None, - )] + turns = [ + TurnTrace( + turn_index=0, + input_tokens=in_tok, + output_tokens=out_tok, + wall_clock_s=end_time - start_time, + cost_usd=cost if cost else None, + ) + ] # Backfill tokens from result when turns have zero tokens if turns and in_tok > 0 and out_tok > 0: @@ -514,9 +519,7 @@ class AgenticRunner: if total_turn_in == 0 and total_turn_out == 0: turns[0].input_tokens = in_tok turns[0].output_tokens = out_tok - turns[0].wall_clock_s = ( - turns[0].wall_clock_s or (end_time - start_time) - ) + turns[0].wall_clock_s = turns[0].wall_clock_s or (end_time - start_time) if cost and turns[0].cost_usd is None: turns[0].cost_usd = cost @@ -526,6 +529,7 @@ class AgenticRunner: turn.input_tokens > 0 or turn.output_tokens > 0 ): from openjarvis.evals.core.pricing import compute_turn_cost + turn.cost_usd = compute_turn_cost( model, turn.input_tokens, turn.output_tokens ) @@ -538,16 +542,10 @@ class AgenticRunner: # Extract MBU from telemetry readings mbu_values = [ - getattr(s, "gpu_memory_bandwidth_utilization_pct", None) - for s in readings + getattr(s, "gpu_memory_bandwidth_utilization_pct", None) for s in readings ] - mbu_values = [ - v for v in mbu_values - if v is not None and v >= 0 - ] - query_mbu_avg = ( - statistics.mean(mbu_values) if mbu_values else None - ) + mbu_values = [v for v in mbu_values if v is not None and v >= 0] + query_mbu_avg = statistics.mean(mbu_values) if mbu_values else None query_mbu_max = max(mbu_values) if mbu_values else None trace = QueryTrace( @@ -585,10 +583,7 @@ class AgenticRunner: """ start_ns = int(start_s * 1e9) end_ns = int(end_s * 1e9) - window = [ - r for r in readings - if start_ns <= r.timestamp_ns <= end_ns - ] + window = [r for r in readings if start_ns <= r.timestamp_ns <= end_ns] gpu_energy = _compute_energy_delta(window, "gpu_energy_j") cpu_energy = _compute_energy_delta(window, "cpu_energy_j") avg_gpu_power = _compute_power_avg(window, "gpu_power_w") @@ -629,12 +624,14 @@ class AgenticRunner: wall_clock = 0.0 if current_turn_start is not None: wall_clock = event.timestamp - current_turn_start - current_action_spans.append({ - "action_type": "lm_inference", - "start_s": current_turn_start, - "end_s": event.timestamp, - "duration_s": wall_clock, - }) + current_action_spans.append( + { + "action_type": "lm_inference", + "start_s": current_turn_start, + "end_s": event.timestamp, + "duration_s": wall_clock, + } + ) input_tokens = event.metadata.get("prompt_tokens", 0) output_tokens = event.metadata.get("completion_tokens", 0) @@ -645,13 +642,17 @@ class AgenticRunner: action_breakdown = [] for span in current_action_spans: energy = self._action_energy_from_readings( - readings, span["start_s"], span["end_s"], + readings, + span["start_s"], + span["end_s"], + ) + action_breakdown.append( + { + "action_type": span["action_type"], + "duration_s": span["duration_s"], + **energy, + } ) - action_breakdown.append({ - "action_type": span["action_type"], - "duration_s": span["duration_s"], - **energy, - }) turn = TurnTrace( turn_index=current_turn_index, @@ -683,21 +684,25 @@ class AgenticRunner: elif etype == EventType.TOOL_CALL_END: tool_name = event.metadata.get("tool", "unknown") current_tools.append(tool_name) - current_tool_calls.append({ - "name": tool_name, - "arguments": current_tool_args.pop(tool_name, {}), - "result": event.metadata.get("result", ""), - }) + current_tool_calls.append( + { + "name": tool_name, + "arguments": current_tool_args.pop(tool_name, {}), + "result": event.metadata.get("result", ""), + } + ) start_ts = tool_start_times.pop(tool_name, None) if start_ts is not None: duration = event.timestamp - start_ts current_tool_latencies[tool_name] = duration - current_action_spans.append({ - "action_type": f"tool_call:{tool_name}", - "start_s": start_ts, - "end_s": event.timestamp, - "duration_s": duration, - }) + current_action_spans.append( + { + "action_type": f"tool_call:{tool_name}", + "start_s": start_ts, + "end_s": event.timestamp, + "duration_s": duration, + } + ) # Synthetic turn if events but no complete LM_START/END pair if not turns and events: @@ -724,9 +729,7 @@ class AgenticRunner: if not readings or not trace.turns: return trace - has_turn_energy = any( - t.gpu_energy_joules is not None for t in trace.turns - ) + has_turn_energy = any(t.gpu_energy_joules is not None for t in trace.turns) if has_turn_energy: return trace @@ -770,8 +773,11 @@ class AgenticRunner: "num_turns": trace.num_turns, } for key in ( - "repo", "base_commit", "dataset_name", - "is_resolved", "test_results", + "repo", + "base_commit", + "dataset_name", + "is_resolved", + "test_results", ): val = record.metadata.get(key) if val is not None: diff --git a/src/openjarvis/evals/core/config.py b/src/openjarvis/evals/core/config.py index ab864c71..7067477a 100644 --- a/src/openjarvis/evals/core/config.py +++ b/src/openjarvis/evals/core/config.py @@ -35,14 +35,31 @@ VALID_BACKENDS = {"jarvis-direct", "jarvis-agent"} # Known benchmark names (used for warnings, not hard validation) KNOWN_BENCHMARKS = { - "supergpqa", "gpqa", "mmlu-pro", "math500", "natural-reasoning", "hle", - "simpleqa", "wildchat", "ipw", - "gaia", "frames", "swebench", "swefficiency", - "terminalbench", "terminalbench-native", - "email_triage", "morning_brief", "research_mining", - "knowledge_base", "coding_task", - "coding_assistant", "security_scanner", "daily_digest", - "doc_qa", "browser_assistant", + "supergpqa", + "gpqa", + "mmlu-pro", + "math500", + "natural-reasoning", + "hle", + "simpleqa", + "wildchat", + "ipw", + "gaia", + "frames", + "swebench", + "swefficiency", + "terminalbench", + "terminalbench-native", + "email_triage", + "morning_brief", + "research_mining", + "knowledge_base", + "coding_task", + "coding_assistant", + "security_scanner", + "daily_digest", + "doc_qa", + "browser_assistant", "pinchbench", } @@ -123,36 +140,32 @@ def load_eval_config(path: str | Path) -> EvalSuiteConfig: for m in models_raw: if not m.get("name"): raise EvalConfigError("Each [[models]] entry must have a 'name' field") - models.append(ModelConfig( - name=m["name"], - engine=m.get("engine"), - provider=m.get("provider"), - temperature=float(m["temperature"]) if "temperature" in m else None, - max_tokens=int(m["max_tokens"]) if "max_tokens" in m else None, - param_count_b=float(m.get("param_count_b", 0.0)), - active_params_b=( - float(m["active_params_b"]) - if "active_params_b" in m - else None - ), - gpu_peak_tflops=float(m.get("gpu_peak_tflops", 0.0)), - gpu_peak_bandwidth_gb_s=float(m.get("gpu_peak_bandwidth_gb_s", 0.0)), - num_gpus=int(m.get("num_gpus", 1)), - )) + models.append( + ModelConfig( + name=m["name"], + engine=m.get("engine"), + provider=m.get("provider"), + temperature=float(m["temperature"]) if "temperature" in m else None, + max_tokens=int(m["max_tokens"]) if "max_tokens" in m else None, + param_count_b=float(m.get("param_count_b", 0.0)), + active_params_b=( + float(m["active_params_b"]) if "active_params_b" in m else None + ), + gpu_peak_tflops=float(m.get("gpu_peak_tflops", 0.0)), + gpu_peak_bandwidth_gb_s=float(m.get("gpu_peak_bandwidth_gb_s", 0.0)), + num_gpus=int(m.get("num_gpus", 1)), + ) + ) # Parse [[benchmarks]] benchmarks_raw = raw.get("benchmarks", []) if not benchmarks_raw: - raise EvalConfigError( - "Config must define at least one [[benchmarks]] entry" - ) + raise EvalConfigError("Config must define at least one [[benchmarks]] entry") benchmarks: List[BenchmarkConfig] = [] for b in benchmarks_raw: if not b.get("name"): - raise EvalConfigError( - "Each [[benchmarks]] entry must have a 'name' field" - ) + raise EvalConfigError("Each [[benchmarks]] entry must have a 'name' field") backend = b.get("backend", "jarvis-direct") if backend not in VALID_BACKENDS: @@ -166,18 +179,20 @@ def load_eval_config(path: str | Path) -> EvalSuiteConfig: logger.warning("Unknown benchmark name: '%s'", bench_name) tools_raw = b.get("tools", []) - benchmarks.append(BenchmarkConfig( - name=bench_name, - backend=backend, - max_samples=int(b["max_samples"]) if "max_samples" in b else None, - split=b.get("split"), - agent=b.get("agent"), - tools=list(tools_raw), - judge_model=b.get("judge_model"), - temperature=float(b["temperature"]) if "temperature" in b else None, - max_tokens=int(b["max_tokens"]) if "max_tokens" in b else None, - subset=b.get("subset"), - )) + benchmarks.append( + BenchmarkConfig( + name=bench_name, + backend=backend, + max_samples=int(b["max_samples"]) if "max_samples" in b else None, + split=b.get("split"), + agent=b.get("agent"), + tools=list(tools_raw), + judge_model=b.get("judge_model"), + temperature=float(b["temperature"]) if "temperature" in b else None, + max_tokens=int(b["max_tokens"]) if "max_tokens" in b else None, + subset=b.get("subset"), + ) + ) return EvalSuiteConfig( meta=meta, @@ -245,35 +260,37 @@ def expand_suite(suite: EvalSuiteConfig) -> List[RunConfig]: # Judge engine: suite.judge.engine > "cloud" judge_engine = suite.judge.engine or "cloud" - configs.append(RunConfig( - benchmark=bench.name, - backend=bench.backend, - model=model.name, - max_samples=bench.max_samples, - max_workers=suite.run.max_workers, - temperature=temperature, - max_tokens=max_tokens, - judge_model=judge_model, - judge_engine=judge_engine, - engine_key=model.engine, - agent_name=bench.agent, - tools=list(bench.tools), - output_path=output_path, - seed=suite.run.seed, - dataset_split=bench.split, - dataset_subset=bench.subset, - telemetry=suite.run.telemetry, - gpu_metrics=suite.run.gpu_metrics, - metadata=model_meta, - warmup_samples=suite.run.warmup_samples, - wandb_project=suite.run.wandb_project, - wandb_entity=suite.run.wandb_entity, - wandb_tags=suite.run.wandb_tags, - wandb_group=suite.run.wandb_group, - sheets_spreadsheet_id=suite.run.sheets_spreadsheet_id, - sheets_worksheet=suite.run.sheets_worksheet, - sheets_credentials_path=suite.run.sheets_credentials_path, - )) + configs.append( + RunConfig( + benchmark=bench.name, + backend=bench.backend, + model=model.name, + max_samples=bench.max_samples, + max_workers=suite.run.max_workers, + temperature=temperature, + max_tokens=max_tokens, + judge_model=judge_model, + judge_engine=judge_engine, + engine_key=model.engine, + agent_name=bench.agent, + tools=list(bench.tools), + output_path=output_path, + seed=suite.run.seed, + dataset_split=bench.split, + dataset_subset=bench.subset, + telemetry=suite.run.telemetry, + gpu_metrics=suite.run.gpu_metrics, + metadata=model_meta, + warmup_samples=suite.run.warmup_samples, + wandb_project=suite.run.wandb_project, + wandb_entity=suite.run.wandb_entity, + wandb_tags=suite.run.wandb_tags, + wandb_group=suite.run.wandb_group, + sheets_spreadsheet_id=suite.run.sheets_spreadsheet_id, + sheets_worksheet=suite.run.sheets_worksheet, + sheets_credentials_path=suite.run.sheets_credentials_path, + ) + ) return configs diff --git a/src/openjarvis/evals/core/dataset.py b/src/openjarvis/evals/core/dataset.py index 6503ddd6..12f95baa 100644 --- a/src/openjarvis/evals/core/dataset.py +++ b/src/openjarvis/evals/core/dataset.py @@ -34,7 +34,8 @@ class DatasetProvider(ABC): """Return the number of loaded records.""" def create_task_env( - self, record: EvalRecord, + self, + record: EvalRecord, ) -> Optional[AbstractContextManager]: """Return a task environment context manager, or None.""" return None diff --git a/src/openjarvis/evals/core/display.py b/src/openjarvis/evals/core/display.py index 182d9c5e..e26a5bc8 100644 --- a/src/openjarvis/evals/core/display.py +++ b/src/openjarvis/evals/core/display.py @@ -125,8 +125,10 @@ def print_metrics_table(console: Console, summary: RunSummary) -> None: _add_metric_row(table, "Power (W)", summary.power_stats) _add_metric_row(table, "GPU Util (%)", summary.gpu_utilization_stats, decimals=1) _add_metric_row( - table, "Energy/OutTok (J)", - summary.energy_per_output_token_stats, decimals=6, + table, + "Energy/OutTok (J)", + summary.energy_per_output_token_stats, + decimals=6, ) _add_metric_row(table, "Throughput/Watt", summary.throughput_per_watt_stats) _add_metric_row(table, "MFU (%)", summary.mfu_stats, decimals=2) @@ -203,35 +205,43 @@ def print_accuracy_panel(console: Console, summary: RunSummary) -> None: lines.append(f" {subj:<20s} {acc:.1%} ({correct}/{scored})") body = "\n".join(lines) panel = Panel( - body, title="[bold]Accuracy[/bold]", - border_style="green", expand=False, + body, + title="[bold]Accuracy[/bold]", + border_style="green", + expand=False, ) console.print(panel) def print_latency_table(console: Console, summary: RunSummary) -> None: """Print latency, throughput, and token stats table.""" - table = _stats_table("Latency & Throughput", [ - ("Latency (s)", summary.latency_stats, 2), - ("TTFT (s)", summary.ttft_stats, 3), - ("Throughput (tok/s)", summary.throughput_stats, 1), - ("Avg Input Tokens", summary.input_token_stats, 1), - ("Avg Output Tokens", summary.output_token_stats, 1), - ]) + table = _stats_table( + "Latency & Throughput", + [ + ("Latency (s)", summary.latency_stats, 2), + ("TTFT (s)", summary.ttft_stats, 3), + ("Throughput (tok/s)", summary.throughput_stats, 1), + ("Avg Input Tokens", summary.input_token_stats, 1), + ("Avg Output Tokens", summary.output_token_stats, 1), + ], + ) if table.row_count > 0: console.print(table) def print_energy_table(console: Console, summary: RunSummary) -> None: """Print energy, efficiency, and IPJ/IPW table.""" - table = _stats_table("Energy & Efficiency", [ - ("Energy (J)", summary.energy_stats, 1), - ("Power (W)", summary.power_stats, 1), - ("GPU Util (%)", summary.gpu_utilization_stats, 1), - ("Energy/OutTok (J)", summary.energy_per_output_token_stats, 6), - ("MFU (%)", summary.mfu_stats, 3), - ("MBU (%)", summary.mbu_stats, 3), - ]) + table = _stats_table( + "Energy & Efficiency", + [ + ("Energy (J)", summary.energy_stats, 1), + ("Power (W)", summary.power_stats, 1), + ("GPU Util (%)", summary.gpu_utilization_stats, 1), + ("Energy/OutTok (J)", summary.energy_per_output_token_stats, 6), + ("MFU (%)", summary.mfu_stats, 3), + ("MBU (%)", summary.mbu_stats, 3), + ], + ) if table.row_count > 0: console.print(table) # Headline: IPW, IPJ, Total Energy @@ -258,9 +268,7 @@ def print_trace_summary(console: Console, summary: RunSummary) -> None: return total_steps = sum(s.get("count", 0) for s in sts.values()) avg_per_sample = ( - total_steps / summary.scored_samples - if summary.scored_samples > 0 - else 0 + total_steps / summary.scored_samples if summary.scored_samples > 0 else 0 ) table = Table( @@ -270,8 +278,7 @@ def print_trace_summary(console: Console, summary: RunSummary) -> None: border_style="bright_blue", title_style="bold cyan", caption=( - f"Total Steps: {total_steps}" - f" | Avg Steps/Sample: {avg_per_sample:.1f}" + f"Total Steps: {total_steps} | Avg Steps/Sample: {avg_per_sample:.1f}" ), ) table.add_column("Step Type", style="cyan", no_wrap=True) diff --git a/src/openjarvis/evals/core/environment.py b/src/openjarvis/evals/core/environment.py index b7b5e323..664215ec 100644 --- a/src/openjarvis/evals/core/environment.py +++ b/src/openjarvis/evals/core/environment.py @@ -34,7 +34,8 @@ class EnvironmentProvider(ABC): @abstractmethod def validate( - self, record: EvalRecord, + self, + record: EvalRecord, ) -> Tuple[bool, Dict[str, Any]]: """Check environment state against expected outcome. diff --git a/src/openjarvis/evals/core/export.py b/src/openjarvis/evals/core/export.py index b2377cf6..69780379 100644 --- a/src/openjarvis/evals/core/export.py +++ b/src/openjarvis/evals/core/export.py @@ -36,12 +36,10 @@ def _compute_efficiency( resolved = sum(1 for t in scored if t.is_resolved is True) accuracy = resolved / len(scored) if scored else None gpu_powers = [ - t.avg_gpu_power_watts for t in traces - if t.avg_gpu_power_watts is not None + t.avg_gpu_power_watts for t in traces if t.avg_gpu_power_watts is not None ] cpu_powers = [ - t.avg_cpu_power_watts for t in traces - if t.avg_cpu_power_watts is not None + t.avg_cpu_power_watts for t in traces if t.avg_cpu_power_watts is not None ] avg_gpu_power = statistics.mean(gpu_powers) if gpu_powers else None avg_cpu_power = statistics.mean(cpu_powers) if cpu_powers else None @@ -51,16 +49,8 @@ def _compute_efficiency( "total_cpu_energy_joules": total_cpu_energy, "avg_gpu_power_watts": avg_gpu_power, "avg_cpu_power_watts": avg_cpu_power, - "ipj": ( - accuracy / total_gpu_energy - if accuracy and total_gpu_energy - else None - ), - "ipw": ( - accuracy / avg_gpu_power - if accuracy and avg_gpu_power - else None - ), + "ipj": (accuracy / total_gpu_energy if accuracy and total_gpu_energy else None), + "ipw": (accuracy / avg_gpu_power if accuracy and avg_gpu_power else None), } @@ -77,14 +67,15 @@ def _compute_normalized( trim_count = max(1, math.floor(n * 0.05)) sorted_traces = sorted(traces, key=lambda t: t.total_wall_clock_s) - trimmed = sorted_traces[trim_count: n - trim_count] + trimmed = sorted_traces[trim_count : n - trim_count] if not trimmed: return None # Recompute aggregate energy on trimmed set gpu_energy_values = [ - t.total_gpu_energy_joules for t in trimmed + t.total_gpu_energy_joules + for t in trimmed if t.total_gpu_energy_joules is not None ] total_gpu_energy = sum(gpu_energy_values) if gpu_energy_values else None @@ -92,7 +83,8 @@ def _compute_normalized( cpu_energy_values: list[float] = [] for trace in trimmed: cpu_vals = [ - turn.cpu_energy_joules for turn in trace.turns + turn.cpu_energy_joules + for turn in trace.turns if turn.cpu_energy_joules is not None ] if cpu_vals: @@ -188,6 +180,7 @@ def _hardware_info_dict() -> dict[str, Any]: """Detect hardware and return a JSON-serializable dict.""" try: from openjarvis.core.config import detect_hardware + hw = detect_hardware() info: dict[str, Any] = { "platform": hw.platform, @@ -237,7 +230,8 @@ def export_summary_json( total_wall_clock_s = sum(t.total_wall_clock_s for t in traces) gpu_energy_values = [ - t.total_gpu_energy_joules for t in traces + t.total_gpu_energy_joules + for t in traces if t.total_gpu_energy_joules is not None ] total_gpu_energy = sum(gpu_energy_values) if gpu_energy_values else None @@ -245,7 +239,8 @@ def export_summary_json( cpu_energy_values: list[float] = [] for trace in traces: cpu_vals = [ - turn.cpu_energy_joules for turn in trace.turns + turn.cpu_energy_joules + for turn in trace.turns if turn.cpu_energy_joules is not None ] if cpu_vals: @@ -255,10 +250,7 @@ def export_summary_json( resolved = sum(1 for t in traces if t.is_resolved is True) unresolved = sum(1 for t in traces if t.is_resolved is False) - cost_values = [ - t.total_cost_usd for t in traces - if t.total_cost_usd is not None - ] + cost_values = [t.total_cost_usd for t in traces if t.total_cost_usd is not None] total_cost = sum(cost_values) if cost_values else None avg_turns = total_turns / total_queries if total_queries > 0 else 0 @@ -309,9 +301,7 @@ def export_summary_json( } accuracy = ( - resolved / (resolved + unresolved) - if (resolved + unresolved) > 0 - else None + resolved / (resolved + unresolved) if (resolved + unresolved) > 0 else None ) efficiency = _compute_efficiency(traces, total_gpu_energy, total_cpu_energy) @@ -336,7 +326,8 @@ def export_summary_json( entry = action_totals[atype] entry["count"] += 1 entry["total_duration_s"] += action.get( - "duration_s", 0.0, + "duration_s", + 0.0, ) gpu_e = action.get("gpu_energy_joules") if gpu_e is not None: diff --git a/src/openjarvis/evals/core/runner.py b/src/openjarvis/evals/core/runner.py index 9f4d82e6..e81789b5 100644 --- a/src/openjarvis/evals/core/runner.py +++ b/src/openjarvis/evals/core/runner.py @@ -102,6 +102,7 @@ class EvalRunner: # default iter_episodes() that wraps each record in its own episode, # so hasattr() is always True — we must check for a real override. from openjarvis.evals.core.dataset import DatasetProvider as _DP + try: _overrides_episodes = ( type(self._dataset).iter_episodes is not _DP.iter_episodes @@ -120,18 +121,20 @@ class EvalRunner: # Detect if dataset provides task environments (e.g. PinchBench) try: self._has_task_env = ( - type(self._dataset).create_task_env - is not _DP.create_task_env + type(self._dataset).create_task_env is not _DP.create_task_env ) except AttributeError: self._has_task_env = False records = list(self._dataset.iter_records()) LOGGER.info( - "Running %s: %d samples, backend=%s, model=%s, workers=%d, " - "episode_mode=%s", - cfg.benchmark, len(records), cfg.backend, cfg.model, - cfg.max_workers, cfg.episode_mode, + "Running %s: %d samples, backend=%s, model=%s, workers=%d, episode_mode=%s", + cfg.benchmark, + len(records), + cfg.backend, + cfg.model, + cfg.max_workers, + cfg.episode_mode, ) # --- Warmup phase (discard results) --- @@ -155,7 +158,8 @@ class EvalRunner: except Exception as exc: LOGGER.warning( "Tracker %s.on_run_start failed: %s", - type(tracker).__name__, exc, + type(tracker).__name__, + exc, ) total = len(records) @@ -173,9 +177,7 @@ class EvalRunner: progress_callback(len(self._results), total) else: with ThreadPoolExecutor(max_workers=cfg.max_workers) as pool: - futures = { - pool.submit(self._process_one, r): r for r in records - } + futures = {pool.submit(self._process_one, r): r for r in records} for future in as_completed(futures): result = future.result() self._results.append(result) @@ -197,14 +199,16 @@ class EvalRunner: except Exception as exc: LOGGER.warning( "Tracker %s.on_summary failed: %s", - type(tracker).__name__, exc, + type(tracker).__name__, + exc, ) try: tracker.on_run_end() except Exception as exc: LOGGER.warning( "Tracker %s.on_run_end failed: %s", - type(tracker).__name__, exc, + type(tracker).__name__, + exc, ) # Write summary JSON alongside JSONL @@ -253,6 +257,7 @@ class EvalRunner: if getattr(self, "_has_task_env", False): from contextlib import nullcontext + task_env = self._dataset.create_task_env(record) ctx = task_env if task_env is not None else nullcontext() with ctx: @@ -261,22 +266,23 @@ class EvalRunner: **gen_kwargs, ) full = full or {} - all_tool_results = list(full.get( - "tool_results", [], - )) + all_tool_results = list( + full.get( + "tool_results", + [], + ) + ) # Multi-session: execute remaining sessions sessions = record.metadata.get("sessions", []) - if ( - record.metadata.get("multi_session") - and len(sessions) > 1 - ): + if record.metadata.get("multi_session") and len(sessions) > 1: for session in sessions[1:]: prompt = session.get("prompt", "") if not prompt: continue sfull = self._backend.generate_full( - prompt, **gen_kwargs, + prompt, + **gen_kwargs, ) sfull = sfull or {} all_tool_results.extend( @@ -287,7 +293,8 @@ class EvalRunner: # Score INSIDE context so workspace files still exist content = full.get("content", "") is_correct, scoring_meta = self._scorer.score( - record, content, + record, + content, ) else: full = self._backend.generate_full( @@ -297,7 +304,8 @@ class EvalRunner: full = full or {} content = full.get("content", "") is_correct, scoring_meta = self._scorer.score( - record, content, + record, + content, ) usage = full.get("usage", {}) @@ -340,9 +348,7 @@ class EvalRunner: # Extract derived and ITL metrics from _telemetry dict _telem = full.get("_telemetry", {}) - energy_per_out_tok = _telem.get( - "energy_per_output_token_joules", 0.0 - ) + energy_per_out_tok = _telem.get("energy_per_output_token_joules", 0.0) throughput_per_w = _telem.get("throughput_per_watt", 0.0) mean_itl = _telem.get("mean_itl_ms", 0.0) @@ -406,9 +412,9 @@ class EvalRunner: # default implementation that returns None, so hasattr() is always True — # we must check for a real override to avoid calling env.reset() on None. from openjarvis.evals.core.dataset import DatasetProvider + has_task_env = ( - type(self._dataset).create_task_env - is not DatasetProvider.create_task_env + type(self._dataset).create_task_env is not DatasetProvider.create_task_env ) for episode in self._dataset.iter_episodes(): @@ -417,12 +423,14 @@ class EvalRunner: for record in episode: if has_task_env: result = self._process_interactive( - record, successful_examples, + record, + successful_examples, ) else: # Inject prior examples into prompt, then single-shot augmented = self._inject_examples( - record, successful_examples, + record, + successful_examples, ) result = self._process_one(augmented) @@ -437,9 +445,13 @@ class EvalRunner: "answer": result.model_answer, } # Attach full interaction history if recorded - interaction = result.scoring_metadata.get( - "_interaction_history", - ) if result.scoring_metadata else None + interaction = ( + result.scoring_metadata.get( + "_interaction_history", + ) + if result.scoring_metadata + else None + ) if interaction: example["interaction_history"] = interaction successful_examples.append(example) @@ -482,8 +494,7 @@ class EvalRunner: else: # Fallback: problem + answer summary example_text += ( - f"Task: {ex['problem'][:800]}\n" - f"Solution: {ex['answer'][:800]}\n\n" + f"Task: {ex['problem'][:800]}\nSolution: {ex['answer'][:800]}\n\n" ) example_text += "## Current Task\n\n" @@ -549,9 +560,7 @@ class EvalRunner: # PreviousSampleUtilizationCallback which replays the complete # chat history from prior successful sessions. if prior_examples: - examples_text = ( - "Here are examples of previously completed tasks:\n\n" - ) + examples_text = "Here are examples of previously completed tasks:\n\n" for i, ex in enumerate(prior_examples, 1): history = ex.get("interaction_history") if history and isinstance(history, list): @@ -573,10 +582,12 @@ class EvalRunner: f"Solution: {ex['answer'][:800]}\n\n" ) messages.append({"role": "user", "content": examples_text}) - messages.append({ - "role": "assistant", - "content": "I've reviewed the examples. Ready.", - }) + messages.append( + { + "role": "assistant", + "content": "I've reviewed the examples. Ready.", + } + ) # Initial task message — always use the full problem text which # contains the system prompt, schema, AND task instruction. @@ -648,7 +659,8 @@ class EvalRunner: except Exception as exc: LOGGER.error( "Interactive processing failed for %s: %s", - record.record_id, exc, + record.record_id, + exc, ) return EvalResult( record_id=record.record_id, @@ -720,15 +732,18 @@ class EvalRunner: except Exception as exc: LOGGER.error( "Failed to serialize result %s to JSON: %s — writing error record", - result.record_id, exc, + result.record_id, + exc, + ) + line = json.dumps( + { + "record_id": result.record_id, + "benchmark": self._config.benchmark, + "model": self._config.model, + "is_correct": result.is_correct, + "error": f"serialization_error: {exc}", + } ) - line = json.dumps({ - "record_id": result.record_id, - "benchmark": self._config.benchmark, - "model": self._config.model, - "is_correct": result.is_correct, - "error": f"serialization_error: {exc}", - }) self._output_file.write(line + "\n") self._output_file.flush() @@ -739,7 +754,8 @@ class EvalRunner: except Exception as exc: LOGGER.warning( "Tracker %s.on_result failed: %s", - type(tracker).__name__, exc, + type(tracker).__name__, + exc, ) def _resolve_output_path(self) -> Optional[Path]: @@ -802,12 +818,10 @@ class EvalRunner: energy_vals = [r.energy_joules for r in results if r.energy_joules > 0] power_vals = [r.power_watts for r in results if r.power_watts > 0] gpu_util_vals = [ - r.gpu_utilization_pct for r in results - if r.gpu_utilization_pct > 0 + r.gpu_utilization_pct for r in results if r.gpu_utilization_pct > 0 ] throughput_vals = [ - r.throughput_tok_per_sec for r in results - if r.throughput_tok_per_sec > 0 + r.throughput_tok_per_sec for r in results if r.throughput_tok_per_sec > 0 ] mfu_vals = [r.mfu_pct for r in results if r.mfu_pct > 0] mbu_vals = [r.mbu_pct for r in results if r.mbu_pct > 0] @@ -818,15 +832,11 @@ class EvalRunner: for r in results if r.energy_per_output_token_joules > 0 ] - tpw_vals = [ - r.throughput_per_watt - for r in results if r.throughput_per_watt > 0 - ] + tpw_vals = [r.throughput_per_watt for r in results if r.throughput_per_watt > 0] itl_vals = [r.mean_itl_ms for r in results if r.mean_itl_ms > 0] input_tok_vals = [r.prompt_tokens for r in results if r.prompt_tokens > 0] output_tok_vals = [ - r.completion_tokens for r in results - if r.completion_tokens > 0 + r.completion_tokens for r in results if r.completion_tokens > 0 ] total_energy = sum(r.energy_joules for r in results) @@ -839,19 +849,14 @@ class EvalRunner: "accuracy": round(accuracy, 4), "total_energy_joules": round(total_energy, 6), "avg_power_watts": round(avg_power, 4), - "ipj": ( - round(accuracy / total_energy, 6) - if total_energy > 0 else None - ), - "ipw": ( - round(accuracy / avg_power, 6) - if avg_power > 0 else None - ), + "ipj": (round(accuracy / total_energy, 6) if total_energy > 0 else None), + "ipw": (round(accuracy / avg_power, 6) if avg_power > 0 else None), } # Compute normalized statistics (trim 5% outliers by latency) normalized_stats, normalized_eff = _compute_normalized_stats( - results, accuracy, + results, + accuracy, ) return RunSummary( @@ -911,7 +916,7 @@ def _compute_normalized_stats( trim_count = max(1, math.floor(n * 0.05)) sorted_results = sorted(results, key=lambda r: r.latency_seconds) - trimmed = sorted_results[trim_count: n - trim_count] + trimmed = sorted_results[trim_count : n - trim_count] if not trimmed: return None, None @@ -924,8 +929,7 @@ def _compute_normalized_stats( t_energy_vals = [r.energy_joules for r in trimmed if r.energy_joules > 0] t_power_vals = [r.power_watts for r in trimmed if r.power_watts > 0] t_throughput_vals = [ - r.throughput_tok_per_sec for r in trimmed - if r.throughput_tok_per_sec > 0 + r.throughput_tok_per_sec for r in trimmed if r.throughput_tok_per_sec > 0 ] t_mbu_vals = [r.mbu_pct for r in trimmed if r.mbu_pct > 0] @@ -951,14 +955,8 @@ def _compute_normalized_stats( "accuracy": round(t_accuracy, 4), "total_energy_joules": round(t_total_energy, 6), "avg_power_watts": round(t_avg_power, 4), - "ipj": ( - round(t_accuracy / t_total_energy, 6) - if t_total_energy > 0 else None - ), - "ipw": ( - round(t_accuracy / t_avg_power, 6) - if t_avg_power > 0 else None - ), + "ipj": (round(t_accuracy / t_total_energy, 6) if t_total_energy > 0 else None), + "ipw": (round(t_accuracy / t_avg_power, 6) if t_avg_power > 0 else None), } return norm_stats, norm_eff diff --git a/src/openjarvis/evals/core/scorer.py b/src/openjarvis/evals/core/scorer.py index c67ae628..cb95690a 100644 --- a/src/openjarvis/evals/core/scorer.py +++ b/src/openjarvis/evals/core/scorer.py @@ -16,7 +16,9 @@ class Scorer(ABC): @abstractmethod def score( - self, record: EvalRecord, model_answer: str, + self, + record: EvalRecord, + model_answer: str, ) -> Tuple[Optional[bool], Dict[str, Any]]: """Score a model answer against the reference. diff --git a/src/openjarvis/evals/core/trace.py b/src/openjarvis/evals/core/trace.py index 356f593b..f2d230f5 100644 --- a/src/openjarvis/evals/core/trace.py +++ b/src/openjarvis/evals/core/trace.py @@ -115,8 +115,7 @@ class QueryTrace: @property def total_gpu_energy_joules(self) -> Optional[float]: values = [ - t.gpu_energy_joules for t in self.turns - if t.gpu_energy_joules is not None + t.gpu_energy_joules for t in self.turns if t.gpu_energy_joules is not None ] if values: return sum(values) @@ -125,8 +124,7 @@ class QueryTrace: @property def total_cpu_energy_joules(self) -> Optional[float]: values = [ - t.cpu_energy_joules for t in self.turns - if t.cpu_energy_joules is not None + t.cpu_energy_joules for t in self.turns if t.cpu_energy_joules is not None ] if values: return sum(values) @@ -146,7 +144,8 @@ class QueryTrace: def avg_gpu_power_watts(self) -> Optional[float]: """Mean GPU power across turns; falls back to query-level power.""" values = [ - t.gpu_power_avg_watts for t in self.turns + t.gpu_power_avg_watts + for t in self.turns if t.gpu_power_avg_watts is not None ] if values: @@ -157,7 +156,8 @@ class QueryTrace: def avg_cpu_power_watts(self) -> Optional[float]: """Mean CPU power across turns; falls back to query-level power.""" values = [ - t.cpu_power_avg_watts for t in self.turns + t.cpu_power_avg_watts + for t in self.turns if t.cpu_power_avg_watts is not None ] if values: @@ -246,31 +246,33 @@ class QueryTrace: rows = [] for trace in traces: - rows.append({ - "query_id": trace.query_id, - "workload_type": trace.workload_type, - "query_text": trace.query_text, - "response_text": trace.response_text, - "num_turns": trace.num_turns, - "total_input_tokens": trace.total_input_tokens, - "total_output_tokens": trace.total_output_tokens, - "total_tool_calls": trace.total_tool_calls, - "total_wall_clock_s": trace.total_wall_clock_s, - "total_gpu_energy_joules": trace.total_gpu_energy_joules, - "total_cpu_energy_joules": trace.total_cpu_energy_joules, - "total_tokens": trace.total_tokens, - "total_cost_usd": trace.total_cost_usd, - "avg_gpu_power_watts": trace.avg_gpu_power_watts, - "avg_cpu_power_watts": trace.avg_cpu_power_watts, - "throughput_tokens_per_sec": trace.throughput_tokens_per_sec, - "energy_per_token_joules": trace.energy_per_token_joules, - "completed": trace.completed, - "timed_out": trace.timed_out, - "is_resolved": trace.is_resolved, - "query_mbu_avg_pct": trace.query_mbu_avg_pct, - "query_mbu_max_pct": trace.query_mbu_max_pct, - "trace_json": json.dumps(trace.to_dict()), - }) + rows.append( + { + "query_id": trace.query_id, + "workload_type": trace.workload_type, + "query_text": trace.query_text, + "response_text": trace.response_text, + "num_turns": trace.num_turns, + "total_input_tokens": trace.total_input_tokens, + "total_output_tokens": trace.total_output_tokens, + "total_tool_calls": trace.total_tool_calls, + "total_wall_clock_s": trace.total_wall_clock_s, + "total_gpu_energy_joules": trace.total_gpu_energy_joules, + "total_cpu_energy_joules": trace.total_cpu_energy_joules, + "total_tokens": trace.total_tokens, + "total_cost_usd": trace.total_cost_usd, + "avg_gpu_power_watts": trace.avg_gpu_power_watts, + "avg_cpu_power_watts": trace.avg_cpu_power_watts, + "throughput_tokens_per_sec": trace.throughput_tokens_per_sec, + "energy_per_token_joules": trace.energy_per_token_joules, + "completed": trace.completed, + "timed_out": trace.timed_out, + "is_resolved": trace.is_resolved, + "query_mbu_avg_pct": trace.query_mbu_avg_pct, + "query_mbu_max_pct": trace.query_mbu_max_pct, + "trace_json": json.dumps(trace.to_dict()), + } + ) return Dataset.from_list(rows) diff --git a/src/openjarvis/evals/datasets/ama_bench.py b/src/openjarvis/evals/datasets/ama_bench.py index 41475359..870ad397 100644 --- a/src/openjarvis/evals/datasets/ama_bench.py +++ b/src/openjarvis/evals/datasets/ama_bench.py @@ -130,7 +130,8 @@ class AMABenchDataset(DatasetProvider): return [dict(row) for row in rows] def _row_to_episode( - self, row: Dict[str, Any], + self, + row: Dict[str, Any], ) -> List[EvalRecord]: """Convert one AMA-Bench episode row to EvalRecord(s).""" episode_id = str(row.get("episode_id", "")).strip() @@ -139,11 +140,15 @@ class AMABenchDataset(DatasetProvider): trajectory = row.get("trajectory") if not isinstance(trajectory, list): - raise ValueError(f"AMA-Bench episode {episode_id}: trajectory must be a list") + raise ValueError( + f"AMA-Bench episode {episode_id}: trajectory must be a list" + ) qa_pairs = row.get("qa_pairs") if not isinstance(qa_pairs, list) or not qa_pairs: - raise ValueError(f"AMA-Bench episode {episode_id}: qa_pairs must be a non-empty list") + raise ValueError( + f"AMA-Bench episode {episode_id}: qa_pairs must be a non-empty list" + ) task = str(row.get("task", "")).strip() domain = str(row.get("domain", "")).strip() or "general" @@ -159,12 +164,15 @@ class AMABenchDataset(DatasetProvider): if len(trajectory_text) > max_chars: original_len = len(trajectory_text) trajectory_text = self._truncate_trajectory_text( - trajectory_text, max_chars, + trajectory_text, + max_chars, ) LOGGER.info( "AMA-Bench episode %s: trajectory truncated from %d to %d chars " "(first 50%% + last 50%% of budget kept per Appendix B)", - episode_id, original_len, len(trajectory_text), + episode_id, + original_len, + len(trajectory_text), ) records: List[EvalRecord] = [] @@ -194,31 +202,33 @@ class AMABenchDataset(DatasetProvider): f"## Question\n{question}" ) - records.append(EvalRecord( - record_id=( - f"ama-{episode_id}-{q_uuid}" - if q_uuid - else f"ama-{episode_id}-q{question_index}" - ), - problem=problem, - reference=answer, - category="agentic", - subject=subject, - metadata={ - "episode_id": episode_id, - "task": task, - "task_type": task_type, - "source": source, - "domain": domain, - "success": success, - "num_turns": num_turns, - "total_tokens": total_tokens, - "question_index": question_index, - "question_uuid": q_uuid, - "question_type": q_type, - "capability": subject, - }, - )) + records.append( + EvalRecord( + record_id=( + f"ama-{episode_id}-{q_uuid}" + if q_uuid + else f"ama-{episode_id}-q{question_index}" + ), + problem=problem, + reference=answer, + category="agentic", + subject=subject, + metadata={ + "episode_id": episode_id, + "task": task, + "task_type": task_type, + "source": source, + "domain": domain, + "success": success, + "num_turns": num_turns, + "total_tokens": total_tokens, + "question_index": question_index, + "question_uuid": q_uuid, + "question_type": q_type, + "capability": subject, + }, + ) + ) return records @@ -240,7 +250,8 @@ class AMABenchDataset(DatasetProvider): @staticmethod def _truncate_trajectory_text( - trajectory_text: str, max_chars: int, + trajectory_text: str, + max_chars: int, ) -> str: """Truncate formatted trajectory text by keeping first 50% + last 50% of the character budget, discarding the middle. @@ -254,9 +265,7 @@ class AMABenchDataset(DatasetProvider): if budget_each <= 0: return trajectory_text[:max_chars] return ( - trajectory_text[:budget_each] - + separator - + trajectory_text[-budget_each:] + trajectory_text[:budget_each] + separator + trajectory_text[-budget_each:] ) diff --git a/src/openjarvis/evals/datasets/browser_assistant.py b/src/openjarvis/evals/datasets/browser_assistant.py index 0bcad917..f27b352f 100644 --- a/src/openjarvis/evals/datasets/browser_assistant.py +++ b/src/openjarvis/evals/datasets/browser_assistant.py @@ -81,7 +81,10 @@ _EASY_TASKS: List[Dict[str, Any]] = [ { "question": "How many attention heads does GPT-4 use (as reported in public documentation)?", "expected_facts": [ - {"fact": "architecture details not publicly disclosed by OpenAI", "type": "semantic"}, + { + "fact": "architecture details not publicly disclosed by OpenAI", + "type": "semantic", + }, ], }, { @@ -116,9 +119,18 @@ _MEDIUM_TASKS: List[Dict[str, Any]] = [ { "question": "Compare vLLM and TensorRT-LLM for serving large language models.", "expected_facts": [ - {"fact": "vLLM uses PagedAttention for memory efficiency", "type": "semantic"}, - {"fact": "TensorRT-LLM optimizes for NVIDIA GPUs specifically", "type": "semantic"}, - {"fact": "vLLM is easier to set up, TensorRT-LLM has higher throughput on supported hardware", "type": "semantic"}, + { + "fact": "vLLM uses PagedAttention for memory efficiency", + "type": "semantic", + }, + { + "fact": "TensorRT-LLM optimizes for NVIDIA GPUs specifically", + "type": "semantic", + }, + { + "fact": "vLLM is easier to set up, TensorRT-LLM has higher throughput on supported hardware", + "type": "semantic", + }, ], }, { @@ -133,8 +145,14 @@ _MEDIUM_TASKS: List[Dict[str, Any]] = [ "question": "Compare PostgreSQL and MySQL for JSON document storage.", "expected_facts": [ {"fact": "PostgreSQL has JSONB with GIN indexes", "type": "semantic"}, - {"fact": "MySQL has JSON type with generated columns for indexing", "type": "semantic"}, - {"fact": "PostgreSQL JSONB is generally more feature-rich for JSON operations", "type": "semantic"}, + { + "fact": "MySQL has JSON type with generated columns for indexing", + "type": "semantic", + }, + { + "fact": "PostgreSQL JSONB is generally more feature-rich for JSON operations", + "type": "semantic", + }, ], }, { @@ -142,15 +160,24 @@ _MEDIUM_TASKS: List[Dict[str, Any]] = [ "expected_facts": [ {"fact": "Podman is daemonless", "type": "semantic"}, {"fact": "Podman runs rootless by default", "type": "semantic"}, - {"fact": "Docker uses a client-server architecture with dockerd", "type": "semantic"}, + { + "fact": "Docker uses a client-server architecture with dockerd", + "type": "semantic", + }, ], }, { "question": "Compare Terraform and Pulumi for infrastructure as code.", "expected_facts": [ - {"fact": "Terraform uses HCL, Pulumi uses general-purpose languages", "type": "semantic"}, + { + "fact": "Terraform uses HCL, Pulumi uses general-purpose languages", + "type": "semantic", + }, {"fact": "both support multiple cloud providers", "type": "semantic"}, - {"fact": "Pulumi has native support for Python, TypeScript, Go, C#", "type": "semantic"}, + { + "fact": "Pulumi has native support for Python, TypeScript, Go, C#", + "type": "semantic", + }, ], }, { @@ -164,9 +191,15 @@ _MEDIUM_TASKS: List[Dict[str, Any]] = [ { "question": "Compare FastAPI, Flask, and Django for building REST APIs.", "expected_facts": [ - {"fact": "FastAPI is async-first with automatic OpenAPI docs", "type": "semantic"}, + { + "fact": "FastAPI is async-first with automatic OpenAPI docs", + "type": "semantic", + }, {"fact": "Flask is lightweight and flexible", "type": "semantic"}, - {"fact": "Django includes ORM, admin, and batteries-included approach", "type": "semantic"}, + { + "fact": "Django includes ORM, admin, and batteries-included approach", + "type": "semantic", + }, ], }, { @@ -187,33 +220,63 @@ _HARD_TASKS: List[Dict[str, Any]] = [ "question": "What is the current state of the art for code generation benchmarks (HumanEval, MBPP)? Which models lead and what are their scores?", "expected_facts": [ {"fact": "recent top models score 90%+ on HumanEval", "type": "semantic"}, - {"fact": "MBPP scores are generally lower than HumanEval", "type": "semantic"}, + { + "fact": "MBPP scores are generally lower than HumanEval", + "type": "semantic", + }, {"fact": "mentions specific model names and scores", "type": "semantic"}, ], }, { "question": "Explain the trade-offs between different attention mechanisms: multi-head attention, grouped-query attention, and multi-query attention.", "expected_facts": [ - {"fact": "MHA uses separate K,V heads per attention head", "type": "semantic"}, - {"fact": "GQA groups multiple query heads to share K,V heads", "type": "semantic"}, - {"fact": "MQA uses single K,V head for all query heads", "type": "semantic"}, + { + "fact": "MHA uses separate K,V heads per attention head", + "type": "semantic", + }, + { + "fact": "GQA groups multiple query heads to share K,V heads", + "type": "semantic", + }, + { + "fact": "MQA uses single K,V head for all query heads", + "type": "semantic", + }, {"fact": "GQA balances quality and inference speed", "type": "semantic"}, ], }, { "question": "What are the latest developments in mixture-of-experts (MoE) architectures for LLMs? Compare Mixtral, DeepSeek-V2, and Grok-1.", "expected_facts": [ - {"fact": "Mixtral 8x7B routes to 2 of 8 experts per token", "type": "semantic"}, - {"fact": "DeepSeek-V2 uses fine-grained expert segmentation", "type": "semantic"}, - {"fact": "MoE reduces compute cost per token vs dense models", "type": "semantic"}, + { + "fact": "Mixtral 8x7B routes to 2 of 8 experts per token", + "type": "semantic", + }, + { + "fact": "DeepSeek-V2 uses fine-grained expert segmentation", + "type": "semantic", + }, + { + "fact": "MoE reduces compute cost per token vs dense models", + "type": "semantic", + }, ], }, { "question": "How do different quantization methods (GPTQ, AWQ, GGUF/GGML, bitsandbytes) compare for LLM inference?", "expected_facts": [ - {"fact": "GPTQ uses post-training quantization with calibration data", "type": "semantic"}, - {"fact": "AWQ preserves salient weights for better quality", "type": "semantic"}, - {"fact": "GGUF is the llama.cpp format for CPU and mixed inference", "type": "semantic"}, + { + "fact": "GPTQ uses post-training quantization with calibration data", + "type": "semantic", + }, + { + "fact": "AWQ preserves salient weights for better quality", + "type": "semantic", + }, + { + "fact": "GGUF is the llama.cpp format for CPU and mixed inference", + "type": "semantic", + }, {"fact": "bitsandbytes supports QLoRA for fine-tuning", "type": "semantic"}, ], }, @@ -221,17 +284,35 @@ _HARD_TASKS: List[Dict[str, Any]] = [ "question": "What are the security implications of running LLMs in production? Cover prompt injection, data exfiltration, and model poisoning.", "expected_facts": [ {"fact": "prompt injection can bypass system prompts", "type": "semantic"}, - {"fact": "data exfiltration via tool use or function calling", "type": "semantic"}, - {"fact": "model poisoning through training data contamination", "type": "semantic"}, - {"fact": "mitigations include input filtering and output scanning", "type": "semantic"}, + { + "fact": "data exfiltration via tool use or function calling", + "type": "semantic", + }, + { + "fact": "model poisoning through training data contamination", + "type": "semantic", + }, + { + "fact": "mitigations include input filtering and output scanning", + "type": "semantic", + }, ], }, { "question": "Compare the architectures and capabilities of major embedding models: OpenAI ada-002/3, Cohere embed-v3, and open-source alternatives like BGE and E5.", "expected_facts": [ - {"fact": "OpenAI text-embedding-3 supports Matryoshka dimensions", "type": "semantic"}, - {"fact": "Cohere embed-v3 supports multiple input types", "type": "semantic"}, - {"fact": "BGE and E5 are open-source alternatives with competitive performance", "type": "semantic"}, + { + "fact": "OpenAI text-embedding-3 supports Matryoshka dimensions", + "type": "semantic", + }, + { + "fact": "Cohere embed-v3 supports multiple input types", + "type": "semantic", + }, + { + "fact": "BGE and E5 are open-source alternatives with competitive performance", + "type": "semantic", + }, ], }, { @@ -246,10 +327,22 @@ _HARD_TASKS: List[Dict[str, Any]] = [ { "question": "How do different RAG (Retrieval-Augmented Generation) strategies compare? Cover naive RAG, advanced RAG with re-ranking, and GraphRAG.", "expected_facts": [ - {"fact": "naive RAG does simple similarity search then generate", "type": "semantic"}, - {"fact": "re-ranking improves retrieval precision with cross-encoder", "type": "semantic"}, - {"fact": "GraphRAG uses knowledge graphs for structured retrieval", "type": "semantic"}, - {"fact": "hybrid search combines dense and sparse retrieval", "type": "semantic"}, + { + "fact": "naive RAG does simple similarity search then generate", + "type": "semantic", + }, + { + "fact": "re-ranking improves retrieval precision with cross-encoder", + "type": "semantic", + }, + { + "fact": "GraphRAG uses knowledge graphs for structured retrieval", + "type": "semantic", + }, + { + "fact": "hybrid search combines dense and sparse retrieval", + "type": "semantic", + }, ], }, { @@ -257,17 +350,32 @@ _HARD_TASKS: List[Dict[str, Any]] = [ "expected_facts": [ {"fact": "LoRA adds low-rank adapter matrices", "type": "semantic"}, {"fact": "QLoRA combines quantization with LoRA", "type": "semantic"}, - {"fact": "DoRA decomposes weights into magnitude and direction", "type": "semantic"}, - {"fact": "full fine-tuning updates all parameters but needs more memory", "type": "semantic"}, + { + "fact": "DoRA decomposes weights into magnitude and direction", + "type": "semantic", + }, + { + "fact": "full fine-tuning updates all parameters but needs more memory", + "type": "semantic", + }, ], }, { "question": "Explain the evolution of position encoding in transformers: absolute, RoPE, ALiBi, and YaRN. What are the trade-offs for long-context extension?", "expected_facts": [ - {"fact": "absolute position embeddings are fixed length", "type": "semantic"}, - {"fact": "RoPE encodes relative position through rotation", "type": "semantic"}, + { + "fact": "absolute position embeddings are fixed length", + "type": "semantic", + }, + { + "fact": "RoPE encodes relative position through rotation", + "type": "semantic", + }, {"fact": "ALiBi adds linear bias to attention scores", "type": "semantic"}, - {"fact": "YaRN extends context through interpolation of RoPE", "type": "semantic"}, + { + "fact": "YaRN extends context through interpolation of RoPE", + "type": "semantic", + }, ], }, ] @@ -305,29 +413,27 @@ class BrowserAssistantDataset(DatasetProvider): ) exact_facts = [ - f["fact"] for f in task["expected_facts"] - if f["type"] == "exact" + f["fact"] for f in task["expected_facts"] if f["type"] == "exact" ] semantic_facts = [ - f["fact"] for f in task["expected_facts"] - if f["type"] == "semantic" + f["fact"] for f in task["expected_facts"] if f["type"] == "semantic" ] - self._records.append(EvalRecord( - record_id=f"browser-assistant-{idx:03d}", - problem=prompt, - reference="; ".join( - f["fact"] for f in task["expected_facts"] - ), - category="agentic", - subject=diff, - metadata={ - "question": task["question"], - "expected_facts": task["expected_facts"], - "exact_facts": exact_facts, - "semantic_facts": semantic_facts, - }, - )) + self._records.append( + EvalRecord( + record_id=f"browser-assistant-{idx:03d}", + problem=prompt, + reference="; ".join(f["fact"] for f in task["expected_facts"]), + category="agentic", + subject=diff, + metadata={ + "question": task["question"], + "expected_facts": task["expected_facts"], + "exact_facts": exact_facts, + "semantic_facts": semantic_facts, + }, + ) + ) def iter_records(self) -> Iterable[EvalRecord]: return iter(self._records) diff --git a/src/openjarvis/evals/datasets/coding_assistant.py b/src/openjarvis/evals/datasets/coding_assistant.py index f0e83cda..4625dfca 100644 --- a/src/openjarvis/evals/datasets/coding_assistant.py +++ b/src/openjarvis/evals/datasets/coding_assistant.py @@ -64,7 +64,13 @@ _EASY_TASKS: List[Dict[str, Any]] = [ "def test_empty_page():\n" " assert paginate(list(range(10)), 5, 3) == []\n" ), - "bugs": [{"description": "Off-by-one: end = start + per_page - 1 should be start + per_page", "file": "solution.py", "line": 3}], + "bugs": [ + { + "description": "Off-by-one: end = start + per_page - 1 should be start + per_page", + "file": "solution.py", + "line": 3, + } + ], "originally_failing_tests": ["test_first_page", "test_second_page"], "originally_passing_tests": ["test_last_page", "test_empty_page"], }, @@ -91,20 +97,20 @@ _EASY_TASKS: List[Dict[str, Any]] = [ "def test_empty():\n" " assert is_palindrome('') is True\n" ), - "bugs": [{"description": "Unnecessary len(s) > 1 check rejects single chars and empty strings", "file": "solution.py", "line": 3}], + "bugs": [ + { + "description": "Unnecessary len(s) > 1 check rejects single chars and empty strings", + "file": "solution.py", + "line": 3, + } + ], "originally_failing_tests": ["test_single_char", "test_empty"], "originally_passing_tests": ["test_racecar", "test_hello"], }, { "bug_report": "The celsius_to_fahrenheit function returns wrong values.", - "buggy_code": ( - "def celsius_to_fahrenheit(c):\n" - " return c * 9 / 5 + 23\n" - ), - "fixed_code": ( - "def celsius_to_fahrenheit(c):\n" - " return c * 9 / 5 + 32\n" - ), + "buggy_code": ("def celsius_to_fahrenheit(c):\n return c * 9 / 5 + 23\n"), + "fixed_code": ("def celsius_to_fahrenheit(c):\n return c * 9 / 5 + 32\n"), "test_code": ( "from solution import celsius_to_fahrenheit\n\n" "def test_freezing():\n" @@ -114,7 +120,13 @@ _EASY_TASKS: List[Dict[str, Any]] = [ "def test_body_temp():\n" " assert abs(celsius_to_fahrenheit(37) - 98.6) < 0.01\n" ), - "bugs": [{"description": "Wrong constant: +23 should be +32", "file": "solution.py", "line": 2}], + "bugs": [ + { + "description": "Wrong constant: +23 should be +32", + "file": "solution.py", + "line": 2, + } + ], "originally_failing_tests": ["test_freezing", "test_boiling", "test_body_temp"], "originally_passing_tests": [], }, @@ -151,7 +163,13 @@ _EASY_TASKS: List[Dict[str, Any]] = [ "def test_deeply_nested():\n" " assert flatten([[[1]], [[2]], [[3]]]) == [1, 2, 3]\n" ), - "bugs": [{"description": "append should be extend for recursive flattening", "file": "solution.py", "line": 5}], + "bugs": [ + { + "description": "append should be extend for recursive flattening", + "file": "solution.py", + "line": 5, + } + ], "originally_failing_tests": ["test_nested", "test_deeply_nested"], "originally_passing_tests": ["test_flat", "test_empty"], }, @@ -194,9 +212,19 @@ _EASY_TASKS: List[Dict[str, Any]] = [ "def test_not_found():\n" " assert binary_search([1, 3, 5, 7, 9], 4) == -1\n" ), - "bugs": [{"description": "lo starts at 1 instead of 0, skipping first element", "file": "solution.py", "line": 2}], + "bugs": [ + { + "description": "lo starts at 1 instead of 0, skipping first element", + "file": "solution.py", + "line": 2, + } + ], "originally_failing_tests": ["test_find_first"], - "originally_passing_tests": ["test_find_middle", "test_find_last", "test_not_found"], + "originally_passing_tests": [ + "test_find_middle", + "test_find_last", + "test_not_found", + ], }, { "bug_report": "The clamp function doesn't work when value equals min_val.", @@ -231,9 +259,21 @@ _EASY_TASKS: List[Dict[str, Any]] = [ "def test_at_max():\n" " assert clamp(10, 0, 10) == 10\n" ), - "bugs": [{"description": "Actually no bug — this is a control task to test false positive rate", "file": "solution.py", "line": 0}], + "bugs": [ + { + "description": "Actually no bug — this is a control task to test false positive rate", + "file": "solution.py", + "line": 0, + } + ], "originally_failing_tests": [], - "originally_passing_tests": ["test_below", "test_above", "test_within", "test_at_min", "test_at_max"], + "originally_passing_tests": [ + "test_below", + "test_above", + "test_within", + "test_at_min", + "test_at_max", + ], }, { "bug_report": "The count_vowels function misses uppercase vowels.", @@ -264,7 +304,13 @@ _EASY_TASKS: List[Dict[str, Any]] = [ "def test_empty():\n" " assert count_vowels('') == 0\n" ), - "bugs": [{"description": "Does not handle uppercase — should lowercase first", "file": "solution.py", "line": 3}], + "bugs": [ + { + "description": "Does not handle uppercase — should lowercase first", + "file": "solution.py", + "line": 3, + } + ], "originally_failing_tests": ["test_uppercase", "test_mixed"], "originally_passing_tests": ["test_lowercase", "test_empty"], }, @@ -303,7 +349,13 @@ _EASY_TASKS: List[Dict[str, Any]] = [ "def test_two_elements():\n" " assert max_profit([1, 5]) == 4\n" ), - "bugs": [{"description": "Subtraction order reversed: min_price - price should be price - min_price", "file": "solution.py", "line": 7}], + "bugs": [ + { + "description": "Subtraction order reversed: min_price - price should be price - min_price", + "file": "solution.py", + "line": 7, + } + ], "originally_failing_tests": ["test_basic", "test_two_elements"], "originally_passing_tests": ["test_declining", "test_single"], }, @@ -333,7 +385,13 @@ _EASY_TASKS: List[Dict[str, Any]] = [ "def test_already_title():\n" " assert title_case('Hello') == 'Hello'\n" ), - "bugs": [{"description": "Does not handle hyphenated words", "file": "solution.py", "line": 2}], + "bugs": [ + { + "description": "Does not handle hyphenated words", + "file": "solution.py", + "line": 2, + } + ], "originally_failing_tests": ["test_hyphenated"], "originally_passing_tests": ["test_simple", "test_already_title"], }, @@ -364,7 +422,13 @@ _EASY_TASKS: List[Dict[str, Any]] = [ "def test_type_error():\n" " assert safe_divide(10, 'a') == 0\n" ), - "bugs": [{"description": "Catches TypeError but not ZeroDivisionError", "file": "solution.py", "line": 4}], + "bugs": [ + { + "description": "Catches TypeError but not ZeroDivisionError", + "file": "solution.py", + "line": 4, + } + ], "originally_failing_tests": ["test_zero_div", "test_zero_div_custom"], "originally_passing_tests": ["test_normal", "test_type_error"], }, @@ -438,8 +502,16 @@ _MEDIUM_TASKS: List[Dict[str, Any]] = [ " assert c.get(1) == 10\n" ), "bugs": [ - {"description": "order.pop() removes last (MRU) instead of order.pop(0) for LRU", "file": "solution.py", "line": 17}, - {"description": "get/put don't remove old position before re-appending", "file": "solution.py", "line": 9}, + { + "description": "order.pop() removes last (MRU) instead of order.pop(0) for LRU", + "file": "solution.py", + "line": 17, + }, + { + "description": "get/put don't remove old position before re-appending", + "file": "solution.py", + "line": 9, + }, ], "originally_failing_tests": ["test_basic", "test_update"], "originally_passing_tests": [], @@ -488,7 +560,13 @@ _MEDIUM_TASKS: List[Dict[str, Any]] = [ "def test_duplicates():\n" " assert merge_sorted([1, 2], [2, 3]) == [1, 2, 2, 3]\n" ), - "bugs": [{"description": "Duplicate result.extend(a[i:]) on line 13 causes extra elements", "file": "solution.py", "line": 13}], + "bugs": [ + { + "description": "Duplicate result.extend(a[i:]) on line 13 causes extra elements", + "file": "solution.py", + "line": 13, + } + ], "originally_failing_tests": ["test_basic", "test_duplicates"], "originally_passing_tests": ["test_one_empty", "test_both_empty"], }, @@ -535,7 +613,13 @@ _MEDIUM_TASKS: List[Dict[str, Any]] = [ " b = [[4], [5], [6]]\n" " assert matrix_multiply(a, b) == [[32]]\n" ), - "bugs": [{"description": "Wrong index: b[j][k] should be b[k][j]", "file": "solution.py", "line": 10}], + "bugs": [ + { + "description": "Wrong index: b[j][k] should be b[k][j]", + "file": "solution.py", + "line": 10, + } + ], "originally_failing_tests": ["test_basic", "test_non_square"], "originally_passing_tests": ["test_identity"], }, @@ -582,7 +666,13 @@ _MEDIUM_TASKS: List[Dict[str, Any]] = [ "def test_no_repeats():\n" " assert run_length_encode('abc') == 'a1b1c1'\n" ), - "bugs": [{"description": "In else branch, appends s[i] (next char) instead of s[i-1] (current char)", "file": "solution.py", "line": 10}], + "bugs": [ + { + "description": "In else branch, appends s[i] (next char) instead of s[i-1] (current char)", + "file": "solution.py", + "line": 10, + } + ], "originally_failing_tests": ["test_basic", "test_no_repeats"], "originally_passing_tests": ["test_single", "test_empty"], }, @@ -627,9 +717,20 @@ _MEDIUM_TASKS: List[Dict[str, Any]] = [ "def test_extra_close():\n" " assert validate_brackets(')') is False\n" ), - "bugs": [{"description": "Does not check that closing bracket matches the top of stack", "file": "solution.py", "line": 10}], + "bugs": [ + { + "description": "Does not check that closing bracket matches the top of stack", + "file": "solution.py", + "line": 10, + } + ], "originally_failing_tests": ["test_mismatched"], - "originally_passing_tests": ["test_valid", "test_unclosed", "test_empty", "test_extra_close"], + "originally_passing_tests": [ + "test_valid", + "test_unclosed", + "test_empty", + "test_extra_close", + ], }, { "bug_report": "The group_by function loses entries when multiple items have the same key.", @@ -660,7 +761,13 @@ _MEDIUM_TASKS: List[Dict[str, Any]] = [ "def test_empty():\n" " assert group_by([], lambda x: x) == {}\n" ), - "bugs": [{"description": "Overwrites group with [item] instead of appending to list", "file": "solution.py", "line": 5}], + "bugs": [ + { + "description": "Overwrites group with [item] instead of appending to list", + "file": "solution.py", + "line": 5, + } + ], "originally_failing_tests": ["test_basic", "test_strings"], "originally_passing_tests": ["test_empty"], }, @@ -697,8 +804,18 @@ _MEDIUM_TASKS: List[Dict[str, Any]] = [ "def test_window_too_large():\n" " assert moving_average([1], 5) == []\n" ), - "bugs": [{"description": "Off-by-one in range: should be len(data) - window + 1", "file": "solution.py", "line": 5}], - "originally_failing_tests": ["test_basic", "test_window_equals_length", "test_window_one"], + "bugs": [ + { + "description": "Off-by-one in range: should be len(data) - window + 1", + "file": "solution.py", + "line": 5, + } + ], + "originally_failing_tests": [ + "test_basic", + "test_window_equals_length", + "test_window_one", + ], "originally_passing_tests": ["test_window_too_large"], }, { @@ -763,7 +880,13 @@ _MEDIUM_TASKS: List[Dict[str, Any]] = [ " t = Trie()\n" " assert t.search('hello') is False\n" ), - "bugs": [{"description": "search returns False instead of node.is_end", "file": "solution.py", "line": 24}], + "bugs": [ + { + "description": "search returns False instead of node.is_end", + "file": "solution.py", + "line": 24, + } + ], "originally_failing_tests": ["test_insert_search"], "originally_passing_tests": ["test_search_missing", "test_search_not_inserted"], }, @@ -803,7 +926,13 @@ _MEDIUM_TASKS: List[Dict[str, Any]] = [ " expected = {'a': {'b': {'c': 3, 'd': 2}}}\n" " assert deep_merge(base, over) == expected\n" ), - "bugs": [{"description": "Shallow merge: overwrites nested dicts instead of recursively merging", "file": "solution.py", "line": 4}], + "bugs": [ + { + "description": "Shallow merge: overwrites nested dicts instead of recursively merging", + "file": "solution.py", + "line": 4, + } + ], "originally_failing_tests": ["test_nested", "test_deeply_nested"], "originally_passing_tests": ["test_flat", "test_override"], }, @@ -864,9 +993,18 @@ _MEDIUM_TASKS: List[Dict[str, Any]] = [ " debounced(2) # fires after wait\n" " assert len(calls) == 2\n" ), - "bugs": [{"description": "Missing time check: calls func every time instead of only after wait_seconds", "file": "solution.py", "line": 9}], + "bugs": [ + { + "description": "Missing time check: calls func every time instead of only after wait_seconds", + "file": "solution.py", + "line": 9, + } + ], "originally_failing_tests": ["test_debounce_suppresses"], - "originally_passing_tests": ["test_debounce_fires_first", "test_debounce_fires_after_wait"], + "originally_passing_tests": [ + "test_debounce_fires_first", + "test_debounce_fires_after_wait", + ], }, ] @@ -908,16 +1046,19 @@ _HARD_TASKS: List[Dict[str, Any]] = [ " # 19.99 * 1.0875 * 0.95 = 20.6496...\n" " assert financial_round(19.99, 0.0875, 0.05) == 20.65\n" ), - "bugs": [{"description": "Float arithmetic causes precision errors — should use Decimal", "file": "solution.py", "line": 2}], + "bugs": [ + { + "description": "Float arithmetic causes precision errors — should use Decimal", + "file": "solution.py", + "line": 2, + } + ], "originally_failing_tests": ["test_precision", "test_rounding_boundary"], "originally_passing_tests": ["test_basic", "test_no_discount"], }, { "bug_report": "The csv_parser doesn't handle quoted fields with commas inside.", - "buggy_code": ( - "def parse_csv_line(line):\n" - " return line.split(',')\n" - ), + "buggy_code": ("def parse_csv_line(line):\n return line.split(',')\n"), "fixed_code": ( "import csv\nimport io\n\n" "def parse_csv_line(line):\n" @@ -935,7 +1076,13 @@ _HARD_TASKS: List[Dict[str, Any]] = [ "def test_empty_fields():\n" " assert parse_csv_line('a,,c') == ['a', '', 'c']\n" ), - "bugs": [{"description": "Naive split(',') doesn't handle quoted fields", "file": "solution.py", "line": 2}], + "bugs": [ + { + "description": "Naive split(',') doesn't handle quoted fields", + "file": "solution.py", + "line": 2, + } + ], "originally_failing_tests": ["test_quoted_comma"], "originally_passing_tests": ["test_simple", "test_empty_fields"], }, @@ -1003,7 +1150,13 @@ _HARD_TASKS: List[Dict[str, Any]] = [ " except RuntimeError:\n" " pass\n" ), - "bugs": [{"description": "Success path doesn't return immediately and sleeps unnecessarily; runs all attempts", "file": "solution.py", "line": 8}], + "bugs": [ + { + "description": "Success path doesn't return immediately and sleeps unnecessarily; runs all attempts", + "file": "solution.py", + "line": 8, + } + ], "originally_failing_tests": ["test_succeeds_first_try"], "originally_passing_tests": ["test_exhausts_retries"], }, @@ -1040,7 +1193,13 @@ _HARD_TASKS: List[Dict[str, Any]] = [ " result = parse_url_params('http://example.com?data=a=b')\n" " assert result['data'] == 'a=b'\n" ), - "bugs": [{"description": "split('=') breaks on values containing '='; doesn't decode URL encoding", "file": "solution.py", "line": 7}], + "bugs": [ + { + "description": "split('=') breaks on values containing '='; doesn't decode URL encoding", + "file": "solution.py", + "line": 7, + } + ], "originally_failing_tests": ["test_encoded_value", "test_value_with_equals"], "originally_passing_tests": ["test_basic", "test_no_params"], }, @@ -1093,8 +1252,17 @@ _HARD_TASKS: List[Dict[str, Any]] = [ " time.sleep(0.1)\n" " assert rl.allow() is True\n" ), - "bugs": [{"description": "Never removes old requests; always appends even when over limit", "file": "solution.py", "line": 10}], - "originally_failing_tests": ["test_blocks_over_limit", "test_allows_after_window"], + "bugs": [ + { + "description": "Never removes old requests; always appends even when over limit", + "file": "solution.py", + "line": 10, + } + ], + "originally_failing_tests": [ + "test_blocks_over_limit", + "test_allows_after_window", + ], "originally_passing_tests": ["test_allows_within_limit"], }, { @@ -1148,7 +1316,13 @@ _HARD_TASKS: List[Dict[str, Any]] = [ " greet(name='Alice', greeting='hey')\n" " assert len(calls) == 2\n" ), - "bugs": [{"description": "Wrapper doesn't accept or cache kwargs", "file": "solution.py", "line": 3}], + "bugs": [ + { + "description": "Wrapper doesn't accept or cache kwargs", + "file": "solution.py", + "line": 3, + } + ], "originally_failing_tests": ["test_kwargs", "test_different_kwargs"], "originally_passing_tests": ["test_positional"], }, @@ -1194,7 +1368,13 @@ _HARD_TASKS: List[Dict[str, Any]] = [ " ee = EventEmitter()\n" " ee.emit('missing') # should not raise\n" ), - "bugs": [{"description": "on() overwrites listener instead of appending to list", "file": "solution.py", "line": 6}], + "bugs": [ + { + "description": "on() overwrites listener instead of appending to list", + "file": "solution.py", + "line": 6, + } + ], "originally_failing_tests": ["test_multiple_listeners"], "originally_passing_tests": ["test_single_listener", "test_no_listeners"], }, @@ -1240,7 +1420,13 @@ _HARD_TASKS: List[Dict[str, Any]] = [ " result = json_flatten(obj)\n" " assert result == {'a.b.c[0]': 1, 'a.b.c[1]': 2}\n" ), - "bugs": [{"description": "Does not handle list values, only dicts", "file": "solution.py", "line": 5}], + "bugs": [ + { + "description": "Does not handle list values, only dicts", + "file": "solution.py", + "line": 5, + } + ], "originally_failing_tests": ["test_with_array", "test_deeply_nested"], "originally_passing_tests": ["test_flat", "test_nested"], }, @@ -1295,7 +1481,13 @@ _HARD_TASKS: List[Dict[str, Any]] = [ " assert len(results) == 2\n" " assert all(r[0] == 'error' for r in results)\n" ), - "bugs": [{"description": "Silently swallows exceptions instead of recording them", "file": "solution.py", "line": 9}], + "bugs": [ + { + "description": "Silently swallows exceptions instead of recording them", + "file": "solution.py", + "line": 9, + } + ], "originally_failing_tests": ["test_one_fails", "test_all_fail"], "originally_passing_tests": ["test_all_succeed"], }, @@ -1354,7 +1546,13 @@ _HARD_TASKS: List[Dict[str, Any]] = [ " with pytest.raises(ValueError, match='Cycle'):\n" " topological_sort(graph)\n" ), - "bugs": [{"description": "No cycle detection — infinite recursion on cyclic graphs", "file": "solution.py", "line": 5}], + "bugs": [ + { + "description": "No cycle detection — infinite recursion on cyclic graphs", + "file": "solution.py", + "line": 5, + } + ], "originally_failing_tests": ["test_cycle_detection"], "originally_passing_tests": ["test_linear", "test_diamond"], }, diff --git a/src/openjarvis/evals/datasets/coding_task.py b/src/openjarvis/evals/datasets/coding_task.py index 3679e5d1..8d8bbd85 100644 --- a/src/openjarvis/evals/datasets/coding_task.py +++ b/src/openjarvis/evals/datasets/coding_task.py @@ -200,7 +200,7 @@ _TASKS = [ { "spec": "Implement a function that serializes and deserializes a binary tree. The tree node has val, left, right attributes.", "signature": "def serialize(root) -> str\ndef deserialize(data: str) -> TreeNode", - "examples": 'serialize a tree [1,2,3,null,null,4,5] -> some string\ndeserialize that string -> same tree', + "examples": "serialize a tree [1,2,3,null,null,4,5] -> some string\ndeserialize that string -> same tree", "test_cases": "class TreeNode:\n def __init__(self, val=0, left=None, right=None):\n self.val, self.left, self.right = val, left, right\nroot = TreeNode(1, TreeNode(2), TreeNode(3, TreeNode(4), TreeNode(5)))\ns = serialize(root)\nnew_root = deserialize(s)\nassert new_root.val == 1\nassert new_root.left.val == 2\nassert new_root.right.val == 3\nassert new_root.right.left.val == 4", "reference": "class TreeNode:\n def __init__(self, val=0, left=None, right=None):\n self.val, self.left, self.right = val, left, right\ndef serialize(root):\n if not root: return 'null'\n return f'{root.val},{serialize(root.left)},{serialize(root.right)}'\ndef deserialize(data):\n vals = iter(data.split(','))\n def build():\n v = next(vals)\n if v == 'null': return None\n node = TreeNode(int(v))\n node.left = build()\n node.right = build()\n return node\n return build()", }, @@ -260,17 +260,19 @@ class CodingTaskDataset(DatasetProvider): signature=task["signature"], examples=task["examples"], ) - self._records.append(EvalRecord( - record_id=f"coding-task-{idx}", - problem=prompt, - reference=task["reference"], - category="use-case", - subject="coding_task", - metadata={ - "test_cases": task["test_cases"], - "signature": task["signature"], - }, - )) + self._records.append( + EvalRecord( + record_id=f"coding-task-{idx}", + problem=prompt, + reference=task["reference"], + category="use-case", + subject="coding_task", + metadata={ + "test_cases": task["test_cases"], + "signature": task["signature"], + }, + ) + ) def iter_records(self) -> Iterable[EvalRecord]: return iter(self._records) diff --git a/src/openjarvis/evals/datasets/daily_digest.py b/src/openjarvis/evals/datasets/daily_digest.py index 19d6da13..44549d76 100644 --- a/src/openjarvis/evals/datasets/daily_digest.py +++ b/src/openjarvis/evals/datasets/daily_digest.py @@ -97,7 +97,12 @@ _EASY_TASKS: List[Dict[str, Any]] = [ "todos": "- Finalize Q4 roadmap draft\n- Send release notes for v3.2", "messages": "- Slack from @eng-lead: 'v3.2 deployed to staging, all green'\n- Email from VP: 'Need roadmap by EOD Monday'", "news_interests": "Fintech regulations, payment APIs", - "must_mention": ["stakeholder demo", "v3.2 staging", "Q4 roadmap", "EOD Monday deadline"], + "must_mention": [ + "stakeholder demo", + "v3.2 staging", + "Q4 roadmap", + "EOD Monday deadline", + ], "priority_order": ["stakeholder demo", "v3.2 staging", "Q4 roadmap"], }, { @@ -119,7 +124,11 @@ _EASY_TASKS: List[Dict[str, Any]] = [ "todos": "- Fix crash in offline mode (Sentry issue #891)\n- Implement push notification deep links", "messages": "- Slack from QA: 'Sentry #891 affecting 2% of users on iOS 17'\n- App Store Connect: 'v4.1 approved for release'", "news_interests": "Swift, iOS", - "must_mention": ["Sentry #891 crash", "v4.1 approved", "push notification deep links"], + "must_mention": [ + "Sentry #891 crash", + "v4.1 approved", + "push notification deep links", + ], "priority_order": ["Sentry #891 crash", "v4.1 approved"], }, { @@ -141,7 +150,11 @@ _EASY_TASKS: List[Dict[str, Any]] = [ "todos": "- Run regression suite for v5.0\n- Update test plan for payment module", "messages": "- Jira: 'RELEASE-50: v5.0 release candidate tagged'\n- Slack from dev: 'Payment module refactored, old tests may break'", "news_interests": "Test automation, Playwright", - "must_mention": ["v5.0 regression suite", "payment module tests", "release readiness"], + "must_mention": [ + "v5.0 regression suite", + "payment module tests", + "release readiness", + ], "priority_order": ["v5.0 regression suite", "payment module tests"], }, { @@ -152,7 +165,11 @@ _EASY_TASKS: List[Dict[str, Any]] = [ "todos": "- Write post-mortem for Wednesday's outage\n- Update runbook for database failover", "messages": "- Datadog: 'API p99 latency back to normal (12ms)'\n- Manager: 'Post-mortem draft needed before Monday'", "news_interests": "SRE practices, observability", - "must_mention": ["post-mortem draft", "API latency normal", "capacity planning"], + "must_mention": [ + "post-mortem draft", + "API latency normal", + "capacity planning", + ], "priority_order": ["post-mortem draft", "capacity planning"], }, ] @@ -170,8 +187,18 @@ _MEDIUM_TASKS: List[Dict[str, Any]] = [ "todos": "- Finalize RFC for webhook retry redesign\n- Review perf regression in checkout API\n- Respond to security review findings", "messages": "- Slack from @payments-lead: 'Webhook failures up 3x since Friday deploy'\n- Email from VP Eng: 'RFC deadline extended to Wednesday'\n- Jira INFRA-2847: 'Checkout API p99 jumped from 200ms to 800ms'", "news_interests": "distributed systems, payment processing", - "must_mention": ["webhook failures 3x", "INFRA-2847 checkout latency", "RFC deadline Wednesday", "architecture review", "security review findings"], - "priority_order": ["webhook failures 3x", "INFRA-2847 checkout latency", "RFC deadline"], + "must_mention": [ + "webhook failures 3x", + "INFRA-2847 checkout latency", + "RFC deadline Wednesday", + "architecture review", + "security review findings", + ], + "priority_order": [ + "webhook failures 3x", + "INFRA-2847 checkout latency", + "RFC deadline", + ], }, { "role": "ML Engineer", @@ -181,7 +208,13 @@ _MEDIUM_TASKS: List[Dict[str, Any]] = [ "todos": "- Investigate 5% CTR drop in product recommendations\n- Prepare training data for v3 model\n- Review feature store PR from junior engineer", "messages": "- Slack from product: 'Recommendation CTR dropped on mobile — exec visibility'\n- Airflow alert: 'DAG user_features failed at 03:00, 2 retries exhausted'\n- Email from research: 'New embedding approach shows 12% improvement in offline eval'", "news_interests": "recommendation systems, transformer architectures", - "must_mention": ["CTR drop", "Airflow DAG failure", "v3 model training data", "A/B test review", "new embedding approach"], + "must_mention": [ + "CTR drop", + "Airflow DAG failure", + "v3 model training data", + "A/B test review", + "new embedding approach", + ], "priority_order": ["CTR drop", "Airflow DAG failure", "A/B test review"], }, { @@ -192,8 +225,18 @@ _MEDIUM_TASKS: List[Dict[str, Any]] = [ "todos": "- Complete service mesh POC documentation\n- Fix flaky integration tests in CI pipeline\n- Review IAM policy changes for staging env", "messages": "- Slack from SRE: 'CI pipeline blocked — flaky test_payment_flow failing 40% of runs'\n- Email from CTO: 'Need vendor recommendation by Friday for budget approval'\n- GitHub: '3 new Dependabot PRs — 1 critical (lodash prototype pollution)'", "news_interests": "service mesh, platform engineering", - "must_mention": ["CI flaky test", "vendor recommendation Friday", "lodash critical CVE", "migration planning", "IAM policy review"], - "priority_order": ["CI flaky test", "lodash critical CVE", "vendor recommendation"], + "must_mention": [ + "CI flaky test", + "vendor recommendation Friday", + "lodash critical CVE", + "migration planning", + "IAM policy review", + ], + "priority_order": [ + "CI flaky test", + "lodash critical CVE", + "vendor recommendation", + ], }, { "role": "Tech Lead", @@ -203,8 +246,18 @@ _MEDIUM_TASKS: List[Dict[str, Any]] = [ "todos": "- Audit PHI access logs for September\n- Design API for lab results integration\n- Mentor new hire on HIPAA data handling", "messages": "- Compliance team: 'September PHI audit due Oct 15 — need your section by Oct 10'\n- PM: 'Patient portal launch moved up to Nov 1'\n- QA: 'Found PHI leak in staging logs — ticket SEC-445'", "news_interests": "healthcare APIs, HIPAA compliance", - "must_mention": ["PHI leak SEC-445", "audit due Oct 10", "patient portal Nov 1", "lab results API", "HIPAA compliance review"], - "priority_order": ["PHI leak SEC-445", "audit due Oct 10", "patient portal Nov 1"], + "must_mention": [ + "PHI leak SEC-445", + "audit due Oct 10", + "patient portal Nov 1", + "lab results API", + "HIPAA compliance review", + ], + "priority_order": [ + "PHI leak SEC-445", + "audit due Oct 10", + "patient portal Nov 1", + ], }, { "role": "Backend Lead", @@ -214,8 +267,18 @@ _MEDIUM_TASKS: List[Dict[str, Any]] = [ "todos": "- Sign off on deploy for order management refactor\n- Review load test results (target: 10x normal traffic)\n- Update on-call rotation for holiday season", "messages": "- Load test results: 'System handles 7x but OOM at 8x on order-service'\n- Slack from payments: 'New Stripe API version — migration needed by Dec 1'\n- PagerDuty: 'Memory leak alert on cart-service, auto-scaled to 8 pods'", "news_interests": "e-commerce scaling, distributed systems", - "must_mention": ["OOM at 8x load", "cart-service memory leak", "Stripe migration Dec 1", "Black Friday capacity", "deploy review"], - "priority_order": ["OOM at 8x load", "cart-service memory leak", "Black Friday capacity"], + "must_mention": [ + "OOM at 8x load", + "cart-service memory leak", + "Stripe migration Dec 1", + "Black Friday capacity", + "deploy review", + ], + "priority_order": [ + "OOM at 8x load", + "cart-service memory leak", + "Black Friday capacity", + ], }, { "role": "Data Engineer", @@ -225,8 +288,17 @@ _MEDIUM_TASKS: List[Dict[str, Any]] = [ "todos": "- Debug Spark job OOM on customer_360 pipeline\n- Implement CDC for new orders table\n- Review dbt model changes from analytics team", "messages": "- Airflow: 'customer_360 DAG failed — Spark executor OOM'\n- Marketing: 'Attribution numbers look off by 15% since last week'\n- Slack from manager: 'Quarterly data infrastructure review next Tuesday, prep needed'", "news_interests": "data engineering, Apache Spark", - "must_mention": ["Spark OOM customer_360", "attribution 15% discrepancy", "CDC orders table", "infrastructure review prep"], - "priority_order": ["Spark OOM", "attribution discrepancy", "infrastructure review"], + "must_mention": [ + "Spark OOM customer_360", + "attribution 15% discrepancy", + "CDC orders table", + "infrastructure review prep", + ], + "priority_order": [ + "Spark OOM", + "attribution discrepancy", + "infrastructure review", + ], }, { "role": "iOS Lead", @@ -236,8 +308,18 @@ _MEDIUM_TASKS: List[Dict[str, Any]] = [ "todos": "- Fix crash in background refresh (Sentry #1204)\n- Prepare submission for iOS 18 compatibility update\n- Review accessibility audit findings", "messages": "- App Store Connect: 'v6.2.1 rejected — missing privacy manifest for tracking'\n- Sentry: '#1204 crash rate increased to 0.8% (was 0.2%)'\n- Apple developer news: 'Privacy manifest deadline extended to Oct 31'", "news_interests": "iOS development, SwiftUI", - "must_mention": ["v6.2.1 rejected", "privacy manifest deadline Oct 31", "Sentry #1204 crash", "accessibility audit", "SwiftUI migration"], - "priority_order": ["v6.2.1 rejected", "Sentry #1204 crash", "privacy manifest deadline"], + "must_mention": [ + "v6.2.1 rejected", + "privacy manifest deadline Oct 31", + "Sentry #1204 crash", + "accessibility audit", + "SwiftUI migration", + ], + "priority_order": [ + "v6.2.1 rejected", + "Sentry #1204 crash", + "privacy manifest deadline", + ], }, { "role": "VP of Engineering", @@ -247,7 +329,13 @@ _MEDIUM_TASKS: List[Dict[str, Any]] = [ "todos": "- Finalize engineering section of board deck\n- Approve Q1 hiring plan (8 headcount)\n- Review SOC 2 Type II readiness", "messages": "- CEO: 'Board meeting moved to Oct 20, deck due Oct 17'\n- HR: 'Director candidate strong — competing offer from FAANG, need to move fast'\n- CTO: 'SOC 2 auditor found 2 medium findings, remediation needed'", "news_interests": "engineering leadership, scaling teams", - "must_mention": ["board deck due Oct 17", "director candidate competing offer", "SOC 2 findings", "Q1 hiring plan", "engineering all-hands"], + "must_mention": [ + "board deck due Oct 17", + "director candidate competing offer", + "SOC 2 findings", + "Q1 hiring plan", + "engineering all-hands", + ], "priority_order": ["director candidate", "board deck Oct 17", "SOC 2 findings"], }, { @@ -258,8 +346,18 @@ _MEDIUM_TASKS: List[Dict[str, Any]] = [ "todos": "- Fix video player buffering on slow connections\n- Implement quiz auto-save feature\n- Update staging with latest migrations", "messages": "- Support: '15 teachers reported video playback issues today'\n- PM: 'Quiz module launch is hard deadline — Nov 15'\n- Slack from DevOps: 'Staging DB needs migration, currently 3 behind'", "news_interests": "video streaming, education technology", - "must_mention": ["video playback issues", "quiz module Nov 15", "staging migrations behind", "feature demo", "tech debt review"], - "priority_order": ["video playback issues", "quiz module Nov 15", "staging migrations"], + "must_mention": [ + "video playback issues", + "quiz module Nov 15", + "staging migrations behind", + "feature demo", + "tech debt review", + ], + "priority_order": [ + "video playback issues", + "quiz module Nov 15", + "staging migrations", + ], }, { "role": "Infrastructure Engineer", @@ -269,8 +367,18 @@ _MEDIUM_TASKS: List[Dict[str, Any]] = [ "todos": "- Prepare DR test runbook for Q4 drill\n- Migrate legacy monitoring to Grafana\n- Review firewall rule change requests", "messages": "- AWS: 'Scheduled maintenance us-east-1: Oct 25, 02:00-06:00 UTC'\n- Compliance: 'Annual DR test must complete before Dec 31'\n- Slack from SRE: 'Grafana migration 60% complete, need help with alert rules'", "news_interests": "cloud infrastructure, disaster recovery", - "must_mention": ["AWS maintenance Oct 25", "DR test before Dec 31", "Grafana migration 60%", "firewall rule review", "change advisory board"], - "priority_order": ["AWS maintenance Oct 25", "DR test planning", "Grafana migration"], + "must_mention": [ + "AWS maintenance Oct 25", + "DR test before Dec 31", + "Grafana migration 60%", + "firewall rule review", + "change advisory board", + ], + "priority_order": [ + "AWS maintenance Oct 25", + "DR test planning", + "Grafana migration", + ], }, ] @@ -287,8 +395,21 @@ _HARD_TASKS: List[Dict[str, Any]] = [ "todos": "- Complete post-mortem document for payment outage (5h downtime, $2.3M impact)\n- Implement circuit breaker for payment gateway\n- Review and merge hotfix for connection pool exhaustion\n- Update incident response playbook with lessons learned", "messages": "- PagerDuty: 'CRITICAL — Payment success rate dropped to 45% at 02:14, recovered at 07:30'\n- CEO: 'Need customer communication plan by noon'\n- VP Eng: 'Board wants root cause by Wednesday'\n- Slack from payments team: 'Connection pool fix in PR #891, needs urgent review'\n- Datadog: '3 new anomaly alerts on order-service since 08:00'\n- Customer success: '47 enterprise customers opened tickets about Friday outage'", "news_interests": "SRE, payment systems, incident management", - "must_mention": ["payment outage $2.3M", "PR #891 connection pool", "customer communication noon", "board root cause Wednesday", "new anomaly alerts", "47 enterprise tickets", "post-mortem"], - "priority_order": ["customer communication noon", "PR #891 connection pool", "anomaly alerts", "board root cause Wednesday"], + "must_mention": [ + "payment outage $2.3M", + "PR #891 connection pool", + "customer communication noon", + "board root cause Wednesday", + "new anomaly alerts", + "47 enterprise tickets", + "post-mortem", + ], + "priority_order": [ + "customer communication noon", + "PR #891 connection pool", + "anomaly alerts", + "board root cause Wednesday", + ], }, { "role": "CISO", @@ -298,8 +419,21 @@ _HARD_TASKS: List[Dict[str, Any]] = [ "todos": "- Coordinate response to detected data exfiltration attempt\n- Prepare board briefing on security incident\n- Review forensic analysis from IR team\n- Determine notification obligations under GDPR/CCPA", "messages": "- SOC: 'Detected unusual data transfer — 2.3GB to external IP from staging-db-02 at 01:47'\n- IR team: 'Attack vector identified: compromised service account via phished credentials'\n- Legal: 'If PII confirmed, 72h GDPR notification clock starts now'\n- CEO: 'No external communication until legal approves messaging'\n- AWS GuardDuty: 'UnauthorizedAccess:IAMUser/MaliciousIPCaller on staging account'\n- HR: 'Phished employee identified, account disabled'", "news_interests": "cybersecurity, data breach response, compliance", - "must_mention": ["2.3GB exfiltration", "phished credentials", "GDPR 72h notification", "board briefing", "legal messaging approval", "GuardDuty alert", "compromised service account"], - "priority_order": ["exfiltration incident", "GDPR notification", "board briefing", "legal approval"], + "must_mention": [ + "2.3GB exfiltration", + "phished credentials", + "GDPR 72h notification", + "board briefing", + "legal messaging approval", + "GuardDuty alert", + "compromised service account", + ], + "priority_order": [ + "exfiltration incident", + "GDPR notification", + "board briefing", + "legal approval", + ], }, { "role": "Engineering Director", @@ -309,8 +443,20 @@ _HARD_TASKS: List[Dict[str, Any]] = [ "todos": "- Resolve broken Salesforce integration (blocking $4M deal)\n- Prepare Q1 roadmap proposal for exec review\n- Address team morale issues in Platform team (3 resignations in 2 months)\n- Review compensation adjustments for retention", "messages": "- Salesforce partner: 'API breaking change in v58 — our integration returns 400s since midnight'\n- Sales VP: '$4M Acme deal closes Friday, integration must work by Thursday EOD'\n- HR: 'Exit interview themes: lack of growth, below-market comp'\n- CTO: 'Q1 roadmap proposal due Friday, include AI strategy'\n- Slack from platform-lead: 'Two more engineers gave notice yesterday'\n- GitHub: 'Salesforce integration fix in draft PR, needs API key rotation'", "news_interests": "engineering management, API integrations", - "must_mention": ["Salesforce API breaking change", "$4M deal Thursday deadline", "platform team resignations", "Q1 roadmap Friday", "compensation adjustments", "API key rotation"], - "priority_order": ["Salesforce integration", "$4M deal deadline", "platform team resignations", "Q1 roadmap"], + "must_mention": [ + "Salesforce API breaking change", + "$4M deal Thursday deadline", + "platform team resignations", + "Q1 roadmap Friday", + "compensation adjustments", + "API key rotation", + ], + "priority_order": [ + "Salesforce integration", + "$4M deal deadline", + "platform team resignations", + "Q1 roadmap", + ], }, { "role": "Lead SRE", @@ -320,8 +466,21 @@ _HARD_TASKS: List[Dict[str, Any]] = [ "todos": "- Coordinate failover to backup CDN\n- Estimate customer impact (MAU affected, SLA credits)\n- Prepare status page communications\n- Document timeline for incident report", "messages": "- CDN provider: 'Global edge outage affecting EU and APAC, ETA 4h for full recovery'\n- Monitoring: 'Video start failures at 78% (normal: 2%), 3.2M users affected'\n- Customer success: 'ESPN, Netflix, Disney+ all escalating — SLA breach imminent'\n- CEO: 'Status page must update every 30 minutes'\n- Finance: 'Estimated SLA credits: $850K if not resolved by noon'\n- Engineering: 'Backup CDN can handle 60% of traffic, degraded quality'", "news_interests": "CDN, video streaming, incident management", - "must_mention": ["CDN outage EU APAC", "3.2M users affected", "SLA credits $850K", "backup CDN 60%", "status page updates 30min", "ETA 4h recovery", "ESPN Netflix Disney escalation"], - "priority_order": ["CDN failover", "status page updates", "customer escalations", "SLA credit estimate"], + "must_mention": [ + "CDN outage EU APAC", + "3.2M users affected", + "SLA credits $850K", + "backup CDN 60%", + "status page updates 30min", + "ETA 4h recovery", + "ESPN Netflix Disney escalation", + ], + "priority_order": [ + "CDN failover", + "status page updates", + "customer escalations", + "SLA credit estimate", + ], }, { "role": "CTO", @@ -331,8 +490,20 @@ _HARD_TASKS: List[Dict[str, Any]] = [ "todos": "- Address GPU cost overrun ($180K over budget this quarter)\n- Prepare technical due diligence materials for Series B\n- Evaluate build vs buy for vector database\n- Respond to acquisition interest from BigTech", "messages": "- CFO: 'GPU costs 40% over budget — need cost reduction plan by Monday'\n- Lead investor: 'Series B term sheet ready, need tech DD materials by next Friday'\n- BigTech BD: 'Acquisition conversation request — CEO says explore quietly'\n- AWS: 'Reserved instance commitment expires Nov 30'\n- ML team: 'New model achieves SOTA on benchmark but requires 2x GPU'\n- VP Eng: 'Team burnout is real — 60h average weeks for past month'", "news_interests": "AI infrastructure, venture capital, GPU optimization", - "must_mention": ["GPU cost overrun $180K", "Series B DD materials", "acquisition interest", "RI expiration Nov 30", "SOTA model 2x GPU", "team burnout 60h weeks"], - "priority_order": ["GPU cost reduction Monday", "Series B DD", "acquisition exploration", "team burnout"], + "must_mention": [ + "GPU cost overrun $180K", + "Series B DD materials", + "acquisition interest", + "RI expiration Nov 30", + "SOTA model 2x GPU", + "team burnout 60h weeks", + ], + "priority_order": [ + "GPU cost reduction Monday", + "Series B DD", + "acquisition exploration", + "team burnout", + ], }, { "role": "Principal Engineer", @@ -342,8 +513,20 @@ _HARD_TASKS: List[Dict[str, Any]] = [ "todos": "- Investigate 50ms latency spike in order matching engine\n- Review SEC Rule 15c3-5 compliance for new algorithm\n- Design circuit breaker for third-party market data feeds\n- Prepare tech brief on microsecond timestamping upgrade", "messages": "- Trading desk: 'Order matching 50ms slower since Friday deploy — losing $40K/day in slippage'\n- Compliance: 'SEC audit next month, need algorithm documentation by Nov 20'\n- Vendor: 'Market data feed v3 deprecating Dec 15, migration required'\n- CTO: 'Board approved microsecond upgrade budget, need timeline'\n- Monitoring: 'Memory leak detected in risk engine — growing 50MB/hour'\n- DevOps: 'Friday deploy rollback available but needs 2h maintenance window'", "news_interests": "low-latency systems, financial technology", - "must_mention": ["order matching latency $40K/day", "SEC audit Nov 20", "market data feed migration Dec 15", "memory leak risk engine", "deploy rollback option", "microsecond upgrade timeline"], - "priority_order": ["order matching latency", "memory leak risk engine", "SEC audit documentation", "market data migration"], + "must_mention": [ + "order matching latency $40K/day", + "SEC audit Nov 20", + "market data feed migration Dec 15", + "memory leak risk engine", + "deploy rollback option", + "microsecond upgrade timeline", + ], + "priority_order": [ + "order matching latency", + "memory leak risk engine", + "SEC audit documentation", + "market data migration", + ], }, { "role": "VP Platform", @@ -353,8 +536,21 @@ _HARD_TASKS: List[Dict[str, Any]] = [ "todos": "- Coordinate DDoS mitigation (attack started 22:00 last night)\n- Prepare customer impact assessment\n- Evaluate WAF vendor proposals (Cloudflare vs Akamai)\n- Draft risk committee presentation", "messages": "- NOC: 'DDoS attack ongoing — 340Gbps, Cloudflare mitigating 95% but 5% getting through'\n- Fortune 500 client: 'Our SLA guarantees 99.99%, currently at 99.2% for November'\n- Legal: 'Three customers sent breach of contract notices'\n- Cloudflare TAM: 'Can upgrade to Enterprise+ with 1Tbps mitigation, $50K/month'\n- CFO: 'Board wants cyber insurance claim assessment'\n- CISO: 'Attack signature matches known botnet, FBI notified'", "news_interests": "DDoS mitigation, cloud security, SaaS operations", - "must_mention": ["DDoS 340Gbps", "Fortune 500 SLA breach", "breach of contract notices", "Cloudflare upgrade $50K", "FBI notified", "cyber insurance claim", "risk committee prep"], - "priority_order": ["DDoS mitigation", "Fortune 500 SLA", "breach of contract", "cyber insurance"], + "must_mention": [ + "DDoS 340Gbps", + "Fortune 500 SLA breach", + "breach of contract notices", + "Cloudflare upgrade $50K", + "FBI notified", + "cyber insurance claim", + "risk committee prep", + ], + "priority_order": [ + "DDoS mitigation", + "Fortune 500 SLA", + "breach of contract", + "cyber insurance", + ], }, { "role": "Head of ML", @@ -364,8 +560,20 @@ _HARD_TASKS: List[Dict[str, Any]] = [ "todos": "- Investigate false negative in pedestrian detection model\n- Prepare safety data package for NHTSA\n- Review validation results for model v4.2\n- Coordinate OTA update for affected vehicles", "messages": "- Safety team: 'Vehicle #4892 near-miss incident — pedestrian not detected at dusk'\n- NHTSA: 'Requesting incident data within 48 hours per Standing General Order'\n- ML validation: 'v4.2 shows 0.3% regression in low-light pedestrian detection'\n- Legal: 'Do not communicate externally until safety review complete'\n- Fleet ops: '12,000 vehicles on v4.1, OTA update requires 72h rollout'\n- PR: 'Reuters reporter asking about safety incident reports'", "news_interests": "autonomous vehicles, ML safety, computer vision", - "must_mention": ["pedestrian detection near-miss", "NHTSA 48h data request", "v4.2 low-light regression", "12000 vehicles OTA", "Reuters inquiry", "legal communication hold"], - "priority_order": ["NHTSA data request", "pedestrian detection investigation", "OTA update planning", "legal hold"], + "must_mention": [ + "pedestrian detection near-miss", + "NHTSA 48h data request", + "v4.2 low-light regression", + "12000 vehicles OTA", + "Reuters inquiry", + "legal communication hold", + ], + "priority_order": [ + "NHTSA data request", + "pedestrian detection investigation", + "OTA update planning", + "legal hold", + ], }, { "role": "VP Engineering", @@ -375,8 +583,21 @@ _HARD_TASKS: List[Dict[str, Any]] = [ "todos": "- Resolve core banking system freeze (customer transactions blocked)\n- Prepare OCC examination response materials\n- Coordinate Oracle emergency support (database deadlock)\n- Assess customer impact and communication plan", "messages": "- NOC: 'Core banking frozen at 06:15 — Oracle RAC deadlock, 450K transactions queued'\n- OCC examiner: 'Examination starts Monday, technology resilience focus area'\n- Customer ops: '2,300 customers called about failed transactions'\n- Oracle support: 'Severity 1 case opened, engineer assigned ETA 2h'\n- CFO: 'Regulatory fine risk if transactions not processed by EOD'\n- CISO: 'No security incident indicated, pure technology failure'", "news_interests": "banking technology, database systems, regulatory compliance", - "must_mention": ["core banking freeze", "450K queued transactions", "OCC examination Monday", "Oracle RAC deadlock", "2300 customer calls", "regulatory fine risk EOD", "Oracle engineer ETA 2h"], - "priority_order": ["Oracle deadlock resolution", "regulatory fine risk", "customer communication", "OCC examination prep"], + "must_mention": [ + "core banking freeze", + "450K queued transactions", + "OCC examination Monday", + "Oracle RAC deadlock", + "2300 customer calls", + "regulatory fine risk EOD", + "Oracle engineer ETA 2h", + ], + "priority_order": [ + "Oracle deadlock resolution", + "regulatory fine risk", + "customer communication", + "OCC examination prep", + ], }, { "role": "Director of Engineering", @@ -386,8 +607,20 @@ _HARD_TASKS: List[Dict[str, Any]] = [ "todos": "- Finalize FDA 510(k) pre-submission package\n- Review clinical trial results for diagnostic AI accuracy\n- Address model drift detected in production deployment\n- Prepare for Monday's IRB ethics review", "messages": "- FDA liaison: 'Pre-submission meeting confirmed Nov 21, final materials due Nov 18'\n- Clinical team: 'Trial results: 94.2% sensitivity, 91.8% specificity — below 95% target for sensitivity'\n- ML ops: 'Model drift alert — AUC dropped from 0.96 to 0.89 over 30 days'\n- Legal: 'IRB flagged consent form language, revision needed before Monday'\n- Research lead: 'Promising approach to improve sensitivity — needs 2 weeks'\n- Quality: 'CAPA-2847 open: production model version mismatch between regions'", "news_interests": "medical AI, FDA regulation, clinical trials", - "must_mention": ["FDA materials due Nov 18", "sensitivity below target 94.2%", "model drift AUC 0.89", "IRB consent revision Monday", "CAPA-2847 version mismatch", "2-week sensitivity improvement"], - "priority_order": ["model drift", "IRB consent revision", "FDA materials", "sensitivity improvement decision"], + "must_mention": [ + "FDA materials due Nov 18", + "sensitivity below target 94.2%", + "model drift AUC 0.89", + "IRB consent revision Monday", + "CAPA-2847 version mismatch", + "2-week sensitivity improvement", + ], + "priority_order": [ + "model drift", + "IRB consent revision", + "FDA materials", + "sensitivity improvement decision", + ], }, ] @@ -429,19 +662,21 @@ class DailyDigestDataset(DatasetProvider): news_interests=task["news_interests"], ) - self._records.append(EvalRecord( - record_id=f"daily-digest-{idx:03d}", - problem=prompt, - reference="; ".join(task["must_mention"]), - category="agentic", - subject=diff, - metadata={ - "role": task["role"], - "company": task["company"], - "must_mention": task["must_mention"], - "priority_order": task["priority_order"], - }, - )) + self._records.append( + EvalRecord( + record_id=f"daily-digest-{idx:03d}", + problem=prompt, + reference="; ".join(task["must_mention"]), + category="agentic", + subject=diff, + metadata={ + "role": task["role"], + "company": task["company"], + "must_mention": task["must_mention"], + "priority_order": task["priority_order"], + }, + ) + ) def iter_records(self) -> Iterable[EvalRecord]: return iter(self._records) diff --git a/src/openjarvis/evals/datasets/deepplanning.py b/src/openjarvis/evals/datasets/deepplanning.py index 6ab38dab..03e2dd2d 100644 --- a/src/openjarvis/evals/datasets/deepplanning.py +++ b/src/openjarvis/evals/datasets/deepplanning.py @@ -110,15 +110,16 @@ class DeepPlanningDataset(DatasetProvider): from datasets import load_dataset except ImportError: raise ImportError( - "datasets required for DeepPlanning. " - "Install with: pip install datasets" + "datasets required for DeepPlanning. Install with: pip install datasets" ) logger.info("Downloading Qwen/DeepPlanning from HuggingFace...") # Loading triggers the download even though we don't use the result load_dataset("Qwen/DeepPlanning", split="train") def _extract_cases( - self, tar_path: Path, level: int, + self, + tar_path: Path, + level: int, ) -> List[EvalRecord]: """Extract shopping cases from a tar.gz archive.""" records: List[EvalRecord] = [] @@ -152,7 +153,10 @@ class DeepPlanningDataset(DatasetProvider): products_text = pf.read().decode("utf-8").strip() rec = self._case_to_record( - data, level, case_name, products_text, + data, + level, + case_name, + products_text, ) if rec: records.append(rec) @@ -185,15 +189,9 @@ class DeepPlanningDataset(DatasetProvider): reference = "; ".join(ref_parts) if ref_parts else "" # Count constraints - n_constraints = sum( - len(m.get("features", [])) for m in meta_info - ) + n_constraints = sum(len(m.get("features", [])) for m in meta_info) - difficulty = ( - "easy" if level == 1 - else "medium" if level == 2 - else "hard" - ) + difficulty = "easy" if level == 1 else "medium" if level == 2 else "hard" # Build product catalog section catalog_section = "" diff --git a/src/openjarvis/evals/datasets/doc_qa.py b/src/openjarvis/evals/datasets/doc_qa.py index ffd97021..407a6fd8 100644 --- a/src/openjarvis/evals/datasets/doc_qa.py +++ b/src/openjarvis/evals/datasets/doc_qa.py @@ -36,21 +36,42 @@ _EASY_TASKS: List[Dict[str, Any]] = [ { "question": "What is the purpose of PostgreSQL's VACUUM command?", "documents": [ - {"title": "PostgreSQL: VACUUM", "content": "VACUUM reclaims storage occupied by dead tuples. In normal PostgreSQL operation, tuples that are deleted or obsoleted by an update are not physically removed from their table; they remain present until a VACUUM is done. Therefore it's necessary to do VACUUM periodically, especially on frequently-updated tables."}, - {"title": "PostgreSQL: CREATE INDEX", "content": "CREATE INDEX constructs an index on the specified column(s) of the specified relation. Indexes are primarily used to enhance database performance. An index defined on a table column that is part of a join condition can also significantly speed up queries with joins."}, - {"title": "PostgreSQL: EXPLAIN", "content": "EXPLAIN displays the execution plan that the PostgreSQL planner generates for the supplied statement. The execution plan shows how the table(s) referenced by the statement will be scanned."}, + { + "title": "PostgreSQL: VACUUM", + "content": "VACUUM reclaims storage occupied by dead tuples. In normal PostgreSQL operation, tuples that are deleted or obsoleted by an update are not physically removed from their table; they remain present until a VACUUM is done. Therefore it's necessary to do VACUUM periodically, especially on frequently-updated tables.", + }, + { + "title": "PostgreSQL: CREATE INDEX", + "content": "CREATE INDEX constructs an index on the specified column(s) of the specified relation. Indexes are primarily used to enhance database performance. An index defined on a table column that is part of a join condition can also significantly speed up queries with joins.", + }, + { + "title": "PostgreSQL: EXPLAIN", + "content": "EXPLAIN displays the execution plan that the PostgreSQL planner generates for the supplied statement. The execution plan shows how the table(s) referenced by the statement will be scanned.", + }, ], "required_facts": [ {"fact": "reclaims storage from dead tuples", "source_doc_index": 0}, - {"fact": "deleted or updated tuples remain until VACUUM", "source_doc_index": 0}, + { + "fact": "deleted or updated tuples remain until VACUUM", + "source_doc_index": 0, + }, ], }, { "question": "How does Redis persistence work with RDB snapshots?", "documents": [ - {"title": "Redis: RDB Persistence", "content": "RDB persistence performs point-in-time snapshots of your dataset at specified intervals. Redis forks a child process to write the dataset to a temporary file on disk, then atomically replaces the old file. This allows Redis to benefit from copy-on-write semantics. RDB files are compact and perfect for backups."}, - {"title": "Redis: AOF Persistence", "content": "AOF persistence logs every write operation received by the server. These operations can then be replayed again at server startup, reconstructing the original dataset. Commands are logged using the same format as the Redis protocol itself."}, - {"title": "Redis: Memory Management", "content": "Redis manages memory using a configurable maxmemory directive. When the limit is reached, Redis applies an eviction policy to remove keys. Available policies include volatile-lru, allkeys-lru, volatile-random, allkeys-random, and noeviction."}, + { + "title": "Redis: RDB Persistence", + "content": "RDB persistence performs point-in-time snapshots of your dataset at specified intervals. Redis forks a child process to write the dataset to a temporary file on disk, then atomically replaces the old file. This allows Redis to benefit from copy-on-write semantics. RDB files are compact and perfect for backups.", + }, + { + "title": "Redis: AOF Persistence", + "content": "AOF persistence logs every write operation received by the server. These operations can then be replayed again at server startup, reconstructing the original dataset. Commands are logged using the same format as the Redis protocol itself.", + }, + { + "title": "Redis: Memory Management", + "content": "Redis manages memory using a configurable maxmemory directive. When the limit is reached, Redis applies an eviction policy to remove keys. Available policies include volatile-lru, allkeys-lru, volatile-random, allkeys-random, and noeviction.", + }, ], "required_facts": [ {"fact": "point-in-time snapshots at intervals", "source_doc_index": 0}, @@ -61,9 +82,18 @@ _EASY_TASKS: List[Dict[str, Any]] = [ { "question": "What are Kubernetes network policies used for?", "documents": [ - {"title": "K8s: Network Policies", "content": "NetworkPolicies are an application-centric construct which allow you to specify how a pod is allowed to communicate with various network entities. The entities that a Pod can communicate with are identified through a combination of namespaceSelector, podSelector, and ipBlock. By default, pods are non-isolated and accept traffic from any source."}, - {"title": "K8s: Services", "content": "A Service is an abstract way to expose an application running on a set of Pods as a network service. Kubernetes gives Pods their own IP addresses and a single DNS name for a set of Pods, and can load-balance across them."}, - {"title": "K8s: Ingress", "content": "Ingress exposes HTTP and HTTPS routes from outside the cluster to services within the cluster. Traffic routing is controlled by rules defined on the Ingress resource."}, + { + "title": "K8s: Network Policies", + "content": "NetworkPolicies are an application-centric construct which allow you to specify how a pod is allowed to communicate with various network entities. The entities that a Pod can communicate with are identified through a combination of namespaceSelector, podSelector, and ipBlock. By default, pods are non-isolated and accept traffic from any source.", + }, + { + "title": "K8s: Services", + "content": "A Service is an abstract way to expose an application running on a set of Pods as a network service. Kubernetes gives Pods their own IP addresses and a single DNS name for a set of Pods, and can load-balance across them.", + }, + { + "title": "K8s: Ingress", + "content": "Ingress exposes HTTP and HTTPS routes from outside the cluster to services within the cluster. Traffic routing is controlled by rules defined on the Ingress resource.", + }, ], "required_facts": [ {"fact": "specify how pods communicate", "source_doc_index": 0}, @@ -74,9 +104,18 @@ _EASY_TASKS: List[Dict[str, Any]] = [ { "question": "How do FastAPI dependencies work?", "documents": [ - {"title": "FastAPI: Dependencies", "content": "FastAPI has a very powerful but intuitive Dependency Injection system. It is designed to be very simple to use, and to make it very easy for any developer to integrate other components with FastAPI. Dependencies can be declared as function parameters using Depends(). FastAPI will call the dependency function, get the result, and pass it to your function."}, - {"title": "FastAPI: Path Operations", "content": "You can declare path operation functions with Python type hints. FastAPI uses these type hints to validate data, serialize output, and generate documentation automatically."}, - {"title": "FastAPI: Middleware", "content": "Middleware is a function that works with every request before it is processed by any specific path operation. It also handles every response before returning it. You add middleware using app.add_middleware()."}, + { + "title": "FastAPI: Dependencies", + "content": "FastAPI has a very powerful but intuitive Dependency Injection system. It is designed to be very simple to use, and to make it very easy for any developer to integrate other components with FastAPI. Dependencies can be declared as function parameters using Depends(). FastAPI will call the dependency function, get the result, and pass it to your function.", + }, + { + "title": "FastAPI: Path Operations", + "content": "You can declare path operation functions with Python type hints. FastAPI uses these type hints to validate data, serialize output, and generate documentation automatically.", + }, + { + "title": "FastAPI: Middleware", + "content": "Middleware is a function that works with every request before it is processed by any specific path operation. It also handles every response before returning it. You add middleware using app.add_middleware().", + }, ], "required_facts": [ {"fact": "dependency injection system", "source_doc_index": 0}, @@ -86,9 +125,18 @@ _EASY_TASKS: List[Dict[str, Any]] = [ { "question": "What is Python's pathlib module used for?", "documents": [ - {"title": "Python: pathlib", "content": "The pathlib module offers classes representing filesystem paths with semantics appropriate for different operating systems. Path classes are divided between pure paths, which provide purely computational operations without I/O, and concrete paths, which inherit from pure paths but also provide I/O operations. Path objects can be used with os.scandir() and other functions expecting string paths."}, - {"title": "Python: os.path", "content": "The os.path module implements some useful functions on pathnames. This module is always available. Unlike pathlib, os.path operates on strings rather than path objects."}, - {"title": "Python: shutil", "content": "The shutil module offers a number of high-level operations on files and collections of files. In particular, functions are provided which support file copying and removal."}, + { + "title": "Python: pathlib", + "content": "The pathlib module offers classes representing filesystem paths with semantics appropriate for different operating systems. Path classes are divided between pure paths, which provide purely computational operations without I/O, and concrete paths, which inherit from pure paths but also provide I/O operations. Path objects can be used with os.scandir() and other functions expecting string paths.", + }, + { + "title": "Python: os.path", + "content": "The os.path module implements some useful functions on pathnames. This module is always available. Unlike pathlib, os.path operates on strings rather than path objects.", + }, + { + "title": "Python: shutil", + "content": "The shutil module offers a number of high-level operations on files and collections of files. In particular, functions are provided which support file copying and removal.", + }, ], "required_facts": [ {"fact": "classes representing filesystem paths", "source_doc_index": 0}, @@ -98,9 +146,18 @@ _EASY_TASKS: List[Dict[str, Any]] = [ { "question": "How does nginx reverse proxy configuration work?", "documents": [ - {"title": "nginx: Reverse Proxy", "content": "To configure a basic reverse proxy, use the proxy_pass directive inside a location block. For example, proxy_pass http://backend; forwards all requests to the backend server group. nginx can also modify request headers using proxy_set_header, and buffer responses using proxy_buffering."}, - {"title": "nginx: Load Balancing", "content": "nginx can distribute traffic across multiple servers using upstream blocks. Methods include round-robin (default), least_conn (fewest active connections), and ip_hash (session persistence based on client IP)."}, - {"title": "nginx: SSL/TLS", "content": "To enable HTTPS, configure ssl_certificate and ssl_certificate_key directives in the server block. nginx supports TLS 1.2 and 1.3 by default."}, + { + "title": "nginx: Reverse Proxy", + "content": "To configure a basic reverse proxy, use the proxy_pass directive inside a location block. For example, proxy_pass http://backend; forwards all requests to the backend server group. nginx can also modify request headers using proxy_set_header, and buffer responses using proxy_buffering.", + }, + { + "title": "nginx: Load Balancing", + "content": "nginx can distribute traffic across multiple servers using upstream blocks. Methods include round-robin (default), least_conn (fewest active connections), and ip_hash (session persistence based on client IP).", + }, + { + "title": "nginx: SSL/TLS", + "content": "To enable HTTPS, configure ssl_certificate and ssl_certificate_key directives in the server block. nginx supports TLS 1.2 and 1.3 by default.", + }, ], "required_facts": [ {"fact": "proxy_pass directive in location block", "source_doc_index": 0}, @@ -110,9 +167,18 @@ _EASY_TASKS: List[Dict[str, Any]] = [ { "question": "What are Python asyncio tasks?", "documents": [ - {"title": "Python: asyncio Tasks", "content": "Tasks are used to schedule coroutines concurrently. When a coroutine is wrapped into a Task with functions like asyncio.create_task(), the coroutine is automatically scheduled to run soon. Tasks are used to run coroutines in event loops. If a coroutine awaits a Future, the Task suspends the execution of the coroutine and waits for the completion of the Future."}, - {"title": "Python: asyncio Event Loop", "content": "The event loop is the core of every asyncio application. Event loops run asynchronous tasks and callbacks, perform network IO operations, and run subprocesses. Application developers should typically use the high-level asyncio functions, such as asyncio.run()."}, - {"title": "Python: threading", "content": "The threading module constructs higher-level threading interfaces on top of the lower level _thread module. The Thread class represents an activity that is run in a separate thread of control."}, + { + "title": "Python: asyncio Tasks", + "content": "Tasks are used to schedule coroutines concurrently. When a coroutine is wrapped into a Task with functions like asyncio.create_task(), the coroutine is automatically scheduled to run soon. Tasks are used to run coroutines in event loops. If a coroutine awaits a Future, the Task suspends the execution of the coroutine and waits for the completion of the Future.", + }, + { + "title": "Python: asyncio Event Loop", + "content": "The event loop is the core of every asyncio application. Event loops run asynchronous tasks and callbacks, perform network IO operations, and run subprocesses. Application developers should typically use the high-level asyncio functions, such as asyncio.run().", + }, + { + "title": "Python: threading", + "content": "The threading module constructs higher-level threading interfaces on top of the lower level _thread module. The Thread class represents an activity that is run in a separate thread of control.", + }, ], "required_facts": [ {"fact": "schedule coroutines concurrently", "source_doc_index": 0}, @@ -122,22 +188,43 @@ _EASY_TASKS: List[Dict[str, Any]] = [ { "question": "What is Kubernetes RBAC?", "documents": [ - {"title": "K8s: RBAC", "content": "Role-based access control (RBAC) is a method of regulating access to computer or network resources based on the roles of individual users. RBAC uses rbac.authorization.k8s.io API group to drive authorization decisions. A Role sets permissions within a namespace, while a ClusterRole is non-namespaced. RoleBinding binds a Role to subjects (users, groups, or service accounts)."}, - {"title": "K8s: Service Accounts", "content": "A ServiceAccount provides an identity for processes that run in a Pod. When you create a pod, if you do not specify a service account, it is automatically assigned the default service account in the same namespace."}, - {"title": "K8s: Pod Security", "content": "Pod Security Standards define three different policies to broadly cover the security spectrum. These policies are cumulative and range from highly-permissive to highly-restrictive."}, + { + "title": "K8s: RBAC", + "content": "Role-based access control (RBAC) is a method of regulating access to computer or network resources based on the roles of individual users. RBAC uses rbac.authorization.k8s.io API group to drive authorization decisions. A Role sets permissions within a namespace, while a ClusterRole is non-namespaced. RoleBinding binds a Role to subjects (users, groups, or service accounts).", + }, + { + "title": "K8s: Service Accounts", + "content": "A ServiceAccount provides an identity for processes that run in a Pod. When you create a pod, if you do not specify a service account, it is automatically assigned the default service account in the same namespace.", + }, + { + "title": "K8s: Pod Security", + "content": "Pod Security Standards define three different policies to broadly cover the security spectrum. These policies are cumulative and range from highly-permissive to highly-restrictive.", + }, ], "required_facts": [ {"fact": "role-based access control", "source_doc_index": 0}, - {"fact": "Role for namespace ClusterRole for cluster", "source_doc_index": 0}, + { + "fact": "Role for namespace ClusterRole for cluster", + "source_doc_index": 0, + }, {"fact": "RoleBinding binds to subjects", "source_doc_index": 0}, ], }, { "question": "How does Redis clustering work?", "documents": [ - {"title": "Redis: Cluster", "content": "Redis Cluster provides a way to run a Redis installation where data is automatically sharded across multiple Redis nodes. It uses hash slots — there are 16384 hash slots in Redis Cluster. Every node in a Redis Cluster is responsible for a subset of the hash slots. Redis Cluster provides automatic failover when a master node becomes unreachable."}, - {"title": "Redis: Sentinel", "content": "Redis Sentinel provides high availability for Redis. Sentinel monitors Redis instances, notifies administrators of failures, and performs automatic failover. Sentinel is a separate process from the Redis server."}, - {"title": "Redis: Replication", "content": "Redis replication allows replica nodes to be exact copies of master nodes. A replica automatically reconnects to the master every time the link breaks and attempts to be an exact copy of it."}, + { + "title": "Redis: Cluster", + "content": "Redis Cluster provides a way to run a Redis installation where data is automatically sharded across multiple Redis nodes. It uses hash slots — there are 16384 hash slots in Redis Cluster. Every node in a Redis Cluster is responsible for a subset of the hash slots. Redis Cluster provides automatic failover when a master node becomes unreachable.", + }, + { + "title": "Redis: Sentinel", + "content": "Redis Sentinel provides high availability for Redis. Sentinel monitors Redis instances, notifies administrators of failures, and performs automatic failover. Sentinel is a separate process from the Redis server.", + }, + { + "title": "Redis: Replication", + "content": "Redis replication allows replica nodes to be exact copies of master nodes. A replica automatically reconnects to the master every time the link breaks and attempts to be an exact copy of it.", + }, ], "required_facts": [ {"fact": "data sharded across nodes", "source_doc_index": 0}, @@ -148,13 +235,25 @@ _EASY_TASKS: List[Dict[str, Any]] = [ { "question": "What is the Python logging module's basic architecture?", "documents": [ - {"title": "Python: logging", "content": "The logging module provides a flexible framework for emitting log messages from Python programs. The basic classes are Logger, Handler, Filter, and Formatter. Loggers expose the interface that application code directly uses. Handlers send log records to the appropriate destination. Filters provide fine-grained control. Formatters specify the layout of log records in the final output."}, - {"title": "Python: warnings", "content": "Warning messages are typically issued in situations where it is useful to alert the user of some condition in a program. The warnings module provides functions to issue warnings and to filter them."}, - {"title": "Python: traceback", "content": "The traceback module provides a standard interface to extract, format and print stack traces of Python programs. It exactly mimics the behavior of the Python interpreter when it prints a stack trace."}, + { + "title": "Python: logging", + "content": "The logging module provides a flexible framework for emitting log messages from Python programs. The basic classes are Logger, Handler, Filter, and Formatter. Loggers expose the interface that application code directly uses. Handlers send log records to the appropriate destination. Filters provide fine-grained control. Formatters specify the layout of log records in the final output.", + }, + { + "title": "Python: warnings", + "content": "Warning messages are typically issued in situations where it is useful to alert the user of some condition in a program. The warnings module provides functions to issue warnings and to filter them.", + }, + { + "title": "Python: traceback", + "content": "The traceback module provides a standard interface to extract, format and print stack traces of Python programs. It exactly mimics the behavior of the Python interpreter when it prints a stack trace.", + }, ], "required_facts": [ {"fact": "Logger Handler Filter Formatter", "source_doc_index": 0}, - {"fact": "Loggers expose interface for application code", "source_doc_index": 0}, + { + "fact": "Loggers expose interface for application code", + "source_doc_index": 0, + }, ], }, ] @@ -167,26 +266,53 @@ _MEDIUM_TASKS: List[Dict[str, Any]] = [ { "question": "Compare RDB and AOF persistence strategies in Redis. When should you use each?", "documents": [ - {"title": "Redis: RDB Persistence", "content": "RDB persistence performs point-in-time snapshots at specified intervals. It produces compact single-file backups, perfect for disaster recovery. RDB maximizes Redis performance since the only work the Redis parent process needs to do is forking. However, RDB is not good if you need to minimize data loss — you might lose several minutes of data if Redis stops working."}, - {"title": "Redis: AOF Persistence", "content": "AOF logs every write operation. The AOF can be configured to fsync every second (default), every query, or never. With every-second fsync, you lose at most one second of data. The AOF file is append-only, so no seeks or corruption problems. Redis can automatically rewrite the AOF in background when it gets too big."}, - {"title": "Redis: Configuration", "content": "Redis configuration can be set via redis.conf file or CONFIG SET command at runtime. Key persistence settings: save for RDB, appendonly yes/no for AOF, appendfsync always/everysec/no for AOF sync policy."}, - {"title": "Redis: Performance Tuning", "content": "For best performance, disable persistence entirely if data loss is acceptable. If persistence is needed, RDB with infrequent saves offers better throughput than AOF. For critical data, use both RDB and AOF together."}, + { + "title": "Redis: RDB Persistence", + "content": "RDB persistence performs point-in-time snapshots at specified intervals. It produces compact single-file backups, perfect for disaster recovery. RDB maximizes Redis performance since the only work the Redis parent process needs to do is forking. However, RDB is not good if you need to minimize data loss — you might lose several minutes of data if Redis stops working.", + }, + { + "title": "Redis: AOF Persistence", + "content": "AOF logs every write operation. The AOF can be configured to fsync every second (default), every query, or never. With every-second fsync, you lose at most one second of data. The AOF file is append-only, so no seeks or corruption problems. Redis can automatically rewrite the AOF in background when it gets too big.", + }, + { + "title": "Redis: Configuration", + "content": "Redis configuration can be set via redis.conf file or CONFIG SET command at runtime. Key persistence settings: save for RDB, appendonly yes/no for AOF, appendfsync always/everysec/no for AOF sync policy.", + }, + { + "title": "Redis: Performance Tuning", + "content": "For best performance, disable persistence entirely if data loss is acceptable. If persistence is needed, RDB with infrequent saves offers better throughput than AOF. For critical data, use both RDB and AOF together.", + }, ], "required_facts": [ {"fact": "RDB creates point-in-time snapshots", "source_doc_index": 0}, {"fact": "RDB may lose minutes of data", "source_doc_index": 0}, {"fact": "AOF logs every write operation", "source_doc_index": 1}, - {"fact": "AOF loses at most one second with default fsync", "source_doc_index": 1}, + { + "fact": "AOF loses at most one second with default fsync", + "source_doc_index": 1, + }, {"fact": "use both together for critical data", "source_doc_index": 3}, ], }, { "question": "How do Kubernetes Services and Ingress work together to expose applications?", "documents": [ - {"title": "K8s: Services", "content": "A Service is an abstraction that defines a logical set of Pods and a policy to access them. Service types include ClusterIP (internal only), NodePort (exposes on each node's IP), and LoadBalancer (provisions external load balancer). Services use label selectors to identify target pods."}, - {"title": "K8s: Ingress", "content": "Ingress exposes HTTP/HTTPS routes from outside the cluster to Services. An Ingress may be configured to give Services externally-reachable URLs, load balance traffic, terminate SSL/TLS, and offer name-based virtual hosting. You need an Ingress controller to satisfy an Ingress."}, - {"title": "K8s: Ingress Controllers", "content": "An Ingress controller is responsible for fulfilling the Ingress, usually with a load balancer. Popular controllers include NGINX Ingress Controller, Traefik, and HAProxy. Without a controller, Ingress resources have no effect."}, - {"title": "K8s: DNS", "content": "Kubernetes creates DNS records for Services and Pods. A Service named my-svc in namespace my-ns can be reached at my-svc.my-ns.svc.cluster.local. The DNS server watches the Kubernetes API for new Services."}, + { + "title": "K8s: Services", + "content": "A Service is an abstraction that defines a logical set of Pods and a policy to access them. Service types include ClusterIP (internal only), NodePort (exposes on each node's IP), and LoadBalancer (provisions external load balancer). Services use label selectors to identify target pods.", + }, + { + "title": "K8s: Ingress", + "content": "Ingress exposes HTTP/HTTPS routes from outside the cluster to Services. An Ingress may be configured to give Services externally-reachable URLs, load balance traffic, terminate SSL/TLS, and offer name-based virtual hosting. You need an Ingress controller to satisfy an Ingress.", + }, + { + "title": "K8s: Ingress Controllers", + "content": "An Ingress controller is responsible for fulfilling the Ingress, usually with a load balancer. Popular controllers include NGINX Ingress Controller, Traefik, and HAProxy. Without a controller, Ingress resources have no effect.", + }, + { + "title": "K8s: DNS", + "content": "Kubernetes creates DNS records for Services and Pods. A Service named my-svc in namespace my-ns can be reached at my-svc.my-ns.svc.cluster.local. The DNS server watches the Kubernetes API for new Services.", + }, ], "required_facts": [ {"fact": "Service defines logical set of Pods", "source_doc_index": 0}, @@ -198,25 +324,58 @@ _MEDIUM_TASKS: List[Dict[str, Any]] = [ { "question": "Explain PostgreSQL replication: streaming replication vs logical replication.", "documents": [ - {"title": "PostgreSQL: Streaming Replication", "content": "Streaming replication allows a standby server to stay more up-to-date than is possible with file-based log shipping. The standby connects to the primary and receives WAL records as they are generated, without waiting for a WAL file to be filled. Streaming replication is asynchronous by default but can be configured as synchronous."}, - {"title": "PostgreSQL: Logical Replication", "content": "Logical replication uses a publish/subscribe model. The publisher defines publications, and subscribers define subscriptions. It allows fine-grained control over which tables and operations are replicated. Unlike streaming replication, logical replication can replicate across different major versions."}, - {"title": "PostgreSQL: WAL", "content": "Write-Ahead Logging (WAL) is the method PostgreSQL uses to ensure data integrity. Changes are first written to a log before being applied to data files. WAL enables both crash recovery and replication."}, - {"title": "PostgreSQL: Backup", "content": "pg_basebackup makes a binary copy of the database cluster files. It creates a base backup that can be used as a starting point for streaming replication or point-in-time recovery."}, + { + "title": "PostgreSQL: Streaming Replication", + "content": "Streaming replication allows a standby server to stay more up-to-date than is possible with file-based log shipping. The standby connects to the primary and receives WAL records as they are generated, without waiting for a WAL file to be filled. Streaming replication is asynchronous by default but can be configured as synchronous.", + }, + { + "title": "PostgreSQL: Logical Replication", + "content": "Logical replication uses a publish/subscribe model. The publisher defines publications, and subscribers define subscriptions. It allows fine-grained control over which tables and operations are replicated. Unlike streaming replication, logical replication can replicate across different major versions.", + }, + { + "title": "PostgreSQL: WAL", + "content": "Write-Ahead Logging (WAL) is the method PostgreSQL uses to ensure data integrity. Changes are first written to a log before being applied to data files. WAL enables both crash recovery and replication.", + }, + { + "title": "PostgreSQL: Backup", + "content": "pg_basebackup makes a binary copy of the database cluster files. It creates a base backup that can be used as a starting point for streaming replication or point-in-time recovery.", + }, ], "required_facts": [ - {"fact": "streaming replication sends WAL records continuously", "source_doc_index": 0}, - {"fact": "streaming can be synchronous or asynchronous", "source_doc_index": 0}, + { + "fact": "streaming replication sends WAL records continuously", + "source_doc_index": 0, + }, + { + "fact": "streaming can be synchronous or asynchronous", + "source_doc_index": 0, + }, {"fact": "logical uses publish subscribe model", "source_doc_index": 1}, - {"fact": "logical can replicate across major versions", "source_doc_index": 1}, + { + "fact": "logical can replicate across major versions", + "source_doc_index": 1, + }, ], }, { "question": "How do FastAPI middleware and dependencies differ? When should you use each?", "documents": [ - {"title": "FastAPI: Middleware", "content": "Middleware processes every request before it reaches path operations, and every response before it is sent back. Middleware functions receive the request and a call_next function. Common uses: CORS headers, request timing, authentication checks on all routes. Middleware runs for ALL requests."}, - {"title": "FastAPI: Dependencies", "content": "Dependencies are declared per-path-operation or per-router using Depends(). They can have their own dependencies (sub-dependencies). Dependencies can yield values (for cleanup), raise exceptions, and access the request. They run only for the specific endpoints that declare them."}, - {"title": "FastAPI: Security", "content": "FastAPI provides security utilities built on top of dependencies. OAuth2PasswordBearer is a dependency that extracts a token from the Authorization header. You can combine multiple security schemes using dependency injection."}, - {"title": "FastAPI: Background Tasks", "content": "You can define background tasks to run after returning a response. BackgroundTasks can be declared as a parameter in path operation functions or dependencies."}, + { + "title": "FastAPI: Middleware", + "content": "Middleware processes every request before it reaches path operations, and every response before it is sent back. Middleware functions receive the request and a call_next function. Common uses: CORS headers, request timing, authentication checks on all routes. Middleware runs for ALL requests.", + }, + { + "title": "FastAPI: Dependencies", + "content": "Dependencies are declared per-path-operation or per-router using Depends(). They can have their own dependencies (sub-dependencies). Dependencies can yield values (for cleanup), raise exceptions, and access the request. They run only for the specific endpoints that declare them.", + }, + { + "title": "FastAPI: Security", + "content": "FastAPI provides security utilities built on top of dependencies. OAuth2PasswordBearer is a dependency that extracts a token from the Authorization header. You can combine multiple security schemes using dependency injection.", + }, + { + "title": "FastAPI: Background Tasks", + "content": "You can define background tasks to run after returning a response. BackgroundTasks can be declared as a parameter in path operation functions or dependencies.", + }, ], "required_facts": [ {"fact": "middleware runs for all requests", "source_doc_index": 0}, @@ -228,43 +387,103 @@ _MEDIUM_TASKS: List[Dict[str, Any]] = [ { "question": "How do Python asyncio event loops and tasks coordinate concurrent execution?", "documents": [ - {"title": "Python: asyncio Event Loop", "content": "The event loop runs asynchronous tasks and callbacks, performs network IO operations, and runs subprocesses. asyncio.run() creates a new event loop, runs the given coroutine, and closes the loop. Only one event loop can run per thread. The event loop uses cooperative multitasking — tasks must yield control voluntarily."}, - {"title": "Python: asyncio Tasks", "content": "Tasks wrap coroutines and schedule them on the event loop. asyncio.create_task() schedules a coroutine for concurrent execution. asyncio.gather() runs multiple awaitables concurrently and returns results when all complete. Tasks can be cancelled using task.cancel()."}, - {"title": "Python: asyncio Synchronization", "content": "asyncio provides Lock, Event, Condition, and Semaphore for coordinating concurrent tasks. Unlike threading primitives, these are not thread-safe and must be used within the same event loop."}, - {"title": "Python: concurrent.futures", "content": "The concurrent.futures module provides ThreadPoolExecutor and ProcessPoolExecutor. asyncio can integrate with these using loop.run_in_executor() to run blocking functions without blocking the event loop."}, + { + "title": "Python: asyncio Event Loop", + "content": "The event loop runs asynchronous tasks and callbacks, performs network IO operations, and runs subprocesses. asyncio.run() creates a new event loop, runs the given coroutine, and closes the loop. Only one event loop can run per thread. The event loop uses cooperative multitasking — tasks must yield control voluntarily.", + }, + { + "title": "Python: asyncio Tasks", + "content": "Tasks wrap coroutines and schedule them on the event loop. asyncio.create_task() schedules a coroutine for concurrent execution. asyncio.gather() runs multiple awaitables concurrently and returns results when all complete. Tasks can be cancelled using task.cancel().", + }, + { + "title": "Python: asyncio Synchronization", + "content": "asyncio provides Lock, Event, Condition, and Semaphore for coordinating concurrent tasks. Unlike threading primitives, these are not thread-safe and must be used within the same event loop.", + }, + { + "title": "Python: concurrent.futures", + "content": "The concurrent.futures module provides ThreadPoolExecutor and ProcessPoolExecutor. asyncio can integrate with these using loop.run_in_executor() to run blocking functions without blocking the event loop.", + }, ], "required_facts": [ - {"fact": "cooperative multitasking tasks yield control", "source_doc_index": 0}, - {"fact": "create_task schedules concurrent execution", "source_doc_index": 1}, - {"fact": "gather runs multiple awaitables concurrently", "source_doc_index": 1}, + { + "fact": "cooperative multitasking tasks yield control", + "source_doc_index": 0, + }, + { + "fact": "create_task schedules concurrent execution", + "source_doc_index": 1, + }, + { + "fact": "gather runs multiple awaitables concurrently", + "source_doc_index": 1, + }, {"fact": "run_in_executor for blocking functions", "source_doc_index": 3}, ], }, { "question": "How does nginx load balancing interact with SSL termination?", "documents": [ - {"title": "nginx: Load Balancing", "content": "nginx distributes traffic using upstream blocks with methods: round-robin (default), least_conn, ip_hash, and hash. Health checks can detect failed backends. The max_fails and fail_timeout parameters control when a server is marked unavailable."}, - {"title": "nginx: SSL/TLS Termination", "content": "SSL termination at nginx means nginx handles TLS handshakes and decryption, then forwards plain HTTP to backend servers. This offloads CPU-intensive cryptographic operations from application servers. Configure with ssl_certificate and ssl_certificate_key in the server block."}, - {"title": "nginx: Proxy Settings", "content": "proxy_set_header X-Forwarded-Proto $scheme passes the original protocol to backends. proxy_set_header X-Real-IP $remote_addr passes the client's real IP. These headers are important when SSL is terminated at the proxy layer."}, - {"title": "nginx: HTTP/2", "content": "nginx supports HTTP/2 with the http2 parameter on the listen directive. HTTP/2 enables multiplexing, header compression, and server push. HTTP/2 requires TLS in most browsers."}, + { + "title": "nginx: Load Balancing", + "content": "nginx distributes traffic using upstream blocks with methods: round-robin (default), least_conn, ip_hash, and hash. Health checks can detect failed backends. The max_fails and fail_timeout parameters control when a server is marked unavailable.", + }, + { + "title": "nginx: SSL/TLS Termination", + "content": "SSL termination at nginx means nginx handles TLS handshakes and decryption, then forwards plain HTTP to backend servers. This offloads CPU-intensive cryptographic operations from application servers. Configure with ssl_certificate and ssl_certificate_key in the server block.", + }, + { + "title": "nginx: Proxy Settings", + "content": "proxy_set_header X-Forwarded-Proto $scheme passes the original protocol to backends. proxy_set_header X-Real-IP $remote_addr passes the client's real IP. These headers are important when SSL is terminated at the proxy layer.", + }, + { + "title": "nginx: HTTP/2", + "content": "nginx supports HTTP/2 with the http2 parameter on the listen directive. HTTP/2 enables multiplexing, header compression, and server push. HTTP/2 requires TLS in most browsers.", + }, ], "required_facts": [ - {"fact": "upstream blocks with round-robin least_conn ip_hash", "source_doc_index": 0}, - {"fact": "SSL termination offloads crypto from backends", "source_doc_index": 1}, - {"fact": "X-Forwarded-Proto passes original protocol", "source_doc_index": 2}, + { + "fact": "upstream blocks with round-robin least_conn ip_hash", + "source_doc_index": 0, + }, + { + "fact": "SSL termination offloads crypto from backends", + "source_doc_index": 1, + }, + { + "fact": "X-Forwarded-Proto passes original protocol", + "source_doc_index": 2, + }, ], }, { "question": "Compare PostgreSQL B-tree and GIN indexes. When is each appropriate?", "documents": [ - {"title": "PostgreSQL: B-tree Indexes", "content": "B-tree indexes are the default index type. They can handle equality and range queries on data that can be sorted. B-tree indexes work with all data types that have a well-defined ordering. They support queries with operators <, <=, =, >=, >, BETWEEN, IN, IS NULL, and IS NOT NULL."}, - {"title": "PostgreSQL: GIN Indexes", "content": "GIN (Generalized Inverted Index) indexes are designed for composite values where queries search for elements within the composite value. Common uses: full-text search (tsvector), JSONB containment (@>), and array operations (&&, @>). GIN indexes are slower to build than B-tree but faster for lookups on composite types."}, - {"title": "PostgreSQL: Index Maintenance", "content": "Indexes consume storage and slow down writes. Each INSERT/UPDATE/DELETE must update every index on the table. PostgreSQL's HOT (Heap Only Tuple) updates can avoid index updates when indexed columns are not modified. Use REINDEX to rebuild corrupted indexes."}, - {"title": "PostgreSQL: Query Planning", "content": "The query planner decides whether to use an index based on table statistics. ANALYZE updates these statistics. A sequential scan may be chosen over an index scan if the query will access a large portion of the table."}, + { + "title": "PostgreSQL: B-tree Indexes", + "content": "B-tree indexes are the default index type. They can handle equality and range queries on data that can be sorted. B-tree indexes work with all data types that have a well-defined ordering. They support queries with operators <, <=, =, >=, >, BETWEEN, IN, IS NULL, and IS NOT NULL.", + }, + { + "title": "PostgreSQL: GIN Indexes", + "content": "GIN (Generalized Inverted Index) indexes are designed for composite values where queries search for elements within the composite value. Common uses: full-text search (tsvector), JSONB containment (@>), and array operations (&&, @>). GIN indexes are slower to build than B-tree but faster for lookups on composite types.", + }, + { + "title": "PostgreSQL: Index Maintenance", + "content": "Indexes consume storage and slow down writes. Each INSERT/UPDATE/DELETE must update every index on the table. PostgreSQL's HOT (Heap Only Tuple) updates can avoid index updates when indexed columns are not modified. Use REINDEX to rebuild corrupted indexes.", + }, + { + "title": "PostgreSQL: Query Planning", + "content": "The query planner decides whether to use an index based on table statistics. ANALYZE updates these statistics. A sequential scan may be chosen over an index scan if the query will access a large portion of the table.", + }, ], "required_facts": [ - {"fact": "B-tree handles equality and range queries", "source_doc_index": 0}, - {"fact": "GIN for composite values like full-text JSONB arrays", "source_doc_index": 1}, + { + "fact": "B-tree handles equality and range queries", + "source_doc_index": 0, + }, + { + "fact": "GIN for composite values like full-text JSONB arrays", + "source_doc_index": 1, + }, {"fact": "GIN slower to build faster for lookups", "source_doc_index": 1}, {"fact": "indexes slow down writes", "source_doc_index": 2}, ], @@ -272,14 +491,32 @@ _MEDIUM_TASKS: List[Dict[str, Any]] = [ { "question": "How do Kubernetes pod scheduling and resource management work together?", "documents": [ - {"title": "K8s: Scheduling", "content": "The scheduler assigns Pods to Nodes based on resource requirements, affinity/anti-affinity rules, taints and tolerations, and topology spread constraints. Pods can specify nodeSelector to constrain which nodes they can run on. The scheduler filters nodes that don't meet requirements, then ranks remaining nodes."}, - {"title": "K8s: Resource Management", "content": "Containers can specify CPU and memory requests and limits. Requests are what the container is guaranteed. Limits are the maximum allowed. If a container exceeds its memory limit, it is OOM-killed. CPU limits are enforced via CFS throttling."}, - {"title": "K8s: Priority and Preemption", "content": "PriorityClasses define pod priority. Higher-priority pods can preempt (evict) lower-priority pods when no node has enough resources. The preempted pods are given a graceful termination period."}, - {"title": "K8s: Horizontal Pod Autoscaler", "content": "HPA automatically scales the number of pod replicas based on observed CPU utilization, memory usage, or custom metrics. It queries the metrics API every 15 seconds by default."}, + { + "title": "K8s: Scheduling", + "content": "The scheduler assigns Pods to Nodes based on resource requirements, affinity/anti-affinity rules, taints and tolerations, and topology spread constraints. Pods can specify nodeSelector to constrain which nodes they can run on. The scheduler filters nodes that don't meet requirements, then ranks remaining nodes.", + }, + { + "title": "K8s: Resource Management", + "content": "Containers can specify CPU and memory requests and limits. Requests are what the container is guaranteed. Limits are the maximum allowed. If a container exceeds its memory limit, it is OOM-killed. CPU limits are enforced via CFS throttling.", + }, + { + "title": "K8s: Priority and Preemption", + "content": "PriorityClasses define pod priority. Higher-priority pods can preempt (evict) lower-priority pods when no node has enough resources. The preempted pods are given a graceful termination period.", + }, + { + "title": "K8s: Horizontal Pod Autoscaler", + "content": "HPA automatically scales the number of pod replicas based on observed CPU utilization, memory usage, or custom metrics. It queries the metrics API every 15 seconds by default.", + }, ], "required_facts": [ - {"fact": "scheduler uses affinity taints nodeSelector", "source_doc_index": 0}, - {"fact": "requests are guaranteed limits are maximum", "source_doc_index": 1}, + { + "fact": "scheduler uses affinity taints nodeSelector", + "source_doc_index": 0, + }, + { + "fact": "requests are guaranteed limits are maximum", + "source_doc_index": 1, + }, {"fact": "OOM-killed if memory limit exceeded", "source_doc_index": 1}, {"fact": "priority classes enable preemption", "source_doc_index": 2}, ], @@ -287,31 +524,70 @@ _MEDIUM_TASKS: List[Dict[str, Any]] = [ { "question": "How do FastAPI WebSockets differ from regular HTTP endpoints?", "documents": [ - {"title": "FastAPI: WebSocket", "content": "FastAPI supports WebSocket endpoints using @app.websocket() decorator. Unlike HTTP, WebSocket provides full-duplex communication — both client and server can send messages at any time. The connection remains open until explicitly closed. You use await websocket.accept() to accept the connection, and await websocket.receive_text() / await websocket.send_text() for messaging."}, - {"title": "FastAPI: HTTP Endpoints", "content": "Regular HTTP endpoints use @app.get(), @app.post(), etc. Each request creates a new connection, the server processes it, and returns a response. HTTP is stateless — each request is independent. FastAPI automatically generates OpenAPI documentation for HTTP endpoints."}, - {"title": "FastAPI: Testing", "content": "TestClient from Starlette supports testing both HTTP and WebSocket endpoints. For WebSocket testing, use 'with client.websocket_connect(\"/ws\") as ws:' to establish a test connection."}, - {"title": "FastAPI: Events", "content": "Lifespan events allow running code on application startup and shutdown. Use the lifespan parameter with an async context manager. This replaces the deprecated on_event decorator."}, + { + "title": "FastAPI: WebSocket", + "content": "FastAPI supports WebSocket endpoints using @app.websocket() decorator. Unlike HTTP, WebSocket provides full-duplex communication — both client and server can send messages at any time. The connection remains open until explicitly closed. You use await websocket.accept() to accept the connection, and await websocket.receive_text() / await websocket.send_text() for messaging.", + }, + { + "title": "FastAPI: HTTP Endpoints", + "content": "Regular HTTP endpoints use @app.get(), @app.post(), etc. Each request creates a new connection, the server processes it, and returns a response. HTTP is stateless — each request is independent. FastAPI automatically generates OpenAPI documentation for HTTP endpoints.", + }, + { + "title": "FastAPI: Testing", + "content": "TestClient from Starlette supports testing both HTTP and WebSocket endpoints. For WebSocket testing, use 'with client.websocket_connect(\"/ws\") as ws:' to establish a test connection.", + }, + { + "title": "FastAPI: Events", + "content": "Lifespan events allow running code on application startup and shutdown. Use the lifespan parameter with an async context manager. This replaces the deprecated on_event decorator.", + }, ], "required_facts": [ - {"fact": "WebSocket provides full-duplex communication", "source_doc_index": 0}, + { + "fact": "WebSocket provides full-duplex communication", + "source_doc_index": 0, + }, {"fact": "connection remains open until closed", "source_doc_index": 0}, - {"fact": "HTTP is stateless each request independent", "source_doc_index": 1}, - {"fact": "OpenAPI docs generated for HTTP not WebSocket", "source_doc_index": 1}, + { + "fact": "HTTP is stateless each request independent", + "source_doc_index": 1, + }, + { + "fact": "OpenAPI docs generated for HTTP not WebSocket", + "source_doc_index": 1, + }, ], }, { "question": "Explain Redis memory eviction policies and when to use maxmemory.", "documents": [ - {"title": "Redis: Memory Management", "content": "When maxmemory is set and the limit is reached, Redis applies an eviction policy. volatile-lru removes least recently used keys with an expire set. allkeys-lru removes any LRU key. volatile-ttl removes keys with shortest TTL. noeviction returns errors on write commands when memory is full."}, - {"title": "Redis: Configuration", "content": "Set maxmemory in redis.conf or via CONFIG SET maxmemory . Without maxmemory, Redis on 64-bit systems has no memory limit and will use all available RAM. On 32-bit systems, there is an implicit 3GB limit."}, - {"title": "Redis: Data Types", "content": "Redis supports strings, lists, sets, sorted sets, hashes, streams, and more. Each data type has internal encoding optimizations. Small sets use ziplist encoding, larger sets use hashtable encoding."}, - {"title": "Redis: Monitoring", "content": "INFO memory command shows used_memory, used_memory_rss, mem_fragmentation_ratio, and evicted_keys count. A high fragmentation ratio indicates memory fragmentation. MEMORY DOCTOR provides recommendations."}, + { + "title": "Redis: Memory Management", + "content": "When maxmemory is set and the limit is reached, Redis applies an eviction policy. volatile-lru removes least recently used keys with an expire set. allkeys-lru removes any LRU key. volatile-ttl removes keys with shortest TTL. noeviction returns errors on write commands when memory is full.", + }, + { + "title": "Redis: Configuration", + "content": "Set maxmemory in redis.conf or via CONFIG SET maxmemory . Without maxmemory, Redis on 64-bit systems has no memory limit and will use all available RAM. On 32-bit systems, there is an implicit 3GB limit.", + }, + { + "title": "Redis: Data Types", + "content": "Redis supports strings, lists, sets, sorted sets, hashes, streams, and more. Each data type has internal encoding optimizations. Small sets use ziplist encoding, larger sets use hashtable encoding.", + }, + { + "title": "Redis: Monitoring", + "content": "INFO memory command shows used_memory, used_memory_rss, mem_fragmentation_ratio, and evicted_keys count. A high fragmentation ratio indicates memory fragmentation. MEMORY DOCTOR provides recommendations.", + }, ], "required_facts": [ - {"fact": "volatile-lru removes LRU keys with expire", "source_doc_index": 0}, + { + "fact": "volatile-lru removes LRU keys with expire", + "source_doc_index": 0, + }, {"fact": "noeviction returns errors on writes", "source_doc_index": 0}, {"fact": "without maxmemory uses all available RAM", "source_doc_index": 1}, - {"fact": "INFO memory shows usage and fragmentation", "source_doc_index": 3}, + { + "fact": "INFO memory shows usage and fragmentation", + "source_doc_index": 3, + }, ], }, ] @@ -324,128 +600,311 @@ _HARD_TASKS: List[Dict[str, Any]] = [ { "question": "Design a high-availability PostgreSQL setup with both streaming and logical replication. What are the trade-offs?", "documents": [ - {"title": "PostgreSQL: Streaming Replication", "content": "Streaming replication transmits WAL records from primary to standbys in real-time. Synchronous mode guarantees no data loss but adds write latency (typically 1-5ms per commit). Asynchronous mode has minimal latency impact but may lose recent transactions on failover. A standby can serve read-only queries."}, - {"title": "PostgreSQL: Logical Replication", "content": "Logical replication decodes WAL into logical changes and applies them on subscribers. It supports selective table replication, cross-version upgrades, and publishing to multiple subscribers. It cannot replicate DDL, sequences, or large objects. Logical replication has higher CPU overhead than streaming."}, - {"title": "PostgreSQL: Connection Pooling", "content": "PgBouncer or Pgpool-II can pool connections to reduce overhead. PgBouncer supports transaction-mode pooling where connections are returned to the pool after each transaction. This is essential for applications with many short-lived connections."}, - {"title": "PostgreSQL: Partitioning", "content": "Table partitioning divides a large table into smaller physical pieces. Range partitioning and list partitioning are the most common. Partition pruning eliminates irrelevant partitions during query planning. Partitioning can improve both query performance and maintenance operations like VACUUM."}, - {"title": "PostgreSQL: pg_stat_replication", "content": "The pg_stat_replication view shows one row per WAL sender process, with information about replication state, sent_lsn, write_lsn, flush_lsn, and replay_lsn. The difference between sent_lsn and replay_lsn indicates replication lag."}, - {"title": "PostgreSQL: Failover", "content": "pg_promote() promotes a standby to primary. Tools like Patroni or repmgr automate failover with consensus-based leader election. After failover, old primary must be rebuilt as a standby using pg_rewind or pg_basebackup."}, + { + "title": "PostgreSQL: Streaming Replication", + "content": "Streaming replication transmits WAL records from primary to standbys in real-time. Synchronous mode guarantees no data loss but adds write latency (typically 1-5ms per commit). Asynchronous mode has minimal latency impact but may lose recent transactions on failover. A standby can serve read-only queries.", + }, + { + "title": "PostgreSQL: Logical Replication", + "content": "Logical replication decodes WAL into logical changes and applies them on subscribers. It supports selective table replication, cross-version upgrades, and publishing to multiple subscribers. It cannot replicate DDL, sequences, or large objects. Logical replication has higher CPU overhead than streaming.", + }, + { + "title": "PostgreSQL: Connection Pooling", + "content": "PgBouncer or Pgpool-II can pool connections to reduce overhead. PgBouncer supports transaction-mode pooling where connections are returned to the pool after each transaction. This is essential for applications with many short-lived connections.", + }, + { + "title": "PostgreSQL: Partitioning", + "content": "Table partitioning divides a large table into smaller physical pieces. Range partitioning and list partitioning are the most common. Partition pruning eliminates irrelevant partitions during query planning. Partitioning can improve both query performance and maintenance operations like VACUUM.", + }, + { + "title": "PostgreSQL: pg_stat_replication", + "content": "The pg_stat_replication view shows one row per WAL sender process, with information about replication state, sent_lsn, write_lsn, flush_lsn, and replay_lsn. The difference between sent_lsn and replay_lsn indicates replication lag.", + }, + { + "title": "PostgreSQL: Failover", + "content": "pg_promote() promotes a standby to primary. Tools like Patroni or repmgr automate failover with consensus-based leader election. After failover, old primary must be rebuilt as a standby using pg_rewind or pg_basebackup.", + }, ], "required_facts": [ - {"fact": "synchronous streaming guarantees no data loss adds latency", "source_doc_index": 0}, + { + "fact": "synchronous streaming guarantees no data loss adds latency", + "source_doc_index": 0, + }, {"fact": "standby serves read-only queries", "source_doc_index": 0}, {"fact": "logical cannot replicate DDL sequences", "source_doc_index": 1}, {"fact": "logical has higher CPU overhead", "source_doc_index": 1}, {"fact": "Patroni or repmgr for automated failover", "source_doc_index": 5}, - {"fact": "replication lag via sent_lsn vs replay_lsn", "source_doc_index": 4}, + { + "fact": "replication lag via sent_lsn vs replay_lsn", + "source_doc_index": 4, + }, ], }, { "question": "How would you implement a zero-downtime Kubernetes deployment with proper health checks and rollback?", "documents": [ - {"title": "K8s: Rolling Updates", "content": "Deployments support rolling updates via maxSurge and maxUnavailable parameters. maxSurge controls how many extra pods can be created. maxUnavailable controls how many pods can be unavailable during update. Setting maxUnavailable to 0 and maxSurge to 1 ensures no downtime but slower rollout."}, - {"title": "K8s: Health Probes", "content": "livenessProbe restarts containers that are deadlocked. readinessProbe removes pods from Service endpoints when not ready. startupProbe disables liveness/readiness checks until the app has started. All support httpGet, tcpSocket, exec, and gRPC probe types. initialDelaySeconds, periodSeconds, and failureThreshold control timing."}, - {"title": "K8s: Rollback", "content": "kubectl rollout undo deployment/ reverts to the previous revision. kubectl rollout history shows all revisions. minReadySeconds specifies how long a new pod should be ready before it's considered available. progressDeadlineSeconds sets the maximum time for a deployment to make progress."}, - {"title": "K8s: Pod Disruption Budgets", "content": "PodDisruptionBudget (PDB) limits the number of pods of a replicated application that are down simultaneously from voluntary disruptions. It specifies either minAvailable or maxUnavailable."}, - {"title": "K8s: ConfigMaps", "content": "ConfigMaps decouple configuration from container images. They can be consumed as environment variables, command-line arguments, or configuration files in a volume. Changes to ConfigMaps do not trigger pod restarts by default."}, - {"title": "K8s: Resource Quotas", "content": "ResourceQuota constrains aggregate resource consumption per namespace. It can limit total CPU, memory, storage, and object counts. LimitRange sets default limits for containers that don't specify their own."}, + { + "title": "K8s: Rolling Updates", + "content": "Deployments support rolling updates via maxSurge and maxUnavailable parameters. maxSurge controls how many extra pods can be created. maxUnavailable controls how many pods can be unavailable during update. Setting maxUnavailable to 0 and maxSurge to 1 ensures no downtime but slower rollout.", + }, + { + "title": "K8s: Health Probes", + "content": "livenessProbe restarts containers that are deadlocked. readinessProbe removes pods from Service endpoints when not ready. startupProbe disables liveness/readiness checks until the app has started. All support httpGet, tcpSocket, exec, and gRPC probe types. initialDelaySeconds, periodSeconds, and failureThreshold control timing.", + }, + { + "title": "K8s: Rollback", + "content": "kubectl rollout undo deployment/ reverts to the previous revision. kubectl rollout history shows all revisions. minReadySeconds specifies how long a new pod should be ready before it's considered available. progressDeadlineSeconds sets the maximum time for a deployment to make progress.", + }, + { + "title": "K8s: Pod Disruption Budgets", + "content": "PodDisruptionBudget (PDB) limits the number of pods of a replicated application that are down simultaneously from voluntary disruptions. It specifies either minAvailable or maxUnavailable.", + }, + { + "title": "K8s: ConfigMaps", + "content": "ConfigMaps decouple configuration from container images. They can be consumed as environment variables, command-line arguments, or configuration files in a volume. Changes to ConfigMaps do not trigger pod restarts by default.", + }, + { + "title": "K8s: Resource Quotas", + "content": "ResourceQuota constrains aggregate resource consumption per namespace. It can limit total CPU, memory, storage, and object counts. LimitRange sets default limits for containers that don't specify their own.", + }, ], "required_facts": [ - {"fact": "maxSurge and maxUnavailable control rolling update", "source_doc_index": 0}, - {"fact": "readinessProbe removes from endpoints when not ready", "source_doc_index": 1}, - {"fact": "startupProbe disables other probes until started", "source_doc_index": 1}, - {"fact": "rollout undo reverts to previous revision", "source_doc_index": 2}, - {"fact": "PodDisruptionBudget limits voluntary disruptions", "source_doc_index": 3}, + { + "fact": "maxSurge and maxUnavailable control rolling update", + "source_doc_index": 0, + }, + { + "fact": "readinessProbe removes from endpoints when not ready", + "source_doc_index": 1, + }, + { + "fact": "startupProbe disables other probes until started", + "source_doc_index": 1, + }, + { + "fact": "rollout undo reverts to previous revision", + "source_doc_index": 2, + }, + { + "fact": "PodDisruptionBudget limits voluntary disruptions", + "source_doc_index": 3, + }, ], }, { "question": "Design a Redis caching layer with persistence guarantees. How do you handle cache warming and eviction?", "documents": [ - {"title": "Redis: Cache Patterns", "content": "Cache-aside (lazy loading): application checks cache first, on miss reads from database and populates cache. Write-through: application writes to both cache and database simultaneously. Write-behind: application writes to cache, which asynchronously writes to database. Each pattern has different consistency guarantees."}, - {"title": "Redis: RDB + AOF", "content": "Using both RDB and AOF together provides the strongest persistence guarantees. On restart, Redis uses AOF to reconstruct data (it's more complete). RDB serves as a compact backup. Configure with 'save 900 1' for RDB and 'appendonly yes' with 'appendfsync everysec' for AOF."}, - {"title": "Redis: Pipeline", "content": "Pipelining sends multiple commands without waiting for replies. This reduces round-trip time from one RTT per command to one RTT per batch. MULTI/EXEC provides atomic transactions. Pipeline + MULTI combines batching with atomicity."}, - {"title": "Redis: Memory Management", "content": "allkeys-lfu evicts least frequently used keys. volatile-lfu evicts LFU keys with TTL set. LFU is better than LRU for caches with hot/cold key patterns. Configure with maxmemory-policy. Redis 4.0+ supports LFU with configurable decay time."}, - {"title": "Redis: Pub/Sub", "content": "PUBLISH/SUBSCRIBE allows message passing between Redis clients. Messages are fire-and-forget — if no subscriber is listening, the message is lost. For durable messaging, use Redis Streams instead."}, - {"title": "Redis: Keyspace Notifications", "content": "Redis can emit notifications for key events (set, del, expired, evicted). Configure with notify-keyspace-events. Applications can subscribe to these events for cache invalidation or monitoring."}, + { + "title": "Redis: Cache Patterns", + "content": "Cache-aside (lazy loading): application checks cache first, on miss reads from database and populates cache. Write-through: application writes to both cache and database simultaneously. Write-behind: application writes to cache, which asynchronously writes to database. Each pattern has different consistency guarantees.", + }, + { + "title": "Redis: RDB + AOF", + "content": "Using both RDB and AOF together provides the strongest persistence guarantees. On restart, Redis uses AOF to reconstruct data (it's more complete). RDB serves as a compact backup. Configure with 'save 900 1' for RDB and 'appendonly yes' with 'appendfsync everysec' for AOF.", + }, + { + "title": "Redis: Pipeline", + "content": "Pipelining sends multiple commands without waiting for replies. This reduces round-trip time from one RTT per command to one RTT per batch. MULTI/EXEC provides atomic transactions. Pipeline + MULTI combines batching with atomicity.", + }, + { + "title": "Redis: Memory Management", + "content": "allkeys-lfu evicts least frequently used keys. volatile-lfu evicts LFU keys with TTL set. LFU is better than LRU for caches with hot/cold key patterns. Configure with maxmemory-policy. Redis 4.0+ supports LFU with configurable decay time.", + }, + { + "title": "Redis: Pub/Sub", + "content": "PUBLISH/SUBSCRIBE allows message passing between Redis clients. Messages are fire-and-forget — if no subscriber is listening, the message is lost. For durable messaging, use Redis Streams instead.", + }, + { + "title": "Redis: Keyspace Notifications", + "content": "Redis can emit notifications for key events (set, del, expired, evicted). Configure with notify-keyspace-events. Applications can subscribe to these events for cache invalidation or monitoring.", + }, ], "required_facts": [ - {"fact": "cache-aside checks cache first reads DB on miss", "source_doc_index": 0}, - {"fact": "write-through writes to both simultaneously", "source_doc_index": 0}, - {"fact": "RDB plus AOF together for strongest persistence", "source_doc_index": 1}, - {"fact": "LFU better than LRU for hot cold patterns", "source_doc_index": 3}, - {"fact": "keyspace notifications for cache invalidation", "source_doc_index": 5}, + { + "fact": "cache-aside checks cache first reads DB on miss", + "source_doc_index": 0, + }, + { + "fact": "write-through writes to both simultaneously", + "source_doc_index": 0, + }, + { + "fact": "RDB plus AOF together for strongest persistence", + "source_doc_index": 1, + }, + { + "fact": "LFU better than LRU for hot cold patterns", + "source_doc_index": 3, + }, + { + "fact": "keyspace notifications for cache invalidation", + "source_doc_index": 5, + }, ], }, { "question": "How would you build a production FastAPI application with proper middleware, dependency injection, and error handling?", "documents": [ - {"title": "FastAPI: Middleware Stack", "content": "Middleware executes in order of registration. Common middleware stack: CORSMiddleware (CORS headers), TrustedHostMiddleware (host header validation), GZipMiddleware (response compression). Custom middleware can measure request duration, add request IDs, and handle rate limiting. Middleware exceptions bypass exception handlers."}, - {"title": "FastAPI: Dependency Injection", "content": "Dependencies can be scoped to path operations, routers, or the entire app. Use yield dependencies for setup/teardown (e.g., database sessions). Dependencies form a DAG — FastAPI resolves them in the correct order. Sub-dependencies are cached per-request by default."}, - {"title": "FastAPI: Exception Handling", "content": "HTTPException returns HTTP error responses. Custom exception handlers can be registered with @app.exception_handler(). RequestValidationError handles Pydantic validation failures. Override the default handlers to customize error response format. Unhandled exceptions return 500 with no details in production."}, - {"title": "FastAPI: Authentication", "content": "OAuth2PasswordBearer extracts bearer tokens. HTTPBearer validates Authorization headers. Security schemes appear in OpenAPI docs. Combine with dependencies for role-based access control."}, - {"title": "FastAPI: Static Files", "content": "StaticFiles middleware serves static files from a directory. Mount with app.mount('/static', StaticFiles(directory='static'), name='static'). Supports HTML, CSS, JS, and other file types."}, - {"title": "FastAPI: Response Models", "content": "response_model parameter filters output to match the declared model. response_model_exclude_unset removes default values from response. This prevents leaking internal fields and ensures consistent API contracts."}, + { + "title": "FastAPI: Middleware Stack", + "content": "Middleware executes in order of registration. Common middleware stack: CORSMiddleware (CORS headers), TrustedHostMiddleware (host header validation), GZipMiddleware (response compression). Custom middleware can measure request duration, add request IDs, and handle rate limiting. Middleware exceptions bypass exception handlers.", + }, + { + "title": "FastAPI: Dependency Injection", + "content": "Dependencies can be scoped to path operations, routers, or the entire app. Use yield dependencies for setup/teardown (e.g., database sessions). Dependencies form a DAG — FastAPI resolves them in the correct order. Sub-dependencies are cached per-request by default.", + }, + { + "title": "FastAPI: Exception Handling", + "content": "HTTPException returns HTTP error responses. Custom exception handlers can be registered with @app.exception_handler(). RequestValidationError handles Pydantic validation failures. Override the default handlers to customize error response format. Unhandled exceptions return 500 with no details in production.", + }, + { + "title": "FastAPI: Authentication", + "content": "OAuth2PasswordBearer extracts bearer tokens. HTTPBearer validates Authorization headers. Security schemes appear in OpenAPI docs. Combine with dependencies for role-based access control.", + }, + { + "title": "FastAPI: Static Files", + "content": "StaticFiles middleware serves static files from a directory. Mount with app.mount('/static', StaticFiles(directory='static'), name='static'). Supports HTML, CSS, JS, and other file types.", + }, + { + "title": "FastAPI: Response Models", + "content": "response_model parameter filters output to match the declared model. response_model_exclude_unset removes default values from response. This prevents leaking internal fields and ensures consistent API contracts.", + }, ], "required_facts": [ - {"fact": "middleware exceptions bypass exception handlers", "source_doc_index": 0}, + { + "fact": "middleware exceptions bypass exception handlers", + "source_doc_index": 0, + }, {"fact": "yield dependencies for setup teardown", "source_doc_index": 1}, {"fact": "sub-dependencies cached per request", "source_doc_index": 1}, - {"fact": "RequestValidationError for Pydantic failures", "source_doc_index": 2}, + { + "fact": "RequestValidationError for Pydantic failures", + "source_doc_index": 2, + }, {"fact": "response_model filters output", "source_doc_index": 5}, ], }, { "question": "Design a secure Kubernetes cluster network architecture with RBAC, network policies, and pod security.", "documents": [ - {"title": "K8s: Network Policies", "content": "Network policies use label selectors to control pod-to-pod traffic. A default-deny policy blocks all ingress traffic to pods in a namespace. Egress policies can restrict outbound traffic. Network policies require a CNI plugin that supports them (Calico, Cilium, Weave Net). Policies are additive — if any policy allows traffic, it is allowed."}, - {"title": "K8s: RBAC", "content": "Use least-privilege principle: create specific Roles with minimal verbs (get, list, watch, create, update, delete). Avoid ClusterRoleBinding to cluster-admin. Use namespaced RoleBindings when possible. Audit RBAC permissions with 'kubectl auth can-i --list'. ServiceAccounts should have automountServiceAccountToken: false when not needed."}, - {"title": "K8s: Pod Security Standards", "content": "Pod Security Admission enforces Privileged (unrestricted), Baseline (minimally restrictive), and Restricted (hardened) profiles. Restricted profile prohibits running as root, requires read-only root filesystem, drops ALL capabilities, and requires seccomp profile. Apply per-namespace with labels."}, - {"title": "K8s: Secrets", "content": "Secrets store sensitive data like passwords and tokens. By default, Secrets are stored unencrypted in etcd. Enable encryption at rest using EncryptionConfiguration. Prefer external secret stores (Vault, AWS Secrets Manager) with CSI driver or external-secrets operator."}, - {"title": "K8s: Admission Controllers", "content": "Admission controllers intercept API requests after authentication but before persistence. ValidatingWebhookConfiguration rejects non-compliant resources. MutatingWebhookConfiguration can modify resources (e.g., inject sidecars, add labels). OPA Gatekeeper or Kyverno provide policy-as-code."}, - {"title": "K8s: Namespaces", "content": "Namespaces provide scope for names and a mechanism to divide cluster resources. Use namespaces to separate environments (dev, staging, prod) or teams. ResourceQuotas and LimitRanges are applied per-namespace."}, + { + "title": "K8s: Network Policies", + "content": "Network policies use label selectors to control pod-to-pod traffic. A default-deny policy blocks all ingress traffic to pods in a namespace. Egress policies can restrict outbound traffic. Network policies require a CNI plugin that supports them (Calico, Cilium, Weave Net). Policies are additive — if any policy allows traffic, it is allowed.", + }, + { + "title": "K8s: RBAC", + "content": "Use least-privilege principle: create specific Roles with minimal verbs (get, list, watch, create, update, delete). Avoid ClusterRoleBinding to cluster-admin. Use namespaced RoleBindings when possible. Audit RBAC permissions with 'kubectl auth can-i --list'. ServiceAccounts should have automountServiceAccountToken: false when not needed.", + }, + { + "title": "K8s: Pod Security Standards", + "content": "Pod Security Admission enforces Privileged (unrestricted), Baseline (minimally restrictive), and Restricted (hardened) profiles. Restricted profile prohibits running as root, requires read-only root filesystem, drops ALL capabilities, and requires seccomp profile. Apply per-namespace with labels.", + }, + { + "title": "K8s: Secrets", + "content": "Secrets store sensitive data like passwords and tokens. By default, Secrets are stored unencrypted in etcd. Enable encryption at rest using EncryptionConfiguration. Prefer external secret stores (Vault, AWS Secrets Manager) with CSI driver or external-secrets operator.", + }, + { + "title": "K8s: Admission Controllers", + "content": "Admission controllers intercept API requests after authentication but before persistence. ValidatingWebhookConfiguration rejects non-compliant resources. MutatingWebhookConfiguration can modify resources (e.g., inject sidecars, add labels). OPA Gatekeeper or Kyverno provide policy-as-code.", + }, + { + "title": "K8s: Namespaces", + "content": "Namespaces provide scope for names and a mechanism to divide cluster resources. Use namespaces to separate environments (dev, staging, prod) or teams. ResourceQuotas and LimitRanges are applied per-namespace.", + }, ], "required_facts": [ {"fact": "default-deny policy blocks all ingress", "source_doc_index": 0}, {"fact": "policies are additive", "source_doc_index": 0}, {"fact": "least privilege with minimal verbs", "source_doc_index": 1}, {"fact": "automountServiceAccountToken false", "source_doc_index": 1}, - {"fact": "restricted profile prohibits root drops capabilities", "source_doc_index": 2}, + { + "fact": "restricted profile prohibits root drops capabilities", + "source_doc_index": 2, + }, {"fact": "secrets unencrypted in etcd by default", "source_doc_index": 3}, ], }, { "question": "How would you implement comprehensive Python logging with structured output, rotation, and monitoring integration?", "documents": [ - {"title": "Python: logging Configuration", "content": "dictConfig() provides the most flexible configuration. Loggers form a hierarchy based on dot-separated names. The root logger catches all unhandled log records. Each logger can have multiple handlers with different levels. Propagation sends records up the hierarchy — set propagate=False to prevent duplicate logs."}, - {"title": "Python: logging Handlers", "content": "RotatingFileHandler rotates logs when they reach a size limit (maxBytes, backupCount). TimedRotatingFileHandler rotates based on time intervals. SocketHandler sends records to a network socket. QueueHandler + QueueListener enable non-blocking logging with a separate thread for I/O."}, - {"title": "Python: logging Formatters", "content": "Custom formatters can output JSON for structured logging. Use LogRecord attributes: %(asctime)s, %(name)s, %(levelname)s, %(message)s, %(pathname)s, %(lineno)d. python-json-logger library provides JsonFormatter. Include request IDs and trace context for distributed tracing."}, - {"title": "Python: logging Filters", "content": "Filters can modify or suppress log records. A filter's filter() method returns True to allow the record. Filters can add extra fields to records (e.g., user_id, request_id). Attach filters to loggers or handlers."}, - {"title": "Python: warnings Integration", "content": "logging.captureWarnings(True) redirects Python warnings to the logging system. This allows consistent handling of both warnings and log messages through the same pipeline."}, - {"title": "Python: contextvars", "content": "contextvars provides context-local storage for async code. ContextVar objects store per-task values. Unlike threading.local(), contextvars works correctly with asyncio. Use it to propagate request IDs through async call chains."}, + { + "title": "Python: logging Configuration", + "content": "dictConfig() provides the most flexible configuration. Loggers form a hierarchy based on dot-separated names. The root logger catches all unhandled log records. Each logger can have multiple handlers with different levels. Propagation sends records up the hierarchy — set propagate=False to prevent duplicate logs.", + }, + { + "title": "Python: logging Handlers", + "content": "RotatingFileHandler rotates logs when they reach a size limit (maxBytes, backupCount). TimedRotatingFileHandler rotates based on time intervals. SocketHandler sends records to a network socket. QueueHandler + QueueListener enable non-blocking logging with a separate thread for I/O.", + }, + { + "title": "Python: logging Formatters", + "content": "Custom formatters can output JSON for structured logging. Use LogRecord attributes: %(asctime)s, %(name)s, %(levelname)s, %(message)s, %(pathname)s, %(lineno)d. python-json-logger library provides JsonFormatter. Include request IDs and trace context for distributed tracing.", + }, + { + "title": "Python: logging Filters", + "content": "Filters can modify or suppress log records. A filter's filter() method returns True to allow the record. Filters can add extra fields to records (e.g., user_id, request_id). Attach filters to loggers or handlers.", + }, + { + "title": "Python: warnings Integration", + "content": "logging.captureWarnings(True) redirects Python warnings to the logging system. This allows consistent handling of both warnings and log messages through the same pipeline.", + }, + { + "title": "Python: contextvars", + "content": "contextvars provides context-local storage for async code. ContextVar objects store per-task values. Unlike threading.local(), contextvars works correctly with asyncio. Use it to propagate request IDs through async call chains.", + }, ], "required_facts": [ {"fact": "dictConfig for flexible configuration", "source_doc_index": 0}, {"fact": "propagate False prevents duplicate logs", "source_doc_index": 0}, - {"fact": "RotatingFileHandler with maxBytes backupCount", "source_doc_index": 1}, + { + "fact": "RotatingFileHandler with maxBytes backupCount", + "source_doc_index": 1, + }, {"fact": "QueueHandler for non-blocking logging", "source_doc_index": 1}, {"fact": "JSON formatters for structured logging", "source_doc_index": 2}, - {"fact": "contextvars for request ID propagation in async", "source_doc_index": 5}, + { + "fact": "contextvars for request ID propagation in async", + "source_doc_index": 5, + }, ], }, { "question": "Explain how to optimize PostgreSQL query performance using EXPLAIN ANALYZE, indexes, and configuration tuning.", "documents": [ - {"title": "PostgreSQL: EXPLAIN ANALYZE", "content": "EXPLAIN ANALYZE actually runs the query and shows real execution times alongside estimates. Key metrics: actual time (startup..total), rows (estimated vs actual), loops. Bitmap Heap Scan indicates the planner chose an index but needs to recheck. Sort Method shows if sort spilled to disk. Buffers option shows shared/local hit/read counts for I/O analysis."}, - {"title": "PostgreSQL: Index Types", "content": "B-tree for equality/range (default). Hash for equality-only (rarely useful). GiST for geometric, full-text, and range types. GIN for full-text search, JSONB, arrays. BRIN for large tables with natural ordering (timestamps). Partial indexes (WHERE clause) reduce index size. Expression indexes allow indexing computed values."}, - {"title": "PostgreSQL: Configuration", "content": "shared_buffers: typically 25% of system RAM. effective_cache_size: total expected filesystem cache (50-75% of RAM). work_mem: per-operation sort/hash memory. maintenance_work_mem: for VACUUM, CREATE INDEX. random_page_cost: lower for SSDs (1.1 vs 4.0 default). Setting these correctly can improve query plans dramatically."}, - {"title": "PostgreSQL: Table Statistics", "content": "ANALYZE collects statistics about column value distributions. default_statistics_target controls granularity (default 100, increase for skewed data). pg_stats view shows most common values, histogram bounds, null fraction, and n_distinct. Stale statistics lead to poor query plans."}, - {"title": "PostgreSQL: Partitioning", "content": "Partition pruning eliminates irrelevant partitions during planning. Constraint exclusion achieves similar results for inherited tables. Partitioning helps when tables exceed available memory or when queries consistently filter on the partition key."}, - {"title": "PostgreSQL: Lock Monitoring", "content": "pg_stat_activity shows currently running queries. pg_locks shows held and awaited locks. Lock contention can cause query timeouts. Use statement_timeout to prevent runaway queries. pg_stat_statements tracks query performance over time."}, + { + "title": "PostgreSQL: EXPLAIN ANALYZE", + "content": "EXPLAIN ANALYZE actually runs the query and shows real execution times alongside estimates. Key metrics: actual time (startup..total), rows (estimated vs actual), loops. Bitmap Heap Scan indicates the planner chose an index but needs to recheck. Sort Method shows if sort spilled to disk. Buffers option shows shared/local hit/read counts for I/O analysis.", + }, + { + "title": "PostgreSQL: Index Types", + "content": "B-tree for equality/range (default). Hash for equality-only (rarely useful). GiST for geometric, full-text, and range types. GIN for full-text search, JSONB, arrays. BRIN for large tables with natural ordering (timestamps). Partial indexes (WHERE clause) reduce index size. Expression indexes allow indexing computed values.", + }, + { + "title": "PostgreSQL: Configuration", + "content": "shared_buffers: typically 25% of system RAM. effective_cache_size: total expected filesystem cache (50-75% of RAM). work_mem: per-operation sort/hash memory. maintenance_work_mem: for VACUUM, CREATE INDEX. random_page_cost: lower for SSDs (1.1 vs 4.0 default). Setting these correctly can improve query plans dramatically.", + }, + { + "title": "PostgreSQL: Table Statistics", + "content": "ANALYZE collects statistics about column value distributions. default_statistics_target controls granularity (default 100, increase for skewed data). pg_stats view shows most common values, histogram bounds, null fraction, and n_distinct. Stale statistics lead to poor query plans.", + }, + { + "title": "PostgreSQL: Partitioning", + "content": "Partition pruning eliminates irrelevant partitions during planning. Constraint exclusion achieves similar results for inherited tables. Partitioning helps when tables exceed available memory or when queries consistently filter on the partition key.", + }, + { + "title": "PostgreSQL: Lock Monitoring", + "content": "pg_stat_activity shows currently running queries. pg_locks shows held and awaited locks. Lock contention can cause query timeouts. Use statement_timeout to prevent runaway queries. pg_stat_statements tracks query performance over time.", + }, ], "required_facts": [ - {"fact": "EXPLAIN ANALYZE shows real execution times", "source_doc_index": 0}, + { + "fact": "EXPLAIN ANALYZE shows real execution times", + "source_doc_index": 0, + }, {"fact": "Buffers option shows IO hit read counts", "source_doc_index": 0}, - {"fact": "BRIN for large tables with natural ordering", "source_doc_index": 1}, + { + "fact": "BRIN for large tables with natural ordering", + "source_doc_index": 1, + }, {"fact": "partial indexes reduce index size", "source_doc_index": 1}, {"fact": "shared_buffers 25% of RAM", "source_doc_index": 2}, {"fact": "random_page_cost lower for SSDs", "source_doc_index": 2}, @@ -454,34 +913,82 @@ _HARD_TASKS: List[Dict[str, Any]] = [ { "question": "How would you implement a robust nginx configuration for a microservices architecture with rate limiting, caching, and health checks?", "documents": [ - {"title": "nginx: Rate Limiting", "content": "limit_req_zone defines a shared memory zone for rate limiting. limit_req applies the limit to a location. burst parameter allows temporary spikes. nodelay processes burst requests immediately rather than queuing. limit_req_status sets the HTTP status code for rejected requests (default 503). Use $binary_remote_addr for per-IP limiting."}, - {"title": "nginx: Caching", "content": "proxy_cache_path defines the cache directory, zone size, and eviction settings. proxy_cache activates caching for a location. proxy_cache_valid sets cache duration per status code. proxy_cache_bypass allows skipping cache based on conditions. proxy_cache_use_stale serves stale content during backend errors or updates."}, - {"title": "nginx: Health Checks", "content": "Passive health checks mark backends as unavailable after max_fails within fail_timeout period. Active health checks (nginx Plus or third-party modules) send periodic requests to a health endpoint. health_check interval=5s passes=3 fails=2 configures active checks."}, - {"title": "nginx: Load Balancing Advanced", "content": "upstream blocks support server weights for proportional distribution. backup servers receive traffic only when all primary servers are unavailable. least_conn with weights combines connection-aware and weighted distribution. keepalive directive maintains persistent connections to upstreams."}, - {"title": "nginx: Logging", "content": "access_log and error_log configure logging. log_format defines custom log formats. access_log can use variables for conditional logging. Buffer and flush parameters control I/O performance. JSON log format enables structured log processing."}, - {"title": "nginx: Security Headers", "content": "add_header X-Frame-Options DENY prevents clickjacking. add_header Content-Security-Policy restricts resource loading. add_header Strict-Transport-Security enables HSTS. add_header X-Content-Type-Options nosniff prevents MIME sniffing."}, + { + "title": "nginx: Rate Limiting", + "content": "limit_req_zone defines a shared memory zone for rate limiting. limit_req applies the limit to a location. burst parameter allows temporary spikes. nodelay processes burst requests immediately rather than queuing. limit_req_status sets the HTTP status code for rejected requests (default 503). Use $binary_remote_addr for per-IP limiting.", + }, + { + "title": "nginx: Caching", + "content": "proxy_cache_path defines the cache directory, zone size, and eviction settings. proxy_cache activates caching for a location. proxy_cache_valid sets cache duration per status code. proxy_cache_bypass allows skipping cache based on conditions. proxy_cache_use_stale serves stale content during backend errors or updates.", + }, + { + "title": "nginx: Health Checks", + "content": "Passive health checks mark backends as unavailable after max_fails within fail_timeout period. Active health checks (nginx Plus or third-party modules) send periodic requests to a health endpoint. health_check interval=5s passes=3 fails=2 configures active checks.", + }, + { + "title": "nginx: Load Balancing Advanced", + "content": "upstream blocks support server weights for proportional distribution. backup servers receive traffic only when all primary servers are unavailable. least_conn with weights combines connection-aware and weighted distribution. keepalive directive maintains persistent connections to upstreams.", + }, + { + "title": "nginx: Logging", + "content": "access_log and error_log configure logging. log_format defines custom log formats. access_log can use variables for conditional logging. Buffer and flush parameters control I/O performance. JSON log format enables structured log processing.", + }, + { + "title": "nginx: Security Headers", + "content": "add_header X-Frame-Options DENY prevents clickjacking. add_header Content-Security-Policy restricts resource loading. add_header Strict-Transport-Security enables HSTS. add_header X-Content-Type-Options nosniff prevents MIME sniffing.", + }, ], "required_facts": [ {"fact": "limit_req_zone with binary_remote_addr", "source_doc_index": 0}, {"fact": "burst and nodelay for spike handling", "source_doc_index": 0}, - {"fact": "proxy_cache_use_stale serves during errors", "source_doc_index": 1}, - {"fact": "passive checks use max_fails fail_timeout", "source_doc_index": 2}, + { + "fact": "proxy_cache_use_stale serves during errors", + "source_doc_index": 1, + }, + { + "fact": "passive checks use max_fails fail_timeout", + "source_doc_index": 2, + }, {"fact": "backup servers for failover", "source_doc_index": 3}, ], }, { "question": "Design a Redis-based distributed locking system with proper expiration, retry logic, and fencing tokens.", "documents": [ - {"title": "Redis: Distributed Locks (Redlock)", "content": "SET resource_name my_random_value NX PX 30000 acquires a lock atomically. NX ensures only-if-not-exists. PX sets millisecond expiry. The random value is used for safe release: only delete if the current value matches (use Lua script for atomicity). Redlock algorithm acquires locks on N/2+1 independent Redis instances for fault tolerance."}, - {"title": "Redis: Lua Scripting", "content": "EVAL runs Lua scripts atomically on the server. Scripts can read and write multiple keys atomically. Use EVALSHA with script caching for performance. Lua scripts block other commands during execution — keep them short. Scripts have access to redis.call() and redis.pcall() functions."}, - {"title": "Redis: Pub/Sub", "content": "SUBSCRIBE/PUBLISH enables real-time messaging. Channel-based messaging is fire-and-forget. Pattern subscriptions (PSUBSCRIBE) match multiple channels. Pub/Sub does not persist messages. Use Redis Streams for durable messaging with consumer groups."}, - {"title": "Redis: Transactions", "content": "MULTI/EXEC executes commands atomically. WATCH provides optimistic locking — if a watched key changes before EXEC, the transaction is aborted. Transactions do not support rollback — all or nothing execution. DISCARD cancels a transaction."}, - {"title": "Redis: Cluster", "content": "In Redis Cluster, locks must account for hash slot assignment. Keys in the same slot are on the same node. Use hash tags {resource} to control slot assignment. During failover, locks may be lost if async replication hasn't caught up."}, - {"title": "Redis: Monitoring", "content": "SLOWLOG records queries exceeding a time threshold. CLIENT LIST shows connected clients. MONITOR streams all commands received — use sparingly as it impacts performance. INFO commandstats shows per-command statistics."}, + { + "title": "Redis: Distributed Locks (Redlock)", + "content": "SET resource_name my_random_value NX PX 30000 acquires a lock atomically. NX ensures only-if-not-exists. PX sets millisecond expiry. The random value is used for safe release: only delete if the current value matches (use Lua script for atomicity). Redlock algorithm acquires locks on N/2+1 independent Redis instances for fault tolerance.", + }, + { + "title": "Redis: Lua Scripting", + "content": "EVAL runs Lua scripts atomically on the server. Scripts can read and write multiple keys atomically. Use EVALSHA with script caching for performance. Lua scripts block other commands during execution — keep them short. Scripts have access to redis.call() and redis.pcall() functions.", + }, + { + "title": "Redis: Pub/Sub", + "content": "SUBSCRIBE/PUBLISH enables real-time messaging. Channel-based messaging is fire-and-forget. Pattern subscriptions (PSUBSCRIBE) match multiple channels. Pub/Sub does not persist messages. Use Redis Streams for durable messaging with consumer groups.", + }, + { + "title": "Redis: Transactions", + "content": "MULTI/EXEC executes commands atomically. WATCH provides optimistic locking — if a watched key changes before EXEC, the transaction is aborted. Transactions do not support rollback — all or nothing execution. DISCARD cancels a transaction.", + }, + { + "title": "Redis: Cluster", + "content": "In Redis Cluster, locks must account for hash slot assignment. Keys in the same slot are on the same node. Use hash tags {resource} to control slot assignment. During failover, locks may be lost if async replication hasn't caught up.", + }, + { + "title": "Redis: Monitoring", + "content": "SLOWLOG records queries exceeding a time threshold. CLIENT LIST shows connected clients. MONITOR streams all commands received — use sparingly as it impacts performance. INFO commandstats shows per-command statistics.", + }, ], "required_facts": [ - {"fact": "SET NX PX for atomic lock acquisition with expiry", "source_doc_index": 0}, - {"fact": "random value for safe release with Lua script", "source_doc_index": 0}, + { + "fact": "SET NX PX for atomic lock acquisition with expiry", + "source_doc_index": 0, + }, + { + "fact": "random value for safe release with Lua script", + "source_doc_index": 0, + }, {"fact": "Redlock on N/2+1 instances", "source_doc_index": 0}, {"fact": "Lua scripts execute atomically", "source_doc_index": 1}, {"fact": "WATCH for optimistic locking", "source_doc_index": 3}, @@ -491,17 +998,41 @@ _HARD_TASKS: List[Dict[str, Any]] = [ { "question": "How do you implement a complete observability stack for a Python asyncio application with logging, tracing, and metrics?", "documents": [ - {"title": "Python: Structured Logging", "content": "python-json-logger outputs JSON log records. structlog provides contextualized logging with processors. Both integrate with the standard logging module. Use contextvars to propagate trace IDs through async call chains. Log at structured fields: service, environment, trace_id, span_id, user_id, duration_ms."}, - {"title": "Python: OpenTelemetry", "content": "opentelemetry-api provides the Tracer and Meter interfaces. opentelemetry-sdk implements them. TracerProvider creates Tracers. Spans represent operations with start/end times, attributes, and events. Context propagation across async boundaries uses contextvars. Exporters send data to Jaeger, Zipkin, or OTLP collectors."}, - {"title": "Python: Prometheus Metrics", "content": "prometheus_client provides Counter, Gauge, Histogram, and Summary metric types. Counter only goes up (request counts, error counts). Histogram tracks value distributions with configurable buckets. start_http_server() exposes /metrics endpoint. Use labels for dimensions (method, path, status_code)."}, - {"title": "Python: asyncio Instrumentation", "content": "asyncio.Task has get_name()/set_name() for identification. Task groups (asyncio.TaskGroup) manage related tasks. loop.slow_callback_duration controls slow callback warnings. asyncio debug mode logs coroutines that take too long. Custom task factories can inject tracing context."}, - {"title": "Python: Health Checks", "content": "Implement /healthz and /readyz endpoints. Health checks should verify database connectivity, cache availability, and external service reachability. Use timeouts to prevent health check hangs. Separate liveness (is the process alive?) from readiness (can it serve traffic?)."}, - {"title": "Python: Error Tracking", "content": "Sentry SDK captures exceptions with full stack traces and context. breadcrumbs track events leading to an error. Integrations auto-instrument popular frameworks. Set sample_rate to control volume. tags and extra context help with debugging."}, + { + "title": "Python: Structured Logging", + "content": "python-json-logger outputs JSON log records. structlog provides contextualized logging with processors. Both integrate with the standard logging module. Use contextvars to propagate trace IDs through async call chains. Log at structured fields: service, environment, trace_id, span_id, user_id, duration_ms.", + }, + { + "title": "Python: OpenTelemetry", + "content": "opentelemetry-api provides the Tracer and Meter interfaces. opentelemetry-sdk implements them. TracerProvider creates Tracers. Spans represent operations with start/end times, attributes, and events. Context propagation across async boundaries uses contextvars. Exporters send data to Jaeger, Zipkin, or OTLP collectors.", + }, + { + "title": "Python: Prometheus Metrics", + "content": "prometheus_client provides Counter, Gauge, Histogram, and Summary metric types. Counter only goes up (request counts, error counts). Histogram tracks value distributions with configurable buckets. start_http_server() exposes /metrics endpoint. Use labels for dimensions (method, path, status_code).", + }, + { + "title": "Python: asyncio Instrumentation", + "content": "asyncio.Task has get_name()/set_name() for identification. Task groups (asyncio.TaskGroup) manage related tasks. loop.slow_callback_duration controls slow callback warnings. asyncio debug mode logs coroutines that take too long. Custom task factories can inject tracing context.", + }, + { + "title": "Python: Health Checks", + "content": "Implement /healthz and /readyz endpoints. Health checks should verify database connectivity, cache availability, and external service reachability. Use timeouts to prevent health check hangs. Separate liveness (is the process alive?) from readiness (can it serve traffic?).", + }, + { + "title": "Python: Error Tracking", + "content": "Sentry SDK captures exceptions with full stack traces and context. breadcrumbs track events leading to an error. Integrations auto-instrument popular frameworks. Set sample_rate to control volume. tags and extra context help with debugging.", + }, ], "required_facts": [ - {"fact": "structlog or json-logger for structured logging", "source_doc_index": 0}, + { + "fact": "structlog or json-logger for structured logging", + "source_doc_index": 0, + }, {"fact": "contextvars for trace ID propagation", "source_doc_index": 0}, - {"fact": "OpenTelemetry Spans with attributes and events", "source_doc_index": 1}, + { + "fact": "OpenTelemetry Spans with attributes and events", + "source_doc_index": 1, + }, {"fact": "Prometheus Counter Gauge Histogram types", "source_doc_index": 2}, {"fact": "separate liveness from readiness checks", "source_doc_index": 4}, {"fact": "Sentry captures exceptions with context", "source_doc_index": 5}, @@ -550,18 +1081,20 @@ class DocQADataset(DatasetProvider): # Build reference from required facts ref_parts = [f["fact"] for f in task["required_facts"]] - self._records.append(EvalRecord( - record_id=f"doc-qa-{idx:03d}", - problem=prompt, - reference="; ".join(ref_parts), - category="agentic", - subject=diff, - metadata={ - "question": task["question"], - "documents": task["documents"], - "required_facts": task["required_facts"], - }, - )) + self._records.append( + EvalRecord( + record_id=f"doc-qa-{idx:03d}", + problem=prompt, + reference="; ".join(ref_parts), + category="agentic", + subject=diff, + metadata={ + "question": task["question"], + "documents": task["documents"], + "required_facts": task["required_facts"], + }, + ) + ) def iter_records(self) -> Iterable[EvalRecord]: return iter(self._records) diff --git a/src/openjarvis/evals/datasets/email_triage.py b/src/openjarvis/evals/datasets/email_triage.py index 366da967..daf03196 100644 --- a/src/openjarvis/evals/datasets/email_triage.py +++ b/src/openjarvis/evals/datasets/email_triage.py @@ -307,23 +307,22 @@ class EmailTriageDataset(DatasetProvider): date=email["date"], body=email["body"], ) - reference = ( - f"urgency: {email['urgency']}\n" - f"category: {email['category']}" + reference = f"urgency: {email['urgency']}\ncategory: {email['category']}" + self._records.append( + EvalRecord( + record_id=f"email-triage-{idx}", + problem=prompt, + reference=reference, + category="use-case", + subject="email_triage", + metadata={ + "urgency": email["urgency"], + "category": email["category"], + "sender": email["sender"], + "subject_line": email["subject"], + }, + ) ) - self._records.append(EvalRecord( - record_id=f"email-triage-{idx}", - problem=prompt, - reference=reference, - category="use-case", - subject="email_triage", - metadata={ - "urgency": email["urgency"], - "category": email["category"], - "sender": email["sender"], - "subject_line": email["subject"], - }, - )) def iter_records(self) -> Iterable[EvalRecord]: return iter(self._records) diff --git a/src/openjarvis/evals/datasets/frames.py b/src/openjarvis/evals/datasets/frames.py index 8c4df51d..936afd5a 100644 --- a/src/openjarvis/evals/datasets/frames.py +++ b/src/openjarvis/evals/datasets/frames.py @@ -76,7 +76,9 @@ class FRAMESDataset(DatasetProvider): return len(self._records) def _convert_row( - self, raw: MutableMapping[str, object], idx: int, + self, + raw: MutableMapping[str, object], + idx: int, ) -> Optional[EvalRecord]: question = str( raw.get("Prompt") or raw.get("prompt") or raw.get("question") or "" @@ -97,7 +99,9 @@ class FRAMESDataset(DatasetProvider): # Extract wiki links wiki_links_raw = raw.get("wiki_links", raw.get("wikipedia_links", [])) if isinstance(wiki_links_raw, str): - wiki_links = [link.strip() for link in wiki_links_raw.split(",") if link.strip()] + wiki_links = [ + link.strip() for link in wiki_links_raw.split(",") if link.strip() + ] elif isinstance(wiki_links_raw, list): wiki_links = [str(link) for link in wiki_links_raw] else: @@ -112,7 +116,8 @@ class FRAMESDataset(DatasetProvider): ) problem = _DEFAULT_INPUT_PROMPT.format( - question=question, wiki_context=wiki_context, + question=question, + wiki_context=wiki_context, ) subject = reasoning if reasoning else "general" diff --git a/src/openjarvis/evals/datasets/gpqa.py b/src/openjarvis/evals/datasets/gpqa.py index 9102f985..226af88d 100644 --- a/src/openjarvis/evals/datasets/gpqa.py +++ b/src/openjarvis/evals/datasets/gpqa.py @@ -72,7 +72,9 @@ class GPQADataset(DatasetProvider): return len(self._records) def _convert_row( - self, raw: MutableMapping[str, object], idx: int, + self, + raw: MutableMapping[str, object], + idx: int, ) -> Optional[EvalRecord]: # Field names vary across dataset versions. question = str( @@ -86,9 +88,12 @@ class GPQADataset(DatasetProvider): # Gather distractor answers. distractors: List[str] = [] for key in ( - "Incorrect Answer 1", "incorrect_answer_1", - "Incorrect Answer 2", "incorrect_answer_2", - "Incorrect Answer 3", "incorrect_answer_3", + "Incorrect Answer 1", + "incorrect_answer_1", + "Incorrect Answer 2", + "incorrect_answer_2", + "Incorrect Answer 3", + "incorrect_answer_3", ): val = raw.get(key) if val is not None: @@ -103,21 +108,19 @@ class GPQADataset(DatasetProvider): options = [correct_answer] + distractors[:3] subdomain = str( - raw.get("Subdomain") - or raw.get("subdomain") - or "", + raw.get("Subdomain") or raw.get("subdomain") or "", ).strip() domain = str( - raw.get("High-level domain") - or raw.get("domain") - or "", + raw.get("High-level domain") or raw.get("domain") or "", ).strip() subject = subdomain or domain or "general" prompt_parts = [ - question, "", + question, + "", "Options:", - _format_options(options), "", + _format_options(options), + "", "Provide only the letter of the correct answer (A, B, C, or D).", ] problem = "\n".join(part for part in prompt_parts if part).strip() diff --git a/src/openjarvis/evals/datasets/hle.py b/src/openjarvis/evals/datasets/hle.py index a44afd51..e7d44372 100644 --- a/src/openjarvis/evals/datasets/hle.py +++ b/src/openjarvis/evals/datasets/hle.py @@ -76,7 +76,9 @@ class HLEDataset(DatasetProvider): return False def _convert_row( - self, raw: MutableMapping[str, object], idx: int, + self, + raw: MutableMapping[str, object], + idx: int, ) -> Optional[EvalRecord]: # Skip multimodal rows when text_only is enabled if self._text_only and self._is_multimodal(raw): @@ -84,36 +86,31 @@ class HLEDataset(DatasetProvider): # Extract question text question_text = str( - raw.get("question") - or raw.get("instruction") - or raw.get("prompt") - or "" + raw.get("question") or raw.get("instruction") or raw.get("prompt") or "" ).strip() if not question_text: return None # Extract reference answer reference = str( - raw.get("answer") - or raw.get("gold_answer") - or raw.get("response") - or "" + raw.get("answer") or raw.get("gold_answer") or raw.get("response") or "" ).strip() if not reference: return None # Extract category - category_value = str( - raw.get("category") - or raw.get("subject") - or raw.get("type") + category_value = ( + str( + raw.get("category") + or raw.get("subject") + or raw.get("type") + or "general" + ).strip() or "general" - ).strip() or "general" + ) # Extract task_id for the record_id - task_id = str( - raw.get("id") or raw.get("task_id") or f"hle_{idx}" - ).strip() + task_id = str(raw.get("id") or raw.get("task_id") or f"hle_{idx}").strip() # Use question directly (no wrapper template) problem = question_text diff --git a/src/openjarvis/evals/datasets/ipw_mixed.py b/src/openjarvis/evals/datasets/ipw_mixed.py index 1cc85d8f..5bad3af9 100644 --- a/src/openjarvis/evals/datasets/ipw_mixed.py +++ b/src/openjarvis/evals/datasets/ipw_mixed.py @@ -66,7 +66,9 @@ class IPWDataset(DatasetProvider): self._records.append(record) LOGGER.info( - "IPW dataset loaded: %d records from %s", len(self._records), data_path, + "IPW dataset loaded: %d records from %s", + len(self._records), + data_path, ) def iter_records(self) -> Iterable[EvalRecord]: @@ -166,14 +168,11 @@ class IPWDataset(DatasetProvider): @staticmethod def _convert_row( - raw: MutableMapping[str, Any], idx: int, + raw: MutableMapping[str, Any], + idx: int, ) -> Optional[EvalRecord]: - problem = str( - raw.get("problem") or raw.get("prompt") or "" - ).strip() - answer = str( - raw.get("answer") or raw.get("expected_answer") or "" - ).strip() + problem = str(raw.get("problem") or raw.get("prompt") or "").strip() + answer = str(raw.get("answer") or raw.get("expected_answer") or "").strip() subject = str(raw.get("subject") or "general").strip() or "general" # Require non-empty problem, answer, and subject diff --git a/src/openjarvis/evals/datasets/knowledge_base.py b/src/openjarvis/evals/datasets/knowledge_base.py index d2bb1c2f..26984e7c 100644 --- a/src/openjarvis/evals/datasets/knowledge_base.py +++ b/src/openjarvis/evals/datasets/knowledge_base.py @@ -206,14 +206,16 @@ class KnowledgeBaseDataset(DatasetProvider): documents=rec["documents"], question=rec["question"], ) - self._records.append(EvalRecord( - record_id=f"knowledge-base-{idx}", - problem=prompt, - reference=rec["answer"], - category="use-case", - subject="knowledge_base", - metadata={}, - )) + self._records.append( + EvalRecord( + record_id=f"knowledge-base-{idx}", + problem=prompt, + reference=rec["answer"], + category="use-case", + subject="knowledge_base", + metadata={}, + ) + ) def iter_records(self) -> Iterable[EvalRecord]: return iter(self._records) diff --git a/src/openjarvis/evals/datasets/lifelong_agent.py b/src/openjarvis/evals/datasets/lifelong_agent.py index b67e12fa..a8ea6bc2 100644 --- a/src/openjarvis/evals/datasets/lifelong_agent.py +++ b/src/openjarvis/evals/datasets/lifelong_agent.py @@ -59,7 +59,7 @@ _DB_SYSTEM_PROMPT = ( "operate with one SQL statement at a time.\n" "Every time you can only execute one SQL statement. I will execute " "the SQL statement on the MySQL database and return the result to you.\n\n" - "If the question cannot be answered, respond with \"N/A\".\n\n" + 'If the question cannot be answered, respond with "N/A".\n\n' "To execute SQL, respond with:\n" "Action: Operation\n" "```sql\n\n```\n\n" @@ -173,10 +173,7 @@ class LifelongAgentDataset(DatasetProvider): split: Optional[str] = None, seed: Optional[int] = None, ) -> None: - subsets = ( - list(_VALID_SUBSETS) if self._subset == "all" - else [self._subset] - ) + subsets = list(_VALID_SUBSETS) if self._subset == "all" else [self._subset] self._records = [] load_failures: List[str] = [] @@ -209,15 +206,16 @@ class LifelongAgentDataset(DatasetProvider): subset_count += 1 logger.info( - "LifelongAgentBench[%s]: loaded %d records", subset, subset_count, + "LifelongAgentBench[%s]: loaded %d records", + subset, + subset_count, ) # Fail loudly if nothing loaded if not self._records: raise RuntimeError( "LifelongAgentBench: loaded 0 records across all subsets. " - "The benchmark cannot run. Failures:\n" - + "\n".join(load_failures) + "The benchmark cannot run. Failures:\n" + "\n".join(load_failures) ) # NOTE: Do NOT shuffle records. The original LifelongAgentBench @@ -228,7 +226,8 @@ class LifelongAgentDataset(DatasetProvider): if seed is not None: logger.info( "LifelongAgentBench: seed=%d ignored — lifelong protocol " - "requires strict sample_index ordering.", seed, + "requires strict sample_index ordering.", + seed, ) if max_samples is not None: @@ -269,6 +268,7 @@ class LifelongAgentDataset(DatasetProvider): from openjarvis.evals.environments.lifelong_agent_env import ( create_task_environment, ) + return create_task_environment( record, sparql_endpoint=self._sparql_endpoint, @@ -283,9 +283,7 @@ class LifelongAgentDataset(DatasetProvider): try: import datasets # noqa: F401 except ImportError: - missing.append( - "The 'datasets' library is required: pip install datasets" - ) + missing.append("The 'datasets' library is required: pip install datasets") return missing # ------------------------------------------------------------------ @@ -326,7 +324,11 @@ class LifelongAgentDataset(DatasetProvider): "FAILED to load %s[%s] from HuggingFace: %s\n" "Tried both data_files='%s/*.parquet' and name='%s'.\n" "Check your network connection and 'datasets' installation.", - _HF_REPO_ID, subset, exc2, subset, subset, + _HF_REPO_ID, + subset, + exc2, + subset, + subset, ) return [] diff --git a/src/openjarvis/evals/datasets/loghub.py b/src/openjarvis/evals/datasets/loghub.py index 31601b01..6ddbcf4b 100644 --- a/src/openjarvis/evals/datasets/loghub.py +++ b/src/openjarvis/evals/datasets/loghub.py @@ -65,8 +65,7 @@ class LogHubDataset(DatasetProvider): ) self._subset = subset self._cache_dir = ( - Path(cache_dir) if cache_dir - else Path.home() / ".cache" / "loghub" + Path(cache_dir) if cache_dir else Path.home() / ".cache" / "loghub" ) self._records: List[EvalRecord] = [] @@ -118,7 +117,9 @@ class LogHubDataset(DatasetProvider): ) def _load_session_mode( - self, data_dir: Path, meta: Dict[str, Any], + self, + data_dir: Path, + meta: Dict[str, Any], ) -> List[EvalRecord]: """Load HDFS-style session-based records (group by block_id).""" log_path = data_dir / meta["log_file"] @@ -157,24 +158,28 @@ class LogHubDataset(DatasetProvider): f"({len(lines)} lines):\n```\n{log_text}\n```" ) - records.append(EvalRecord( - record_id=f"loghub-{self._subset}-{bid}", - problem=problem, - reference=reference, - category="agentic", - subject=self._subset, - metadata={ - "block_id": bid, - "num_lines": len(lines), - "dataset": self._subset, - "label": label, - }, - )) + records.append( + EvalRecord( + record_id=f"loghub-{self._subset}-{bid}", + problem=problem, + reference=reference, + category="agentic", + subject=self._subset, + metadata={ + "block_id": bid, + "num_lines": len(lines), + "dataset": self._subset, + "label": label, + }, + ) + ) return records def _load_window_mode( - self, data_dir: Path, meta: Dict[str, Any], + self, + data_dir: Path, + meta: Dict[str, Any], ) -> List[EvalRecord]: """Load BGL/Thunderbird-style windowed records.""" log_path = data_dir / meta["log_file"] @@ -204,19 +209,21 @@ class LogHubDataset(DatasetProvider): f"({len(window)} lines):\n```\n{log_text}\n```" ) - records.append(EvalRecord( - record_id=f"loghub-{self._subset}-w{window_idx}", - problem=problem, - reference=reference, - category="agentic", - subject=self._subset, - metadata={ - "window_idx": window_idx, - "num_lines": len(window), - "dataset": self._subset, - "has_anomaly": has_anomaly, - }, - )) + records.append( + EvalRecord( + record_id=f"loghub-{self._subset}-w{window_idx}", + problem=problem, + reference=reference, + category="agentic", + subject=self._subset, + metadata={ + "window_idx": window_idx, + "num_lines": len(window), + "dataset": self._subset, + "has_anomaly": has_anomaly, + }, + ) + ) window = [] has_anomaly = False @@ -231,19 +238,21 @@ class LogHubDataset(DatasetProvider): f"Log window {window_idx} " f"({len(window)} lines):\n```\n{log_text}\n```" ) - records.append(EvalRecord( - record_id=f"loghub-{self._subset}-w{window_idx}", - problem=problem, - reference=reference, - category="agentic", - subject=self._subset, - metadata={ - "window_idx": window_idx, - "num_lines": len(window), - "dataset": self._subset, - "has_anomaly": has_anomaly, - }, - )) + records.append( + EvalRecord( + record_id=f"loghub-{self._subset}-w{window_idx}", + problem=problem, + reference=reference, + category="agentic", + subject=self._subset, + metadata={ + "window_idx": window_idx, + "num_lines": len(window), + "dataset": self._subset, + "has_anomaly": has_anomaly, + }, + ) + ) return records diff --git a/src/openjarvis/evals/datasets/math500.py b/src/openjarvis/evals/datasets/math500.py index 9178e74f..6c8ac957 100644 --- a/src/openjarvis/evals/datasets/math500.py +++ b/src/openjarvis/evals/datasets/math500.py @@ -68,26 +68,25 @@ class MATH500Dataset(DatasetProvider): return len(self._records) def _convert_row( - self, raw: MutableMapping[str, object], idx: int, + self, + raw: MutableMapping[str, object], + idx: int, ) -> Optional[EvalRecord]: # Extract problem text - problem_text = str( - raw.get("problem") or raw.get("question") or "" - ).strip() + problem_text = str(raw.get("problem") or raw.get("question") or "").strip() if not problem_text: return None # Extract reference answer - reference = str( - raw.get("answer") or raw.get("solution") or "" - ).strip() + reference = str(raw.get("answer") or raw.get("solution") or "").strip() if not reference: return None # Extract subject - subject = str( - raw.get("subject") or raw.get("type") or "Mathematics" - ).strip() or "Mathematics" + subject = ( + str(raw.get("subject") or raw.get("type") or "Mathematics").strip() + or "Mathematics" + ) # Build prompt problem = _PROMPT_TEMPLATE.format(problem=problem_text) diff --git a/src/openjarvis/evals/datasets/mmlu_pro.py b/src/openjarvis/evals/datasets/mmlu_pro.py index 761d6669..9b3bdb69 100644 --- a/src/openjarvis/evals/datasets/mmlu_pro.py +++ b/src/openjarvis/evals/datasets/mmlu_pro.py @@ -71,7 +71,9 @@ class MMLUProDataset(DatasetProvider): return len(self._records) def _convert_row( - self, raw: MutableMapping[str, object], idx: int, + self, + raw: MutableMapping[str, object], + idx: int, ) -> Optional[EvalRecord]: question = str(raw.get("question") or "").strip() options_raw = raw.get("options") or [] @@ -84,9 +86,11 @@ class MMLUProDataset(DatasetProvider): return None prompt_parts = [ - question, "", + question, + "", "Options:", - _format_options(options), "", + _format_options(options), + "", "Respond with the correct letter.", ] problem = "\n".join(part for part in prompt_parts if part).strip() diff --git a/src/openjarvis/evals/datasets/morning_brief.py b/src/openjarvis/evals/datasets/morning_brief.py index 1fa79d3c..c9549906 100644 --- a/src/openjarvis/evals/datasets/morning_brief.py +++ b/src/openjarvis/evals/datasets/morning_brief.py @@ -196,14 +196,16 @@ class MorningBriefDataset(DatasetProvider): news_topics=ctx["news_topics"], messages=ctx["messages"], ) - self._records.append(EvalRecord( - record_id=f"morning-brief-{idx}", - problem=prompt, - reference=ctx["key_priorities"], - category="use-case", - subject="morning_brief", - metadata={"date": ctx["date"]}, - )) + self._records.append( + EvalRecord( + record_id=f"morning-brief-{idx}", + problem=prompt, + reference=ctx["key_priorities"], + category="use-case", + subject="morning_brief", + metadata={"date": ctx["date"]}, + ) + ) def iter_records(self) -> Iterable[EvalRecord]: return iter(self._records) diff --git a/src/openjarvis/evals/datasets/natural_reasoning.py b/src/openjarvis/evals/datasets/natural_reasoning.py index 5ce98075..79c74bcc 100644 --- a/src/openjarvis/evals/datasets/natural_reasoning.py +++ b/src/openjarvis/evals/datasets/natural_reasoning.py @@ -68,29 +68,30 @@ class NaturalReasoningDataset(DatasetProvider): return len(self._records) def _convert_row( - self, raw: MutableMapping[str, object], idx: int, + self, + raw: MutableMapping[str, object], + idx: int, ) -> Optional[EvalRecord]: # Extract question text - question_text = str( - raw.get("question") or raw.get("problem") or "" - ).strip() + question_text = str(raw.get("question") or raw.get("problem") or "").strip() if not question_text: return None # Extract reference answer - reference = str( - raw.get("answer") or raw.get("solution") or "" - ).strip() + reference = str(raw.get("answer") or raw.get("solution") or "").strip() if not reference: return None # Extract subject/category - subject = str( - raw.get("category") - or raw.get("field") - or raw.get("source") + subject = ( + str( + raw.get("category") + or raw.get("field") + or raw.get("source") + or "General" + ).strip() or "General" - ).strip() or "General" + ) # Build prompt problem = _PROMPT_TEMPLATE.format(question=question_text) diff --git a/src/openjarvis/evals/datasets/paperarena.py b/src/openjarvis/evals/datasets/paperarena.py index 786bad8a..9aa9bea4 100644 --- a/src/openjarvis/evals/datasets/paperarena.py +++ b/src/openjarvis/evals/datasets/paperarena.py @@ -41,8 +41,7 @@ class PaperArenaDataset(DatasetProvider): cache_dir: Optional[str] = None, ) -> None: self._cache_dir = ( - Path(cache_dir) if cache_dir - else Path.home() / ".cache" / "paperarena" + Path(cache_dir) if cache_dir else Path.home() / ".cache" / "paperarena" ) self._records: List[EvalRecord] = [] @@ -143,10 +142,10 @@ class PaperArenaDataset(DatasetProvider): prompt += f"## Question\n{question}" if options and question_type == "MC": - options_text = "\n".join( - f" {k}) {v}" for k, v in options.items() - ) if isinstance(options, dict) else "\n".join( - f" {chr(65 + i)}) {o}" for i, o in enumerate(options) + options_text = ( + "\n".join(f" {k}) {v}" for k, v in options.items()) + if isinstance(options, dict) + else "\n".join(f" {chr(65 + i)}) {o}" for i, o in enumerate(options)) ) prompt += f"\n\nOptions:\n{options_text}" diff --git a/src/openjarvis/evals/datasets/pinchbench.py b/src/openjarvis/evals/datasets/pinchbench.py index 9c084bc4..f059d8b5 100644 --- a/src/openjarvis/evals/datasets/pinchbench.py +++ b/src/openjarvis/evals/datasets/pinchbench.py @@ -102,7 +102,9 @@ class PinchBenchDataset(DatasetProvider): def verify_requirements(self) -> List[str]: issues: List[str] = [] if self._local_path is None and shutil.which("git") is None: - issues.append("git binary not found. Install git to clone PinchBench tasks.") + issues.append( + "git binary not found. Install git to clone PinchBench tasks." + ) if self._repo_dir.exists() and not (self._repo_dir / "tasks").is_dir(): issues.append( f"PinchBench cache at {self._repo_dir} is corrupted (missing tasks/). " @@ -114,7 +116,9 @@ class PinchBenchDataset(DatasetProvider): """Clone the repo if not already cached. Returns repo dir.""" if self._local_path is not None: if not self._local_path.exists(): - raise FileNotFoundError(f"PinchBench path not found: {self._local_path}") + raise FileNotFoundError( + f"PinchBench path not found: {self._local_path}" + ) return self._local_path if not self._repo_dir.exists(): @@ -200,6 +204,7 @@ class PinchBenchDataset(DatasetProvider): def create_task_env(self, record: EvalRecord): from openjarvis.evals.execution.pinchbench_env import PinchBenchTaskEnv + return PinchBenchTaskEnv( record, judge_backend=getattr(self, "_judge_backend", None), diff --git a/src/openjarvis/evals/datasets/research_mining.py b/src/openjarvis/evals/datasets/research_mining.py index 499ddd8c..fbdf39be 100644 --- a/src/openjarvis/evals/datasets/research_mining.py +++ b/src/openjarvis/evals/datasets/research_mining.py @@ -245,14 +245,16 @@ class ResearchMiningDataset(DatasetProvider): domain=q["domain"], scope=q["scope"], ) - self._records.append(EvalRecord( - record_id=f"research-mining-{idx}", - problem=prompt, - reference=q["key_facts"], - category="use-case", - subject="research_mining", - metadata={"domain": q["domain"], "scope": q["scope"]}, - )) + self._records.append( + EvalRecord( + record_id=f"research-mining-{idx}", + problem=prompt, + reference=q["key_facts"], + category="use-case", + subject="research_mining", + metadata={"domain": q["domain"], "scope": q["scope"]}, + ) + ) def iter_records(self) -> Iterable[EvalRecord]: return iter(self._records) diff --git a/src/openjarvis/evals/datasets/security_scanner.py b/src/openjarvis/evals/datasets/security_scanner.py index d5252710..b743c963 100644 --- a/src/openjarvis/evals/datasets/security_scanner.py +++ b/src/openjarvis/evals/datasets/security_scanner.py @@ -46,12 +46,18 @@ _EASY_TASKS: List[Dict[str, Any]] = [ " uid = request.args.get('id')\n" " conn = sqlite3.connect('app.db')\n" " cursor = conn.cursor()\n" - " cursor.execute(f\"SELECT * FROM users WHERE id = {uid}\")\n" + ' cursor.execute(f"SELECT * FROM users WHERE id = {uid}")\n' " return str(cursor.fetchone())\n" ), }, "vulnerabilities": [ - {"file": "app.py", "line": 11, "type": "sql_injection", "severity": "CRITICAL", "description": "f-string SQL query with unsanitized user input"}, + { + "file": "app.py", + "line": 11, + "type": "sql_injection", + "severity": "CRITICAL", + "description": "f-string SQL query with unsanitized user input", + }, ], "safe_patterns": ["Flask app instantiation", "sqlite3.connect usage"], }, @@ -65,10 +71,34 @@ _EASY_TASKS: List[Dict[str, Any]] = [ ), }, "vulnerabilities": [ - {"file": "config.py", "line": 1, "type": "hardcoded_secret", "severity": "CRITICAL", "description": "Database password hardcoded in source"}, - {"file": "config.py", "line": 2, "type": "hardcoded_secret", "severity": "HIGH", "description": "Secret key hardcoded in source"}, - {"file": "config.py", "line": 4, "type": "hardcoded_secret", "severity": "CRITICAL", "description": "API key hardcoded in source"}, - {"file": "config.py", "line": 3, "type": "debug_enabled", "severity": "MEDIUM", "description": "Debug mode enabled in production config"}, + { + "file": "config.py", + "line": 1, + "type": "hardcoded_secret", + "severity": "CRITICAL", + "description": "Database password hardcoded in source", + }, + { + "file": "config.py", + "line": 2, + "type": "hardcoded_secret", + "severity": "HIGH", + "description": "Secret key hardcoded in source", + }, + { + "file": "config.py", + "line": 4, + "type": "hardcoded_secret", + "severity": "CRITICAL", + "description": "API key hardcoded in source", + }, + { + "file": "config.py", + "line": 3, + "type": "debug_enabled", + "severity": "MEDIUM", + "description": "Debug mode enabled in production config", + }, ], "safe_patterns": [], }, @@ -82,7 +112,13 @@ _EASY_TASKS: List[Dict[str, Any]] = [ ), }, "vulnerabilities": [ - {"file": "utils.py", "line": 4, "type": "command_injection", "severity": "CRITICAL", "description": "subprocess.call with shell=True and unsanitized input"}, + { + "file": "utils.py", + "line": 4, + "type": "command_injection", + "severity": "CRITICAL", + "description": "subprocess.call with shell=True and unsanitized input", + }, ], "safe_patterns": [], }, @@ -97,7 +133,13 @@ _EASY_TASKS: List[Dict[str, Any]] = [ ), }, "vulnerabilities": [ - {"file": "auth.py", "line": 4, "type": "weak_crypto", "severity": "HIGH", "description": "MD5 used for password hashing — use bcrypt or argon2"}, + { + "file": "auth.py", + "line": 4, + "type": "weak_crypto", + "severity": "HIGH", + "description": "MD5 used for password hashing — use bcrypt or argon2", + }, ], "safe_patterns": ["verify_password comparison logic"], }, @@ -113,7 +155,13 @@ _EASY_TASKS: List[Dict[str, Any]] = [ ), }, "vulnerabilities": [ - {"file": "api.py", "line": 8, "type": "insecure_deserialization", "severity": "CRITICAL", "description": "pickle.loads on untrusted request data allows arbitrary code execution"}, + { + "file": "api.py", + "line": 8, + "type": "insecure_deserialization", + "severity": "CRITICAL", + "description": "pickle.loads on untrusted request data allows arbitrary code execution", + }, ], "safe_patterns": ["Flask route decorator"], }, @@ -129,7 +177,13 @@ _EASY_TASKS: List[Dict[str, Any]] = [ ), }, "vulnerabilities": [ - {"file": "server.py", "line": 8, "type": "xss", "severity": "HIGH", "description": "render_template_string with unescaped user input enables XSS/SSTI"}, + { + "file": "server.py", + "line": 8, + "type": "xss", + "severity": "HIGH", + "description": "render_template_string with unescaped user input enables XSS/SSTI", + }, ], "safe_patterns": ["Default parameter value"], }, @@ -145,7 +199,13 @@ _EASY_TASKS: List[Dict[str, Any]] = [ ), }, "vulnerabilities": [ - {"file": "download.py", "line": 9, "type": "path_traversal", "severity": "HIGH", "description": "No validation on filename allows path traversal (../../etc/passwd)"}, + { + "file": "download.py", + "line": 9, + "type": "path_traversal", + "severity": "HIGH", + "description": "No validation on filename allows path traversal (../../etc/passwd)", + }, ], "safe_patterns": ["os.path.join usage"], }, @@ -161,7 +221,13 @@ _EASY_TASKS: List[Dict[str, Any]] = [ ), }, "vulnerabilities": [ - {"file": "cors_app.py", "line": 5, "type": "misconfiguration", "severity": "MEDIUM", "description": "CORS allows all origins — exposes API to any domain"}, + { + "file": "cors_app.py", + "line": 5, + "type": "misconfiguration", + "severity": "MEDIUM", + "description": "CORS allows all origins — exposes API to any domain", + }, ], "safe_patterns": ["Flask-CORS import pattern"], }, @@ -177,7 +243,13 @@ _EASY_TASKS: List[Dict[str, Any]] = [ ), }, "vulnerabilities": [ - {"file": "logging_app.py", "line": 6, "type": "sensitive_data_exposure", "severity": "HIGH", "description": "Credit card number logged in plaintext"}, + { + "file": "logging_app.py", + "line": 6, + "type": "sensitive_data_exposure", + "severity": "HIGH", + "description": "Credit card number logged in plaintext", + }, ], "safe_patterns": ["logging.getLogger pattern"], }, @@ -192,8 +264,20 @@ _EASY_TASKS: List[Dict[str, Any]] = [ ), }, "vulnerabilities": [ - {"file": "jwt_app.py", "line": 4, "type": "weak_crypto", "severity": "CRITICAL", "description": "JWT using algorithm='none' disables signature verification"}, - {"file": "jwt_app.py", "line": 7, "type": "weak_crypto", "severity": "CRITICAL", "description": "JWT decode allows 'none' algorithm — attacker can forge tokens"}, + { + "file": "jwt_app.py", + "line": 4, + "type": "weak_crypto", + "severity": "CRITICAL", + "description": "JWT using algorithm='none' disables signature verification", + }, + { + "file": "jwt_app.py", + "line": 7, + "type": "weak_crypto", + "severity": "CRITICAL", + "description": "JWT decode allows 'none' algorithm — attacker can forge tokens", + }, ], "safe_patterns": [], }, @@ -224,9 +308,27 @@ _MEDIUM_TASKS: List[Dict[str, Any]] = [ ), }, "vulnerabilities": [ - {"file": "app.py", "line": 14, "type": "sql_injection", "severity": "CRITICAL", "description": "f-string SQL with user-supplied username and password"}, - {"file": "app.py", "line": 17, "type": "open_redirect", "severity": "MEDIUM", "description": "Unvalidated redirect URL from request.args"}, - {"file": "app.py", "line": 6, "type": "hardcoded_secret", "severity": "LOW", "description": "Default secret key in fallback (acceptable for dev)"}, + { + "file": "app.py", + "line": 14, + "type": "sql_injection", + "severity": "CRITICAL", + "description": "f-string SQL with user-supplied username and password", + }, + { + "file": "app.py", + "line": 17, + "type": "open_redirect", + "severity": "MEDIUM", + "description": "Unvalidated redirect URL from request.args", + }, + { + "file": "app.py", + "line": 6, + "type": "hardcoded_secret", + "severity": "LOW", + "description": "Default secret key in fallback (acceptable for dev)", + }, ], "safe_patterns": ["os.environ.get for secret key (with fallback)"], }, @@ -246,8 +348,20 @@ _MEDIUM_TASKS: List[Dict[str, Any]] = [ ), }, "vulnerabilities": [ - {"file": "api.py", "line": 9, "type": "insecure_deserialization", "severity": "CRITICAL", "description": "yaml.load without Loader allows arbitrary code execution"}, - {"file": "api.py", "line": 14, "type": "xxe", "severity": "HIGH", "description": "XML parsing without disabling external entities"}, + { + "file": "api.py", + "line": 9, + "type": "insecure_deserialization", + "severity": "CRITICAL", + "description": "yaml.load without Loader allows arbitrary code execution", + }, + { + "file": "api.py", + "line": 14, + "type": "xxe", + "severity": "HIGH", + "description": "XML parsing without disabling external entities", + }, ], "safe_patterns": ["jsonify usage", "Flask route decorators"], }, @@ -266,8 +380,20 @@ _MEDIUM_TASKS: List[Dict[str, Any]] = [ ), }, "vulnerabilities": [ - {"file": "models.py", "line": 5, "type": "path_traversal", "severity": "HIGH", "description": "No sanitization of filename allows path traversal"}, - {"file": "models.py", "line": 10, "type": "resource_leak", "severity": "MEDIUM", "description": "File descriptor from mkstemp never closed"}, + { + "file": "models.py", + "line": 5, + "type": "path_traversal", + "severity": "HIGH", + "description": "No sanitization of filename allows path traversal", + }, + { + "file": "models.py", + "line": 10, + "type": "resource_leak", + "severity": "MEDIUM", + "description": "File descriptor from mkstemp never closed", + }, ], "safe_patterns": ["tempfile.mkstemp usage pattern"], }, @@ -285,9 +411,27 @@ _MEDIUM_TASKS: List[Dict[str, Any]] = [ ), }, "vulnerabilities": [ - {"file": "crypto_utils.py", "line": 4, "type": "hardcoded_secret", "severity": "CRITICAL", "description": "Encryption key hardcoded in source"}, - {"file": "crypto_utils.py", "line": 8, "type": "weak_crypto", "severity": "HIGH", "description": "AES-ECB mode is insecure — use CBC or GCM"}, - {"file": "crypto_utils.py", "line": 5, "type": "weak_crypto", "severity": "HIGH", "description": "Static IV of all zeros is insecure"}, + { + "file": "crypto_utils.py", + "line": 4, + "type": "hardcoded_secret", + "severity": "CRITICAL", + "description": "Encryption key hardcoded in source", + }, + { + "file": "crypto_utils.py", + "line": 8, + "type": "weak_crypto", + "severity": "HIGH", + "description": "AES-ECB mode is insecure — use CBC or GCM", + }, + { + "file": "crypto_utils.py", + "line": 5, + "type": "weak_crypto", + "severity": "HIGH", + "description": "Static IV of all zeros is insecure", + }, ], "safe_patterns": ["base64 encoding of ciphertext"], }, @@ -311,8 +455,20 @@ _MEDIUM_TASKS: List[Dict[str, Any]] = [ ), }, "vulnerabilities": [ - {"file": "session.py", "line": 12, "type": "misconfiguration", "severity": "MEDIUM", "description": "Cookie set without secure, httponly, or samesite flags"}, - {"file": "session.py", "line": 18, "type": "insecure_deserialization", "severity": "MEDIUM", "description": "Base64-encoded cookie is not signed — client can tamper with preferences"}, + { + "file": "session.py", + "line": 12, + "type": "misconfiguration", + "severity": "MEDIUM", + "description": "Cookie set without secure, httponly, or samesite flags", + }, + { + "file": "session.py", + "line": 18, + "type": "insecure_deserialization", + "severity": "MEDIUM", + "description": "Base64-encoded cookie is not signed — client can tamper with preferences", + }, ], "safe_patterns": ["json.dumps/loads for serialization (not pickle)"], }, @@ -330,8 +486,20 @@ _MEDIUM_TASKS: List[Dict[str, Any]] = [ ), }, "vulnerabilities": [ - {"file": "admin.py", "line": 6, "type": "broken_access_control", "severity": "CRITICAL", "description": "Admin endpoint has no authentication or authorization check"}, - {"file": "admin.py", "line": 10, "type": "xss", "severity": "MEDIUM", "description": "User ID reflected in response without escaping"}, + { + "file": "admin.py", + "line": 6, + "type": "broken_access_control", + "severity": "CRITICAL", + "description": "Admin endpoint has no authentication or authorization check", + }, + { + "file": "admin.py", + "line": 10, + "type": "xss", + "severity": "MEDIUM", + "description": "User ID reflected in response without escaping", + }, ], "safe_patterns": ["POST method for destructive action"], }, @@ -353,10 +521,25 @@ _MEDIUM_TASKS: List[Dict[str, Any]] = [ ), }, "vulnerabilities": [ - {"file": "email_sender.py", "line": 6, "type": "hardcoded_secret", "severity": "CRITICAL", "description": "SMTP password hardcoded in source"}, - {"file": "email_sender.py", "line": 13, "type": "misconfiguration", "severity": "HIGH", "description": "SMTP connection without TLS (use SMTP_SSL or starttls)"}, + { + "file": "email_sender.py", + "line": 6, + "type": "hardcoded_secret", + "severity": "CRITICAL", + "description": "SMTP password hardcoded in source", + }, + { + "file": "email_sender.py", + "line": 13, + "type": "misconfiguration", + "severity": "HIGH", + "description": "SMTP connection without TLS (use SMTP_SSL or starttls)", + }, + ], + "safe_patterns": [ + "MIMEText for email construction", + "context manager for SMTP", ], - "safe_patterns": ["MIMEText for email construction", "context manager for SMTP"], }, { "project_files": { @@ -373,8 +556,20 @@ _MEDIUM_TASKS: List[Dict[str, Any]] = [ ), }, "vulnerabilities": [ - {"file": "cache.py", "line": 4, "type": "misconfiguration", "severity": "HIGH", "description": "Redis connection without authentication or TLS"}, - {"file": "cache.py", "line": 14, "type": "broken_access_control", "severity": "MEDIUM", "description": "flushall exposed without access control — could wipe entire cache"}, + { + "file": "cache.py", + "line": 4, + "type": "misconfiguration", + "severity": "HIGH", + "description": "Redis connection without authentication or TLS", + }, + { + "file": "cache.py", + "line": 14, + "type": "broken_access_control", + "severity": "MEDIUM", + "description": "flushall exposed without access control — could wipe entire cache", + }, ], "safe_patterns": ["json serialization for cache values", "TTL on cached items"], }, @@ -393,8 +588,20 @@ _MEDIUM_TASKS: List[Dict[str, Any]] = [ ), }, "vulnerabilities": [ - {"file": "file_processor.py", "line": 7, "type": "command_injection", "severity": "CRITICAL", "description": "shell=True with f-string filepath allows command injection"}, - {"file": "file_processor.py", "line": 5, "type": "insufficient_validation", "severity": "MEDIUM", "description": "Extension check is bypassable (e.g., file.txt; rm -rf /)"}, + { + "file": "file_processor.py", + "line": 7, + "type": "command_injection", + "severity": "CRITICAL", + "description": "shell=True with f-string filepath allows command injection", + }, + { + "file": "file_processor.py", + "line": 5, + "type": "insufficient_validation", + "severity": "MEDIUM", + "description": "Extension check is bypassable (e.g., file.txt; rm -rf /)", + }, ], "safe_patterns": ["subprocess with list args (count_lines) is safe pattern"], }, @@ -418,8 +625,20 @@ _MEDIUM_TASKS: List[Dict[str, Any]] = [ ), }, "vulnerabilities": [ - {"file": "rate_limiter.py", "line": 9, "type": "ip_spoofing", "severity": "HIGH", "description": "X-Forwarded-For header is client-controlled — rate limit can be bypassed"}, - {"file": "rate_limiter.py", "line": 5, "type": "resource_leak", "severity": "MEDIUM", "description": "In-memory dict grows unbounded — no cleanup of old entries"}, + { + "file": "rate_limiter.py", + "line": 9, + "type": "ip_spoofing", + "severity": "HIGH", + "description": "X-Forwarded-For header is client-controlled — rate limit can be bypassed", + }, + { + "file": "rate_limiter.py", + "line": 5, + "type": "resource_leak", + "severity": "MEDIUM", + "description": "In-memory dict grows unbounded — no cleanup of old entries", + }, ], "safe_patterns": ["Rate limiting concept", "before_request pattern"], }, @@ -449,10 +668,25 @@ _HARD_TASKS: List[Dict[str, Any]] = [ ), }, "vulnerabilities": [ - {"file": "auth.py", "line": 18, "type": "timing_attack", "severity": "MEDIUM", "description": "String comparison (==) instead of hmac.compare_digest for signature — vulnerable to timing attack"}, - {"file": "auth.py", "line": 7, "type": "insufficient_validation", "severity": "LOW", "description": "No token expiration check — tokens valid forever"}, + { + "file": "auth.py", + "line": 18, + "type": "timing_attack", + "severity": "MEDIUM", + "description": "String comparison (==) instead of hmac.compare_digest for signature — vulnerable to timing attack", + }, + { + "file": "auth.py", + "line": 7, + "type": "insufficient_validation", + "severity": "LOW", + "description": "No token expiration check — tokens valid forever", + }, + ], + "safe_patterns": [ + "hmac.new with SHA-256 is correct HMAC construction", + "Token format is reasonable", ], - "safe_patterns": ["hmac.new with SHA-256 is correct HMAC construction", "Token format is reasonable"], }, { "project_files": { @@ -468,10 +702,25 @@ _HARD_TASKS: List[Dict[str, Any]] = [ ), }, "vulnerabilities": [ - {"file": "sanitizer.py", "line": 8, "type": "xss", "severity": "HIGH", "description": "Regex-based HTML sanitization is bypassable (e.g., , )"}, - {"file": "sanitizer.py", "line": 10, "type": "xss", "severity": "HIGH", "description": "Event handler regex can be bypassed with newlines or encoding"}, + { + "file": "sanitizer.py", + "line": 8, + "type": "xss", + "severity": "HIGH", + "description": "Regex-based HTML sanitization is bypassable (e.g., , )", + }, + { + "file": "sanitizer.py", + "line": 10, + "type": "xss", + "severity": "HIGH", + "description": "Event handler regex can be bypassed with newlines or encoding", + }, + ], + "safe_patterns": [ + "html module import (though not used)", + "ALLOWED_TAGS list (good intent)", ], - "safe_patterns": ["html module import (though not used)", "ALLOWED_TAGS list (good intent)"], }, { "project_files": { @@ -497,9 +746,19 @@ _HARD_TASKS: List[Dict[str, Any]] = [ ), }, "vulnerabilities": [ - {"file": "db.py", "line": 21, "type": "sql_injection", "severity": "CRITICAL", "description": "search_users uses f-string SQL with unsanitized query parameter"}, + { + "file": "db.py", + "line": 21, + "type": "sql_injection", + "severity": "CRITICAL", + "description": "search_users uses f-string SQL with unsanitized query parameter", + }, + ], + "safe_patterns": [ + "get_user uses parameterized query (?)", + "Context manager for connection", + "conn.close in finally", ], - "safe_patterns": ["get_user uses parameterized query (?)", "Context manager for connection", "conn.close in finally"], }, { "project_files": { @@ -513,7 +772,13 @@ _HARD_TASKS: List[Dict[str, Any]] = [ ), }, "vulnerabilities": [ - {"file": "password_reset.py", "line": 6, "type": "weak_crypto", "severity": "HIGH", "description": "Reset token is predictable — based on email + timestamp, not cryptographic random"}, + { + "file": "password_reset.py", + "line": 6, + "type": "weak_crypto", + "severity": "HIGH", + "description": "Reset token is predictable — based on email + timestamp, not cryptographic random", + }, ], "safe_patterns": ["secrets.token_urlsafe for API key generation is correct"], }, @@ -536,8 +801,20 @@ _HARD_TASKS: List[Dict[str, Any]] = [ ), }, "vulnerabilities": [ - {"file": "middleware.py", "line": 14, "type": "ssrf", "severity": "HIGH", "description": "SSRF check is bypassable — doesn't handle 0.0.0.0, IPv6 ::1, or DNS rebinding"}, - {"file": "middleware.py", "line": 14, "type": "ssrf", "severity": "MEDIUM", "description": "URL parsing via split is fragile — use urllib.parse"}, + { + "file": "middleware.py", + "line": 14, + "type": "ssrf", + "severity": "HIGH", + "description": "SSRF check is bypassable — doesn't handle 0.0.0.0, IPv6 ::1, or DNS rebinding", + }, + { + "file": "middleware.py", + "line": 14, + "type": "ssrf", + "severity": "MEDIUM", + "description": "URL parsing via split is fragile — use urllib.parse", + }, ], "safe_patterns": ["IP blocklist concept", "SSRF protection attempt"], }, @@ -559,10 +836,25 @@ _HARD_TASKS: List[Dict[str, Any]] = [ ), }, "vulnerabilities": [ - {"file": "session_store.py", "line": 15, "type": "path_traversal", "severity": "HIGH", "description": "session_id not validated — ../../etc/passwd traversal possible"}, - {"file": "session_store.py", "line": 5, "type": "misconfiguration", "severity": "MEDIUM", "description": "/tmp is world-readable — session files accessible to other users"}, + { + "file": "session_store.py", + "line": 15, + "type": "path_traversal", + "severity": "HIGH", + "description": "session_id not validated — ../../etc/passwd traversal possible", + }, + { + "file": "session_store.py", + "line": 5, + "type": "misconfiguration", + "severity": "MEDIUM", + "description": "/tmp is world-readable — session files accessible to other users", + }, + ], + "safe_patterns": [ + "os.urandom(32) for session ID generation is cryptographically strong", + "json serialization (not pickle)", ], - "safe_patterns": ["os.urandom(32) for session ID generation is cryptographically strong", "json serialization (not pickle)"], }, { "project_files": { @@ -582,10 +874,25 @@ _HARD_TASKS: List[Dict[str, Any]] = [ ), }, "vulnerabilities": [ - {"file": "webhook.py", "line": 13, "type": "timing_attack", "severity": "MEDIUM", "description": "String != for HMAC comparison is vulnerable to timing attack — use hmac.compare_digest"}, - {"file": "webhook.py", "line": 6, "type": "misconfiguration", "severity": "LOW", "description": "Empty string default for WEBHOOK_SECRET — webhook verification disabled if env var missing"}, + { + "file": "webhook.py", + "line": 13, + "type": "timing_attack", + "severity": "MEDIUM", + "description": "String != for HMAC comparison is vulnerable to timing attack — use hmac.compare_digest", + }, + { + "file": "webhook.py", + "line": 6, + "type": "misconfiguration", + "severity": "LOW", + "description": "Empty string default for WEBHOOK_SECRET — webhook verification disabled if env var missing", + }, + ], + "safe_patterns": [ + "HMAC-SHA256 for webhook verification", + "os.environ.get for secrets", ], - "safe_patterns": ["HMAC-SHA256 for webhook verification", "os.environ.get for secrets"], }, { "project_files": { @@ -606,9 +913,18 @@ _HARD_TASKS: List[Dict[str, Any]] = [ ), }, "vulnerabilities": [ - {"file": "upload.py", "line": 15, "type": "path_traversal", "severity": "HIGH", "description": "f.filename not sanitized — path traversal via ../../../etc/cron.d/evil"}, + { + "file": "upload.py", + "line": 15, + "type": "path_traversal", + "severity": "HIGH", + "description": "f.filename not sanitized — path traversal via ../../../etc/cron.d/evil", + }, + ], + "safe_patterns": [ + "MIME type validation via python-magic", + "Allowlist approach for file types", ], - "safe_patterns": ["MIME type validation via python-magic", "Allowlist approach for file types"], }, { "project_files": { @@ -633,10 +949,25 @@ _HARD_TASKS: List[Dict[str, Any]] = [ ), }, "vulnerabilities": [ - {"file": "export.py", "line": 9, "type": "broken_access_control", "severity": "HIGH", "description": "No authorization check — any user can export any other user's data via user_id parameter"}, - {"file": "export.py", "line": 18, "type": "header_injection", "severity": "MEDIUM", "description": "Unsanitized filename in Content-Disposition header allows response header injection"}, + { + "file": "export.py", + "line": 9, + "type": "broken_access_control", + "severity": "HIGH", + "description": "No authorization check — any user can export any other user's data via user_id parameter", + }, + { + "file": "export.py", + "line": 18, + "type": "header_injection", + "severity": "MEDIUM", + "description": "Unsanitized filename in Content-Disposition header allows response header injection", + }, + ], + "safe_patterns": [ + "csv.writer for CSV generation", + "Content-Disposition header for download", ], - "safe_patterns": ["csv.writer for CSV generation", "Content-Disposition header for download"], }, { "project_files": { @@ -657,9 +988,18 @@ _HARD_TASKS: List[Dict[str, Any]] = [ ), }, "vulnerabilities": [ - {"file": "search.py", "line": 10, "type": "redos", "severity": "HIGH", "description": "User-supplied regex compiled without timeout — ReDoS vulnerability (e.g., (a+)+$ on long input)"}, + { + "file": "search.py", + "line": 10, + "type": "redos", + "severity": "HIGH", + "description": "User-supplied regex compiled without timeout — ReDoS vulnerability (e.g., (a+)+$ on long input)", + }, + ], + "safe_patterns": [ + "re.error handling for invalid patterns", + "List comprehension for filtering", ], - "safe_patterns": ["re.error handling for invalid patterns", "List comprehension for filtering"], }, ] diff --git a/src/openjarvis/evals/datasets/simpleqa.py b/src/openjarvis/evals/datasets/simpleqa.py index c309d84a..cfa109fe 100644 --- a/src/openjarvis/evals/datasets/simpleqa.py +++ b/src/openjarvis/evals/datasets/simpleqa.py @@ -70,14 +70,12 @@ class SimpleQADataset(DatasetProvider): return len(self._records) def _convert_row( - self, raw: MutableMapping[str, object], idx: int, + self, + raw: MutableMapping[str, object], + idx: int, ) -> Optional[EvalRecord]: - question = str( - raw.get("problem") or raw.get("question") or "" - ).strip() - answer = str( - raw.get("answer") or raw.get("gold_answer") or "" - ).strip() + question = str(raw.get("problem") or raw.get("question") or "").strip() + answer = str(raw.get("answer") or raw.get("gold_answer") or "").strip() if not question or not answer: return None diff --git a/src/openjarvis/evals/datasets/supergpqa.py b/src/openjarvis/evals/datasets/supergpqa.py index f698ad65..e8c519d3 100644 --- a/src/openjarvis/evals/datasets/supergpqa.py +++ b/src/openjarvis/evals/datasets/supergpqa.py @@ -71,7 +71,9 @@ class SuperGPQADataset(DatasetProvider): return len(self._records) def _convert_row( - self, raw: MutableMapping[str, object], idx: int, + self, + raw: MutableMapping[str, object], + idx: int, ) -> Optional[EvalRecord]: question = str(raw.get("question") or "").strip() options_raw = raw.get("options") or [] @@ -79,20 +81,25 @@ class SuperGPQADataset(DatasetProvider): answer_letter = str(raw.get("answer_letter") or "").strip().upper() answer_text = str(raw.get("answer") or "").strip() - subject = str( - raw.get("subfield") - or raw.get("field") - or raw.get("discipline") + subject = ( + str( + raw.get("subfield") + or raw.get("field") + or raw.get("discipline") + or "general" + ).strip() or "general" - ).strip() or "general" + ) if not question or not options or not answer_letter: return None prompt_parts = [ - question, "", + question, + "", "Options:", - _format_options(options), "", + _format_options(options), + "", "Respond with the correct letter only.", ] problem = "\n".join(part for part in prompt_parts if part).strip() diff --git a/src/openjarvis/evals/datasets/swebench.py b/src/openjarvis/evals/datasets/swebench.py index 220693b5..1117649f 100644 --- a/src/openjarvis/evals/datasets/swebench.py +++ b/src/openjarvis/evals/datasets/swebench.py @@ -111,7 +111,9 @@ class SWEBenchDataset(DatasetProvider): return len(self._records) def _convert_row( - self, raw: MutableMapping[str, object], idx: int, + self, + raw: MutableMapping[str, object], + idx: int, ) -> Optional[EvalRecord]: instance_id = str(raw.get("instance_id") or "") repo = str(raw.get("repo") or "") diff --git a/src/openjarvis/evals/datasets/swefficiency.py b/src/openjarvis/evals/datasets/swefficiency.py index 17ccc19a..41f9932c 100644 --- a/src/openjarvis/evals/datasets/swefficiency.py +++ b/src/openjarvis/evals/datasets/swefficiency.py @@ -108,7 +108,9 @@ class SWEfficiencyDataset(DatasetProvider): return len(self._records) def _convert_row( - self, raw: MutableMapping[str, object], idx: int, + self, + raw: MutableMapping[str, object], + idx: int, ) -> Optional[EvalRecord]: instance_id = str(raw.get("instance_id") or "") repo = str(raw.get("repo") or "") diff --git a/src/openjarvis/evals/datasets/terminalbench.py b/src/openjarvis/evals/datasets/terminalbench.py index 2c662315..7931b27c 100644 --- a/src/openjarvis/evals/datasets/terminalbench.py +++ b/src/openjarvis/evals/datasets/terminalbench.py @@ -77,14 +77,13 @@ class TerminalBenchDataset(DatasetProvider): return len(self._records) def _convert_row( - self, raw: MutableMapping[str, object], idx: int, + self, + raw: MutableMapping[str, object], + idx: int, ) -> Optional[EvalRecord]: # Try multiple field name variants for question question = str( - raw.get("prompt") - or raw.get("question") - or raw.get("instruction") - or "" + raw.get("prompt") or raw.get("question") or raw.get("instruction") or "" ).strip() # Try multiple field name variants for answer diff --git a/src/openjarvis/evals/datasets/terminalbench_native.py b/src/openjarvis/evals/datasets/terminalbench_native.py index c4b88fdd..016c4c0d 100644 --- a/src/openjarvis/evals/datasets/terminalbench_native.py +++ b/src/openjarvis/evals/datasets/terminalbench_native.py @@ -28,6 +28,7 @@ def _load_task_yaml(task_dir: Path) -> Dict[str, Any]: return {} try: import yaml + return yaml.safe_load(task_file.read_text()) or {} except ImportError: # Fallback: parse instruction manually @@ -36,7 +37,7 @@ def _load_task_yaml(task_dir: Path) -> Dict[str, Any]: # Extract instruction block if "instruction:" in text: idx = text.index("instruction:") - after = text[idx + len("instruction:"):] + after = text[idx + len("instruction:") :] # Handle YAML block scalar (|-) or plain string after = after.lstrip() if after.startswith("|-"): @@ -52,7 +53,7 @@ def _load_task_yaml(task_dir: Path) -> Dict[str, Any]: for field in ("category", "difficulty", "author_email"): if f"{field}:" in text: idx = text.index(f"{field}:") - val = text[idx + len(field) + 1:].split("\n")[0].strip() + val = text[idx + len(field) + 1 :].split("\n")[0].strip() result[field] = val return result @@ -129,7 +130,9 @@ class TerminalBenchNativeDataset(DatasetProvider): return len(self._records) def _convert_task( - self, task_dir: Path, idx: int, + self, + task_dir: Path, + idx: int, ) -> Optional[EvalRecord]: task_data = _load_task_yaml(task_dir) @@ -158,13 +161,13 @@ class TerminalBenchNativeDataset(DatasetProvider): metadata=metadata, ) - def create_task_env(self, record): """Return a TerminalBenchTaskEnv for the given record.""" try: from openjarvis.evals.execution.terminalbench_env import ( TerminalBenchTaskEnv, ) + return TerminalBenchTaskEnv(record.metadata) except ImportError: return None @@ -175,6 +178,7 @@ class TerminalBenchNativeDataset(DatasetProvider): if not _HAS_TERMINALBENCH: issues.append("terminal-bench package not installed") import shutil + if not shutil.which("docker"): issues.append("docker not found in PATH") return issues diff --git a/src/openjarvis/evals/datasets/webchorearena.py b/src/openjarvis/evals/datasets/webchorearena.py index e101660e..2b72da3e 100644 --- a/src/openjarvis/evals/datasets/webchorearena.py +++ b/src/openjarvis/evals/datasets/webchorearena.py @@ -56,8 +56,7 @@ class WebChoreArenaDataset(DatasetProvider): ) -> None: self._subset = subset # "all", "small", or a site name self._cache_dir = ( - Path(cache_dir) if cache_dir - else Path.home() / ".cache" / "webchorearena" + Path(cache_dir) if cache_dir else Path.home() / ".cache" / "webchorearena" ) self._headless = headless self._records: List[EvalRecord] = [] @@ -84,7 +83,9 @@ class WebChoreArenaDataset(DatasetProvider): self._records = [self._task_to_record(t, i) for i, t in enumerate(tasks)] logger.info( - "WebChoreArena[%s]: loaded %d tasks", self._subset, len(self._records), + "WebChoreArena[%s]: loaded %d tasks", + self._subset, + len(self._records), ) def iter_records(self) -> Iterable[EvalRecord]: @@ -107,11 +108,11 @@ class WebChoreArenaDataset(DatasetProvider): # Core services — eval fails without these _REQUIRED = { - "SHOPPING": "http://localhost:7770", + "SHOPPING": "http://localhost:7770", "SHOPPING_ADMIN": "http://localhost:7780", - "REDDIT": "http://localhost:9999", - "GITLAB": "http://localhost:8023", - "WIKIPEDIA": "http://localhost:8888", + "REDDIT": "http://localhost:9999", + "GITLAB": "http://localhost:8023", + "WIKIPEDIA": "http://localhost:8888", } # Optional — only needed for map tasks; requires a full OpenStreetMap # Docker Compose setup (AWS AMI or manual) @@ -143,7 +144,8 @@ class WebChoreArenaDataset(DatasetProvider): "Optional WebArena service %s (%s) is not reachable — " "map-related tasks will fail. " "See scripts/setup_webchorearena.sh for setup instructions.", - env_var, url, + env_var, + url, ) return issues @@ -154,6 +156,7 @@ class WebChoreArenaDataset(DatasetProvider): from openjarvis.evals.execution.webchorearena_env import ( WebChoreArenaTaskEnv, ) + return WebChoreArenaTaskEnv(record.metadata, headless=self._headless) except ImportError: return None @@ -223,7 +226,9 @@ class WebChoreArenaDataset(DatasetProvider): return ids def _task_to_record( - self, task: Dict[str, Any], idx: int, + self, + task: Dict[str, Any], + idx: int, ) -> EvalRecord: """Convert an original WebChoreArena task config into an EvalRecord.""" task_id = str(task.get("task_id", idx)) @@ -245,7 +250,9 @@ class WebChoreArenaDataset(DatasetProvider): "sites": sites, "start_url": task.get("start_url", ""), "start_url_lite": task.get("start_url_lite", ""), - "storage_state": task.get("storage_state", task.get("strage_state", "")), + "storage_state": task.get( + "storage_state", task.get("strage_state", "") + ), "required_obs": task.get("required_obs", "text"), "type_main": task.get("type_main", ""), "type_sub": task.get("type_sub", ""), diff --git a/src/openjarvis/evals/datasets/workarena.py b/src/openjarvis/evals/datasets/workarena.py index c76f46d2..75c2eb3c 100644 --- a/src/openjarvis/evals/datasets/workarena.py +++ b/src/openjarvis/evals/datasets/workarena.py @@ -41,6 +41,7 @@ except ImportError as _e: def get_all_tasks_agents(**kwargs): # type: ignore[misc] return [] + _VALID_LEVELS = ("l1", "l2", "l3") @@ -84,7 +85,11 @@ class WorkArenaDataset(DatasetProvider): seed: Optional[int] = None, ) -> None: if not _HAS_WORKARENA: - detail = f"\n\nUnderlying error: {_WORKARENA_IMPORT_ERROR}" if _WORKARENA_IMPORT_ERROR else "" + detail = ( + f"\n\nUnderlying error: {_WORKARENA_IMPORT_ERROR}" + if _WORKARENA_IMPORT_ERROR + else "" + ) raise ImportError( "The 'browsergym-workarena' package is required for " "WorkArena evaluation. Install with:\n" @@ -110,7 +115,9 @@ class WorkArenaDataset(DatasetProvider): self._episodes.append([record]) logger.info( - "WorkArena[%s]: loaded %d task instances", self._level, len(self._records), + "WorkArena[%s]: loaded %d task instances", + self._level, + len(self._records), ) def _enumerate_tasks(self) -> List[tuple]: @@ -134,7 +141,10 @@ class WorkArenaDataset(DatasetProvider): return task_tuples def _task_to_record( - self, task_class: type, task_seed: int, idx: int, + self, + task_class: type, + task_seed: int, + idx: int, ) -> EvalRecord: """Convert a (task_class, seed) pair into an EvalRecord.""" task_id = task_class.get_task_id() diff --git a/src/openjarvis/evals/environments/lifelong_agent_env.py b/src/openjarvis/evals/environments/lifelong_agent_env.py index d978e971..8649b24a 100644 --- a/src/openjarvis/evals/environments/lifelong_agent_env.py +++ b/src/openjarvis/evals/environments/lifelong_agent_env.py @@ -37,7 +37,9 @@ MAX_TURNS_OS = 5 def _infer_oracle_result( - func_name: str, current_action: str, next_action: str, + func_name: str, + current_action: str, + next_action: str, ) -> str: """Infer a plausible oracle result from the gold action sequence. @@ -76,13 +78,16 @@ def _infer_oracle_result( # DB Environment # ==================================================================== + def _mysql_available() -> bool: """Check if Docker + MySQL image is available.""" if not shutil.which("docker"): return False try: result = subprocess.run( - ["docker", "info"], capture_output=True, timeout=10, + ["docker", "info"], + capture_output=True, + timeout=10, ) return result.returncode == 0 except (subprocess.TimeoutExpired, OSError): @@ -131,7 +136,8 @@ class DBEnvironment(TaskEnvironment): except Exception as exc: logger.warning( "MySQL setup failed for %s, falling back to SQLite: %s", - record.record_id, exc, + record.record_id, + exc, ) self._use_mysql = False self._mysql_conn = None @@ -152,7 +158,8 @@ class DBEnvironment(TaskEnvironment): # Check for "Action: Answer" / "Final Answer:" answer_match = re.search( r"(?:Action:\s*Answer\s*\n\s*)?Final\s+Answer:\s*(.+)", - agent_response, re.DOTALL | re.IGNORECASE, + agent_response, + re.DOTALL | re.IGNORECASE, ) if answer_match: self._agent_final_answer = answer_match.group(1).strip() @@ -182,9 +189,7 @@ class DBEnvironment(TaskEnvironment): if not rows: return "Result: Empty result set", False desc = cursor.description - col_names = ( - [d[0] for d in desc] if desc else [] - ) + col_names = [d[0] for d in desc] if desc else [] result_lines = [] if col_names: result_lines.append(str(col_names)) @@ -218,7 +223,8 @@ class DBEnvironment(TaskEnvironment): return self._evaluate_direct(meta) def _evaluate_md5( - self, meta: Dict[str, Any], + self, + meta: Dict[str, Any], ) -> Tuple[Optional[bool], Dict[str, Any]]: """Evaluate DML tasks by comparing table state after agent's SQL.""" table_name = self._table_info.get("name", "data") @@ -257,13 +263,15 @@ class DBEnvironment(TaskEnvironment): logger.warning( "MD5 task %s: ground-truth SQL failed on SQLite (%s). " "Falling back to normalized SQL comparison.", - table_name, exc, + table_name, + exc, ) meta["ref_sql_sqlite_error"] = str(exc) meta["fallback"] = "normalized_sql_comparison" from openjarvis.evals.scorers.lifelong_agent_scorer import ( _normalize_sql, ) + # Compare the last DML statement the agent executed against # the ground-truth DML statement. agent_dml = "" @@ -274,14 +282,14 @@ class DBEnvironment(TaskEnvironment): break if not agent_dml: agent_dml = ( - self._agent_sql_history[-1] - if self._agent_sql_history else "" + self._agent_sql_history[-1] if self._agent_sql_history else "" ) norm_agent = _normalize_sql(agent_dml) norm_expected = _normalize_sql(expected_sql) is_correct = norm_agent == norm_expected meta["comparison_detail"] = ( - "normalized_sql_match" if is_correct + "normalized_sql_match" + if is_correct else f"normalized_sql_mismatch: " f"expected={norm_expected!r}, got={norm_agent!r}" ) @@ -295,7 +303,8 @@ class DBEnvironment(TaskEnvironment): return is_correct, meta def _evaluate_direct( - self, meta: Dict[str, Any], + self, + meta: Dict[str, Any], ) -> Tuple[Optional[bool], Dict[str, Any]]: """Evaluate SELECT tasks by comparing result tuples.""" expected_direct = self._answer_info.get("direct") @@ -305,9 +314,7 @@ class DBEnvironment(TaskEnvironment): meta["reason"] = "no_ground_truth_direct" return None, meta - expected_tuples = [ - r if isinstance(r, list) else [r] for r in expected_direct - ] + expected_tuples = [r if isinstance(r, list) else [r] for r in expected_direct] meta["expected_tuples"] = expected_tuples # Strategy 1: Try each SELECT from agent's SQL history (newest first). @@ -322,7 +329,8 @@ class DBEnvironment(TaskEnvironment): actual_rows = [list(row) for row in cursor.fetchall()] ref_conn.close() is_correct, detail = compare_tuple_lists( - expected_tuples, actual_rows, + expected_tuples, + actual_rows, ) if is_correct: meta["actual_tuples"] = actual_rows @@ -337,6 +345,7 @@ class DBEnvironment(TaskEnvironment): from openjarvis.evals.scorers.lifelong_agent_scorer import ( _parse_text_answer, ) + answer_text = self._agent_final_answer if not answer_text.lower().startswith("final answer"): answer_text = f"Final Answer: {answer_text}" @@ -366,8 +375,10 @@ class DBEnvironment(TaskEnvironment): db_name = "lifelong_bench" conn = mysql.connector.connect( - host="127.0.0.1", user="root", - password="password", port=self._mysql_port, + host="127.0.0.1", + user="root", + password="password", + port=self._mysql_port, ) cursor = conn.cursor() cursor.execute(f"DROP DATABASE IF EXISTS `{db_name}`") @@ -382,9 +393,7 @@ class DBEnvironment(TaskEnvironment): if not col_defs: col_defs = ["`value` TEXT"] - cursor.execute( - f"CREATE TABLE `{table_name}` ({', '.join(col_defs)})" - ) + cursor.execute(f"CREATE TABLE `{table_name}` ({', '.join(col_defs)})") if rows and columns: ncols = len(columns) @@ -395,7 +404,8 @@ class DBEnvironment(TaskEnvironment): padded.append(None) try: cursor.execute( - f"INSERT INTO `{table_name}` VALUES ({ph})", padded, + f"INSERT INTO `{table_name}` VALUES ({ph})", + padded, ) except Exception as exc: logger.debug("Skipping row in MySQL table %s: %s", table_name, exc) @@ -406,8 +416,10 @@ class DBEnvironment(TaskEnvironment): # Store a persistent connection for agent interactions self._mysql_conn = mysql.connector.connect( - host="127.0.0.1", user="root", - password="password", port=self._mysql_port, + host="127.0.0.1", + user="root", + password="password", + port=self._mysql_port, database=db_name, ) @@ -485,25 +497,37 @@ class DBEnvironment(TaskEnvironment): self._mysql_container = f"lifelong-db-{self._mysql_port}" subprocess.run( ["docker", "rm", "-f", self._mysql_container], - capture_output=True, timeout=30, + capture_output=True, + timeout=30, ) subprocess.run( [ - "docker", "run", "-d", "--name", self._mysql_container, - "-e", "MYSQL_ROOT_PASSWORD=password", - "-p", f"{self._mysql_port}:3306", + "docker", + "run", + "-d", + "--name", + self._mysql_container, + "-e", + "MYSQL_ROOT_PASSWORD=password", + "-p", + f"{self._mysql_port}:3306", "mysql:8.0", ], - capture_output=True, check=True, timeout=60, + capture_output=True, + check=True, + timeout=60, ) # Wait for MySQL to be ready (up to 30s) for _ in range(30): time.sleep(1) try: import mysql.connector + conn = mysql.connector.connect( - host="127.0.0.1", user="root", - password="password", port=self._mysql_port, + host="127.0.0.1", + user="root", + password="password", + port=self._mysql_port, ) conn.close() break @@ -529,7 +553,8 @@ class DBEnvironment(TaskEnvironment): try: subprocess.run( ["docker", "rm", "-f", self._mysql_container], - capture_output=True, timeout=30, + capture_output=True, + timeout=30, ) except Exception: pass @@ -540,12 +565,16 @@ class DBEnvironment(TaskEnvironment): # KG Environment # ==================================================================== + class _Variable: """Variable in the KG variable store, matching the original's Variable class.""" def __init__( - self, idx: int, program: str, - vtype: str = "entity", callable: bool = True, + self, + idx: int, + program: str, + vtype: str = "entity", + callable: bool = True, ): self.idx = idx self.program = program @@ -638,7 +667,9 @@ class KGEnvironment(TaskEnvironment): # Check for Final Answer with raw entities — fallback format fa_match = re.search( - r"(?i)final\s+answer:\s*(.+)", agent_response, re.DOTALL, + r"(?i)final\s+answer:\s*(.+)", + agent_response, + re.DOTALL, ) if fa_match: self._agent_final_answer = fa_match.group(1).strip() @@ -653,7 +684,9 @@ class KGEnvironment(TaskEnvironment): # Parse API call: Action: func_name(args) action_match = re.search( - r"Action:\s*(\w+)\((.+?)\)", agent_response, re.DOTALL, + r"Action:\s*(\w+)\((.+?)\)", + agent_response, + re.DOTALL, ) if not action_match: return ( @@ -670,8 +703,13 @@ class KGEnvironment(TaskEnvironment): def _execute_api(self, func_name: str, args_str: str) -> str: """Execute a KG API call, matching the original's API interface.""" valid_funcs = { - "get_relations", "get_neighbors", "intersection", - "get_attributes", "argmax", "argmin", "count", + "get_relations", + "get_neighbors", + "intersection", + "get_attributes", + "argmax", + "argmin", + "count", } if func_name not in valid_funcs: return ( @@ -707,22 +745,24 @@ class KGEnvironment(TaskEnvironment): if isinstance(next_action, str): # Extract relation/entity hints from the next call oracle_result = _infer_oracle_result( - func_name, str(oracle), str(next_action), + func_name, + str(oracle), + str(next_action), ) else: oracle_result = next_action.get( - "result", next_action.get("output", ""), + "result", + next_action.get("output", ""), ) else: oracle_result = f"(oracle: action '{oracle}' executed)" - return ( - f"Result stored as #{new_var.idx}.\n" - f"Output: {oracle_result}" - ) + return f"Result stored as #{new_var.idx}.\nOutput: {oracle_result}" # No oracle — simulate with empty results + warning new_var = _Variable( - len(self._variables), f"{func_name}({args_str})", "entity", + len(self._variables), + f"{func_name}({args_str})", + "entity", ) self._variables.append(new_var) @@ -837,7 +877,8 @@ class OSEnvironment(TaskEnvironment): try: result = subprocess.run( ["docker", "image", "inspect", candidate], - capture_output=True, timeout=10, + capture_output=True, + timeout=10, ) if result.returncode == 0: image = candidate @@ -856,14 +897,23 @@ class OSEnvironment(TaskEnvironment): self._container_name = f"lifelong-os-{record.record_id.replace('/', '-')}" subprocess.run( ["docker", "rm", "-f", self._container_name], - capture_output=True, timeout=30, + capture_output=True, + timeout=30, ) subprocess.run( [ - "docker", "run", "-d", "--name", self._container_name, - image, "sleep", "600", + "docker", + "run", + "-d", + "--name", + self._container_name, + image, + "sleep", + "600", ], - capture_output=True, check=True, timeout=60, + capture_output=True, + check=True, + timeout=60, ) # Run initialization command @@ -877,7 +927,8 @@ class OSEnvironment(TaskEnvironment): if init_cmd_str: result = subprocess.run( ["docker", "exec", self._container_name, "bash", "-c", init_cmd_str], - capture_output=True, timeout=self._timeout, + capture_output=True, + timeout=self._timeout, ) if result.returncode != 0: logger.warning( @@ -911,7 +962,8 @@ class OSEnvironment(TaskEnvironment): try: result = subprocess.run( ["docker", "exec", self._container_name, "bash", "-c", cmd], - capture_output=True, timeout=self._timeout, + capture_output=True, + timeout=self._timeout, ) stdout = result.stdout.decode(errors="replace")[:2000] stderr = result.stderr.decode(errors="replace")[:500] @@ -968,7 +1020,8 @@ class OSEnvironment(TaskEnvironment): try: result = subprocess.run( ["docker", "exec", self._container_name, "bash", "-c", eval_cmd_str], - capture_output=True, timeout=self._timeout, + capture_output=True, + timeout=self._timeout, ) meta["eval_exit_code"] = result.returncode meta["eval_stdout"] = result.stdout.decode(errors="replace")[:500] @@ -985,7 +1038,8 @@ class OSEnvironment(TaskEnvironment): try: subprocess.run( ["docker", "rm", "-f", self._container_name], - capture_output=True, timeout=30, + capture_output=True, + timeout=30, ) except Exception: pass @@ -996,6 +1050,7 @@ class OSEnvironment(TaskEnvironment): # Helpers (shared with scorer — avoid duplication) # ==================================================================== + def _build_sqlite_db(table_info: Dict[str, Any]) -> sqlite3.Connection: """Build an in-memory SQLite DB from table_info.""" conn = sqlite3.connect(":memory:") @@ -1031,7 +1086,8 @@ def _build_sqlite_db(table_info: Dict[str, Any]) -> sqlite3.Connection: def _get_table_rows( - conn: Optional[sqlite3.Connection], table_name: str, + conn: Optional[sqlite3.Connection], + table_name: str, ) -> List[List[Any]]: if conn is None: return [] @@ -1040,7 +1096,8 @@ def _get_table_rows( def _compare_table_states( - expected: List[List[Any]], actual: List[List[Any]], + expected: List[List[Any]], + actual: List[List[Any]], ) -> Tuple[bool, str]: if len(expected) != len(actual): return False, f"row_count_mismatch: expected {len(expected)}, got {len(actual)}" @@ -1066,7 +1123,8 @@ def _extract_bash_commands_from_response(text: str) -> List[str]: # Original format: Act: bash\n```bash\n...\n``` for m in re.finditer( r"Act:\s*bash\s*\n\s*```(?:bash)?\s*\n(.*?)\n\s*```", - text, re.DOTALL | re.IGNORECASE, + text, + re.DOTALL | re.IGNORECASE, ): cmd = m.group(1).strip() if cmd: @@ -1077,7 +1135,8 @@ def _extract_bash_commands_from_response(text: str) -> List[str]: # Act: ```bash\n...\n``` for m in re.finditer( r"Act:\s*```(?:bash)?\s*\n(.*?)\n\s*```", - text, re.DOTALL | re.IGNORECASE, + text, + re.DOTALL | re.IGNORECASE, ): cmd = m.group(1).strip() if cmd: @@ -1088,7 +1147,8 @@ def _extract_bash_commands_from_response(text: str) -> List[str]: # ```bash\n...\n``` for m in re.finditer( r"```(?:bash|sh)\s*\n(.*?)\n\s*```", - text, re.DOTALL | re.IGNORECASE, + text, + re.DOTALL | re.IGNORECASE, ): cmd = m.group(1).strip() if cmd: diff --git a/src/openjarvis/evals/execution/pinchbench_env.py b/src/openjarvis/evals/execution/pinchbench_env.py index 9df2c698..d4b36194 100644 --- a/src/openjarvis/evals/execution/pinchbench_env.py +++ b/src/openjarvis/evals/execution/pinchbench_env.py @@ -114,7 +114,9 @@ class PinchBenchTaskEnv: ) except Exception as exc: LOGGER.error( - "Grading failed for %s: %s", self._record.record_id, exc, + "Grading failed for %s: %s", + self._record.record_id, + exc, ) result = {"score": 0.0, "breakdown": {}, "notes": f"Grading error: {exc}"} diff --git a/src/openjarvis/evals/execution/terminalbench_env.py b/src/openjarvis/evals/execution/terminalbench_env.py index 07f9dd98..196b5705 100644 --- a/src/openjarvis/evals/execution/terminalbench_env.py +++ b/src/openjarvis/evals/execution/terminalbench_env.py @@ -133,8 +133,7 @@ class TerminalBenchTaskEnv: test_timeout = task.max_test_timeout_sec test_script_path = ( - DockerComposeManager.CONTAINER_TEST_DIR - / task_paths.run_tests_path.name + DockerComposeManager.CONTAINER_TEST_DIR / task_paths.run_tests_path.name ) try: @@ -144,9 +143,7 @@ class TerminalBenchTaskEnv: max_timeout_sec=test_timeout, ) except TimeoutError: - LOGGER.warning( - "Test command timed out after %.0fs", test_timeout - ) + LOGGER.warning("Test command timed out after %.0fs", test_timeout) results["error"] = "test_timeout" self._metadata["is_resolved"] = False self._metadata["test_results"] = results @@ -159,8 +156,7 @@ class TerminalBenchTaskEnv: try: parser_results = parser.parse(post_test_pane) results["parser_results"] = { - name: status.value - for name, status in parser_results.items() + name: status.value for name, status in parser_results.items() } is_resolved = all( status == UnitTestStatus.PASSED diff --git a/src/openjarvis/evals/execution/webchorearena_env.py b/src/openjarvis/evals/execution/webchorearena_env.py index 68e1c883..3eee9025 100644 --- a/src/openjarvis/evals/execution/webchorearena_env.py +++ b/src/openjarvis/evals/execution/webchorearena_env.py @@ -172,7 +172,8 @@ class WebChoreArenaTaskEnv: responses: List[str] = [] intent = self._task_config.get( - "intent", self._task_config.get("intent_template", ""), + "intent", + self._task_config.get("intent_template", ""), ) for step_idx in range(max_steps): @@ -249,7 +250,9 @@ class WebChoreArenaTaskEnv: LOGGER.info( "WebChoreArena eval: task=%s score=%.2f resolved=%s", - self._metadata.get("task_id", "?"), score, is_resolved, + self._metadata.get("task_id", "?"), + score, + is_resolved, ) # -- StringEvaluator (exact_match, must_include, fuzzy_match) ------ @@ -270,7 +273,8 @@ class WebChoreArenaTaskEnv: must_score = 0.0 for must_value in value: must_score += _must_include( - ref=str(must_value), pred=pred, + ref=str(must_value), + pred=pred, tokenize=(len(value) == 1), ) must_score /= len(value) @@ -293,7 +297,9 @@ class WebChoreArenaTaskEnv: else: fuzzy_value = str(value) score *= self._llm_fuzzy_match( - pred=pred, reference=fuzzy_value, intent=intent, + pred=pred, + reference=fuzzy_value, + intent=intent, ) return score @@ -336,16 +342,13 @@ class WebChoreArenaTaskEnv: pred_base_path, pred_query = _parse_url(current_url) - base_score = float(any( - rbp in pred_base_path for rbp in ref_base_paths - )) + base_score = float(any(rbp in pred_base_path for rbp in ref_base_paths)) query_score = 1.0 for k, possible_values in ref_queries.items(): - query_score *= float(any( - pv in pred_query.get(k, []) - for pv in possible_values - )) + query_score *= float( + any(pv in pred_query.get(k, []) for pv in possible_values) + ) or_scores.append(base_score * query_score) @@ -450,10 +453,12 @@ class WebChoreArenaTaskEnv: scores: List[float] = [] for content in contents: content_or = str(content).split(" |OR| ") - s = float(any( - _must_include(ref=c, pred=selected_element, tokenize=False) - for c in content_or - )) + s = float( + any( + _must_include(ref=c, pred=selected_element, tokenize=False) + for c in content_or + ) + ) scores.append(s) return sum(scores) / len(scores) if scores else 0.0 else: @@ -528,6 +533,7 @@ class WebChoreArenaTaskEnv: """Call an LLM judge for fuzzy/ua matching.""" try: from openjarvis.evals.core.backend import InferenceBackend + backend = InferenceBackend.create_default() return backend.generate( prompt, @@ -557,21 +563,20 @@ class WebChoreArenaTaskEnv: if action_lower.startswith("click"): elem_id = _extract_bracket_arg(action_text) if elem_id: - self._page.locator(f"[data-webarena-id='{elem_id}']").click(timeout=5000) + self._page.locator(f"[data-webarena-id='{elem_id}']").click( + timeout=5000 + ) elif action_lower.startswith("type"): parts = action_text.split("]", 1) elem_id = ( - _extract_bracket_arg(parts[0] + "]") - if "]" in action_text else None - ) - text = ( - parts[1].strip().strip("[]") - if len(parts) > 1 else "" + _extract_bracket_arg(parts[0] + "]") if "]" in action_text else None ) + text = parts[1].strip().strip("[]") if len(parts) > 1 else "" if elem_id: loc = f"[data-webarena-id='{elem_id}']" self._page.locator(loc).fill( - text, timeout=5000, + text, + timeout=5000, ) elif action_lower.startswith("scroll"): if "down" in action_lower: @@ -580,8 +585,7 @@ class WebChoreArenaTaskEnv: self._page.mouse.wheel(0, -300) elif action_lower.startswith("goto"): url = ( - action_text.split(None, 1)[1].strip() - if " " in action_text else "" + action_text.split(None, 1)[1].strip() if " " in action_text else "" ) if url: self._page.goto( @@ -593,17 +597,21 @@ class WebChoreArenaTaskEnv: elif action_lower.startswith("hover"): elem_id = _extract_bracket_arg(action_text) if elem_id: - self._page.locator(f"[data-webarena-id='{elem_id}']").hover(timeout=5000) + self._page.locator(f"[data-webarena-id='{elem_id}']").hover( + timeout=5000 + ) elif action_lower.startswith("press"): key = ( action_text.split(None, 1)[1].strip().strip("[]") - if " " in action_text else "Enter" + if " " in action_text + else "Enter" ) self._page.keyboard.press(key) except Exception as exc: LOGGER.debug( "Action execution error: %s (action: %s)", - exc, action_text[:100], + exc, + action_text[:100], ) self._page.wait_for_timeout(1000) @@ -613,7 +621,10 @@ class WebChoreArenaTaskEnv: # ------------------------------------------------------------------ def _build_step_prompt( - self, intent: str, step_idx: int, max_steps: int, + self, + intent: str, + step_idx: int, + max_steps: int, ) -> str: parts: List[str] = [_SYSTEM_MSG] parts.append(f"## Task\n{intent}") @@ -643,7 +654,9 @@ class WebChoreArenaTaskEnv: """Replace WebArena placeholder hostnames with actual env var values.""" replacements = { "__SHOPPING__": os.environ.get("SHOPPING", "http://localhost:7770"), - "__SHOPPING_ADMIN__": os.environ.get("SHOPPING_ADMIN", "http://localhost:7780/admin"), + "__SHOPPING_ADMIN__": os.environ.get( + "SHOPPING_ADMIN", "http://localhost:7780/admin" + ), "__REDDIT__": os.environ.get("REDDIT", "http://localhost:9999"), "__GITLAB__": os.environ.get("GITLAB", "http://localhost:8023"), "__MAP__": os.environ.get("MAP", "http://localhost:3000"), @@ -691,6 +704,7 @@ def _must_include(ref: str, pred: str, tokenize: bool = False) -> float: if tokenize and len(r) == 1: try: from nltk.tokenize import word_tokenize + tok_pred = word_tokenize(clean_pred) if r in tok_pred: return 1.0 @@ -705,6 +719,7 @@ def _must_include(ref: str, pred: str, tokenize: bool = False) -> float: if tokenize and len(clean_ref) == 1: try: from nltk.tokenize import word_tokenize + tok_pred = word_tokenize(clean_pred) return float(clean_ref in tok_pred) except ImportError: diff --git a/src/openjarvis/evals/execution/workarena_env.py b/src/openjarvis/evals/execution/workarena_env.py index d4b7ecb5..de43db45 100644 --- a/src/openjarvis/evals/execution/workarena_env.py +++ b/src/openjarvis/evals/execution/workarena_env.py @@ -50,10 +50,27 @@ Respond with your reasoning, then the action on its own line. """ _ACTION_NAMES = ( - "click", "dblclick", "fill", "select_option", "check", "uncheck", - "clear", "hover", "press", "focus", "scroll", "goto", "go_back", - "go_forward", "tab_focus", "tab_close", "new_tab", - "send_msg_to_user", "report_infeasible", "noop", "drag_and_drop", + "click", + "dblclick", + "fill", + "select_option", + "check", + "uncheck", + "clear", + "hover", + "press", + "focus", + "scroll", + "goto", + "go_back", + "go_forward", + "tab_focus", + "tab_close", + "new_tab", + "send_msg_to_user", + "report_infeasible", + "noop", + "drag_and_drop", ) _ACTION_PREFIX_RE = re.compile( r"(" + "|".join(_ACTION_NAMES) + r")\s*\(", @@ -117,7 +134,9 @@ class WorkArenaTaskEnv: LOGGER.info( "Setting up WorkArena task: %s (seed=%d, headless=%s)", - task_class.get_task_id(), task_seed, headless, + task_class.get_task_id(), + task_seed, + headless, ) self._env = BrowserEnv( @@ -253,7 +272,8 @@ class WorkArenaTaskEnv: chat_msgs = self._env.chat.messages reward, done, message, info = self._env.task.validate( - self._env.page, chat_msgs, + self._env.page, + chat_msgs, ) is_resolved = reward == 1.0 @@ -269,7 +289,9 @@ class WorkArenaTaskEnv: LOGGER.info( "WorkArena validate: reward=%.1f resolved=%s msg=%s", - reward, is_resolved, message, + reward, + is_resolved, + message, ) return is_resolved, results @@ -330,10 +352,11 @@ class WorkArenaTaskEnv: parse_error = ( "Could not parse a valid action from your previous " "response. Please respond with exactly one action " - "call, e.g. click(\"bid_42\")." + 'call, e.g. click("bid_42").' ) LOGGER.warning( - "Step %d: unparseable action, issuing noop", step_idx, + "Step %d: unparseable action, issuing noop", + step_idx, ) action = "noop()" else: @@ -440,7 +463,7 @@ class WorkArenaTaskEnv: tag = role if bid: - tag += f' [bid={bid}]' + tag += f" [bid={bid}]" if name: tag += f' "{name}"' if value: diff --git a/src/openjarvis/evals/scorers/_checklist.py b/src/openjarvis/evals/scorers/_checklist.py index dd252249..2421bac0 100644 --- a/src/openjarvis/evals/scorers/_checklist.py +++ b/src/openjarvis/evals/scorers/_checklist.py @@ -135,7 +135,9 @@ class ChecklistScorer: return score, details def _parse_response( - self, raw: str, checklist: List[str], + self, + raw: str, + checklist: List[str], ) -> List[Dict[str, Any]]: """Parse the judge response into per-item results.""" details: List[Dict[str, Any]] = [] @@ -151,11 +153,13 @@ class ChecklistScorer: passed = False reasoning = "could not parse judge response" - details.append({ - "item": item, - "passed": passed, - "reasoning": reasoning, - }) + details.append( + { + "item": item, + "passed": passed, + "reasoning": reasoning, + } + ) return details diff --git a/src/openjarvis/evals/scorers/ama_bench_judge.py b/src/openjarvis/evals/scorers/ama_bench_judge.py index 1c2a6a31..e3ecbca8 100644 --- a/src/openjarvis/evals/scorers/ama_bench_judge.py +++ b/src/openjarvis/evals/scorers/ama_bench_judge.py @@ -76,7 +76,9 @@ class AMABenchScorer(LLMJudgeScorer): scorer_id = "ama-bench" def score( - self, record: EvalRecord, model_answer: str, + self, + record: EvalRecord, + model_answer: str, ) -> Tuple[Optional[bool], Dict[str, Any]]: if not model_answer or not model_answer.strip(): return False, {"reason": "empty_response", "f1": 0.0} @@ -97,14 +99,18 @@ class AMABenchScorer(LLMJudgeScorer): ) raw = self._ask_judge( - prompt, system=_JUDGE_SYSTEM, temperature=0.0, max_tokens=128, + prompt, + system=_JUDGE_SYSTEM, + temperature=0.0, + max_tokens=128, ) label = _parse_judge_label(raw) if label is None: LOGGER.warning( "AMA-Bench judge returned unparseable output for %s, " "falling back to F1 threshold: %r", - record.record_id, raw[:200], + record.record_id, + raw[:200], ) is_correct = f1 >= 0.5 return is_correct, { diff --git a/src/openjarvis/evals/scorers/browser_assistant.py b/src/openjarvis/evals/scorers/browser_assistant.py index 2b14dcfd..7597e311 100644 --- a/src/openjarvis/evals/scorers/browser_assistant.py +++ b/src/openjarvis/evals/scorers/browser_assistant.py @@ -75,9 +75,13 @@ def _sources_cited(model_answer: str) -> bool: # Check for common source reference patterns ans_lower = model_answer.lower() source_indicators = [ - "source:", "reference:", "according to", - "official documentation", "official docs", - "cited from", "as stated in", + "source:", + "reference:", + "according to", + "official documentation", + "official docs", + "cited from", + "as stated in", ] return any(ind in ans_lower for ind in source_indicators) @@ -88,13 +92,17 @@ class BrowserAssistantScorer(Scorer): scorer_id = "browser_assistant" def __init__( - self, judge_backend=None, judge_model: str = "", + self, + judge_backend=None, + judge_model: str = "", ) -> None: self._judge_backend = judge_backend self._judge_model = judge_model def score( - self, record: EvalRecord, model_answer: str, + self, + record: EvalRecord, + model_answer: str, ) -> Tuple[Optional[bool], Dict[str, Any]]: if not model_answer or not model_answer.strip(): return False, {"reason": "empty_response"} @@ -110,7 +118,8 @@ class BrowserAssistantScorer(Scorer): exact_details: List[Dict[str, Any]] = [] if exact_facts: exact_score, exact_details = _exact_match_score( - model_answer, exact_facts, + model_answer, + exact_facts, ) # --- Tier 2: Semantic checklist --- @@ -119,17 +128,13 @@ class BrowserAssistantScorer(Scorer): if semantic_facts: if self._judge_backend and self._judge_model: scorer = ChecklistScorer( - self._judge_backend, self._judge_model, + self._judge_backend, + self._judge_model, ) - semantic_score, semantic_details = ( - scorer.score_checklist( - model_answer, - [ - f"The answer covers: {fact}" - for fact in semantic_facts - ], - context=record.problem, - ) + semantic_score, semantic_details = scorer.score_checklist( + model_answer, + [f"The answer covers: {fact}" for fact in semantic_facts], + context=record.problem, ) else: # Heuristic fallback: word-level matching @@ -137,21 +142,15 @@ class BrowserAssistantScorer(Scorer): matched = 0 for fact in semantic_facts: words = normalize_str(fact).split() - word_matches = sum( - 1 for w in words if w in ans_norm - ) - found = ( - word_matches / len(words) >= 0.4 - if words else False - ) + word_matches = sum(1 for w in words if w in ans_norm) + found = word_matches / len(words) >= 0.4 if words else False semantic_details.append( {"item": fact, "passed": found}, ) if found: matched += 1 semantic_score = ( - matched / len(semantic_facts) - if semantic_facts else 1.0 + matched / len(semantic_facts) if semantic_facts else 1.0 ) # --- Quality checklist --- @@ -164,20 +163,20 @@ class BrowserAssistantScorer(Scorer): "Answer directly addresses the question", ] scorer = ChecklistScorer( - self._judge_backend, self._judge_model, + self._judge_backend, + self._judge_model, ) - quality_score, quality_details = ( - scorer.score_checklist( - model_answer, items, context=record.problem, - ) + quality_score, quality_details = scorer.score_checklist( + model_answer, + items, + context=record.problem, ) else: # Heuristic has_numbers = bool(re.search(r"\d+", model_answer)) has_sources = _sources_cited(model_answer) - quality_score = ( - (0.5 if has_numbers else 0.0) - + (0.5 if has_sources else 0.0) + quality_score = (0.5 if has_numbers else 0.0) + ( + 0.5 if has_sources else 0.0 ) # --- Sources --- @@ -213,9 +212,7 @@ class BrowserAssistantScorer(Scorer): ) exact_found = sum(1 for d in exact_details if d["found"]) - semantic_passed = sum( - 1 for d in semantic_details if d.get("passed") - ) + semantic_passed = sum(1 for d in semantic_details if d.get("passed")) is_correct = final_score >= 0.7 diff --git a/src/openjarvis/evals/scorers/coding_assistant.py b/src/openjarvis/evals/scorers/coding_assistant.py index db0ef871..ecc677f2 100644 --- a/src/openjarvis/evals/scorers/coding_assistant.py +++ b/src/openjarvis/evals/scorers/coding_assistant.py @@ -56,7 +56,11 @@ def _extract_test_functions(test_code: str) -> Dict[str, str]: current_name = match.group(1) if match else None current_lines = [line] elif current_name is not None: - if line.strip() and not line.startswith((" ", "\t")) and not stripped.startswith("#"): + if ( + line.strip() + and not line.startswith((" ", "\t")) + and not stripped.startswith("#") + ): # New top-level definition — end of current test tests[current_name] = "\n".join(preamble + current_lines) current_name = None @@ -83,6 +87,7 @@ def _run_single_test(code: str, test_code: str) -> bool: """Run a single test function against the given code. Returns True if passes.""" # Make the code importable as 'solution' import types + mod = types.ModuleType("solution") try: exec(code, mod.__dict__) # noqa: S102 @@ -90,6 +95,7 @@ def _run_single_test(code: str, test_code: str) -> bool: return False import sys + sys.modules["solution"] = mod try: exec(test_code, {"__name__": "__main__"}) # noqa: S102 @@ -109,7 +115,9 @@ class CodingAssistantScorer(Scorer): pass def score( - self, record: EvalRecord, model_answer: str, + self, + record: EvalRecord, + model_answer: str, ) -> Tuple[Optional[bool], Dict[str, Any]]: if not model_answer or not model_answer.strip(): return False, {"reason": "empty_response"} diff --git a/src/openjarvis/evals/scorers/coding_task.py b/src/openjarvis/evals/scorers/coding_task.py index 7329096d..56742d55 100644 --- a/src/openjarvis/evals/scorers/coding_task.py +++ b/src/openjarvis/evals/scorers/coding_task.py @@ -56,9 +56,7 @@ def _run_tests(code: str, test_cases: str) -> Tuple[int, int, str]: # Parse individual test assertions test_lines = [ - line.strip() - for line in test_cases.strip().split("\n") - if line.strip() + line.strip() for line in test_cases.strip().split("\n") if line.strip() ] passed = 0 @@ -106,7 +104,9 @@ class CodingTaskScorer(Scorer): pass def score( - self, record: EvalRecord, model_answer: str, + self, + record: EvalRecord, + model_answer: str, ) -> Tuple[Optional[bool], Dict[str, Any]]: if not model_answer or not model_answer.strip(): return False, {"reason": "empty_response"} diff --git a/src/openjarvis/evals/scorers/daily_digest.py b/src/openjarvis/evals/scorers/daily_digest.py index 08f07170..dc6a81e4 100644 --- a/src/openjarvis/evals/scorers/daily_digest.py +++ b/src/openjarvis/evals/scorers/daily_digest.py @@ -78,13 +78,17 @@ class DailyDigestScorer(Scorer): scorer_id = "daily_digest" def __init__( - self, judge_backend=None, judge_model: str = "", + self, + judge_backend=None, + judge_model: str = "", ) -> None: self._judge_backend = judge_backend self._judge_model = judge_model def score( - self, record: EvalRecord, model_answer: str, + self, + record: EvalRecord, + model_answer: str, ) -> Tuple[Optional[bool], Dict[str, Any]]: if not model_answer or not model_answer.strip(): return False, {"reason": "empty_response"} @@ -97,12 +101,14 @@ class DailyDigestScorer(Scorer): # --- Tier 1: Phrase match --- phrase_score, phrase_details = _phrase_match_score( - model_answer, must_mention, + model_answer, + must_mention, ) # --- Tier 1: Ordering --- order_score, order_details = _ordering_score( - model_answer, priority_order, + model_answer, + priority_order, ) # --- Tier 2: Checklist --- @@ -118,12 +124,13 @@ class DailyDigestScorer(Scorer): "The tone is professional and concise", ] scorer = ChecklistScorer( - self._judge_backend, self._judge_model, + self._judge_backend, + self._judge_model, ) - checklist_score, checklist_details = ( - scorer.score_checklist( - model_answer, items, context=record.problem, - ) + checklist_score, checklist_details = scorer.score_checklist( + model_answer, + items, + context=record.problem, ) else: # Heuristic fallback @@ -133,29 +140,17 @@ class DailyDigestScorer(Scorer): for marker in ["##", "**", "---", "priority", "action"] ) has_actions = any( - kw in ans_lower - for kw in ["action", "todo", "next step", "follow up"] + kw in ans_lower for kw in ["action", "todo", "next step", "follow up"] ) - checklist_score = ( - (0.5 if has_sections else 0.0) - + (0.5 if has_actions else 0.0) + checklist_score = (0.5 if has_sections else 0.0) + ( + 0.5 if has_actions else 0.0 ) # --- Composite score --- - final_score = ( - phrase_score * 0.5 - + order_score * 0.3 - + checklist_score * 0.2 - ) + final_score = phrase_score * 0.5 + order_score * 0.3 + checklist_score * 0.2 - items_mentioned = sum( - 1 for d in phrase_details if d["found"] - ) - is_correct = ( - phrase_score >= 0.8 - and order_score >= 0.5 - and final_score >= 0.7 - ) + items_mentioned = sum(1 for d in phrase_details if d["found"]) + is_correct = phrase_score >= 0.8 and order_score >= 0.5 and final_score >= 0.7 return is_correct, { "match_type": "daily_digest", diff --git a/src/openjarvis/evals/scorers/deepplanning_scorer.py b/src/openjarvis/evals/scorers/deepplanning_scorer.py index 24759370..90c79093 100644 --- a/src/openjarvis/evals/scorers/deepplanning_scorer.py +++ b/src/openjarvis/evals/scorers/deepplanning_scorer.py @@ -44,7 +44,9 @@ class DeepPlanningScorer(LLMJudgeScorer): scorer_id = "deepplanning" def score( - self, record: EvalRecord, model_answer: str, + self, + record: EvalRecord, + model_answer: str, ) -> Tuple[Optional[bool], Dict[str, Any]]: if not model_answer or not model_answer.strip(): return False, {"reason": "empty_response"} diff --git a/src/openjarvis/evals/scorers/doc_qa.py b/src/openjarvis/evals/scorers/doc_qa.py index 84c41b6f..a7de5025 100644 --- a/src/openjarvis/evals/scorers/doc_qa.py +++ b/src/openjarvis/evals/scorers/doc_qa.py @@ -42,11 +42,13 @@ def _fact_match_score( matched = sum(1 for w in words if w in ans_norm) found = matched / len(words) >= 0.6 if words else False - details.append({ - "fact": fact, - "expected_source": fact_entry.get("source_doc_index"), - "found": found, - }) + details.append( + { + "fact": fact, + "expected_source": fact_entry.get("source_doc_index"), + "found": found, + } + ) total = len(required_facts) found_count = sum(1 for d in details if d["found"]) @@ -81,10 +83,12 @@ def _citation_check_score( is_cited = src in cited_indices if is_cited: correct += 1 - details.append({ - "expected_doc_index": src, - "cited": is_cited, - }) + details.append( + { + "expected_doc_index": src, + "cited": is_cited, + } + ) total = len(expected_sources) score = correct / total if total > 0 else 0.0 @@ -98,13 +102,17 @@ class DocQAScorer(Scorer): scorer_id = "doc_qa" def __init__( - self, judge_backend=None, judge_model: str = "", + self, + judge_backend=None, + judge_model: str = "", ) -> None: self._judge_backend = judge_backend self._judge_model = judge_model def score( - self, record: EvalRecord, model_answer: str, + self, + record: EvalRecord, + model_answer: str, ) -> Tuple[Optional[bool], Dict[str, Any]]: if not model_answer or not model_answer.strip(): return False, {"reason": "empty_response"} @@ -115,12 +123,14 @@ class DocQAScorer(Scorer): # --- Tier 1: Fact match --- fact_score, fact_details = _fact_match_score( - model_answer, required_facts, + model_answer, + required_facts, ) # --- Tier 1: Citation check --- citation_score, citation_details = _citation_check_score( - model_answer, required_facts, + model_answer, + required_facts, ) # --- Tier 2: Checklist --- @@ -136,40 +146,28 @@ class DocQAScorer(Scorer): "Answer is well-organized and clear", ] scorer = ChecklistScorer( - self._judge_backend, self._judge_model, + self._judge_backend, + self._judge_model, ) - checklist_score, checklist_details = ( - scorer.score_checklist( - model_answer, items, context=record.problem, - ) + checklist_score, checklist_details = scorer.score_checklist( + model_answer, + items, + context=record.problem, ) else: # Heuristic fallback ans_lower = model_answer.lower() - has_citations = bool( - _CITATION_PATTERN.search(model_answer) - ) - has_structure = any( - m in ans_lower for m in ["##", "**", "- ", "1."] - ) - checklist_score = ( - (0.5 if has_citations else 0.0) - + (0.5 if has_structure else 0.0) + has_citations = bool(_CITATION_PATTERN.search(model_answer)) + has_structure = any(m in ans_lower for m in ["##", "**", "- ", "1."]) + checklist_score = (0.5 if has_citations else 0.0) + ( + 0.5 if has_structure else 0.0 ) # --- Composite score --- - final_score = ( - fact_score * 0.5 - + citation_score * 0.3 - + checklist_score * 0.2 - ) + final_score = fact_score * 0.5 + citation_score * 0.3 + checklist_score * 0.2 facts_found = sum(1 for d in fact_details if d["found"]) - is_correct = ( - fact_score >= 0.8 - and citation_score >= 0.5 - and final_score >= 0.7 - ) + is_correct = fact_score >= 0.8 and citation_score >= 0.5 and final_score >= 0.7 return is_correct, { "match_type": "doc_qa", diff --git a/src/openjarvis/evals/scorers/email_triage.py b/src/openjarvis/evals/scorers/email_triage.py index 0023d74a..8de4e931 100644 --- a/src/openjarvis/evals/scorers/email_triage.py +++ b/src/openjarvis/evals/scorers/email_triage.py @@ -54,7 +54,9 @@ class EmailTriageScorer(LLMJudgeScorer): scorer_id = "email_triage" def score( - self, record: EvalRecord, model_answer: str, + self, + record: EvalRecord, + model_answer: str, ) -> Tuple[Optional[bool], Dict[str, Any]]: if not model_answer or not model_answer.strip(): return False, {"reason": "empty_response"} diff --git a/src/openjarvis/evals/scorers/frames_judge.py b/src/openjarvis/evals/scorers/frames_judge.py index 8f5ba501..a99ec825 100644 --- a/src/openjarvis/evals/scorers/frames_judge.py +++ b/src/openjarvis/evals/scorers/frames_judge.py @@ -47,7 +47,9 @@ class FRAMESScorer(LLMJudgeScorer): scorer_id = "frames" def score( - self, record: EvalRecord, model_answer: str, + self, + record: EvalRecord, + model_answer: str, ) -> Tuple[Optional[bool], Dict[str, Any]]: if not model_answer or not model_answer.strip(): return False, {"reason": "empty_response"} @@ -66,7 +68,9 @@ class FRAMESScorer(LLMJudgeScorer): raw = self._ask_judge(prompt, temperature=0.0, max_tokens=2048) structured_match = re.search( - r"^correct:\s*(yes|no)", raw, re.MULTILINE | re.IGNORECASE, + r"^correct:\s*(yes|no)", + raw, + re.MULTILINE | re.IGNORECASE, ) if structured_match: is_correct = structured_match.group(1).lower() == "yes" @@ -84,7 +88,9 @@ class FRAMESScorer(LLMJudgeScorer): "raw_judge_output": raw, } extracted = re.search( - r"^extracted_final_answer:\s*(.+)", raw, re.MULTILINE, + r"^extracted_final_answer:\s*(.+)", + raw, + re.MULTILINE, ) if extracted: meta["extracted_answer"] = extracted.group(1).strip() diff --git a/src/openjarvis/evals/scorers/gaia_exact.py b/src/openjarvis/evals/scorers/gaia_exact.py index 6050037b..5b1c23a5 100644 --- a/src/openjarvis/evals/scorers/gaia_exact.py +++ b/src/openjarvis/evals/scorers/gaia_exact.py @@ -70,9 +70,7 @@ def exact_match(model_answer: str, ground_truth: str) -> bool: comparisons = [] for ma_elem, gt_elem in zip(ma_elems, gt_elems): if _is_float(gt_elem): - comparisons.append( - _normalize_number_str(ma_elem) == float(gt_elem) - ) + comparisons.append(_normalize_number_str(ma_elem) == float(gt_elem)) else: comparisons.append( _normalize_str(ma_elem, remove_punct=False) @@ -110,7 +108,9 @@ class GAIAScorer(LLMJudgeScorer): scorer_id = "gaia" def score( - self, record: EvalRecord, model_answer: str, + self, + record: EvalRecord, + model_answer: str, ) -> Tuple[Optional[bool], Dict[str, Any]]: if not model_answer or not model_answer.strip(): return False, {"reason": "empty_response"} @@ -133,21 +133,23 @@ class GAIAScorer(LLMJudgeScorer): raw = self._ask_judge(prompt, temperature=0.0, max_tokens=2048) structured_match = re.search( - r"^correct:\s*(yes|no)", raw, re.MULTILINE | re.IGNORECASE, + r"^correct:\s*(yes|no)", + raw, + re.MULTILINE | re.IGNORECASE, ) if structured_match: is_correct = structured_match.group(1).lower() == "yes" else: - is_correct = ( - "CORRECT" in raw.upper() and "INCORRECT" not in raw.upper() - ) + is_correct = "CORRECT" in raw.upper() and "INCORRECT" not in raw.upper() meta: Dict[str, Any] = { "match_type": "llm_fallback", "raw_judge_output": raw, } extracted_match = re.search( - r"^extracted_final_answer:\s*(.+)", raw, re.MULTILINE, + r"^extracted_final_answer:\s*(.+)", + raw, + re.MULTILINE, ) if extracted_match: meta["extracted_answer"] = extracted_match.group(1).strip() diff --git a/src/openjarvis/evals/scorers/gpqa_mcq.py b/src/openjarvis/evals/scorers/gpqa_mcq.py index 5c211130..d8704c99 100644 --- a/src/openjarvis/evals/scorers/gpqa_mcq.py +++ b/src/openjarvis/evals/scorers/gpqa_mcq.py @@ -49,15 +49,19 @@ class GPQAScorer(LLMJudgeScorer): try: raw_response = self._ask_judge( - user_prompt, system=system_prompt, - temperature=0.0, max_tokens=2048, + user_prompt, + system=system_prompt, + temperature=0.0, + max_tokens=2048, ) extracted = raw_response.strip().upper() # Handle "The answer is: A" etc. answer_match = re.search( - r"(?:THE ANSWER IS:?\s*)?([A-Z])", extracted, re.IGNORECASE, + r"(?:THE ANSWER IS:?\s*)?([A-Z])", + extracted, + re.IGNORECASE, ) if answer_match: extracted = answer_match.group(1).upper() @@ -72,7 +76,9 @@ class GPQAScorer(LLMJudgeScorer): return None def score( - self, record: EvalRecord, model_answer: str, + self, + record: EvalRecord, + model_answer: str, ) -> Tuple[Optional[bool], Dict[str, Any]]: ref = record.reference.strip().upper() if not ref: @@ -81,7 +87,9 @@ class GPQAScorer(LLMJudgeScorer): valid_letters = self._valid_letters_from_options(record.metadata) candidate = self._extract_answer_with_llm( - record.problem, model_answer, valid_letters, + record.problem, + model_answer, + valid_letters, ) if not candidate: return None, {"reason": "no_choice_letter_extracted"} diff --git a/src/openjarvis/evals/scorers/hle_judge.py b/src/openjarvis/evals/scorers/hle_judge.py index 0fccd3e0..154ad36c 100644 --- a/src/openjarvis/evals/scorers/hle_judge.py +++ b/src/openjarvis/evals/scorers/hle_judge.py @@ -43,7 +43,9 @@ class HLEScorer(LLMJudgeScorer): scorer_id = "hle" def score( - self, record: EvalRecord, model_answer: str, + self, + record: EvalRecord, + model_answer: str, ) -> Tuple[Optional[bool], Dict[str, Any]]: if not model_answer or not model_answer.strip(): return False, {"reason": "empty_response"} @@ -66,21 +68,23 @@ class HLEScorer(LLMJudgeScorer): raw = self._ask_judge(prompt, temperature=0.0, max_tokens=2048) structured_match = re.search( - r"^correct:\s*(yes|no)", raw, re.MULTILINE | re.IGNORECASE, + r"^correct:\s*(yes|no)", + raw, + re.MULTILINE | re.IGNORECASE, ) if structured_match: is_correct = structured_match.group(1).lower() == "yes" else: - is_correct = ( - "CORRECT" in raw.upper() and "INCORRECT" not in raw.upper() - ) + is_correct = "CORRECT" in raw.upper() and "INCORRECT" not in raw.upper() meta: Dict[str, Any] = { "match_type": "llm_fallback", "raw_judge_output": raw, } extracted = re.search( - r"^extracted_final_answer:\s*(.+)", raw, re.MULTILINE, + r"^extracted_final_answer:\s*(.+)", + raw, + re.MULTILINE, ) if extracted: meta["extracted_answer"] = extracted.group(1).strip() diff --git a/src/openjarvis/evals/scorers/ipw_mixed.py b/src/openjarvis/evals/scorers/ipw_mixed.py index e4d8d132..777d54a6 100644 --- a/src/openjarvis/evals/scorers/ipw_mixed.py +++ b/src/openjarvis/evals/scorers/ipw_mixed.py @@ -50,7 +50,9 @@ class IPWMixedScorer(LLMJudgeScorer): scorer_id = "ipw" def score( - self, record: EvalRecord, model_answer: str, + self, + record: EvalRecord, + model_answer: str, ) -> Tuple[Optional[bool], Dict[str, Any]]: if not model_answer or not model_answer.strip(): return False, {"reason": "empty_response"} @@ -69,7 +71,9 @@ class IPWMixedScorer(LLMJudgeScorer): raw = self._ask_judge(prompt, temperature=0.0, max_tokens=2048) structured_match = re.search( - r"^correct:\s*(yes|no)", raw, re.MULTILINE | re.IGNORECASE, + r"^correct:\s*(yes|no)", + raw, + re.MULTILINE | re.IGNORECASE, ) if structured_match: is_correct = structured_match.group(1).lower() == "yes" @@ -81,7 +85,8 @@ class IPWMixedScorer(LLMJudgeScorer): is_correct = False else: LOGGER.warning( - "Could not parse grade from response: %s", raw[:50], + "Could not parse grade from response: %s", + raw[:50], ) is_correct = False @@ -89,7 +94,9 @@ class IPWMixedScorer(LLMJudgeScorer): "raw_judge_output": raw, } extracted = re.search( - r"^extracted_final_answer:\s*(.+)", raw, re.MULTILINE, + r"^extracted_final_answer:\s*(.+)", + raw, + re.MULTILINE, ) if extracted: meta["extracted_answer"] = extracted.group(1).strip() diff --git a/src/openjarvis/evals/scorers/knowledge_base.py b/src/openjarvis/evals/scorers/knowledge_base.py index 10e0a3e6..e512be38 100644 --- a/src/openjarvis/evals/scorers/knowledge_base.py +++ b/src/openjarvis/evals/scorers/knowledge_base.py @@ -61,7 +61,9 @@ class KnowledgeBaseScorer(LLMJudgeScorer): scorer_id = "knowledge_base" def score( - self, record: EvalRecord, model_answer: str, + self, + record: EvalRecord, + model_answer: str, ) -> Tuple[Optional[bool], Dict[str, Any]]: if not model_answer or not model_answer.strip(): return False, {"reason": "empty_response"} @@ -88,9 +90,7 @@ class KnowledgeBaseScorer(LLMJudgeScorer): re.MULTILINE | re.IGNORECASE, ) is_correct = ( - overall_match.group(1).lower() == "yes" - if overall_match - else False + overall_match.group(1).lower() == "yes" if overall_match else False ) completeness_match = re.search( @@ -99,9 +99,7 @@ class KnowledgeBaseScorer(LLMJudgeScorer): re.MULTILINE | re.IGNORECASE, ) completeness = ( - completeness_match.group(1).lower() - if completeness_match - else "unknown" + completeness_match.group(1).lower() if completeness_match else "unknown" ) return is_correct, { diff --git a/src/openjarvis/evals/scorers/lifelong_agent_scorer.py b/src/openjarvis/evals/scorers/lifelong_agent_scorer.py index 156e3374..ba1f53e4 100644 --- a/src/openjarvis/evals/scorers/lifelong_agent_scorer.py +++ b/src/openjarvis/evals/scorers/lifelong_agent_scorer.py @@ -54,13 +54,25 @@ _SINGLE_SHOT_WARNING = ( ) _TYPE_MAP: Dict[str, str] = { - "INT": "INTEGER", "INTEGER": "INTEGER", "BIGINT": "INTEGER", - "SMALLINT": "INTEGER", "TINYINT": "INTEGER", - "FLOAT": "REAL", "DOUBLE": "REAL", "DECIMAL": "REAL", - "NUMERIC": "REAL", "REAL": "REAL", - "TEXT": "TEXT", "VARCHAR": "TEXT", "CHAR": "TEXT", - "BLOB": "BLOB", "DATE": "TEXT", "DATETIME": "TEXT", - "TIMESTAMP": "TEXT", "BOOLEAN": "INTEGER", "BOOL": "INTEGER", + "INT": "INTEGER", + "INTEGER": "INTEGER", + "BIGINT": "INTEGER", + "SMALLINT": "INTEGER", + "TINYINT": "INTEGER", + "FLOAT": "REAL", + "DOUBLE": "REAL", + "DECIMAL": "REAL", + "NUMERIC": "REAL", + "REAL": "REAL", + "TEXT": "TEXT", + "VARCHAR": "TEXT", + "CHAR": "TEXT", + "BLOB": "BLOB", + "DATE": "TEXT", + "DATETIME": "TEXT", + "TIMESTAMP": "TEXT", + "BOOLEAN": "INTEGER", + "BOOL": "INTEGER", } @@ -83,7 +95,9 @@ class LifelongAgentScorer(Scorer): pass def score( - self, record: EvalRecord, model_answer: str, + self, + record: EvalRecord, + model_answer: str, ) -> Tuple[Optional[bool], Dict[str, Any]]: # If this result came from interactive evaluation, the metadata # already contains the score — pass it through. @@ -117,7 +131,9 @@ class LifelongAgentScorer(Scorer): # ================================================================ def _score_db( - self, record: EvalRecord, model_answer: str, + self, + record: EvalRecord, + model_answer: str, ) -> Tuple[Optional[bool], Dict[str, Any]]: table_info = record.metadata.get("table_info", {}) answer_info = record.metadata.get("answer_info", {}) @@ -136,11 +152,18 @@ class LifelongAgentScorer(Scorer): try: if answer_type == "md5": return self._score_db_md5( - conn, model_answer, table_info, answer_info, skills, + conn, + model_answer, + table_info, + answer_info, + skills, ) else: return self._score_db_direct( - conn, model_answer, answer_info, skills, + conn, + model_answer, + answer_info, + skills, ) finally: conn.close() @@ -205,7 +228,8 @@ class LifelongAgentScorer(Scorer): "MD5 task %s: ground-truth SQL failed on SQLite (%s). " "Falling back to normalized SQL comparison. Use MySQL/Docker " "for faithful evaluation of DML tasks.", - table_name, exc, + table_name, + exc, ) meta["ref_sql_sqlite_error"] = str(exc) # Compare: did the agent execute the same DML statement? @@ -213,7 +237,8 @@ class LifelongAgentScorer(Scorer): norm_expected = _normalize_sql(expected_sql) is_correct = norm_agent == norm_expected meta["comparison_detail"] = ( - "normalized_sql_match" if is_correct + "normalized_sql_match" + if is_correct else f"normalized_sql_mismatch: expected={norm_expected!r}, got={norm_agent!r}" ) meta["fallback"] = "normalized_sql_comparison" @@ -250,9 +275,7 @@ class LifelongAgentScorer(Scorer): meta["reason"] = "no_ground_truth_direct" return None, meta - expected_tuples = [ - r if isinstance(r, list) else [r] for r in expected_direct - ] + expected_tuples = [r if isinstance(r, list) else [r] for r in expected_direct] meta["expected_tuples"] = expected_tuples agent_sql = extract_sql(model_answer) @@ -264,7 +287,8 @@ class LifelongAgentScorer(Scorer): actual_rows = [list(row) for row in cursor.fetchall()] meta["actual_tuples"] = actual_rows is_correct, detail = compare_tuple_lists( - expected_tuples, actual_rows, + expected_tuples, + actual_rows, ) meta["comparison_detail"] = detail meta["strategy"] = "sql_execution" @@ -272,7 +296,8 @@ class LifelongAgentScorer(Scorer): except sqlite3.Error as exc: meta["sql_error"] = str(exc) logger.debug( - "SQL execution failed, trying text parsing: %s", exc, + "SQL execution failed, trying text parsing: %s", + exc, ) parsed = _parse_text_answer(model_answer) @@ -292,7 +317,9 @@ class LifelongAgentScorer(Scorer): # ================================================================ def _score_kg( - self, record: EvalRecord, model_answer: str, + self, + record: EvalRecord, + model_answer: str, ) -> Tuple[Optional[bool], Dict[str, Any]]: expected = record.metadata.get("answer_list", []) skills = record.metadata.get("skills", []) @@ -330,7 +357,9 @@ class LifelongAgentScorer(Scorer): # ================================================================ def _score_os( - self, record: EvalRecord, model_answer: str, + self, + record: EvalRecord, + model_answer: str, ) -> Tuple[Optional[bool], Dict[str, Any]]: skills = record.metadata.get("skills", []) init_command = record.metadata.get("init_command", {}) @@ -369,7 +398,10 @@ class LifelongAgentScorer(Scorer): try: is_correct = _evaluate_os_in_docker( - init_command, agent_commands, eval_command, eval_info, + init_command, + agent_commands, + eval_command, + eval_info, ) meta["docker_eval_completed"] = True return is_correct, meta @@ -384,6 +416,7 @@ class LifelongAgentScorer(Scorer): # DB helpers # ==================================================================== + def build_db(table_info: Dict[str, Any]) -> sqlite3.Connection: """Build an in-memory SQLite DB from table_info. @@ -417,11 +450,15 @@ def build_db(table_info: Dict[str, Any]) -> sqlite3.Connection: padded.append(None) try: conn.execute( - f'INSERT INTO "{table_name}" VALUES ({ph})', padded, + f'INSERT INTO "{table_name}" VALUES ({ph})', + padded, ) except sqlite3.Error as exc: logger.debug( - "Skipping row %d in table %s: %s", row_idx, table_name, exc, + "Skipping row %d in table %s: %s", + row_idx, + table_name, + exc, ) conn.commit() @@ -429,7 +466,8 @@ def build_db(table_info: Dict[str, Any]) -> sqlite3.Connection: def _get_table_rows( - conn: sqlite3.Connection, table_name: str, + conn: sqlite3.Connection, + table_name: str, ) -> List[List[Any]]: cursor = conn.execute( f'SELECT * FROM "{table_name}" ORDER BY rowid', @@ -440,21 +478,19 @@ def _get_table_rows( def _hash_table_state(rows: List[List[Any]]) -> str: row_hashes = [] for row in rows: - concat = ",".join( - str(v) if v is not None else "NULL" for v in row - ) + concat = ",".join(str(v) if v is not None else "NULL" for v in row) row_hashes.append(hashlib.md5(concat.encode()).hexdigest()) row_hashes.sort() return hashlib.md5("".join(row_hashes).encode()).hexdigest() def _compare_table_states( - expected: List[List[Any]], actual: List[List[Any]], + expected: List[List[Any]], + actual: List[List[Any]], ) -> Tuple[bool, str]: if len(expected) != len(actual): return False, ( - f"row_count_mismatch: expected {len(expected)}, " - f"got {len(actual)}" + f"row_count_mismatch: expected {len(expected)}, got {len(actual)}" ) for i, (exp_row, act_row) in enumerate(zip(expected, actual)): if len(exp_row) != len(act_row): @@ -475,20 +511,24 @@ def _compare_table_states( # SQL extraction (matches original's Action: Operation format) # ==================================================================== + def extract_sql(text: str) -> str: text = text.strip() # Action: Operation\n```sql\n...\n``` m = re.search( r"Action:\s*Operation\s*\n\s*```(?:sql)?\s*\n?(.*?)\n?\s*```", - text, re.DOTALL | re.IGNORECASE, + text, + re.DOTALL | re.IGNORECASE, ) if m: return m.group(1).strip() # ```sql ... ``` m = re.search( - r"```sql\s*\n?(.*?)\n?```", text, re.DOTALL | re.IGNORECASE, + r"```sql\s*\n?(.*?)\n?```", + text, + re.DOTALL | re.IGNORECASE, ) if m: return m.group(1).strip() @@ -506,7 +546,7 @@ def extract_sql(text: str) -> str: candidate = m.group(1).strip() answer_pos = re.search(r"(?i)action:\s*answer", candidate) if answer_pos: - candidate = candidate[:answer_pos.start()].strip() + candidate = candidate[: answer_pos.start()].strip() if _looks_like_sql(candidate): return candidate @@ -516,7 +556,7 @@ def extract_sql(text: str) -> str: if _looks_like_sql(stripped): sql_lines = [stripped] start_idx = text.split("\n").index(line) - for cont_line in text.split("\n")[start_idx + 1:]: + for cont_line in text.split("\n")[start_idx + 1 :]: cont = cont_line.strip() if not cont or cont.startswith("Action:") or cont.startswith("Act:"): break @@ -549,10 +589,12 @@ def _normalize_sql(sql: str) -> str: # Text answer parsing (matches DirectTypeAnswerValidator) # ==================================================================== + def _parse_text_answer(text: str) -> Optional[List[List[Any]]]: m = re.search( r"(?:Action:\s*Answer\s*\n\s*)?Final\s+Answer:\s*(.+)", - text, re.DOTALL | re.IGNORECASE, + text, + re.DOTALL | re.IGNORECASE, ) if not m: return None @@ -584,6 +626,7 @@ def _parse_text_answer(text: str) -> Optional[List[List[Any]]]: def _safe_literal_eval(s: str) -> Any: s = re.sub(r"Decimal\('([^']*)'\)", r"\1", s) import ast + return ast.literal_eval(s) @@ -603,6 +646,7 @@ def _try_numeric(s: str) -> Any: # KG helpers (matches original's answer extraction + F1) # ==================================================================== + def extract_kg_answers(text: str) -> List[str]: text = text.strip() @@ -610,7 +654,8 @@ def extract_kg_answers(text: str) -> List[str]: # In single-shot mode we can't resolve variables, so extract any # entity IDs from the surrounding text as a best-effort fallback. var_ref = re.search( - r"Final\s+[Aa]nswer:\s*(?:[Vv]ar(?:iable)?\s*)?#(\d+)", text, + r"Final\s+[Aa]nswer:\s*(?:[Vv]ar(?:iable)?\s*)?#(\d+)", + text, ) if var_ref: # Can't resolve variable in single-shot mode — look for entity IDs @@ -653,13 +698,15 @@ def _normalize_entity(entity: Any) -> str: # OS helpers (matches original's Docker evaluation) # ==================================================================== + def _docker_available() -> bool: if not shutil.which("docker"): return False try: result = subprocess.run( ["docker", "info"], - capture_output=True, timeout=10, + capture_output=True, + timeout=10, ) return result.returncode == 0 except (subprocess.TimeoutExpired, OSError): @@ -672,7 +719,8 @@ def _extract_bash_commands(text: str) -> List[str]: # Original format: Act: bash\n```bash\n...\n``` for m in re.finditer( r"Act:\s*bash\s*\n\s*```(?:bash)?\s*\n(.*?)\n\s*```", - text, re.DOTALL | re.IGNORECASE, + text, + re.DOTALL | re.IGNORECASE, ): cmd = m.group(1).strip() if cmd: @@ -682,7 +730,8 @@ def _extract_bash_commands(text: str) -> List[str]: for m in re.finditer( r"Act:\s*```(?:bash)?\s*\n(.*?)\n\s*```", - text, re.DOTALL | re.IGNORECASE, + text, + re.DOTALL | re.IGNORECASE, ): cmd = m.group(1).strip() if cmd: @@ -692,7 +741,8 @@ def _extract_bash_commands(text: str) -> List[str]: for m in re.finditer( r"```(?:bash|sh)\s*\n(.*?)\n\s*```", - text, re.DOTALL | re.IGNORECASE, + text, + re.DOTALL | re.IGNORECASE, ): cmd = m.group(1).strip() if cmd: @@ -721,7 +771,8 @@ def _evaluate_os_in_docker( try: result = subprocess.run( ["docker", "image", "inspect", "local-os/default"], - capture_output=True, timeout=10, + capture_output=True, + timeout=10, ) if result.returncode == 0: image = "local-os/default" @@ -737,32 +788,44 @@ def _evaluate_os_in_docker( try: subprocess.run( ["docker", "rm", "-f", container_name], - capture_output=True, timeout=30, + capture_output=True, + timeout=30, ) subprocess.run( [ - "docker", "run", "-d", "--name", container_name, - image, "sleep", "300", + "docker", + "run", + "-d", + "--name", + container_name, + image, + "sleep", + "300", ], - capture_output=True, check=True, timeout=60, + capture_output=True, + check=True, + timeout=60, ) init_cmd_str = init_command.get("script", init_command.get("command", "")) if init_cmd_str: result = subprocess.run( ["docker", "exec", container_name, "bash", "-c", init_cmd_str], - capture_output=True, timeout=120, + capture_output=True, + timeout=120, ) if result.returncode != 0: logger.warning( "Init command failed (exit %d): %s", - result.returncode, result.stderr.decode(errors="replace")[:200], + result.returncode, + result.stderr.decode(errors="replace")[:200], ) for cmd in agent_commands: subprocess.run( ["docker", "exec", container_name, "bash", "-c", cmd], - capture_output=True, timeout=120, + capture_output=True, + timeout=120, ) eval_cmd_str = eval_command.get("script", eval_command.get("command", "")) @@ -783,7 +846,8 @@ def _evaluate_os_in_docker( result = subprocess.run( ["docker", "exec", container_name, "bash", "-c", eval_cmd_str], - capture_output=True, timeout=120, + capture_output=True, + timeout=120, ) return result.returncode == 0 @@ -796,7 +860,8 @@ def _evaluate_os_in_docker( finally: subprocess.run( ["docker", "rm", "-f", container_name], - capture_output=True, timeout=30, + capture_output=True, + timeout=30, ) @@ -804,13 +869,14 @@ def _evaluate_os_in_docker( # Value comparison (matches original's DirectTypeAnswerValidator) # ==================================================================== + def compare_tuple_lists( - expected: List[List[Any]], actual: List[List[Any]], + expected: List[List[Any]], + actual: List[List[Any]], ) -> Tuple[bool, str]: if len(expected) != len(actual): return False, ( - f"row_count_mismatch: expected {len(expected)}, " - f"got {len(actual)}" + f"row_count_mismatch: expected {len(expected)}, got {len(actual)}" ) for i, (exp_row, act_row) in enumerate(zip(expected, actual)): if len(exp_row) != len(act_row): @@ -837,8 +903,10 @@ def values_match(expected: Any, actual: Any) -> bool: if expected == 0 and actual == 0: return True return math.isclose( - float(expected), float(actual), - rel_tol=_REL_TOLERANCE, abs_tol=_ABS_TOLERANCE, + float(expected), + float(actual), + rel_tol=_REL_TOLERANCE, + abs_tol=_ABS_TOLERANCE, ) try: @@ -847,7 +915,10 @@ def values_match(expected: Any, actual: Any) -> bool: if exp_f == 0 and act_f == 0: return True return math.isclose( - exp_f, act_f, rel_tol=_REL_TOLERANCE, abs_tol=_ABS_TOLERANCE, + exp_f, + act_f, + rel_tol=_REL_TOLERANCE, + abs_tol=_ABS_TOLERANCE, ) except (ValueError, TypeError): pass diff --git a/src/openjarvis/evals/scorers/loghub_scorer.py b/src/openjarvis/evals/scorers/loghub_scorer.py index a1f692f8..9b4e1a50 100644 --- a/src/openjarvis/evals/scorers/loghub_scorer.py +++ b/src/openjarvis/evals/scorers/loghub_scorer.py @@ -18,7 +18,9 @@ class LogHubScorer(LLMJudgeScorer): scorer_id = "loghub" def score( - self, record: EvalRecord, model_answer: str, + self, + record: EvalRecord, + model_answer: str, ) -> Tuple[Optional[bool], Dict[str, Any]]: if not model_answer or not model_answer.strip(): return False, {"reason": "empty_response"} @@ -50,7 +52,9 @@ class LogHubScorer(LLMJudgeScorer): } def _llm_fallback( - self, record: EvalRecord, model_answer: str, + self, + record: EvalRecord, + model_answer: str, ) -> Tuple[Optional[bool], Dict[str, Any]]: """Use LLM judge when keyword extraction fails.""" prompt = ( diff --git a/src/openjarvis/evals/scorers/mmlu_pro_mcq.py b/src/openjarvis/evals/scorers/mmlu_pro_mcq.py index 951d4dac..75d9bacf 100644 --- a/src/openjarvis/evals/scorers/mmlu_pro_mcq.py +++ b/src/openjarvis/evals/scorers/mmlu_pro_mcq.py @@ -49,15 +49,19 @@ class MMLUProScorer(LLMJudgeScorer): try: raw_response = self._ask_judge( - user_prompt, system=system_prompt, - temperature=0.0, max_tokens=2048, + user_prompt, + system=system_prompt, + temperature=0.0, + max_tokens=2048, ) extracted = raw_response.strip().upper() # Handle "The answer is: A" etc. answer_match = re.search( - r"(?:THE ANSWER IS:?\s*)?([A-Z])", extracted, re.IGNORECASE, + r"(?:THE ANSWER IS:?\s*)?([A-Z])", + extracted, + re.IGNORECASE, ) if answer_match: extracted = answer_match.group(1).upper() @@ -72,7 +76,9 @@ class MMLUProScorer(LLMJudgeScorer): return None def score( - self, record: EvalRecord, model_answer: str, + self, + record: EvalRecord, + model_answer: str, ) -> Tuple[Optional[bool], Dict[str, Any]]: ref = record.reference.strip().upper() if not ref: @@ -81,7 +87,9 @@ class MMLUProScorer(LLMJudgeScorer): valid_letters = self._valid_letters_from_options(record.metadata) candidate = self._extract_answer_with_llm( - record.problem, model_answer, valid_letters, + record.problem, + model_answer, + valid_letters, ) if not candidate: return None, {"reason": "no_choice_letter_extracted"} diff --git a/src/openjarvis/evals/scorers/morning_brief.py b/src/openjarvis/evals/scorers/morning_brief.py index b5bf8a38..0b3fa64c 100644 --- a/src/openjarvis/evals/scorers/morning_brief.py +++ b/src/openjarvis/evals/scorers/morning_brief.py @@ -51,13 +51,19 @@ class MorningBriefScorer(LLMJudgeScorer): scorer_id = "morning_brief" def score( - self, record: EvalRecord, model_answer: str, + self, + record: EvalRecord, + model_answer: str, ) -> Tuple[Optional[bool], Dict[str, Any]]: if not model_answer or not model_answer.strip(): return False, {"reason": "empty_response"} # Truncate problem for the judge prompt to save tokens - problem_excerpt = record.problem[:500] + "..." if len(record.problem) > 500 else record.problem + problem_excerpt = ( + record.problem[:500] + "..." + if len(record.problem) > 500 + else record.problem + ) try: prompt = _JUDGE_PROMPT.format( @@ -69,7 +75,12 @@ class MorningBriefScorer(LLMJudgeScorer): # Extract scores scores = {} - for dim in ("completeness", "prioritization", "conciseness", "actionability"): + for dim in ( + "completeness", + "prioritization", + "conciseness", + "actionability", + ): match = re.search( rf"^{dim}:\s*(\d)", raw, diff --git a/src/openjarvis/evals/scorers/paperarena_judge.py b/src/openjarvis/evals/scorers/paperarena_judge.py index be06d548..66502120 100644 --- a/src/openjarvis/evals/scorers/paperarena_judge.py +++ b/src/openjarvis/evals/scorers/paperarena_judge.py @@ -30,7 +30,9 @@ class PaperArenaScorer(LLMJudgeScorer): scorer_id = "paperarena" def score( - self, record: EvalRecord, model_answer: str, + self, + record: EvalRecord, + model_answer: str, ) -> Tuple[Optional[bool], Dict[str, Any]]: if not model_answer or not model_answer.strip(): return False, {"reason": "empty_response"} @@ -46,7 +48,9 @@ class PaperArenaScorer(LLMJudgeScorer): return self._score_open(record, model_answer) def _score_mc( - self, record: EvalRecord, model_answer: str, + self, + record: EvalRecord, + model_answer: str, ) -> Tuple[Optional[bool], Dict[str, Any]]: """Score multiple-choice via letter extraction.""" ref_letter = record.reference.strip().upper() @@ -60,7 +64,8 @@ class PaperArenaScorer(LLMJudgeScorer): if extracted is None: # Fall back to LLM extraction extracted = self._extract_letter_with_llm( - record.problem, model_answer, + record.problem, + model_answer, ) if extracted is None: @@ -99,7 +104,9 @@ class PaperArenaScorer(LLMJudgeScorer): return None def _extract_letter_with_llm( - self, problem: str, model_answer: str, + self, + problem: str, + model_answer: str, ) -> Optional[str]: """Use LLM to extract answer letter.""" prompt = ( @@ -119,7 +126,9 @@ class PaperArenaScorer(LLMJudgeScorer): return None def _score_open( - self, record: EvalRecord, model_answer: str, + self, + record: EvalRecord, + model_answer: str, ) -> Tuple[Optional[bool], Dict[str, Any]]: """Score CA/OA via LLM judge.""" question = record.problem diff --git a/src/openjarvis/evals/scorers/pinchbench.py b/src/openjarvis/evals/scorers/pinchbench.py index 4325012a..a352f0f4 100644 --- a/src/openjarvis/evals/scorers/pinchbench.py +++ b/src/openjarvis/evals/scorers/pinchbench.py @@ -36,6 +36,7 @@ _TOOL_NAME_MAP: Dict[str, str] = { # Transcript translation # --------------------------------------------------------------------------- + def events_to_transcript(events: List[Any]) -> List[Dict[str, Any]]: """Build PinchBench-format transcript from raw EventRecorder events. @@ -49,26 +50,35 @@ def events_to_transcript(events: List[Any]) -> List[Dict[str, Any]]: if isinstance(etype, str): # Normalize string event types to enum comparison pass - if etype == EventType.TOOL_CALL_START or etype == EventType.TOOL_CALL_START.value: + if ( + etype == EventType.TOOL_CALL_START + or etype == EventType.TOOL_CALL_START.value + ): tool_name = event.metadata.get("tool", "unknown") mapped = _TOOL_NAME_MAP.get(tool_name, tool_name) arguments = event.metadata.get("arguments") or {} - transcript.append({ - "type": "message", - "message": { - "role": "assistant", - "content": [{"type": "toolCall", "name": mapped, "params": arguments}], - }, - }) + transcript.append( + { + "type": "message", + "message": { + "role": "assistant", + "content": [ + {"type": "toolCall", "name": mapped, "params": arguments} + ], + }, + } + ) elif etype == EventType.TOOL_CALL_END or etype == EventType.TOOL_CALL_END.value: result_text = str(event.metadata.get("result", "")) - transcript.append({ - "type": "message", - "message": { - "role": "toolResult", - "content": [{"text": result_text}], - }, - }) + transcript.append( + { + "type": "message", + "message": { + "role": "toolResult", + "content": [{"text": result_text}], + }, + } + ) return transcript @@ -81,31 +91,43 @@ def _trace_to_transcript(trace: Any) -> List[Dict[str, Any]]: if tc is None: continue mapped = _TOOL_NAME_MAP.get(tc["name"], tc["name"]) - transcript.append({ - "type": "message", - "message": { - "role": "assistant", - "content": [{"type": "toolCall", "name": mapped, "params": tc.get("arguments") or {}}], - }, - }) - transcript.append({ - "type": "message", - "message": { - "role": "toolResult", - "content": [{"text": tc.get("result", "")}], - }, - }) + transcript.append( + { + "type": "message", + "message": { + "role": "assistant", + "content": [ + { + "type": "toolCall", + "name": mapped, + "params": tc.get("arguments") or {}, + } + ], + }, + } + ) + transcript.append( + { + "type": "message", + "message": { + "role": "toolResult", + "content": [{"text": tc.get("result", "")}], + }, + } + ) # Capture final assistant text response (for tasks graded on text output) response_text = getattr(trace, "response_text", "") if response_text: - transcript.append({ - "type": "message", - "message": { - "role": "assistant", - "content": [{"type": "text", "text": response_text}], - }, - }) + transcript.append( + { + "type": "message", + "message": { + "role": "assistant", + "content": [{"type": "text", "text": response_text}], + }, + } + ) return transcript @@ -117,20 +139,30 @@ def _tool_results_to_transcript( for tr in tool_results: tool_name = tr.get("tool_name", "unknown") mapped = _TOOL_NAME_MAP.get(tool_name, tool_name) - transcript.append({ - "type": "message", - "message": { - "role": "assistant", - "content": [{"type": "toolCall", "name": mapped, "params": tr.get("arguments", {})}], - }, - }) - transcript.append({ - "type": "message", - "message": { - "role": "toolResult", - "content": [{"text": tr.get("content", "")}], - }, - }) + transcript.append( + { + "type": "message", + "message": { + "role": "assistant", + "content": [ + { + "type": "toolCall", + "name": mapped, + "params": tr.get("arguments", {}), + } + ], + }, + } + ) + transcript.append( + { + "type": "message", + "message": { + "role": "toolResult", + "content": [{"text": tr.get("content", "")}], + }, + } + ) return transcript @@ -138,6 +170,7 @@ def _tool_results_to_transcript( # Transcript summarization (replicates PinchBench _summarize_transcript) # --------------------------------------------------------------------------- + def _summarize_transcript(transcript: List[Dict[str, Any]]) -> str: """Summarize transcript for LLM judge prompt. @@ -176,6 +209,7 @@ def _summarize_transcript(transcript: List[Dict[str, Any]]) -> str: # Judge prompt and response parsing (replicates PinchBench exactly) # --------------------------------------------------------------------------- + def _build_judge_prompt( *, task_prompt: str, @@ -274,8 +308,14 @@ def _parse_judge_response(raw: str) -> Dict[str, Any]: try: total = float(score_match.group(1)) if 0.0 <= total <= 1.0: - LOGGER.warning("Fell back to regex score extraction (total=%.2f)", total) - return {"scores": {}, "total": total, "notes": "Score extracted from prose"} + LOGGER.warning( + "Fell back to regex score extraction (total=%.2f)", total + ) + return { + "scores": {}, + "total": total, + "notes": "Score extracted from prose", + } except ValueError: pass @@ -306,7 +346,9 @@ def _normalize_judge_response(parsed: Dict[str, Any]) -> Dict[str, Any]: break else: if result["scores"]: - values = [v for v in result["scores"].values() if isinstance(v, (int, float))] + values = [ + v for v in result["scores"].values() if isinstance(v, (int, float)) + ] if values: result["total"] = sum(values) / len(values) @@ -333,6 +375,7 @@ def _normalize_judge_response(parsed: Dict[str, Any]) -> Dict[str, Any]: # Grading functions # --------------------------------------------------------------------------- + def _grade_automated( record: EvalRecord, transcript: List[Dict[str, Any]], @@ -352,7 +395,11 @@ def _grade_automated( grade_fn = namespace.get("grade") if not callable(grade_fn): - return {"score": 0.0, "breakdown": {}, "notes": "No grade() function found in automated checks"} + return { + "score": 0.0, + "breakdown": {}, + "notes": "No grade() function found in automated checks", + } try: scores = grade_fn(transcript, workspace_path) @@ -361,7 +408,11 @@ def _grade_automated( return {"score": 0.0, "breakdown": {}, "notes": f"grade() error: {exc}"} if not isinstance(scores, dict) or not scores: - return {"score": 0.0, "breakdown": {}, "notes": "grade() returned empty or non-dict"} + return { + "score": 0.0, + "breakdown": {}, + "notes": "grade() returned empty or non-dict", + } mean_score = sum(scores.values()) / len(scores) return {"score": mean_score, "breakdown": scores, "notes": ""} @@ -391,7 +442,9 @@ def _grade_llm_judge( ) try: - raw = judge_backend.generate(prompt, model=judge_model, temperature=0.0, max_tokens=2048) + raw = judge_backend.generate( + prompt, model=judge_model, temperature=0.0, max_tokens=2048 + ) except Exception as exc: LOGGER.error("LLM judge call failed for %s: %s", record.record_id, exc) return {"score": 0.0, "breakdown": {}, "notes": f"Judge error: {exc}"} @@ -412,15 +465,24 @@ def _grade_hybrid( judge_model: str, ) -> Dict[str, Any]: """Run both automated and LLM judge grading, combine with weights.""" - weights = record.metadata.get("grading_weights") or {"automated": 0.5, "llm_judge": 0.5} + weights = record.metadata.get("grading_weights") or { + "automated": 0.5, + "llm_judge": 0.5, + } auto = _grade_automated(record, transcript, workspace_path) - llm = _grade_llm_judge(record, transcript, workspace_path, judge_backend, judge_model) + llm = _grade_llm_judge( + record, transcript, workspace_path, judge_backend, judge_model + ) auto_w = float(weights.get("automated", 0.5)) llm_w = float(weights.get("llm_judge", 0.5)) total_w = auto_w + llm_w - combined = (auto["score"] * auto_w + llm["score"] * llm_w) / total_w if total_w > 0 else 0.0 + combined = ( + (auto["score"] * auto_w + llm["score"] * llm_w) / total_w + if total_w > 0 + else 0.0 + ) breakdown = { **{f"automated.{k}": v for k, v in auto["breakdown"].items()}, **{f"llm_judge.{k}": v for k, v in llm["breakdown"].items()}, @@ -446,24 +508,35 @@ def grade_pinchbench_task( if grading_type == "automated": return _grade_automated(record, transcript, workspace_path) elif grading_type == "llm_judge": - return _grade_llm_judge(record, transcript, workspace_path, judge_backend, judge_model) + return _grade_llm_judge( + record, transcript, workspace_path, judge_backend, judge_model + ) elif grading_type == "hybrid": - return _grade_hybrid(record, transcript, workspace_path, judge_backend, judge_model) + return _grade_hybrid( + record, transcript, workspace_path, judge_backend, judge_model + ) else: - return {"score": 0.0, "breakdown": {}, "notes": f"Unknown grading type: {grading_type}"} + return { + "score": 0.0, + "breakdown": {}, + "notes": f"Unknown grading type: {grading_type}", + } # --------------------------------------------------------------------------- # Standalone scorer (for EvalRunner non-agentic path) # --------------------------------------------------------------------------- + class PinchBenchScorer(LLMJudgeScorer): """PinchBench scorer for the non-agentic EvalRunner path.""" scorer_id = "pinchbench" def score( - self, record: EvalRecord, model_answer: str, + self, + record: EvalRecord, + model_answer: str, ) -> Tuple[Optional[bool], Dict[str, Any]]: trace = record.metadata.get("query_trace") if trace: @@ -476,13 +549,15 @@ class PinchBenchScorer(LLMJudgeScorer): # Always append final model answer as assistant text message # so grading functions that check for text responses can find it if model_answer: - transcript.append({ - "type": "message", - "message": { - "role": "assistant", - "content": [{"type": "text", "text": model_answer}], - }, - }) + transcript.append( + { + "type": "message", + "message": { + "role": "assistant", + "content": [{"type": "text", "text": model_answer}], + }, + } + ) result = grade_pinchbench_task( record=record, diff --git a/src/openjarvis/evals/scorers/reasoning_judge.py b/src/openjarvis/evals/scorers/reasoning_judge.py index 70d5ac9c..e9c9aced 100644 --- a/src/openjarvis/evals/scorers/reasoning_judge.py +++ b/src/openjarvis/evals/scorers/reasoning_judge.py @@ -107,7 +107,9 @@ class ReasoningJudgeScorer(LLMJudgeScorer): scorer_id = "reasoning_judge" def score( - self, record: EvalRecord, model_answer: str, + self, + record: EvalRecord, + model_answer: str, ) -> Tuple[Optional[bool], Dict[str, Any]]: if not model_answer or not model_answer.strip(): return False, {"reason": "empty_response"} @@ -130,21 +132,23 @@ class ReasoningJudgeScorer(LLMJudgeScorer): raw = self._ask_judge(prompt, temperature=0.0, max_tokens=2048) structured_match = re.search( - r"^correct:\s*(yes|no)", raw, re.MULTILINE | re.IGNORECASE, + r"^correct:\s*(yes|no)", + raw, + re.MULTILINE | re.IGNORECASE, ) if structured_match: is_correct = structured_match.group(1).lower() == "yes" else: - is_correct = ( - "CORRECT" in raw.upper() and "INCORRECT" not in raw.upper() - ) + is_correct = "CORRECT" in raw.upper() and "INCORRECT" not in raw.upper() meta: Dict[str, Any] = { "match_type": "llm_fallback", "raw_judge_output": raw, } extracted = re.search( - r"^extracted_final_answer:\s*(.+)", raw, re.MULTILINE, + r"^extracted_final_answer:\s*(.+)", + raw, + re.MULTILINE, ) if extracted: meta["extracted_answer"] = extracted.group(1).strip() diff --git a/src/openjarvis/evals/scorers/research_mining.py b/src/openjarvis/evals/scorers/research_mining.py index 2682df38..4a7d2416 100644 --- a/src/openjarvis/evals/scorers/research_mining.py +++ b/src/openjarvis/evals/scorers/research_mining.py @@ -51,13 +51,17 @@ class ResearchMiningScorer(LLMJudgeScorer): scorer_id = "research_mining" def score( - self, record: EvalRecord, model_answer: str, + self, + record: EvalRecord, + model_answer: str, ) -> Tuple[Optional[bool], Dict[str, Any]]: if not model_answer or not model_answer.strip(): return False, {"reason": "empty_response"} # Extract original question from the prompt - question = record.problem.split("Research question: ")[-1].split("\n")[0].strip() + question = ( + record.problem.split("Research question: ")[-1].split("\n")[0].strip() + ) try: prompt = _JUDGE_PROMPT.format( diff --git a/src/openjarvis/evals/scorers/security_scanner.py b/src/openjarvis/evals/scorers/security_scanner.py index b1439fc5..33a2697d 100644 --- a/src/openjarvis/evals/scorers/security_scanner.py +++ b/src/openjarvis/evals/scorers/security_scanner.py @@ -24,14 +24,55 @@ LOGGER = logging.getLogger(__name__) # Common aliases for vulnerability types _VULN_TYPE_ALIASES: Dict[str, List[str]] = { "sql_injection": ["sql injection", "sqli", "sql inject"], - "hardcoded_secret": ["hardcoded secret", "hardcoded credential", "hardcoded password", "hardcoded api key", "hard coded", "embedded secret", "embedded credential"], - "command_injection": ["command injection", "os command injection", "shell injection", "subprocess injection"], - "xss": ["xss", "cross site scripting", "cross-site scripting", "reflected xss", "stored xss"], - "path_traversal": ["path traversal", "directory traversal", "lfi", "local file inclusion"], + "hardcoded_secret": [ + "hardcoded secret", + "hardcoded credential", + "hardcoded password", + "hardcoded api key", + "hard coded", + "embedded secret", + "embedded credential", + ], + "command_injection": [ + "command injection", + "os command injection", + "shell injection", + "subprocess injection", + ], + "xss": [ + "xss", + "cross site scripting", + "cross-site scripting", + "reflected xss", + "stored xss", + ], + "path_traversal": [ + "path traversal", + "directory traversal", + "lfi", + "local file inclusion", + ], "ssrf": ["ssrf", "server side request forgery", "server-side request forgery"], - "insecure_deserialization": ["insecure deserialization", "pickle", "yaml load", "unsafe deserialization"], - "weak_crypto": ["weak crypto", "weak cryptography", "md5", "sha1", "ecb mode", "weak hash"], - "timing_attack": ["timing attack", "timing side channel", "constant time", "timing vulnerability"], + "insecure_deserialization": [ + "insecure deserialization", + "pickle", + "yaml load", + "unsafe deserialization", + ], + "weak_crypto": [ + "weak crypto", + "weak cryptography", + "md5", + "sha1", + "ecb mode", + "weak hash", + ], + "timing_attack": [ + "timing attack", + "timing side channel", + "constant time", + "timing vulnerability", + ], "redos": ["redos", "regex denial", "catastrophic backtracking", "regex dos"], "race_condition": ["race condition", "toctou", "time of check"], "xxe": ["xxe", "xml external entity"], @@ -88,7 +129,9 @@ def _count_false_positives( # Look for the pattern and keyword near each other pat_idx = model_norm.find(pattern_norm) if pat_idx >= 0: - surrounding = model_norm[max(0, pat_idx - 200):pat_idx + len(pattern_norm) + 200] + surrounding = model_norm[ + max(0, pat_idx - 200) : pat_idx + len(pattern_norm) + 200 + ] if keyword in surrounding: fp_count += 1 break @@ -106,7 +149,9 @@ class SecurityScannerScorer(Scorer): self._judge_model = judge_model def score( - self, record: EvalRecord, model_answer: str, + self, + record: EvalRecord, + model_answer: str, ) -> Tuple[Optional[bool], Dict[str, Any]]: if not model_answer or not model_answer.strip(): return False, {"reason": "empty_response"} @@ -139,15 +184,17 @@ class SecurityScannerScorer(Scorer): if sev_match: severity_correct += 1 - vuln_details.append({ - "file": vuln["file"], - "type": vuln["type"], - "severity": vuln.get("severity", ""), - "found": found, - "file_mentioned": file_match, - "type_mentioned": type_match, - "severity_correct": sev_match, - }) + vuln_details.append( + { + "file": vuln["file"], + "type": vuln["type"], + "severity": vuln.get("severity", ""), + "found": found, + "file_mentioned": file_match, + "type_mentioned": type_match, + "severity_correct": sev_match, + } + ) total_vulns = len(vulnerabilities) detection_rate = vulns_found / total_vulns if total_vulns > 0 else 0.0 @@ -171,7 +218,9 @@ class SecurityScannerScorer(Scorer): ] scorer = ChecklistScorer(self._judge_backend, self._judge_model) checklist_score, checklist_details = scorer.score_checklist( - model_answer, checklist_items, context=record.problem, + model_answer, + checklist_items, + context=record.problem, ) else: # Without judge, give partial credit based on structural heuristics @@ -181,7 +230,9 @@ class SecurityScannerScorer(Scorer): has_severity = any( kw in model_norm for kw in ["critical", "high", "medium", "low"] ) - checklist_score = (0.5 if has_recommendations else 0.0) + (0.5 if has_severity else 0.0) + checklist_score = (0.5 if has_recommendations else 0.0) + ( + 0.5 if has_severity else 0.0 + ) # --- Composite score --- final_score = ( diff --git a/src/openjarvis/evals/scorers/simpleqa_judge.py b/src/openjarvis/evals/scorers/simpleqa_judge.py index 121e5893..03bf9363 100644 --- a/src/openjarvis/evals/scorers/simpleqa_judge.py +++ b/src/openjarvis/evals/scorers/simpleqa_judge.py @@ -83,7 +83,9 @@ class SimpleQAScorer(LLMJudgeScorer): scorer_id = "simpleqa" def score( - self, record: EvalRecord, model_answer: str, + self, + record: EvalRecord, + model_answer: str, ) -> Tuple[Optional[bool], Dict[str, Any]]: if not model_answer or not model_answer.strip(): return False, {"reason": "empty_response"} @@ -106,21 +108,23 @@ class SimpleQAScorer(LLMJudgeScorer): raw = self._ask_judge(prompt, temperature=0.0, max_tokens=2048) structured_match = re.search( - r"^correct:\s*(yes|no)", raw, re.MULTILINE | re.IGNORECASE, + r"^correct:\s*(yes|no)", + raw, + re.MULTILINE | re.IGNORECASE, ) if structured_match: is_correct = structured_match.group(1).lower() == "yes" else: - is_correct = ( - "CORRECT" in raw.upper() and "INCORRECT" not in raw.upper() - ) + is_correct = "CORRECT" in raw.upper() and "INCORRECT" not in raw.upper() meta: Dict[str, Any] = { "match_type": "llm_fallback", "raw_judge_output": raw, } extracted_match = re.search( - r"^extracted_final_answer:\s*(.+)", raw, re.MULTILINE, + r"^extracted_final_answer:\s*(.+)", + raw, + re.MULTILINE, ) if extracted_match: meta["extracted_answer"] = extracted_match.group(1).strip() diff --git a/src/openjarvis/evals/scorers/supergpqa_mcq.py b/src/openjarvis/evals/scorers/supergpqa_mcq.py index d2dc15d9..da66a003 100644 --- a/src/openjarvis/evals/scorers/supergpqa_mcq.py +++ b/src/openjarvis/evals/scorers/supergpqa_mcq.py @@ -49,15 +49,19 @@ class SuperGPQAScorer(LLMJudgeScorer): try: raw_response = self._ask_judge( - user_prompt, system=system_prompt, - temperature=0.0, max_tokens=2048, + user_prompt, + system=system_prompt, + temperature=0.0, + max_tokens=2048, ) extracted = raw_response.strip().upper() # Handle "The answer is: A" etc. answer_match = re.search( - r"(?:THE ANSWER IS:?\s*)?([A-Z])", extracted, re.IGNORECASE, + r"(?:THE ANSWER IS:?\s*)?([A-Z])", + extracted, + re.IGNORECASE, ) if answer_match: extracted = answer_match.group(1).upper() @@ -72,7 +76,9 @@ class SuperGPQAScorer(LLMJudgeScorer): return None def score( - self, record: EvalRecord, model_answer: str, + self, + record: EvalRecord, + model_answer: str, ) -> Tuple[Optional[bool], Dict[str, Any]]: ref = record.reference.strip().upper() if not ref: @@ -81,7 +87,9 @@ class SuperGPQAScorer(LLMJudgeScorer): valid_letters = self._valid_letters_from_options(record.metadata) candidate = self._extract_answer_with_llm( - record.problem, model_answer, valid_letters, + record.problem, + model_answer, + valid_letters, ) if not candidate: return None, {"reason": "no_choice_letter_extracted"} diff --git a/src/openjarvis/evals/scorers/swebench_structural.py b/src/openjarvis/evals/scorers/swebench_structural.py index 540c7946..15251ddd 100644 --- a/src/openjarvis/evals/scorers/swebench_structural.py +++ b/src/openjarvis/evals/scorers/swebench_structural.py @@ -47,7 +47,9 @@ class SWEBenchScorer(Scorer): self._judge_model = judge_model def score( - self, record: EvalRecord, model_answer: str, + self, + record: EvalRecord, + model_answer: str, ) -> Tuple[Optional[bool], Dict[str, Any]]: if not model_answer or not model_answer.strip(): return False, {"reason": "empty_response"} diff --git a/src/openjarvis/evals/scorers/swefficiency_structural.py b/src/openjarvis/evals/scorers/swefficiency_structural.py index 98efea26..9d592aa8 100644 --- a/src/openjarvis/evals/scorers/swefficiency_structural.py +++ b/src/openjarvis/evals/scorers/swefficiency_structural.py @@ -48,7 +48,9 @@ class SWEfficiencyScorer(Scorer): self._judge_model = judge_model def score( - self, record: EvalRecord, model_answer: str, + self, + record: EvalRecord, + model_answer: str, ) -> Tuple[Optional[bool], Dict[str, Any]]: if not model_answer or not model_answer.strip(): return False, {"reason": "empty_response"} diff --git a/src/openjarvis/evals/scorers/terminalbench_judge.py b/src/openjarvis/evals/scorers/terminalbench_judge.py index daf2a616..231883d2 100644 --- a/src/openjarvis/evals/scorers/terminalbench_judge.py +++ b/src/openjarvis/evals/scorers/terminalbench_judge.py @@ -48,7 +48,9 @@ class TerminalBenchScorer(LLMJudgeScorer): scorer_id = "terminalbench" def score( - self, record: EvalRecord, model_answer: str, + self, + record: EvalRecord, + model_answer: str, ) -> Tuple[Optional[bool], Dict[str, Any]]: if not model_answer or not model_answer.strip(): return False, {"reason": "empty_response"} @@ -67,7 +69,9 @@ class TerminalBenchScorer(LLMJudgeScorer): raw = self._ask_judge(prompt, temperature=0.0, max_tokens=2048) structured_match = re.search( - r"^correct:\s*(yes|no)", raw, re.MULTILINE | re.IGNORECASE, + r"^correct:\s*(yes|no)", + raw, + re.MULTILINE | re.IGNORECASE, ) if structured_match: is_correct = structured_match.group(1).lower() == "yes" @@ -79,7 +83,8 @@ class TerminalBenchScorer(LLMJudgeScorer): is_correct = False else: LOGGER.warning( - "Could not parse grade from response: %s", raw[:50], + "Could not parse grade from response: %s", + raw[:50], ) is_correct = False @@ -87,7 +92,9 @@ class TerminalBenchScorer(LLMJudgeScorer): "raw_judge_output": raw, } extracted = re.search( - r"^extracted_final_answer:\s*(.+)", raw, re.MULTILINE, + r"^extracted_final_answer:\s*(.+)", + raw, + re.MULTILINE, ) if extracted: meta["extracted_answer"] = extracted.group(1).strip() diff --git a/src/openjarvis/evals/scorers/terminalbench_native_structural.py b/src/openjarvis/evals/scorers/terminalbench_native_structural.py index 4fe97f2a..ce999218 100644 --- a/src/openjarvis/evals/scorers/terminalbench_native_structural.py +++ b/src/openjarvis/evals/scorers/terminalbench_native_structural.py @@ -35,7 +35,9 @@ class TerminalBenchNativeScorer(Scorer): self._judge_model = judge_model def score( - self, record: EvalRecord, model_answer: str, + self, + record: EvalRecord, + model_answer: str, ) -> Tuple[Optional[bool], Dict[str, Any]]: meta = record.metadata diff --git a/src/openjarvis/evals/scorers/webchorearena_scorer.py b/src/openjarvis/evals/scorers/webchorearena_scorer.py index 32d8f447..90dd5f0a 100644 --- a/src/openjarvis/evals/scorers/webchorearena_scorer.py +++ b/src/openjarvis/evals/scorers/webchorearena_scorer.py @@ -41,7 +41,9 @@ class WebChoreArenaScorer(Scorer): self._judge_model = judge_model def score( - self, record: EvalRecord, model_answer: str, + self, + record: EvalRecord, + model_answer: str, ) -> Tuple[Optional[bool], Dict[str, Any]]: meta = record.metadata diff --git a/src/openjarvis/evals/scorers/wildchat_judge.py b/src/openjarvis/evals/scorers/wildchat_judge.py index d4f01e11..3763a859 100644 --- a/src/openjarvis/evals/scorers/wildchat_judge.py +++ b/src/openjarvis/evals/scorers/wildchat_judge.py @@ -69,7 +69,9 @@ class WildChatScorer(LLMJudgeScorer): scorer_id = "wildchat" def score( - self, record: EvalRecord, model_answer: str, + self, + record: EvalRecord, + model_answer: str, ) -> Tuple[Optional[bool], Dict[str, Any]]: reference = record.reference if not reference or not reference.strip(): @@ -77,10 +79,14 @@ class WildChatScorer(LLMJudgeScorer): # Two comparisons: (model vs reference) and (reference vs model) verdict1, response1 = self._get_judge_verdict( - record.problem, model_answer, reference, + record.problem, + model_answer, + reference, ) verdict2, response2 = self._get_judge_verdict( - record.problem, reference, model_answer, + record.problem, + reference, + model_answer, ) if verdict1 is None or verdict2 is None: @@ -105,7 +111,10 @@ class WildChatScorer(LLMJudgeScorer): return final_result, meta def _get_judge_verdict( - self, problem: str, response_a: str, response_b: str, + self, + problem: str, + response_a: str, + response_b: str, ) -> Tuple[Optional[str], Optional[str]]: prompt = ( f"<|User Prompt|>\n{problem}\n\n" @@ -117,8 +126,10 @@ class WildChatScorer(LLMJudgeScorer): try: raw = self._ask_judge( - prompt, system=SYSTEM_PROMPT, - temperature=0.0, max_tokens=2048, + prompt, + system=SYSTEM_PROMPT, + temperature=0.0, + max_tokens=2048, ) except Exception as exc: LOGGER.error("WildChat judge call failed: %s", exc) @@ -133,7 +144,8 @@ class WildChatScorer(LLMJudgeScorer): @staticmethod def _verdict_to_bool( - verdict: Optional[str], generated_is_a: bool, + verdict: Optional[str], + generated_is_a: bool, ) -> Optional[bool]: if not verdict: return None diff --git a/src/openjarvis/evals/scorers/workarena_scorer.py b/src/openjarvis/evals/scorers/workarena_scorer.py index 050eaed3..ffb3025a 100644 --- a/src/openjarvis/evals/scorers/workarena_scorer.py +++ b/src/openjarvis/evals/scorers/workarena_scorer.py @@ -36,7 +36,9 @@ class WorkArenaScorer(Scorer): self._judge_model = judge_model def score( - self, record: EvalRecord, model_answer: str, + self, + record: EvalRecord, + model_answer: str, ) -> Tuple[Optional[bool], Dict[str, Any]]: meta = record.metadata diff --git a/src/openjarvis/evals/tests/conftest.py b/src/openjarvis/evals/tests/conftest.py index c2ed646e..5faef6cf 100644 --- a/src/openjarvis/evals/tests/conftest.py +++ b/src/openjarvis/evals/tests/conftest.py @@ -48,8 +48,11 @@ class MockBackend(InferenceBackend): max_tokens: int = 2048, ) -> Dict[str, Any]: content = self.generate( - prompt, model=model, system=system, - temperature=temperature, max_tokens=max_tokens, + prompt, + model=model, + system=system, + temperature=temperature, + max_tokens=max_tokens, ) return { "content": content, @@ -78,7 +81,9 @@ class MockScorer(Scorer): self._result = result def score( - self, record: EvalRecord, model_answer: str, + self, + record: EvalRecord, + model_answer: str, ) -> Tuple[Optional[bool], Dict[str, Any]]: return self._result, {"mock": True} diff --git a/src/openjarvis/evals/tests/test_backends.py b/src/openjarvis/evals/tests/test_backends.py index 7196ee70..7854d2fb 100644 --- a/src/openjarvis/evals/tests/test_backends.py +++ b/src/openjarvis/evals/tests/test_backends.py @@ -99,7 +99,8 @@ class TestJarvisAgentBackend: from openjarvis.evals.backends.jarvis_agent import JarvisAgentBackend backend = JarvisAgentBackend( - engine_key="cloud", agent_name="orchestrator", + engine_key="cloud", + agent_name="orchestrator", tools=["calculator", "think"], ) assert backend.backend_id == "jarvis-agent" diff --git a/src/openjarvis/evals/tests/test_config.py b/src/openjarvis/evals/tests/test_config.py index e03e038c..324297a2 100644 --- a/src/openjarvis/evals/tests/test_config.py +++ b/src/openjarvis/evals/tests/test_config.py @@ -100,13 +100,16 @@ class TestDataclassDefaults: class TestLoadEvalConfig: def test_minimal_config(self, tmp_path): - p = _write_toml(tmp_path, """\ + p = _write_toml( + tmp_path, + """\ [[models]] name = "qwen3:8b" [[benchmarks]] name = "supergpqa" - """) + """, + ) suite = load_eval_config(p) assert len(suite.models) == 1 assert suite.models[0].name == "qwen3:8b" @@ -117,7 +120,9 @@ class TestLoadEvalConfig: assert suite.judge.model == "gpt-5-mini-2025-08-07" def test_full_config(self, tmp_path): - p = _write_toml(tmp_path, """\ + p = _write_toml( + tmp_path, + """\ [meta] name = "test-suite" description = "A test suite" @@ -157,7 +162,8 @@ class TestLoadEvalConfig: agent = "orchestrator" tools = ["calc", "think"] judge_model = "gpt-4o" - """) + """, + ) suite = load_eval_config(p) assert suite.meta.name == "test-suite" assert suite.meta.description == "A test suite" @@ -184,18 +190,24 @@ class TestLoadEvalConfig: assert suite.benchmarks[1].judge_model == "gpt-4o" def test_missing_models_raises(self, tmp_path): - p = _write_toml(tmp_path, """\ + p = _write_toml( + tmp_path, + """\ [[benchmarks]] name = "supergpqa" - """) + """, + ) with pytest.raises(EvalConfigError, match="at least one \\[\\[models\\]\\]"): load_eval_config(p) def test_missing_benchmarks_raises(self, tmp_path): - p = _write_toml(tmp_path, """\ + p = _write_toml( + tmp_path, + """\ [[models]] name = "qwen3:8b" - """) + """, + ) with pytest.raises( EvalConfigError, match="at least one \\[\\[benchmarks\\]\\]", @@ -203,48 +215,61 @@ class TestLoadEvalConfig: load_eval_config(p) def test_model_without_name_raises(self, tmp_path): - p = _write_toml(tmp_path, """\ + p = _write_toml( + tmp_path, + """\ [[models]] engine = "ollama" [[benchmarks]] name = "supergpqa" - """) + """, + ) with pytest.raises(EvalConfigError, match="'name' field"): load_eval_config(p) def test_benchmark_without_name_raises(self, tmp_path): - p = _write_toml(tmp_path, """\ + p = _write_toml( + tmp_path, + """\ [[models]] name = "qwen3:8b" [[benchmarks]] backend = "jarvis-direct" - """) + """, + ) with pytest.raises(EvalConfigError, match="'name' field"): load_eval_config(p) def test_invalid_backend_raises(self, tmp_path): - p = _write_toml(tmp_path, """\ + p = _write_toml( + tmp_path, + """\ [[models]] name = "qwen3:8b" [[benchmarks]] name = "supergpqa" backend = "invalid-backend" - """) + """, + ) with pytest.raises(EvalConfigError, match="Invalid backend"): load_eval_config(p) def test_unknown_benchmark_warns(self, tmp_path, caplog): - p = _write_toml(tmp_path, """\ + p = _write_toml( + tmp_path, + """\ [[models]] name = "qwen3:8b" [[benchmarks]] name = "custom-bench" - """) + """, + ) import logging + with caplog.at_level(logging.WARNING): suite = load_eval_config(p) assert suite.benchmarks[0].name == "custom-bench" @@ -261,22 +286,28 @@ class TestLoadEvalConfig: load_eval_config(p) def test_empty_models_list(self, tmp_path): - p = _write_toml(tmp_path, """\ + p = _write_toml( + tmp_path, + """\ models = [] [[benchmarks]] name = "supergpqa" - """) + """, + ) with pytest.raises(EvalConfigError, match="at least one \\[\\[models\\]\\]"): load_eval_config(p) def test_empty_benchmarks_list(self, tmp_path): - p = _write_toml(tmp_path, """\ + p = _write_toml( + tmp_path, + """\ [[models]] name = "qwen3:8b" benchmarks = [] - """) + """, + ) with pytest.raises( EvalConfigError, match="at least one \\[\\[benchmarks\\]\\]", @@ -284,7 +315,9 @@ class TestLoadEvalConfig: load_eval_config(p) def test_model_hardware_params(self, tmp_path): - p = _write_toml(tmp_path, """\ + p = _write_toml( + tmp_path, + """\ [[models]] name = "GLM-4.7-Flash" engine = "vllm" @@ -296,7 +329,8 @@ class TestLoadEvalConfig: [[benchmarks]] name = "supergpqa" - """) + """, + ) suite = load_eval_config(p) m = suite.models[0] assert m.param_count_b == 30.0 @@ -306,13 +340,16 @@ class TestLoadEvalConfig: assert m.num_gpus == 4 def test_model_hardware_params_defaults(self, tmp_path): - p = _write_toml(tmp_path, """\ + p = _write_toml( + tmp_path, + """\ [[models]] name = "qwen3:8b" [[benchmarks]] name = "supergpqa" - """) + """, + ) suite = load_eval_config(p) m = suite.models[0] assert m.param_count_b == 0.0 @@ -322,7 +359,9 @@ class TestLoadEvalConfig: assert m.num_gpus == 1 def test_telemetry_config(self, tmp_path): - p = _write_toml(tmp_path, """\ + p = _write_toml( + tmp_path, + """\ [run] telemetry = true gpu_metrics = true @@ -332,7 +371,8 @@ class TestLoadEvalConfig: [[benchmarks]] name = "supergpqa" - """) + """, + ) suite = load_eval_config(p) assert suite.run.telemetry is True assert suite.run.gpu_metrics is True @@ -344,13 +384,15 @@ class TestLoadEvalConfig: class TestExampleConfigs: - @pytest.fixture(params=[ - "minimal.toml", - "single-run.toml", - "full-suite.toml", - "glm-4.7-flash-openhands.toml", - "glm-4.7-flash-openhands-remaining.toml", - ]) + @pytest.fixture( + params=[ + "minimal.toml", + "single-run.toml", + "full-suite.toml", + "glm-4.7-flash-openhands.toml", + "glm-4.7-flash-openhands-remaining.toml", + ] + ) def example_config(self, request): configs_dir = Path(__file__).resolve().parent.parent / "configs" return configs_dir / request.param @@ -549,12 +591,17 @@ class TestExpandSuite: def test_metadata_from_model_hardware_params(self): suite = EvalSuiteConfig( - models=[ModelConfig( - name="GLM-4.7-Flash", engine="vllm", - param_count_b=30.0, active_params_b=3.0, - gpu_peak_tflops=312.0, gpu_peak_bandwidth_gb_s=2039.0, - num_gpus=4, - )], + models=[ + ModelConfig( + name="GLM-4.7-Flash", + engine="vllm", + param_count_b=30.0, + active_params_b=3.0, + gpu_peak_tflops=312.0, + gpu_peak_bandwidth_gb_s=2039.0, + num_gpus=4, + ) + ], benchmarks=[BenchmarkConfig(name="supergpqa")], ) configs = expand_suite(suite) @@ -575,11 +622,13 @@ class TestExpandSuite: def test_metadata_partial_hardware_params(self): suite = EvalSuiteConfig( - models=[ModelConfig( - name="m1", - param_count_b=7.0, - gpu_peak_tflops=100.0, - )], + models=[ + ModelConfig( + name="m1", + param_count_b=7.0, + gpu_peak_tflops=100.0, + ) + ], benchmarks=[BenchmarkConfig(name="supergpqa")], ) configs = expand_suite(suite) @@ -647,7 +696,9 @@ class TestCLIConfig: from openjarvis.evals.cli import main - p = _write_toml(tmp_path, """\ + p = _write_toml( + tmp_path, + """\ [meta] name = "test-suite" @@ -656,7 +707,8 @@ class TestCLIConfig: [[benchmarks]] name = "supergpqa" - """) + """, + ) runner = CliRunner() with patch("openjarvis.evals.cli._run_single", side_effect=Exception("mock")): diff --git a/src/openjarvis/evals/tests/test_display.py b/src/openjarvis/evals/tests/test_display.py index 74d84d1d..9293520d 100644 --- a/src/openjarvis/evals/tests/test_display.py +++ b/src/openjarvis/evals/tests/test_display.py @@ -19,12 +19,21 @@ from openjarvis.evals.core.types import MetricStats, RunSummary def _make_summary(**overrides) -> RunSummary: defaults = dict( - benchmark="gaia", category="agentic", backend="jarvis-agent", - model="qwen3:8b", total_samples=100, scored_samples=100, - correct=42, accuracy=0.42, errors=0, - mean_latency_seconds=15.0, total_cost_usd=0.0, - total_energy_joules=9300.0, avg_power_watts=880.0, - total_input_tokens=50000, total_output_tokens=12000, + benchmark="gaia", + category="agentic", + backend="jarvis-agent", + model="qwen3:8b", + total_samples=100, + scored_samples=100, + correct=42, + accuracy=0.42, + errors=0, + mean_latency_seconds=15.0, + total_cost_usd=0.0, + total_energy_joules=9300.0, + avg_power_watts=880.0, + total_input_tokens=50000, + total_output_tokens=12000, per_subject={"level_1": {"accuracy": 0.58, "correct": 40, "scored": 68}}, ) defaults.update(overrides) @@ -33,9 +42,14 @@ def _make_summary(**overrides) -> RunSummary: def _make_stats(mean=10.0) -> MetricStats: return MetricStats( - mean=mean, median=mean * 0.9, min=mean * 0.2, - max=mean * 3.0, std=mean * 0.5, p90=mean * 1.5, - p95=mean * 2.0, p99=mean * 2.5, + mean=mean, + median=mean * 0.9, + min=mean * 0.2, + max=mean * 3.0, + std=mean * 0.5, + p90=mean * 1.5, + p95=mean * 2.0, + p99=mean * 2.5, ) @@ -91,14 +105,18 @@ class TestTraceSummary: summary = _make_summary( trace_step_type_stats={ "generate": { - "count": 580, "avg_duration": 8.2, + "count": 580, + "avg_duration": 8.2, "total_energy": 22000.0, - "avg_input_tokens": 890.0, "avg_output_tokens": 256.0, + "avg_input_tokens": 890.0, + "avg_output_tokens": 256.0, }, "tool_call": { - "count": 420, "avg_duration": 3.1, + "count": 420, + "avg_duration": 3.1, "total_energy": 0.0, - "avg_input_tokens": 0.0, "avg_output_tokens": 0.0, + "avg_input_tokens": 0.0, + "avg_output_tokens": 0.0, }, }, ) diff --git a/src/openjarvis/evals/tests/test_gaia.py b/src/openjarvis/evals/tests/test_gaia.py index 4cce0043..861408f5 100644 --- a/src/openjarvis/evals/tests/test_gaia.py +++ b/src/openjarvis/evals/tests/test_gaia.py @@ -108,9 +108,7 @@ class TestGAIAScorer: def test_llm_fallback_incorrect(self): backend = MockBackend() backend._default_response = ( - "extracted_final_answer: 43\n" - "reasoning: Different number.\n" - "correct: no" + "extracted_final_answer: 43\nreasoning: Different number.\ncorrect: no" ) scorer = GAIAScorer(backend, "gpt-4o") diff --git a/src/openjarvis/evals/tests/test_runner.py b/src/openjarvis/evals/tests/test_runner.py index 38ea7580..1ef19407 100644 --- a/src/openjarvis/evals/tests/test_runner.py +++ b/src/openjarvis/evals/tests/test_runner.py @@ -382,15 +382,20 @@ class TestRunnerTokenStats: """RunSummary should include total token counts.""" records = [ EvalRecord( - record_id=f"r{i}", problem=f"q{i}", - reference="a", category="test", + record_id=f"r{i}", + problem=f"q{i}", + reference="a", + category="test", ) for i in range(3) ] output_path = tmp_path / "results.jsonl" config = RunConfig( - benchmark="test", backend="mock", model="m", - max_workers=1, output_path=str(output_path), + benchmark="test", + backend="mock", + model="m", + max_workers=1, + output_path=str(output_path), ) dataset = MockDataset(records) backend = MockBackend() @@ -408,8 +413,11 @@ class TestRunnerTokenStats: ] output_path = tmp_path / "results.jsonl" config = RunConfig( - benchmark="test", backend="mock", model="m", - max_workers=1, output_path=str(output_path), + benchmark="test", + backend="mock", + model="m", + max_workers=1, + output_path=str(output_path), ) dataset = MockDataset(records) backend = MockBackend() # returns power_watts=250.0 diff --git a/src/openjarvis/evals/tests/test_types.py b/src/openjarvis/evals/tests/test_types.py index 0c0137a0..7aa88b98 100644 --- a/src/openjarvis/evals/tests/test_types.py +++ b/src/openjarvis/evals/tests/test_types.py @@ -21,7 +21,9 @@ from openjarvis.evals.core.types import ( class TestEvalRecord: def test_creation(self): r = EvalRecord( - record_id="r1", problem="What?", reference="42", + record_id="r1", + problem="What?", + reference="42", category="reasoning", ) assert r.record_id == "r1" @@ -33,8 +35,11 @@ class TestEvalRecord: def test_with_subject_and_metadata(self): r = EvalRecord( - record_id="r2", problem="Q", reference="A", - category="chat", subject="greet", + record_id="r2", + problem="Q", + reference="A", + category="chat", + subject="greet", metadata={"key": "val"}, ) assert r.subject == "greet" @@ -64,9 +69,14 @@ class TestEvalResult: def test_full(self): r = EvalResult( - record_id="r1", model_answer="42", is_correct=True, - score=1.0, latency_seconds=1.5, prompt_tokens=100, - completion_tokens=50, cost_usd=0.01, + record_id="r1", + model_answer="42", + is_correct=True, + score=1.0, + latency_seconds=1.5, + prompt_tokens=100, + completion_tokens=50, + cost_usd=0.01, scoring_metadata={"match": "exact"}, ) assert r.is_correct is True @@ -75,11 +85,16 @@ class TestEvalResult: def test_telemetry_fields(self): r = EvalResult( - record_id="r1", model_answer="42", - energy_joules=100.5, power_watts=250.0, - gpu_utilization_pct=45.0, throughput_tok_per_sec=38.5, - mfu_pct=0.018, mbu_pct=27.5, - ipw=0.004, ipj=0.0001, + record_id="r1", + model_answer="42", + energy_joules=100.5, + power_watts=250.0, + gpu_utilization_pct=45.0, + throughput_tok_per_sec=38.5, + mfu_pct=0.018, + mbu_pct=27.5, + ipw=0.004, + ipj=0.0001, ) assert r.energy_joules == 100.5 assert r.power_watts == 250.0 @@ -107,8 +122,11 @@ class TestRunConfig: def test_with_agent(self): c = RunConfig( - benchmark="gaia", backend="jarvis-agent", model="gpt-4o", - engine_key="cloud", agent_name="orchestrator", + benchmark="gaia", + backend="jarvis-agent", + model="gpt-4o", + engine_key="cloud", + agent_name="orchestrator", tools=["calculator", "think"], ) assert c.agent_name == "orchestrator" @@ -117,7 +135,9 @@ class TestRunConfig: def test_with_metadata(self): meta = {"param_count_b": 30.0, "active_params_b": 3.0, "num_gpus": 4} c = RunConfig( - benchmark="supergpqa", backend="jarvis-direct", model="m", + benchmark="supergpqa", + backend="jarvis-direct", + model="m", metadata=meta, ) assert c.metadata["param_count_b"] == 30.0 @@ -153,10 +173,16 @@ class TestMetricStats: class TestRunSummary: def test_creation(self): s = RunSummary( - benchmark="supergpqa", category="reasoning", - backend="jarvis-direct", model="qwen3:8b", - total_samples=100, scored_samples=95, correct=47, - accuracy=0.495, errors=5, mean_latency_seconds=2.1, + benchmark="supergpqa", + category="reasoning", + backend="jarvis-direct", + model="qwen3:8b", + total_samples=100, + scored_samples=95, + correct=47, + accuracy=0.495, + errors=5, + mean_latency_seconds=2.1, total_cost_usd=0.0, per_subject={"math": {"accuracy": 0.5}}, ) @@ -167,10 +193,16 @@ class TestRunSummary: def test_metric_stats_fields(self): stats = MetricStats(mean=0.5, median=0.4, min=0.1, max=0.9, std=0.2) s = RunSummary( - benchmark="test", category="reasoning", - backend="jarvis-direct", model="m", - total_samples=10, scored_samples=10, correct=5, - accuracy=0.5, errors=0, mean_latency_seconds=1.0, + benchmark="test", + category="reasoning", + backend="jarvis-direct", + model="m", + total_samples=10, + scored_samples=10, + correct=5, + accuracy=0.5, + errors=0, + mean_latency_seconds=1.0, total_cost_usd=0.0, accuracy_stats=stats, energy_stats=stats, @@ -191,10 +223,16 @@ class TestRunSummary: def test_metric_stats_defaults_none(self): s = RunSummary( - benchmark="test", category="test", - backend="mock", model="m", - total_samples=0, scored_samples=0, correct=0, - accuracy=0.0, errors=0, mean_latency_seconds=0.0, + benchmark="test", + category="test", + backend="mock", + model="m", + total_samples=0, + scored_samples=0, + correct=0, + accuracy=0.0, + errors=0, + mean_latency_seconds=0.0, total_cost_usd=0.0, ) assert s.accuracy_stats is None @@ -217,8 +255,10 @@ class TestEvalResultTraceFields: def test_trace_fields_set(self): r = EvalResult( - record_id="test", model_answer="hi", - trace_steps=5, trace_energy_joules=100.0, + record_id="test", + model_answer="hi", + trace_steps=5, + trace_energy_joules=100.0, ) assert r.trace_steps == 5 assert r.trace_energy_joules == 100.0 @@ -227,10 +267,17 @@ class TestEvalResultTraceFields: class TestRunSummaryTraceFields: def test_trace_aggregate_fields(self): s = RunSummary( - benchmark="test", category="test", backend="test", - model="test", total_samples=1, scored_samples=1, - correct=1, accuracy=1.0, errors=0, - mean_latency_seconds=1.0, total_cost_usd=0.0, + benchmark="test", + category="test", + backend="test", + model="test", + total_samples=1, + scored_samples=1, + correct=1, + accuracy=1.0, + errors=0, + mean_latency_seconds=1.0, + total_cost_usd=0.0, ) assert s.avg_power_watts == 0.0 assert s.trace_step_type_stats == {} @@ -307,8 +354,11 @@ class TestModelConfig: def test_with_overrides(self): m = ModelConfig( - name="gpt-4o", engine="cloud", provider="openai", - temperature=0.5, max_tokens=4096, + name="gpt-4o", + engine="cloud", + provider="openai", + temperature=0.5, + max_tokens=4096, ) assert m.engine == "cloud" assert m.provider == "openai" @@ -317,9 +367,12 @@ class TestModelConfig: def test_hardware_params(self): m = ModelConfig( - name="GLM-4.7-Flash", engine="vllm", - param_count_b=30.0, active_params_b=3.0, - gpu_peak_tflops=312.0, gpu_peak_bandwidth_gb_s=2039.0, + name="GLM-4.7-Flash", + engine="vllm", + param_count_b=30.0, + active_params_b=3.0, + gpu_peak_tflops=312.0, + gpu_peak_bandwidth_gb_s=2039.0, num_gpus=4, ) assert m.param_count_b == 30.0 @@ -344,10 +397,15 @@ class TestBenchmarkConfig: def test_with_overrides(self): b = BenchmarkConfig( - name="gaia", backend="jarvis-agent", max_samples=50, - split="test", agent="orchestrator", - tools=["calc", "think"], judge_model="custom-judge", - temperature=0.3, max_tokens=1024, + name="gaia", + backend="jarvis-agent", + max_samples=50, + split="test", + agent="orchestrator", + tools=["calc", "think"], + judge_model="custom-judge", + temperature=0.3, + max_tokens=1024, ) assert b.backend == "jarvis-agent" assert b.max_samples == 50 diff --git a/src/openjarvis/evals/trackers/sheets_tracker.py b/src/openjarvis/evals/trackers/sheets_tracker.py index 966be835..47c974cd 100644 --- a/src/openjarvis/evals/trackers/sheets_tracker.py +++ b/src/openjarvis/evals/trackers/sheets_tracker.py @@ -70,8 +70,7 @@ class SheetsTracker(ResultTracker): ) -> None: if gspread is None: raise ImportError( - "gspread is not installed. " - "Install it with: uv sync --extra eval-sheets" + "gspread is not installed. Install it with: uv sync --extra eval-sheets" ) self._spreadsheet_id = spreadsheet_id self._worksheet_name = worksheet @@ -93,7 +92,9 @@ class SheetsTracker(ResultTracker): ws = spreadsheet.worksheet(self._worksheet_name) except gspread.exceptions.WorksheetNotFound: ws = spreadsheet.add_worksheet( - title=self._worksheet_name, rows=1000, cols=len(SHEET_COLUMNS), + title=self._worksheet_name, + rows=1000, + cols=len(SHEET_COLUMNS), ) # Ensure header row exists (idempotent) existing = ws.row_values(1) @@ -115,7 +116,8 @@ class SheetsTracker(ResultTracker): ] if self._credentials_path: creds = Credentials.from_service_account_file( - self._credentials_path, scopes=scopes, + self._credentials_path, + scopes=scopes, ) else: # Fall back to Application Default Credentials diff --git a/src/openjarvis/evals/trackers/wandb_tracker.py b/src/openjarvis/evals/trackers/wandb_tracker.py index 86884ffe..32817051 100644 --- a/src/openjarvis/evals/trackers/wandb_tracker.py +++ b/src/openjarvis/evals/trackers/wandb_tracker.py @@ -44,14 +44,13 @@ class WandbTracker(ResultTracker): ) -> None: if wandb is None: raise ImportError( - "wandb is not installed. " - "Install it with: uv sync --extra eval-wandb" + "wandb is not installed. Install it with: uv sync --extra eval-wandb" ) self._project = project self._entity = entity or None - self._tags: List[str] = [ - t.strip() for t in tags.split(",") if t.strip() - ] if tags else [] + self._tags: List[str] = ( + [t.strip() for t in tags.split(",") if t.strip()] if tags else [] + ) self._group = group or None self._run: Any = None self._step = 0 @@ -134,14 +133,18 @@ class WandbTracker(ResultTracker): flat.update(_flatten_metric_stats("mbu", summary.mbu_stats)) flat.update(_flatten_metric_stats("ipw", summary.ipw_stats)) flat.update(_flatten_metric_stats("ipj", summary.ipj_stats)) - flat.update(_flatten_metric_stats( - "energy_per_output_token", - summary.energy_per_output_token_stats, - )) - flat.update(_flatten_metric_stats( - "throughput_per_watt", - summary.throughput_per_watt_stats, - )) + flat.update( + _flatten_metric_stats( + "energy_per_output_token", + summary.energy_per_output_token_stats, + ) + ) + flat.update( + _flatten_metric_stats( + "throughput_per_watt", + summary.throughput_per_watt_stats, + ) + ) flat.update(_flatten_metric_stats("itl", summary.itl_stats)) wandb.run.summary.update(flat) diff --git a/src/openjarvis/learning/__init__.py b/src/openjarvis/learning/__init__.py index 9623c340..e12715d4 100644 --- a/src/openjarvis/learning/__init__.py +++ b/src/openjarvis/learning/__init__.py @@ -31,11 +31,13 @@ def ensure_registered() -> None: from openjarvis.learning.routing.heuristic_policy import ( ensure_registered as _reg_heuristic, ) + _reg_heuristic() from openjarvis.learning.routing.learned_router import ( ensure_registered as _reg_learned, ) + _reg_learned() # Intelligence training (optional deps) diff --git a/src/openjarvis/learning/agents/agent_evolver.py b/src/openjarvis/learning/agents/agent_evolver.py index 5515e938..d42b9d59 100644 --- a/src/openjarvis/learning/agents/agent_evolver.py +++ b/src/openjarvis/learning/agents/agent_evolver.py @@ -156,9 +156,9 @@ class AgentConfigEvolver: return None # Pick best agent by composite score - best_agent = max( - agent_scores.items(), key=lambda kv: kv[1].composite_score() - )[0] + best_agent = max(agent_scores.items(), key=lambda kv: kv[1].composite_score())[ + 0 + ] # Rank tools by composite score (frequency-weighted quality) ranked_tools = sorted( @@ -240,20 +240,24 @@ class AgentConfigEvolver: pattern = f"{agent_name}.v*.toml" archived = sorted(self._history_dir.glob(pattern)) for idx, archived_path in enumerate(archived, start=1): - versions.append({ - "version": idx, - "path": str(archived_path), - "modified": archived_path.stat().st_mtime, - }) + versions.append( + { + "version": idx, + "path": str(archived_path), + "modified": archived_path.stat().st_mtime, + } + ) # Current version current = self._config_dir / f"{agent_name}.toml" if current.exists(): - versions.append({ - "version": len(versions) + 1, - "path": str(current), - "modified": current.stat().st_mtime, - }) + versions.append( + { + "version": len(versions) + 1, + "path": str(current), + "modified": current.stat().st_mtime, + } + ) return versions diff --git a/src/openjarvis/learning/agents/dspy_optimizer.py b/src/openjarvis/learning/agents/dspy_optimizer.py index e81106cb..bbb338df 100644 --- a/src/openjarvis/learning/agents/dspy_optimizer.py +++ b/src/openjarvis/learning/agents/dspy_optimizer.py @@ -57,8 +57,7 @@ class DSPyAgentOptimizer: return { "status": "skipped", "reason": ( - f"only {len(traces)} traces, " - f"min_traces={self.config.min_traces}" + f"only {len(traces)} traces, min_traces={self.config.min_traces}" ), } @@ -66,8 +65,7 @@ class DSPyAgentOptimizer: return { "status": "error", "reason": ( - "dspy not installed " - "(pip install 'openjarvis[learning-dspy]')" + "dspy not installed (pip install 'openjarvis[learning-dspy]')" ), } @@ -101,9 +99,7 @@ class DSPyAgentOptimizer: examples.append(ex) # Define a simple metric based on trace feedback - def metric( - example: Any, prediction: Any, trace: Any = None - ) -> float: + def metric(example: Any, prediction: Any, trace: Any = None) -> float: for tr in traces: if tr.query == example.question: return tr.feedback if tr.feedback is not None else 0.5 @@ -113,9 +109,7 @@ class DSPyAgentOptimizer: class AgentModule(dspy.Module): def __init__(self_inner: Any) -> None: super().__init__() - self_inner.generate = dspy.ChainOfThought( - "question -> answer" - ) + self_inner.generate = dspy.ChainOfThought("question -> answer") def forward(self_inner: Any, question: str) -> Any: return self_inner.generate(question=question) @@ -146,9 +140,7 @@ class DSPyAgentOptimizer: # Split data for train/test split = max(1, int(len(examples) * 0.8)) train_set = examples[:split] - optimized_program = teleprompter.compile( - program, trainset=train_set - ) + optimized_program = teleprompter.compile(program, trainset=train_set) # Extract optimized parameters result: Dict[str, Any] = {} @@ -167,22 +159,13 @@ class DSPyAgentOptimizer: """Convert DSPy optimization results to TOML-compatible config.""" updates: Dict[str, Any] = {} - if ( - self.config.optimize_system_prompt - and "system_prompt" in optimized - ): + if self.config.optimize_system_prompt and "system_prompt" in optimized: updates["system_prompt"] = optimized["system_prompt"] - if ( - self.config.optimize_few_shot - and "few_shot_examples" in optimized - ): + if self.config.optimize_few_shot and "few_shot_examples" in optimized: updates["few_shot_examples"] = optimized["few_shot_examples"] - if ( - self.config.optimize_tool_descriptions - and "tool_descriptions" in optimized - ): + if self.config.optimize_tool_descriptions and "tool_descriptions" in optimized: updates["tool_descriptions"] = optimized["tool_descriptions"] return updates diff --git a/src/openjarvis/learning/agents/gepa_optimizer.py b/src/openjarvis/learning/agents/gepa_optimizer.py index 20dac8dd..1c3656c3 100644 --- a/src/openjarvis/learning/agents/gepa_optimizer.py +++ b/src/openjarvis/learning/agents/gepa_optimizer.py @@ -19,6 +19,7 @@ logger = logging.getLogger(__name__) # Optional dependency try: import gepa + HAS_GEPA = True except ImportError: HAS_GEPA = False @@ -71,20 +72,22 @@ class OpenJarvisGEPAAdapter: best = max(matching, key=lambda t: t.feedback or 0.0) scores.append(best.feedback or 0.0) if capture_traces: - trace_data.append({ - "query": query, - "result": best.result, - "feedback": best.feedback, - "outcome": best.outcome, - "steps": [ - { - "type": str(s.step_type), - "input": s.input, - "output": s.output, - } - for s in best.steps - ], - }) + trace_data.append( + { + "query": query, + "result": best.result, + "feedback": best.feedback, + "outcome": best.outcome, + "steps": [ + { + "type": str(s.step_type), + "input": s.input, + "output": s.output, + } + for s in best.steps + ], + } + ) else: scores.append(0.0) @@ -126,16 +129,18 @@ class OpenJarvisGEPAAdapter: if is_error: errors.append(step.output["error"]) - dataset.append({ - "query": query, - "result": best.result, - "feedback": best.feedback, - "outcome": best.outcome, - "tool_calls": tool_calls, - "reasoning_steps": reasoning_steps, - "errors": errors, - "components_to_update": components_to_update, - }) + dataset.append( + { + "query": query, + "result": best.result, + "feedback": best.feedback, + "outcome": best.outcome, + "tool_calls": tool_calls, + "reasoning_steps": reasoning_steps, + "errors": errors, + "components_to_update": components_to_update, + } + ) return dataset @@ -170,8 +175,7 @@ class GEPAAgentOptimizer: return { "status": "skipped", "reason": ( - f"only {len(traces)} traces, " - f"min_traces={self.config.min_traces}" + f"only {len(traces)} traces, min_traces={self.config.min_traces}" ), } @@ -179,8 +183,7 @@ class GEPAAgentOptimizer: return { "status": "error", "reason": ( - "gepa not installed" - " (pip install 'openjarvis[learning-gepa]')" + "gepa not installed (pip install 'openjarvis[learning-gepa]')" ), } @@ -206,7 +209,9 @@ class GEPAAgentOptimizer: } def _run_gepa( - self, adapter: OpenJarvisGEPAAdapter, traces: List[Any], + self, + adapter: OpenJarvisGEPAAdapter, + traces: List[Any], ) -> Dict[str, Any]: """Run the GEPA evolutionary optimization loop.""" # Build search space from config flags @@ -236,7 +241,7 @@ class GEPAAgentOptimizer: # Run optimization result = optimizer.optimize( initial_candidate=initial_candidate, - test_cases=queries[:self.config.assessment_batch_size], + test_cases=queries[: self.config.assessment_batch_size], components=components, ) diff --git a/src/openjarvis/learning/agents/skill_discovery.py b/src/openjarvis/learning/agents/skill_discovery.py index 01d20d32..ba9e5d23 100644 --- a/src/openjarvis/learning/agents/skill_discovery.py +++ b/src/openjarvis/learning/agents/skill_discovery.py @@ -10,6 +10,7 @@ from typing import Any, Dict, List, Tuple @dataclass(slots=True) class DiscoveredSkill: """A skill discovered from trace analysis.""" + name: str description: str tool_sequence: List[str] # ordered tool names @@ -72,7 +73,7 @@ class SkillDiscovery: upper = min(self._max_len + 1, len(tool_calls) + 1) for length in range(self._min_len, upper): for start in range(len(tool_calls) - length + 1): - seq = tuple(tool_calls[start:start + length]) + seq = tuple(tool_calls[start : start + length]) sequence_data[seq].append(outcome) if query and len(sequence_inputs[seq]) < 3: sequence_inputs[seq].append(query) @@ -86,14 +87,16 @@ class SkillDiscovery: if freq >= self._min_freq and avg_outcome >= self._min_outcome: name = "_".join(seq) desc = f"Auto-discovered skill: {' -> '.join(seq)} (seen {freq} times)" - discovered.append(DiscoveredSkill( - name=name, - description=desc, - tool_sequence=list(seq), - frequency=freq, - avg_outcome=avg_outcome, - example_inputs=sequence_inputs.get(seq, []), - )) + discovered.append( + DiscoveredSkill( + name=name, + description=desc, + tool_sequence=list(seq), + frequency=freq, + avg_outcome=avg_outcome, + example_inputs=sequence_inputs.get(seq, []), + ) + ) # Sort by frequency * outcome (quality score) discovered.sort(key=lambda s: s.frequency * s.avg_outcome, reverse=True) @@ -123,7 +126,9 @@ class SkillDiscovery: ) if is_tool: name = getattr( - step, "tool_name", getattr(step, "name", ""), + step, + "tool_name", + getattr(step, "name", ""), ) if name: tools.append(name) @@ -150,18 +155,20 @@ class SkillDiscovery: """Convert discovered skills to TOML-compatible manifest dicts.""" manifests = [] for skill in self._discovered: - manifests.append({ - "name": skill.name, - "description": skill.description, - "steps": [ - {"tool": tool, "params": {}} for tool in skill.tool_sequence - ], - "metadata": { - "auto_discovered": True, - "frequency": skill.frequency, - "avg_outcome": skill.avg_outcome, - }, - }) + manifests.append( + { + "name": skill.name, + "description": skill.description, + "steps": [ + {"tool": tool, "params": {}} for tool in skill.tool_sequence + ], + "metadata": { + "auto_discovered": True, + "frequency": skill.frequency, + "avg_outcome": skill.avg_outcome, + }, + } + ) return manifests diff --git a/src/openjarvis/learning/intelligence/grpo_trainer.py b/src/openjarvis/learning/intelligence/grpo_trainer.py index 6ba91bc5..40f19f62 100644 --- a/src/openjarvis/learning/intelligence/grpo_trainer.py +++ b/src/openjarvis/learning/intelligence/grpo_trainer.py @@ -294,9 +294,7 @@ class GRPOTrainer: ) response_ids = gen_ids[0, inputs["input_ids"].shape[1] :] - response = tokenizer.decode( - response_ids, skip_special_tokens=True - ) + response = tokenizer.decode(response_ids, skip_special_tokens=True) # Score with reward function reward = self.reward_fn.score(prompt, response, gt) @@ -305,27 +303,19 @@ class GRPOTrainer: # Compute log probabilities full_ids = gen_ids[0].unsqueeze(0) policy_logits = policy_model(full_ids).logits - policy_lp = torch.nn.functional.log_softmax( - policy_logits, dim=-1 - ) + policy_lp = torch.nn.functional.log_softmax(policy_logits, dim=-1) token_lps = torch.gather( policy_lp[:, :-1, :], 2, full_ids[:, 1:].unsqueeze(-1) ).squeeze(-1) log_probs.append(token_lps.sum()) with torch.no_grad(): - ref_logits = ref_model( - full_ids.to(ref_model.device) - ).logits - ref_lp = torch.nn.functional.log_softmax( - ref_logits, dim=-1 - ) + ref_logits = ref_model(full_ids.to(ref_model.device)).logits + ref_lp = torch.nn.functional.log_softmax(ref_logits, dim=-1) ref_token_lps = torch.gather( ref_lp[:, :-1, :], 2, - full_ids[:, 1:] - .to(ref_model.device) - .unsqueeze(-1), + full_ids[:, 1:].to(ref_model.device).unsqueeze(-1), ).squeeze(-1) ref_lps.append(ref_token_lps.sum()) @@ -334,9 +324,7 @@ class GRPOTrainer: all_ref_log_probs.append(ref_lps) # Compute group-relative advantages and loss - total_loss = torch.tensor( - 0.0, device=policy_model.device, requires_grad=True - ) + total_loss = torch.tensor(0.0, device=policy_model.device, requires_grad=True) for rewards, log_probs, ref_lps in zip( all_rewards, all_log_probs, all_ref_log_probs diff --git a/src/openjarvis/learning/intelligence/orchestrator/environment.py b/src/openjarvis/learning/intelligence/orchestrator/environment.py index 5f200586..7f731164 100644 --- a/src/openjarvis/learning/intelligence/orchestrator/environment.py +++ b/src/openjarvis/learning/intelligence/orchestrator/environment.py @@ -65,14 +65,11 @@ class OrchestratorEnvironment: if action.tool_name not in available: raise ValueError( - f"Tool '{action.tool_name}' not available. " - f"Available: {available}" + f"Tool '{action.tool_name}' not available. Available: {available}" ) if state.num_turns() >= self._max_turns: - raise ValueError( - f"Max turns ({self._max_turns}) exceeded" - ) + raise ValueError(f"Max turns ({self._max_turns}) exceeded") # Execute tool via ToolExecutor tool_call = ToolCall( diff --git a/src/openjarvis/learning/intelligence/orchestrator/grpo_trainer.py b/src/openjarvis/learning/intelligence/orchestrator/grpo_trainer.py index 6949fa20..27e58c59 100644 --- a/src/openjarvis/learning/intelligence/orchestrator/grpo_trainer.py +++ b/src/openjarvis/learning/intelligence/orchestrator/grpo_trainer.py @@ -266,8 +266,7 @@ class OrchestratorGRPOTrainer: # Group-relative advantages mean_r = sum(group_rewards) / len(group_rewards) std_r = ( - sum((r - mean_r) ** 2 for r in group_rewards) - / len(group_rewards) + sum((r - mean_r) ** 2 for r in group_rewards) / len(group_rewards) ) ** 0.5 if std_r > 1e-8: advantages = [(r - mean_r) / std_r for r in group_rewards] @@ -305,9 +304,7 @@ class OrchestratorGRPOTrainer: loss_val = avg_loss.item() if torch.isnan(avg_loss) or torch.isinf(avg_loss): - avg_reward = ( - sum(all_rewards) / len(all_rewards) if all_rewards else 0.0 - ) + avg_reward = sum(all_rewards) / len(all_rewards) if all_rewards else 0.0 return 0.0, float(avg_reward) self.optimizer.zero_grad() @@ -317,11 +314,7 @@ class OrchestratorGRPOTrainer: for param in self.policy.model.parameters(): if param.grad is not None and torch.isnan(param.grad).any(): self.optimizer.zero_grad() - avg_reward = ( - sum(all_rewards) / len(all_rewards) - if all_rewards - else 0.0 - ) + avg_reward = sum(all_rewards) / len(all_rewards) if all_rewards else 0.0 return float(loss_val), float(avg_reward) torch.nn.utils.clip_grad_norm_( @@ -329,16 +322,12 @@ class OrchestratorGRPOTrainer: ) self.optimizer.step() - avg_reward = ( - sum(all_rewards) / len(all_rewards) if all_rewards else 0.0 - ) + avg_reward = sum(all_rewards) / len(all_rewards) if all_rewards else 0.0 return float(loss_val), float(avg_reward) # -- Generation / log-prob helpers --------------------------------------- - def _generate_with_log_probs( - self, prompt: str - ) -> "tuple[str, Any]": + def _generate_with_log_probs(self, prompt: str) -> "tuple[str, Any]": """Generate a response and return ``(text, log_probs)``.""" inputs = self.policy.tokenizer( prompt, @@ -365,9 +354,7 @@ class OrchestratorGRPOTrainer: if len(generated_ids) == 0: return "", torch.tensor(0.0, device=self.device) - text = self.policy.tokenizer.decode( - generated_ids, skip_special_tokens=True - ) + text = self.policy.tokenizer.decode(generated_ids, skip_special_tokens=True) log_probs = [] for token_id, logits in zip(generated_ids, outputs.scores): @@ -383,14 +370,12 @@ class OrchestratorGRPOTrainer: ) return text, total_lp - def _compute_log_probs( - self, prompt: str, response: str - ) -> "torch.Tensor": + def _compute_log_probs(self, prompt: str, response: str) -> "torch.Tensor": """Log-probs of *response* given *prompt* under current policy.""" full = prompt + response - inputs = self.policy.tokenizer( - full, return_tensors="pt", truncation=True - ).to(self.device) + inputs = self.policy.tokenizer(full, return_tensors="pt", truncation=True).to( + self.device + ) prompt_inputs = self.policy.tokenizer( prompt, return_tensors="pt", truncation=True ).to(self.device) @@ -403,16 +388,12 @@ class OrchestratorGRPOTrainer: lps = [] for i in range(start, end - 1): - lp = F.log_softmax(logits[0, i, :], dim=-1)[ - inputs.input_ids[0, i + 1] - ] + lp = F.log_softmax(logits[0, i, :], dim=-1)[inputs.input_ids[0, i + 1]] lps.append(lp) return torch.stack(lps).sum() if lps else torch.tensor(0.0) - def _compute_log_probs_ref( - self, prompt: str, response: str - ) -> "torch.Tensor": + def _compute_log_probs_ref(self, prompt: str, response: str) -> "torch.Tensor": """Log-probs under the frozen reference policy (no grad).""" full = prompt + response inputs = self.ref_policy.tokenizer( @@ -430,9 +411,7 @@ class OrchestratorGRPOTrainer: lps = [] for i in range(start, end - 1): - lp = F.log_softmax(logits[0, i, :], dim=-1)[ - inputs.input_ids[0, i + 1] - ] + lp = F.log_softmax(logits[0, i, :], dim=-1)[inputs.input_ids[0, i + 1]] lps.append(lp) return torch.stack(lps).sum() if lps else torch.tensor(0.0) @@ -490,13 +469,14 @@ def _ensure_registered() -> None: class OrchestratorGRPOPolicy(IntelligenceLearningPolicy): """Wrapper that registers the GRPO trainer as a learning policy.""" - def update( - self, trace_store: Any, **kwargs: object - ) -> Dict[str, Any]: - config = OrchestratorGRPOConfig(**{ - k: v for k, v in kwargs.items() - if k in OrchestratorGRPOConfig.__dataclass_fields__ - }) + def update(self, trace_store: Any, **kwargs: object) -> Dict[str, Any]: + config = OrchestratorGRPOConfig( + **{ + k: v + for k, v in kwargs.items() + if k in OrchestratorGRPOConfig.__dataclass_fields__ + } + ) trainer = OrchestratorGRPOTrainer(config) trainer.train() return {"status": "grpo_training_complete"} diff --git a/src/openjarvis/learning/intelligence/orchestrator/policy_model.py b/src/openjarvis/learning/intelligence/orchestrator/policy_model.py index 49b14379..032122fb 100644 --- a/src/openjarvis/learning/intelligence/orchestrator/policy_model.py +++ b/src/openjarvis/learning/intelligence/orchestrator/policy_model.py @@ -112,9 +112,7 @@ class OrchestratorPolicyModel: model = AutoModelForCausalLM.from_pretrained(model_name, **model_kwargs) - if gradient_checkpointing and hasattr( - model, "gradient_checkpointing_enable" - ): + if gradient_checkpointing and hasattr(model, "gradient_checkpointing_enable"): model.gradient_checkpointing_enable( gradient_checkpointing_kwargs={"use_reentrant": False} ) @@ -176,9 +174,7 @@ class OrchestratorPolicyModel: parts.append(f"Turn {i}:") parts.append(f" Thought: {action.thought}") parts.append(f" Tool: {action.tool_name}") - parts.append( - f" Observation: {observation.content[:100]}..." - ) + parts.append(f" Observation: {observation.content[:100]}...") parts.append("") parts.append("What should you do next?") @@ -207,11 +203,7 @@ class OrchestratorPolicyModel: r"THOUGHT:\s*(.+?)(?:\n|$)", output_text, re.IGNORECASE ) return PolicyOutput( - thought=( - thought_match.group(1).strip() - if thought_match - else "" - ), + thought=(thought_match.group(1).strip() if thought_match else ""), tool_name="", tool_input=final_match.group(1).strip(), is_final_answer=True, @@ -221,9 +213,7 @@ class OrchestratorPolicyModel: thought_match = re.search( r"THOUGHT:\s*(.+?)(?:\n|$)", output_text, re.IGNORECASE ) - tool_match = re.search( - r"TOOL:\s*(.+?)(?:\n|$)", output_text, re.IGNORECASE - ) + tool_match = re.search(r"TOOL:\s*(.+?)(?:\n|$)", output_text, re.IGNORECASE) input_match = re.search( r"INPUT:\s*(.+?)(?:\nTHOUGHT:|\nTOOL:|\nFINAL|\Z)", output_text, @@ -231,9 +221,7 @@ class OrchestratorPolicyModel: ) thought = ( - thought_match.group(1).strip() - if thought_match - else "No thought provided" + thought_match.group(1).strip() if thought_match else "No thought provided" ) tool_name = ( tool_match.group(1).strip() @@ -264,9 +252,7 @@ class OrchestratorPolicyModel: def _generate(self, prompt: str) -> str: """Generate text from the loaded model.""" - inputs = self.tokenizer(prompt, return_tensors="pt").to( - self.model.device - ) + inputs = self.tokenizer(prompt, return_tensors="pt").to(self.model.device) outputs = self.model.generate( **inputs, max_new_tokens=self.max_tokens, @@ -285,12 +271,9 @@ class OrchestratorPolicyModel: self.tokenizer.save_pretrained(path) def __repr__(self) -> str: - model_name = ( - "None" if self.model is None else type(self.model).__name__ - ) + model_name = "None" if self.model is None else type(self.model).__name__ return ( - f"OrchestratorPolicyModel(model={model_name}, " - f"max_tokens={self.max_tokens})" + f"OrchestratorPolicyModel(model={model_name}, max_tokens={self.max_tokens})" ) diff --git a/src/openjarvis/learning/intelligence/orchestrator/prompt_registry.py b/src/openjarvis/learning/intelligence/orchestrator/prompt_registry.py index 5fac2af2..ce30c79f 100644 --- a/src/openjarvis/learning/intelligence/orchestrator/prompt_registry.py +++ b/src/openjarvis/learning/intelligence/orchestrator/prompt_registry.py @@ -104,8 +104,7 @@ TOOL_DESCRIPTIONS: Dict[str, dict] = { { "task": "Find all prime numbers less than 50", "thought": ( - "Need a prime-checking algorithm" - " - code_interpreter is ideal." + "Need a prime-checking algorithm - code_interpreter is ideal." ), "input": ( "def is_prime(n):\n" @@ -278,9 +277,7 @@ def build_system_prompt( by_cat_names: Dict[str, List[str]] = {} for name in tool_names: cat = ( - TOOL_DESCRIPTIONS[name]["category"] - if name in TOOL_DESCRIPTIONS - else "llm" + TOOL_DESCRIPTIONS[name]["category"] if name in TOOL_DESCRIPTIONS else "llm" ) by_cat_names.setdefault(cat, []).append(name) @@ -295,9 +292,7 @@ def build_system_prompt( "- Simple arithmetic/algebra -> calculator (instant, accurate)" ) if "code_interpreter" in tool_names: - math_lines.append( - "- Numerical algorithms -> code_interpreter (programmable)" - ) + math_lines.append("- Numerical algorithms -> code_interpreter (programmable)") if math_lines: guide.append("MATH PROBLEMS:") guide.extend(math_lines) @@ -306,9 +301,7 @@ def build_system_prompt( # Coding code_lines: list[str] = [] if "code_interpreter" in tool_names: - code_lines.append( - "- Algorithm implementation/execution -> code_interpreter" - ) + code_lines.append("- Algorithm implementation/execution -> code_interpreter") if code_lines: guide.append("CODING TASKS:") guide.extend(code_lines) @@ -322,9 +315,7 @@ def build_system_prompt( ) llm_tools = by_cat_names.get("llm", []) if llm_tools: - reasoning_lines.append( - f"- Complex reasoning -> {', '.join(llm_tools)}" - ) + reasoning_lines.append(f"- Complex reasoning -> {', '.join(llm_tools)}") if reasoning_lines: guide.append("REASONING/LOGIC:") guide.extend(reasoning_lines) @@ -336,9 +327,7 @@ def build_system_prompt( general_lines.append("- Current events/recent info -> web_search") memory_tools = by_cat_names.get("memory", []) if memory_tools: - general_lines.append( - f"- Stored knowledge -> {', '.join(memory_tools)}" - ) + general_lines.append(f"- Stored knowledge -> {', '.join(memory_tools)}") if general_lines: guide.append("GENERAL Q&A / FACTUAL:") guide.extend(general_lines) diff --git a/src/openjarvis/learning/intelligence/orchestrator/reward.py b/src/openjarvis/learning/intelligence/orchestrator/reward.py index 040d3b6e..7cf41de5 100644 --- a/src/openjarvis/learning/intelligence/orchestrator/reward.py +++ b/src/openjarvis/learning/intelligence/orchestrator/reward.py @@ -83,12 +83,8 @@ class MultiObjectiveReward: accuracy_reward = 1.0 if episode.correct else 0.0 cost_penalty = episode.total_cost_usd / self.normalizers.cost_scale - energy_penalty = ( - episode.total_energy_joules / self.normalizers.energy_scale - ) - latency_penalty = ( - episode.total_latency_seconds / self.normalizers.latency_scale - ) + energy_penalty = episode.total_energy_joules / self.normalizers.energy_scale + latency_penalty = episode.total_latency_seconds / self.normalizers.latency_scale power_penalty = episode.max_power_watts / self.normalizers.power_scale return ( @@ -104,12 +100,8 @@ class MultiObjectiveReward: accuracy_reward = 1.0 if episode.correct else 0.0 cost_penalty = episode.total_cost_usd / self.normalizers.cost_scale - energy_penalty = ( - episode.total_energy_joules / self.normalizers.energy_scale - ) - latency_penalty = ( - episode.total_latency_seconds / self.normalizers.latency_scale - ) + energy_penalty = episode.total_energy_joules / self.normalizers.energy_scale + latency_penalty = episode.total_latency_seconds / self.normalizers.latency_scale power_penalty = episode.max_power_watts / self.normalizers.power_scale accuracy_component = self.weights.alpha * accuracy_reward diff --git a/src/openjarvis/learning/intelligence/orchestrator/sft_trainer.py b/src/openjarvis/learning/intelligence/orchestrator/sft_trainer.py index 154fd83f..2fe6aa83 100644 --- a/src/openjarvis/learning/intelligence/orchestrator/sft_trainer.py +++ b/src/openjarvis/learning/intelligence/orchestrator/sft_trainer.py @@ -147,9 +147,7 @@ class OrchestratorSFTDataset: "labels": encoding["input_ids"].squeeze(0).clone(), } - def _format_conversation( - self, conversations: List[Dict[str, str]] - ) -> str: + def _format_conversation(self, conversations: List[Dict[str, str]]) -> str: """Format conversation turns into training text.""" if hasattr(self.tokenizer, "apply_chat_template"): try: @@ -164,9 +162,7 @@ class OrchestratorSFTDataset: role = "assistant" elif role == "tool": tool_name = turn.get("name", "tool") - content = ( - f"[Tool '{tool_name}' returned]: {content}" - ) + content = f"[Tool '{tool_name}' returned]: {content}" role = "user" if role in ("user", "assistant", "system"): @@ -192,16 +188,12 @@ class OrchestratorSFTDataset: parts.append(f"<|system|>\n{content}") elif role == "tool": tool_name = turn.get("name", "tool") - parts.append( - f"<|user|>\n[Tool '{tool_name}' returned]: {content}" - ) + parts.append(f"<|user|>\n[Tool '{tool_name}' returned]: {content}") eos = getattr(self.tokenizer, "eos_token", "") or "" return "\n".join(parts) + eos - def iter_batches( - self, batch_size: int - ) -> Iterator[List[Dict[str, Any]]]: + def iter_batches(self, batch_size: int) -> Iterator[List[Dict[str, Any]]]: batch: list[Dict[str, Any]] = [] for i in range(len(self)): batch.append(self[i]) @@ -304,9 +296,7 @@ class OrchestratorSFTTrainer: 1.0 - (step - warmup_steps) / (total_steps - warmup_steps), ) - self.scheduler = torch.optim.lr_scheduler.LambdaLR( - self.optimizer, lr_lambda - ) + self.scheduler = torch.optim.lr_scheduler.LambdaLR(self.optimizer, lr_lambda) def train(self) -> None: """Run the SFT training loop.""" @@ -393,13 +383,14 @@ def _ensure_registered() -> None: class OrchestratorSFTPolicy(IntelligenceLearningPolicy): """Wrapper that registers the SFT trainer as a learning policy.""" - def update( - self, trace_store: Any, **kwargs: object - ) -> Dict[str, Any]: - config = OrchestratorSFTConfig(**{ - k: v for k, v in kwargs.items() - if k in OrchestratorSFTConfig.__dataclass_fields__ - }) + def update(self, trace_store: Any, **kwargs: object) -> Dict[str, Any]: + config = OrchestratorSFTConfig( + **{ + k: v + for k, v in kwargs.items() + if k in OrchestratorSFTConfig.__dataclass_fields__ + } + ) trainer = OrchestratorSFTTrainer(config) trainer.train() return {"status": "sft_training_complete"} diff --git a/src/openjarvis/learning/intelligence/orchestrator/types.py b/src/openjarvis/learning/intelligence/orchestrator/types.py index de664013..f1cb7bb8 100644 --- a/src/openjarvis/learning/intelligence/orchestrator/types.py +++ b/src/openjarvis/learning/intelligence/orchestrator/types.py @@ -57,9 +57,7 @@ def extract_answer(text: str) -> str: return text -def grade_answer( - predicted: str, expected: str, tolerance: float = 1e-6 -) -> bool: +def grade_answer(predicted: str, expected: str, tolerance: float = 1e-6) -> bool: """Grade an answer against expected, with smart matching. Handles: @@ -286,9 +284,7 @@ class EpisodeState: """Return number of turns so far.""" return len(self.history) - def to_episode( - self, task_id: str, ground_truth: str, correct: bool - ) -> Episode: + def to_episode(self, task_id: str, ground_truth: str, correct: bool) -> Episode: """Convert state to Episode for reward computation.""" episode = Episode( task_id=task_id, diff --git a/src/openjarvis/learning/intelligence/sft_trainer.py b/src/openjarvis/learning/intelligence/sft_trainer.py index 3cdbf596..a0a1576f 100644 --- a/src/openjarvis/learning/intelligence/sft_trainer.py +++ b/src/openjarvis/learning/intelligence/sft_trainer.py @@ -115,9 +115,7 @@ class SFTTrainer: ) try: - trainer = LoRATrainer( - lora_config, model_name=self.config.model_name - ) + trainer = LoRATrainer(lora_config, model_name=self.config.model_name) return trainer.train(pairs) except Exception as exc: logger.warning("SFT LoRA training failed: %s", exc) diff --git a/src/openjarvis/learning/learning_orchestrator.py b/src/openjarvis/learning/learning_orchestrator.py index 6e91ff32..6c77ad0a 100644 --- a/src/openjarvis/learning/learning_orchestrator.py +++ b/src/openjarvis/learning/learning_orchestrator.py @@ -66,9 +66,7 @@ class LearningOrchestrator: self._model_name = model_name self._miner = TrainingDataMiner(trace_store, min_quality=min_quality) - self._evolver = AgentConfigEvolver( - trace_store, config_dir=self._config_dir - ) + self._evolver = AgentConfigEvolver(trace_store, config_dir=self._config_dir) # ------------------------------------------------------------------ # public API @@ -132,16 +130,11 @@ class LearningOrchestrator: agent_name = rec.get("recommended_agent", "default") tools = rec.get("recommended_tools", []) max_turns = rec.get("recommended_max_turns", 10) - self._evolver.write_config( - agent_name, tools=tools, max_turns=max_turns - ) + self._evolver.write_config(agent_name, tools=tools, max_turns=max_turns) # 6. LoRA training (optional) result["lora_training"] = None - if ( - self._lora_config is not None - and len(sft_pairs) >= self._min_sft_pairs - ): + if self._lora_config is not None and len(sft_pairs) >= self._min_sft_pairs: lora_result = self._try_lora_training(sft_pairs) result["lora_training"] = lora_result @@ -195,9 +188,7 @@ class LearningOrchestrator: try: model_name = self._model_name or "Qwen/Qwen3-0.6B" - trainer = LoRATrainer( - self._lora_config, model_name=model_name - ) + trainer = LoRATrainer(self._lora_config, model_name=model_name) return trainer.train(sft_pairs) except Exception as exc: logger.warning("LoRA training failed: %s", exc) diff --git a/src/openjarvis/learning/optimize/feedback/collector.py b/src/openjarvis/learning/optimize/feedback/collector.py index 8a4f53b1..a5adfe00 100644 --- a/src/openjarvis/learning/optimize/feedback/collector.py +++ b/src/openjarvis/learning/optimize/feedback/collector.py @@ -30,21 +30,25 @@ class FeedbackCollector: source: str = "api", ) -> None: """Record an explicit numeric score (0-1) for a trace.""" - self._records.append({ - "trace_id": trace_id, - "score": min(max(score, 0.0), 1.0), - "source": source, - "timestamp": time.time(), - }) + self._records.append( + { + "trace_id": trace_id, + "score": min(max(score, 0.0), 1.0), + "source": source, + "timestamp": time.time(), + } + ) def record_thumbs(self, trace_id: str, thumbs_up: bool) -> None: """Record a thumbs-up / thumbs-down signal (converted to 1.0/0.0).""" - self._records.append({ - "trace_id": trace_id, - "score": 1.0 if thumbs_up else 0.0, - "source": "thumbs", - "timestamp": time.time(), - }) + self._records.append( + { + "trace_id": trace_id, + "score": 1.0 if thumbs_up else 0.0, + "source": "thumbs", + "timestamp": time.time(), + } + ) # ------------------------------------------------------------------ # Judge-driven evaluation @@ -78,7 +82,8 @@ class FeedbackCollector: # ------------------------------------------------------------------ def get_records( - self, trace_id: Optional[str] = None, + self, + trace_id: Optional[str] = None, ) -> List[Dict[str, Any]]: """Return stored records, optionally filtered by *trace_id*.""" if trace_id is None: diff --git a/src/openjarvis/learning/optimize/feedback/judge.py b/src/openjarvis/learning/optimize/feedback/judge.py index 24012766..07c7bfa9 100644 --- a/src/openjarvis/learning/optimize/feedback/judge.py +++ b/src/openjarvis/learning/optimize/feedback/judge.py @@ -103,7 +103,8 @@ class TraceJudge: return score, response def batch_evaluate( - self, traces: List[Trace], + self, + traces: List[Trace], ) -> List[Tuple[float, str]]: """Evaluate multiple traces sequentially. diff --git a/src/openjarvis/learning/optimize/llm_optimizer.py b/src/openjarvis/learning/optimize/llm_optimizer.py index f95f851e..50c40267 100644 --- a/src/openjarvis/learning/optimize/llm_optimizer.py +++ b/src/openjarvis/learning/optimize/llm_optimizer.py @@ -52,9 +52,7 @@ class LLMOptimizer: def propose_initial(self) -> TrialConfig: """Propose a reasonable starting config from the search space.""" if self.optimizer_backend is None: - raise ValueError( - "optimizer_backend is required to propose configurations" - ) + raise ValueError("optimizer_backend is required to propose configurations") prompt = self._build_initial_prompt() response = self.optimizer_backend.generate( @@ -74,9 +72,7 @@ class LLMOptimizer: ) -> TrialConfig: """Ask the LLM to propose the next config to evaluate.""" if self.optimizer_backend is None: - raise ValueError( - "optimizer_backend is required to propose configurations" - ) + raise ValueError("optimizer_backend is required to propose configurations") prompt = self._build_propose_prompt(history, traces, frontier_ids=frontier_ids) response = self.optimizer_backend.generate( @@ -98,12 +94,14 @@ class LLMOptimizer: ) -> TrialFeedback: """Ask the LLM to analyze a completed trial. Returns structured feedback.""" if self.optimizer_backend is None: - raise ValueError( - "optimizer_backend is required to analyze trials" - ) + raise ValueError("optimizer_backend is required to analyze trials") prompt = self._build_analyze_prompt( - trial, summary, traces, sample_scores, per_benchmark, + trial, + summary, + traces, + sample_scores, + per_benchmark, ) response = self.optimizer_backend.generate( prompt, @@ -121,15 +119,11 @@ class LLMOptimizer: def _build_initial_prompt(self) -> str: """Construct the prompt for the initial config proposal.""" lines: List[str] = [] - lines.append( - "You are optimizing an OpenJarvis AI system configuration." - ) + lines.append("You are optimizing an OpenJarvis AI system configuration.") lines.append("") lines.append(self.search_space.to_prompt_description()) lines.append("## Objective") - lines.append( - "Maximize accuracy while minimizing latency and cost." - ) + lines.append("Maximize accuracy while minimizing latency and cost.") lines.append("") lines.append("## Your Task") lines.append( @@ -138,12 +132,9 @@ class LLMOptimizer: "accuracy, latency, and cost." ) lines.append("") + lines.append("Return a JSON object inside a ```json code block with:") lines.append( - "Return a JSON object inside a ```json code block with:" - ) - lines.append( - '1. "params": dict of config params (dotted keys matching ' - "the search space)" + '1. "params": dict of config params (dotted keys matching the search space)' ) lines.append( '2. "reasoning": string explaining why this is a good ' @@ -159,9 +150,7 @@ class LLMOptimizer: ) -> str: """Construct the full prompt for propose_next.""" lines: List[str] = [] - lines.append( - "You are optimizing an OpenJarvis AI system configuration." - ) + lines.append("You are optimizing an OpenJarvis AI system configuration.") lines.append("") lines.append(self.search_space.to_prompt_description()) @@ -178,9 +167,7 @@ class LLMOptimizer: lines.append("") lines.append("## Objective") - lines.append( - "Maximize accuracy while minimizing latency and cost." - ) + lines.append("Maximize accuracy while minimizing latency and cost.") lines.append("") lines.append("## Your Task") lines.append( @@ -188,16 +175,12 @@ class LLMOptimizer: "previous trials to improve results." ) lines.append("") + lines.append("Return a JSON object inside a ```json code block with:") lines.append( - "Return a JSON object inside a ```json code block with:" + '1. "params": dict of config params (dotted keys matching the search space)' ) lines.append( - '1. "params": dict of config params (dotted keys matching ' - "the search space)" - ) - lines.append( - '2. "reasoning": string explaining why this config should ' - "improve results" + '2. "reasoning": string explaining why this config should improve results' ) return "\n".join(lines) @@ -242,9 +225,7 @@ class LLMOptimizer: lines.append("## Aggregate Results") lines.append(f"- accuracy: {summary.accuracy:.4f}") - lines.append( - f"- mean_latency_seconds: {summary.mean_latency_seconds:.4f}" - ) + lines.append(f"- mean_latency_seconds: {summary.mean_latency_seconds:.4f}") lines.append(f"- total_cost_usd: {summary.total_cost_usd:.4f}") lines.append(f"- total_samples: {summary.total_samples}") lines.append(f"- scored_samples: {summary.scored_samples}") @@ -299,31 +280,24 @@ class LLMOptimizer: lines.append(f"### Trial {i} (id={result.trial_id}){tag}") lines.append(f"Params: {json.dumps(result.config.params)}") lines.append(f"Accuracy: {result.accuracy:.4f}") - lines.append( - f"Latency: {result.mean_latency_seconds:.4f}s" - ) + lines.append(f"Latency: {result.mean_latency_seconds:.4f}s") lines.append(f"Cost: ${result.total_cost_usd:.4f}") lines.append(f"Energy: {result.total_energy_joules:.4f}J") if result.per_benchmark: bench_parts = [ - f"{b.benchmark}={b.accuracy:.4f}" - for b in result.per_benchmark + f"{b.benchmark}={b.accuracy:.4f}" for b in result.per_benchmark ] lines.append(f"Per-benchmark accuracy: {', '.join(bench_parts)}") if result.summary: s = result.summary if s.throughput_stats: - lines.append( - f"Throughput: {s.throughput_stats.mean:.2f} tok/s" - ) + lines.append(f"Throughput: {s.throughput_stats.mean:.2f} tok/s") if s.ipw_stats: lines.append(f"IPW: {s.ipw_stats.mean:.4f}") if result.structured_feedback: fb = result.structured_feedback if fb.failure_patterns: - lines.append( - f"Failure patterns: {', '.join(fb.failure_patterns)}" - ) + lines.append(f"Failure patterns: {', '.join(fb.failure_patterns)}") if fb.primitive_ratings: ratings = ", ".join( f"{k}={v}" for k, v in sorted(fb.primitive_ratings.items()) @@ -334,9 +308,7 @@ class LLMOptimizer: elif result.analysis: lines.append(f"Analysis: {result.analysis}") if result.failure_modes: - lines.append( - f"Failure modes: {', '.join(result.failure_modes)}" - ) + lines.append(f"Failure modes: {', '.join(result.failure_modes)}") lines.append("") return "\n".join(lines) @@ -355,8 +327,7 @@ class LLMOptimizer: for trace in recent: lines.append( - f"### Trace {trace.trace_id} " - f"(agent={trace.agent}, model={trace.model})" + f"### Trace {trace.trace_id} (agent={trace.agent}, model={trace.model})" ) lines.append(f"Query: {trace.query}") if trace.outcome: @@ -387,8 +358,7 @@ class LLMOptimizer: ) if len(trace.steps) > max_steps_per_trace: lines.append( - f" ... ({len(trace.steps) - max_steps_per_trace} " - "more steps)" + f" ... ({len(trace.steps) - max_steps_per_trace} more steps)" ) result_text = trace.result @@ -408,12 +378,13 @@ class LLMOptimizer: ) -> TrialConfig: """Propose a config that only changes one primitive.""" if self.optimizer_backend is None: - raise ValueError( - "optimizer_backend is required to propose configurations" - ) + raise ValueError("optimizer_backend is required to propose configurations") prompt = self._build_targeted_prompt( - history, base_config, target_primitive, frontier_ids, + history, + base_config, + target_primitive, + frontier_ids, ) response = self.optimizer_backend.generate( prompt, @@ -442,9 +413,7 @@ class LLMOptimizer: ) -> TrialConfig: """Combine best aspects of frontier members into one config.""" if self.optimizer_backend is None: - raise ValueError( - "optimizer_backend is required to propose configurations" - ) + raise ValueError("optimizer_backend is required to propose configurations") prompt = self._build_merge_prompt(candidates, history, frontier_ids) response = self.optimizer_backend.generate( @@ -469,9 +438,7 @@ class LLMOptimizer: ) -> str: """Build prompt for primitive-targeted mutation.""" lines: List[str] = [] - lines.append( - "You are optimizing an OpenJarvis AI system configuration." - ) + lines.append("You are optimizing an OpenJarvis AI system configuration.") lines.append("") lines.append(self.search_space.to_prompt_description()) @@ -508,9 +475,7 @@ class LLMOptimizer: ) -> str: """Build prompt for merging frontier configs.""" lines: List[str] = [] - lines.append( - "You are optimizing an OpenJarvis AI system configuration." - ) + lines.append("You are optimizing an OpenJarvis AI system configuration.") lines.append("") lines.append(self.search_space.to_prompt_description()) @@ -600,7 +565,8 @@ class LLMOptimizer: break except json.JSONDecodeError as exc: logger.debug( - "Failed to parse LLM optimizer JSON response: %s", exc, + "Failed to parse LLM optimizer JSON response: %s", + exc, ) continue @@ -641,9 +607,7 @@ class LLMOptimizer: logger.debug("Failed to parse LLM optimizer JSON response: %s", exc) # Try to extract from a generic ``` code block - code_block_match = re.search( - r"```\s*\n?(.*?)\n?\s*```", response, re.DOTALL - ) + code_block_match = re.search(r"```\s*\n?(.*?)\n?\s*```", response, re.DOTALL) if code_block_match: raw_json = code_block_match.group(1).strip() try: @@ -673,9 +637,7 @@ class LLMOptimizer: reasoning="Failed to parse LLM response.", ) - def _config_from_dict( - self, data: Dict[str, Any], trial_id: str - ) -> TrialConfig: + def _config_from_dict(self, data: Dict[str, Any], trial_id: str) -> TrialConfig: """Build a TrialConfig from a parsed JSON dict. Merges fixed parameters from the search space so that fixed values diff --git a/src/openjarvis/learning/optimize/optimizer.py b/src/openjarvis/learning/optimize/optimizer.py index e7f5911c..82a4ca4b 100644 --- a/src/openjarvis/learning/optimize/optimizer.py +++ b/src/openjarvis/learning/optimize/optimizer.py @@ -47,8 +47,10 @@ def _get_objective_value(trial: TrialResult, obj: ObjectiveSpec) -> float: """Read the metric value from a TrialResult for a given objective.""" # Direct attributes on TrialResult direct = { - "accuracy", "mean_latency_seconds", - "total_cost_usd", "total_energy_joules", + "accuracy", + "mean_latency_seconds", + "total_cost_usd", + "total_energy_joules", } if obj.metric in direct: return getattr(trial, obj.metric, 0.0) @@ -99,12 +101,10 @@ def compute_pareto_frontier( continue # Check if other dominates trial all_ge = all( - trial_vals[j][k] >= trial_vals[i][k] - for k in range(len(objectives)) + trial_vals[j][k] >= trial_vals[i][k] for k in range(len(objectives)) ) any_gt = any( - trial_vals[j][k] > trial_vals[i][k] - for k in range(len(objectives)) + trial_vals[j][k] > trial_vals[i][k] for k in range(len(objectives)) ) if all_ge and any_gt: dominated = True @@ -170,9 +170,7 @@ class OptimizationEngine: benchmark_name = getattr(self.trial_runner, "benchmark", "") benchmark_names: List[str] = [] if isinstance(self.trial_runner, MultiBenchTrialRunner): - benchmark_names = [ - s.benchmark for s in self.trial_runner.benchmark_specs - ] + benchmark_names = [s.benchmark for s in self.trial_runner.benchmark_specs] benchmark_name = "+".join(benchmark_names) optimization_run = OptimizationRun( @@ -228,9 +226,7 @@ class OptimizationEngine: total_energy_joules=result.total_energy_joules, total_samples=result.samples_evaluated, scored_samples=result.samples_evaluated, - correct=int( - result.accuracy * result.samples_evaluated - ), + correct=int(result.accuracy * result.samples_evaluated), errors=0, total_input_tokens=0, total_output_tokens=result.total_tokens, @@ -251,7 +247,8 @@ class OptimizationEngine: # Recompute Pareto frontier optimization_run.pareto_frontier = compute_pareto_frontier( - history, optimization_run.objectives, + history, + optimization_run.objectives, ) frontier_ids = {t.trial_id for t in optimization_run.pareto_frontier} @@ -286,14 +283,13 @@ class OptimizationEngine: if result.structured_feedback: target_primitive = result.structured_feedback.target_primitive - if ( - trial_num % 5 == 0 - and len(optimization_run.pareto_frontier) >= 2 - ): + if trial_num % 5 == 0 and len(optimization_run.pareto_frontier) >= 2: # Merge frontier members periodically candidates = optimization_run.pareto_frontier[:3] config = self.llm_optimizer.propose_merge( - candidates, history, frontier_ids=frontier_ids, + candidates, + history, + frontier_ids=frontier_ids, ) elif target_primitive and trial_num > 2: # Targeted mutation on the suggested primitive @@ -305,7 +301,8 @@ class OptimizationEngine: ) else: config = self.llm_optimizer.propose_next( - history, frontier_ids=frontier_ids, + history, + frontier_ids=frontier_ids, ) optimization_run.status = "completed" @@ -315,9 +312,7 @@ class OptimizationEngine: return optimization_run - def export_best_recipe( - self, run: OptimizationRun, path: Path - ) -> Path: + def export_best_recipe(self, run: OptimizationRun, path: Path) -> Path: """Export the best trial's config as a TOML recipe file. Args: @@ -414,9 +409,7 @@ class OptimizationEngine: return recipe @staticmethod - def _write_toml_fallback( - data: Dict[str, Any], path: Path - ) -> None: + def _write_toml_fallback(data: Dict[str, Any], path: Path) -> None: """Write a simple nested dict as TOML without tomli_w.""" lines: List[str] = [] for section, values in data.items(): @@ -432,8 +425,7 @@ class OptimizationEngine: lines.append(f"{key} = {val}") elif isinstance(val, list): items = ", ".join( - f'"{v}"' if isinstance(v, str) else str(v) - for v in val + f'"{v}"' if isinstance(v, str) else str(v) for v in val ) lines.append(f"{key} = [{items}]") else: diff --git a/src/openjarvis/learning/optimize/personal/scorer.py b/src/openjarvis/learning/optimize/personal/scorer.py index 8ed0bec1..364908a2 100644 --- a/src/openjarvis/learning/optimize/personal/scorer.py +++ b/src/openjarvis/learning/optimize/personal/scorer.py @@ -18,7 +18,9 @@ class PersonalBenchmarkScorer(LLMJudgeScorer): super().__init__(judge_backend, judge_model) def score( - self, record: EvalRecord, model_answer: str, + self, + record: EvalRecord, + model_answer: str, ) -> Tuple[Optional[bool], Dict[str, Any]]: """Compare *model_answer* against *record.reference* using the judge LLM. @@ -37,7 +39,8 @@ class PersonalBenchmarkScorer(LLMJudgeScorer): "then explain your reasoning." ) response = self._ask_judge( - prompt, system="You are an impartial quality judge.", + prompt, + system="You are an impartial quality judge.", ) first_line = response.strip().split("\n")[0].strip().upper() is_correct = first_line.startswith("YES") diff --git a/src/openjarvis/learning/optimize/search_space.py b/src/openjarvis/learning/optimize/search_space.py index 70dbcd2a..5ffa26ef 100644 --- a/src/openjarvis/learning/optimize/search_space.py +++ b/src/openjarvis/learning/optimize/search_space.py @@ -125,9 +125,16 @@ DEFAULT_SEARCH_SPACE = SearchSpace( name="engine.backend", dim_type="categorical", values=[ - "ollama", "vllm", "sglang", "llamacpp", - "mlx", "lmstudio", "exo", "nexa", - "uzu", "apple_fm", + "ollama", + "vllm", + "sglang", + "llamacpp", + "mlx", + "lmstudio", + "exo", + "nexa", + "uzu", + "apple_fm", ], description="Inference engine backend", primitive="engine", diff --git a/src/openjarvis/learning/optimize/store.py b/src/openjarvis/learning/optimize/store.py index 49bb379e..8841384f 100644 --- a/src/openjarvis/learning/optimize/store.py +++ b/src/openjarvis/learning/optimize/store.py @@ -74,19 +74,16 @@ INSERT OR REPLACE INTO trial_results ( _MIGRATE_TRIALS = [ - "ALTER TABLE trial_results ADD COLUMN " - "sample_scores TEXT NOT NULL DEFAULT '[]'", + "ALTER TABLE trial_results ADD COLUMN sample_scores TEXT NOT NULL DEFAULT '[]'", "ALTER TABLE trial_results ADD COLUMN " "structured_feedback TEXT NOT NULL DEFAULT '{}'", - "ALTER TABLE trial_results ADD COLUMN " - "per_benchmark TEXT NOT NULL DEFAULT '[]'", + "ALTER TABLE trial_results ADD COLUMN per_benchmark TEXT NOT NULL DEFAULT '[]'", ] _MIGRATE_RUNS = [ "ALTER TABLE optimization_runs ADD COLUMN " "pareto_frontier_ids TEXT NOT NULL DEFAULT '[]'", - "ALTER TABLE optimization_runs ADD COLUMN " - "benchmarks TEXT NOT NULL DEFAULT '[]'", + "ALTER TABLE optimization_runs ADD COLUMN benchmarks TEXT NOT NULL DEFAULT '[]'", ] @@ -184,40 +181,48 @@ class OptimizationStore: """Persist a single trial result.""" now = time.time() # Serialize sample_scores - scores_json = json.dumps([ - { - "record_id": s.record_id, - "is_correct": s.is_correct, - "score": s.score, - "latency_seconds": s.latency_seconds, - "prompt_tokens": s.prompt_tokens, - "completion_tokens": s.completion_tokens, - "cost_usd": s.cost_usd, - "error": s.error, - "ttft": s.ttft, - "energy_joules": s.energy_joules, - "power_watts": s.power_watts, - "gpu_utilization_pct": s.gpu_utilization_pct, - "throughput_tok_per_sec": s.throughput_tok_per_sec, - "mfu_pct": s.mfu_pct, - "mbu_pct": s.mbu_pct, - "ipw": s.ipw, - "ipj": s.ipj, - "energy_per_output_token_joules": s.energy_per_output_token_joules, - "throughput_per_watt": s.throughput_per_watt, - "mean_itl_ms": s.mean_itl_ms, - } - for s in trial.sample_scores - ]) + scores_json = json.dumps( + [ + { + "record_id": s.record_id, + "is_correct": s.is_correct, + "score": s.score, + "latency_seconds": s.latency_seconds, + "prompt_tokens": s.prompt_tokens, + "completion_tokens": s.completion_tokens, + "cost_usd": s.cost_usd, + "error": s.error, + "ttft": s.ttft, + "energy_joules": s.energy_joules, + "power_watts": s.power_watts, + "gpu_utilization_pct": s.gpu_utilization_pct, + "throughput_tok_per_sec": s.throughput_tok_per_sec, + "mfu_pct": s.mfu_pct, + "mbu_pct": s.mbu_pct, + "ipw": s.ipw, + "ipj": s.ipj, + "energy_per_output_token_joules": s.energy_per_output_token_joules, + "throughput_per_watt": s.throughput_per_watt, + "mean_itl_ms": s.mean_itl_ms, + } + for s in trial.sample_scores + ] + ) # Serialize structured_feedback fb = trial.structured_feedback - fb_json = json.dumps({ - "summary_text": fb.summary_text, - "failure_patterns": fb.failure_patterns, - "primitive_ratings": fb.primitive_ratings, - "suggested_changes": fb.suggested_changes, - "target_primitive": fb.target_primitive, - }) if fb else "{}" + fb_json = ( + json.dumps( + { + "summary_text": fb.summary_text, + "failure_patterns": fb.failure_patterns, + "primitive_ratings": fb.primitive_ratings, + "suggested_changes": fb.suggested_changes, + "target_primitive": fb.target_primitive, + } + ) + if fb + else "{}" + ) self._conn.execute( _INSERT_TRIAL, @@ -238,20 +243,22 @@ class OptimizationStore: ), ) # Serialize per_benchmark - pb_json = json.dumps([ - { - "benchmark": b.benchmark, - "accuracy": b.accuracy, - "mean_latency_seconds": b.mean_latency_seconds, - "total_cost_usd": b.total_cost_usd, - "total_energy_joules": b.total_energy_joules, - "total_tokens": b.total_tokens, - "samples_evaluated": b.samples_evaluated, - "errors": b.errors, - "weight": b.weight, - } - for b in trial.per_benchmark - ]) + pb_json = json.dumps( + [ + { + "benchmark": b.benchmark, + "accuracy": b.accuracy, + "mean_latency_seconds": b.mean_latency_seconds, + "total_cost_usd": b.total_cost_usd, + "total_energy_joules": b.total_energy_joules, + "total_tokens": b.total_tokens, + "samples_evaluated": b.samples_evaluated, + "errors": b.errors, + "weight": b.weight, + } + for b in trial.per_benchmark + ] + ) # Update new columns separately self._conn.execute( @@ -429,7 +436,8 @@ class OptimizationStore: ipw=s.get("ipw", 0.0), ipj=s.get("ipj", 0.0), energy_per_output_token_joules=s.get( - "energy_per_output_token_joules", 0.0, + "energy_per_output_token_joules", + 0.0, ), throughput_per_watt=s.get("throughput_per_watt", 0.0), mean_itl_ms=s.get("mean_itl_ms", 0.0), diff --git a/src/openjarvis/learning/optimize/trial_runner.py b/src/openjarvis/learning/optimize/trial_runner.py index 54a59f72..19588062 100644 --- a/src/openjarvis/learning/optimize/trial_runner.py +++ b/src/openjarvis/learning/optimize/trial_runner.py @@ -83,12 +83,17 @@ class TrialRunner: ) judge_backend = _build_judge_backend(run_config.judge_model) scorer = _build_scorer( - self.benchmark, judge_backend, run_config.judge_model, + self.benchmark, + judge_backend, + run_config.judge_model, ) try: eval_runner = EvalRunner( - run_config, dataset, backend, scorer, + run_config, + dataset, + backend, + scorer, ) summary: RunSummary = eval_runner.run() eval_results = eval_runner.results @@ -238,7 +243,9 @@ class MultiBenchTrialRunner: return self._aggregate(trial, per_benchmark) def _run_terminalbench_native( - self, trial: TrialConfig, spec: BenchmarkSpec, + self, + trial: TrialConfig, + spec: BenchmarkSpec, ) -> BenchmarkScore: """Run terminal-bench natively via Harness with Docker execution.""" import time @@ -268,8 +275,7 @@ class MultiBenchTrialRunner: "dataset_version": "0.1.1", "model_name": litellm_model, "agent_import_path": ( - "openjarvis.evals.backends.tb_agent" - ":OpenJarvisTerminalBenchAgent" + "openjarvis.evals.backends.tb_agent:OpenJarvisTerminalBenchAgent" ), "agent_kwargs": { "model_name": litellm_model, @@ -285,7 +291,8 @@ class MultiBenchTrialRunner: LOGGER.info( "Running terminal-bench native: model=%s, max_tasks=%s", - litellm_model, spec.max_samples, + litellm_model, + spec.max_samples, ) t0 = time.monotonic() @@ -328,7 +335,10 @@ class MultiBenchTrialRunner: LOGGER.info( "terminal-bench native: %d/%d resolved (%.1f%%), %.1fs total", - resolved, total_tasks, accuracy * 100, elapsed, + resolved, + total_tasks, + accuracy * 100, + elapsed, ) return BenchmarkScore( @@ -338,7 +348,8 @@ class MultiBenchTrialRunner: total_tokens=total_input + total_output, samples_evaluated=total_tasks, errors=sum( - 1 for r in tb_results.results + 1 + for r in tb_results.results if r.failure_mode.value not in ("none", "unset") ), weight=spec.weight, diff --git a/src/openjarvis/learning/routing/complexity.py b/src/openjarvis/learning/routing/complexity.py index 7ce77137..3a60c7ac 100644 --- a/src/openjarvis/learning/routing/complexity.py +++ b/src/openjarvis/learning/routing/complexity.py @@ -58,11 +58,11 @@ _THINKING_MODEL_PATTERNS = re.compile( # --------------------------------------------------------------------------- _TOKEN_TIERS = { - "trivial": 1024, # greetings, yes/no, factoid lookups - "simple": 2048, # short answers, definitions - "moderate": 4096, # explanations, summaries - "complex": 8192, # analysis, code generation, multi-step - "very_complex": 16384, # long-form, multi-part reasoning + "trivial": 1024, # greetings, yes/no, factoid lookups + "simple": 2048, # short answers, definitions + "moderate": 4096, # explanations, summaries + "complex": 8192, # analysis, code generation, multi-step + "very_complex": 16384, # long-form, multi-part reasoning } # Thinking models need extra headroom for internal chain-of-thought. @@ -204,9 +204,7 @@ def is_thinking_model(model_name: str) -> bool: return bool(_THINKING_MODEL_PATTERNS.search(model_name)) -def adjust_tokens_for_model( - suggested: int, model_name: Optional[str] = None -) -> int: +def adjust_tokens_for_model(suggested: int, model_name: Optional[str] = None) -> int: """Multiply the token budget when the target is a thinking model.""" if model_name and is_thinking_model(model_name): return suggested * _THINKING_TOKEN_MULTIPLIER @@ -226,9 +224,7 @@ class ComplexityQueryAnalyzer(QueryAnalyzer): to the returned ``RoutingContext``. """ - def analyze( - self, query: str, **kwargs: object - ) -> RoutingContext: + def analyze(self, query: str, **kwargs: object) -> RoutingContext: urgency = kwargs.get("urgency", 0.5) if not isinstance(urgency, (int, float)): urgency = 0.5 diff --git a/src/openjarvis/learning/routing/heuristic_reward.py b/src/openjarvis/learning/routing/heuristic_reward.py index b30e2cbc..4957a157 100644 --- a/src/openjarvis/learning/routing/heuristic_reward.py +++ b/src/openjarvis/learning/routing/heuristic_reward.py @@ -43,9 +43,7 @@ class HeuristicRewardFunction(RewardFunction): latency_score = max(0.0, 1.0 - latency / self.max_latency) cost_score = max(0.0, 1.0 - cost / self.max_cost) - efficiency_score = ( - completion_tokens / total_tokens if total_tokens > 0 else 0.5 - ) + efficiency_score = completion_tokens / total_tokens if total_tokens > 0 else 0.5 reward = ( self.weight_latency * latency_score diff --git a/src/openjarvis/learning/routing/learned_router.py b/src/openjarvis/learning/routing/learned_router.py index 057b54fd..8a7450c6 100644 --- a/src/openjarvis/learning/routing/learned_router.py +++ b/src/openjarvis/learning/routing/learned_router.py @@ -204,8 +204,11 @@ class _ModelScore: """Accumulator for per-model scoring during policy update.""" __slots__ = ( - "count", "successes", "total_latency", - "feedback_sum", "feedback_count", + "count", + "successes", + "total_latency", + "feedback_sum", + "feedback_count", ) def __init__(self) -> None: @@ -218,10 +221,7 @@ class _ModelScore: def composite_score(self) -> float: """Weighted score combining success rate and feedback.""" sr = self.successes / self.count if self.count else 0.0 - fb = ( - self.feedback_sum / self.feedback_count - if self.feedback_count else 0.5 - ) + fb = self.feedback_sum / self.feedback_count if self.feedback_count else 0.5 return 0.6 * sr + 0.4 * fb diff --git a/src/openjarvis/learning/routing/router.py b/src/openjarvis/learning/routing/router.py index 4862c01f..1950e48d 100644 --- a/src/openjarvis/learning/routing/router.py +++ b/src/openjarvis/learning/routing/router.py @@ -126,9 +126,8 @@ class HeuristicRouter(RouterPolicy): # Rule 1: Code detected → prefer model with code/coder in name if context.has_code: - code_model = ( - _find_model_by_tag(available, "code") - or _find_model_by_tag(available, "coder") + code_model = _find_model_by_tag(available, "code") or _find_model_by_tag( + available, "coder" ) if code_model: return code_model diff --git a/src/openjarvis/learning/training/lora.py b/src/openjarvis/learning/training/lora.py index 3dd861e9..587707ef 100644 --- a/src/openjarvis/learning/training/lora.py +++ b/src/openjarvis/learning/training/lora.py @@ -78,9 +78,7 @@ class LoRATrainingConfig: lora_rank: int = 16 lora_alpha: int = 32 lora_dropout: float = 0.05 - target_modules: List[str] = field( - default_factory=lambda: ["q_proj", "v_proj"] - ) + target_modules: List[str] = field(default_factory=lambda: ["q_proj", "v_proj"]) # Training params num_epochs: int = 3 @@ -103,13 +101,9 @@ class LoRATrainingConfig: def __post_init__(self) -> None: if self.lora_rank < 1: - raise ValueError( - f"lora_rank must be >= 1, got {self.lora_rank}" - ) + raise ValueError(f"lora_rank must be >= 1, got {self.lora_rank}") if self.num_epochs < 1: - raise ValueError( - f"num_epochs must be >= 1, got {self.num_epochs}" - ) + raise ValueError(f"num_epochs must be >= 1, got {self.num_epochs}") # --------------------------------------------------------------------------- @@ -156,9 +150,7 @@ class LoRATrainer: # -- Public API ---------------------------------------------------------- - def prepare_dataset( - self, pairs: List[Dict[str, Any]] - ) -> List[Dict[str, Any]]: + def prepare_dataset(self, pairs: List[Dict[str, Any]]) -> List[Dict[str, Any]]: """Convert SFT pairs to tokenized examples. Each returned dict contains ``input_ids``, ``attention_mask``, @@ -182,11 +174,13 @@ class LoRATrainer: padding="max_length", return_tensors="pt", ) - dataset.append({ - "input_ids": encoding["input_ids"].squeeze(0), - "attention_mask": encoding["attention_mask"].squeeze(0), - "text": text, - }) + dataset.append( + { + "input_ids": encoding["input_ids"].squeeze(0), + "attention_mask": encoding["attention_mask"].squeeze(0), + "text": text, + } + ) return dataset @@ -300,8 +294,7 @@ class LoRATrainer: ) except ImportError: logger.warning( - "bitsandbytes not installed; falling back to bf16 " - "(QLoRA disabled)" + "bitsandbytes not installed; falling back to bf16 (QLoRA disabled)" ) if self.device == "cuda" or self.device == "auto": @@ -324,8 +317,7 @@ class LoRATrainer: """Wrap the loaded model with LoRA adapters via peft.""" if not HAS_PEFT: raise ImportError( - "peft is required for LoRA training. " - "Install with: pip install peft" + "peft is required for LoRA training. Install with: pip install peft" ) lora_config = LoraConfig( @@ -390,9 +382,9 @@ class LoRATrainer: optimizer: Any, ) -> float: """Execute a single training step on a micro-batch.""" - input_ids = torch.stack( - [item["input_ids"] for item in batch_items] - ).to(self.device) + input_ids = torch.stack([item["input_ids"] for item in batch_items]).to( + self.device + ) attention_mask = torch.stack( [item["attention_mask"] for item in batch_items] ).to(self.device) diff --git a/src/openjarvis/operators/loader.py b/src/openjarvis/operators/loader.py index a4470b7d..f4fb7f64 100644 --- a/src/openjarvis/operators/loader.py +++ b/src/openjarvis/operators/loader.py @@ -36,7 +36,8 @@ def load_operator(path: str | Path) -> OperatorManifest: temperature = agent_data.get("temperature", op_data.get("temperature", 0.3)) system_prompt = agent_data.get("system_prompt", op_data.get("system_prompt", "")) system_prompt_path = agent_data.get( - "system_prompt_path", op_data.get("system_prompt_path", ""), + "system_prompt_path", + op_data.get("system_prompt_path", ""), ) # Load external system prompt if specified and inline is empty diff --git a/src/openjarvis/optimize/__init__.py b/src/openjarvis/optimize/__init__.py index d79d9c29..f1c5dd81 100644 --- a/src/openjarvis/optimize/__init__.py +++ b/src/openjarvis/optimize/__init__.py @@ -1,3 +1,4 @@ """Backward-compatibility shim -- optimize moved to learning.optimize.""" + from openjarvis.learning.optimize import * # noqa: F401,F403 from openjarvis.learning.optimize import __all__ # noqa: F401 diff --git a/src/openjarvis/optimize/config.py b/src/openjarvis/optimize/config.py index 84dc486b..84c7cf1d 100644 --- a/src/openjarvis/optimize/config.py +++ b/src/openjarvis/optimize/config.py @@ -1,3 +1,4 @@ """Backward-compatibility shim -- optimize.config moved to learning.optimize.config.""" + from openjarvis.learning.optimize.config import * # noqa: F401,F403 from openjarvis.learning.optimize.config import __all__ # noqa: F401 diff --git a/src/openjarvis/optimize/feedback/__init__.py b/src/openjarvis/optimize/feedback/__init__.py index 589fc3cc..009c6a69 100644 --- a/src/openjarvis/optimize/feedback/__init__.py +++ b/src/openjarvis/optimize/feedback/__init__.py @@ -1,3 +1,4 @@ """Backward-compat shim: moved to learning.optimize.""" + from openjarvis.learning.optimize.feedback import * # noqa: F401,F403 from openjarvis.learning.optimize.feedback import __all__ # noqa: F401 diff --git a/src/openjarvis/optimize/feedback/collector.py b/src/openjarvis/optimize/feedback/collector.py index 995c176a..273de082 100644 --- a/src/openjarvis/optimize/feedback/collector.py +++ b/src/openjarvis/optimize/feedback/collector.py @@ -1,3 +1,4 @@ """Backward-compat shim: moved to learning.optimize.""" + from openjarvis.learning.optimize.feedback.collector import * # noqa: F401,F403 from openjarvis.learning.optimize.feedback.collector import __all__ # noqa: F401 diff --git a/src/openjarvis/optimize/feedback/judge.py b/src/openjarvis/optimize/feedback/judge.py index 9010fc3a..1df46313 100644 --- a/src/openjarvis/optimize/feedback/judge.py +++ b/src/openjarvis/optimize/feedback/judge.py @@ -1,4 +1,5 @@ """Backward-compat shim: moved to learning.optimize.""" + from openjarvis.learning.optimize.feedback.judge import * # noqa: F401,F403 from openjarvis.learning.optimize.feedback.judge import ( __all__, # noqa: F401 diff --git a/src/openjarvis/optimize/llm_optimizer.py b/src/openjarvis/optimize/llm_optimizer.py index 1246b280..8db1a0bd 100644 --- a/src/openjarvis/optimize/llm_optimizer.py +++ b/src/openjarvis/optimize/llm_optimizer.py @@ -1,3 +1,4 @@ """Backward-compat shim: moved to learning.optimize.""" + from openjarvis.learning.optimize.llm_optimizer import * # noqa: F401,F403 from openjarvis.learning.optimize.llm_optimizer import __all__ # noqa: F401 diff --git a/src/openjarvis/optimize/optimizer.py b/src/openjarvis/optimize/optimizer.py index 67716608..8270d06b 100644 --- a/src/openjarvis/optimize/optimizer.py +++ b/src/openjarvis/optimize/optimizer.py @@ -1,3 +1,4 @@ """Backward-compat shim: moved to learning.optimize.""" + from openjarvis.learning.optimize.optimizer import * # noqa: F401,F403 from openjarvis.learning.optimize.optimizer import __all__ # noqa: F401 diff --git a/src/openjarvis/optimize/personal/__init__.py b/src/openjarvis/optimize/personal/__init__.py index 5b4a0623..7cea95cd 100644 --- a/src/openjarvis/optimize/personal/__init__.py +++ b/src/openjarvis/optimize/personal/__init__.py @@ -1,3 +1,4 @@ """Backward-compat shim: moved to learning.optimize.""" + from openjarvis.learning.optimize.personal import * # noqa: F401,F403 from openjarvis.learning.optimize.personal import __all__ # noqa: F401 diff --git a/src/openjarvis/optimize/personal/dataset.py b/src/openjarvis/optimize/personal/dataset.py index bde2f478..223960d5 100644 --- a/src/openjarvis/optimize/personal/dataset.py +++ b/src/openjarvis/optimize/personal/dataset.py @@ -1,3 +1,4 @@ """Backward-compat shim: moved to learning.optimize.""" + from openjarvis.learning.optimize.personal.dataset import * # noqa: F401,F403 from openjarvis.learning.optimize.personal.dataset import __all__ # noqa: F401 diff --git a/src/openjarvis/optimize/personal/scorer.py b/src/openjarvis/optimize/personal/scorer.py index 58737002..6db2e551 100644 --- a/src/openjarvis/optimize/personal/scorer.py +++ b/src/openjarvis/optimize/personal/scorer.py @@ -1,3 +1,4 @@ """Backward-compat shim: moved to learning.optimize.""" + from openjarvis.learning.optimize.personal.scorer import * # noqa: F401,F403 from openjarvis.learning.optimize.personal.scorer import __all__ # noqa: F401 diff --git a/src/openjarvis/optimize/personal/synthesizer.py b/src/openjarvis/optimize/personal/synthesizer.py index 0fcc5718..29c810d5 100644 --- a/src/openjarvis/optimize/personal/synthesizer.py +++ b/src/openjarvis/optimize/personal/synthesizer.py @@ -1,3 +1,4 @@ """Backward-compat shim: moved to learning.optimize.""" + from openjarvis.learning.optimize.personal.synthesizer import * # noqa: F401,F403 from openjarvis.learning.optimize.personal.synthesizer import __all__ # noqa: F401 diff --git a/src/openjarvis/optimize/search_space.py b/src/openjarvis/optimize/search_space.py index 3f28b613..6e274cf3 100644 --- a/src/openjarvis/optimize/search_space.py +++ b/src/openjarvis/optimize/search_space.py @@ -1,3 +1,4 @@ """Backward-compat shim: moved to learning.optimize.""" + from openjarvis.learning.optimize.search_space import * # noqa: F401,F403 from openjarvis.learning.optimize.search_space import __all__ # noqa: F401 diff --git a/src/openjarvis/optimize/store.py b/src/openjarvis/optimize/store.py index 6f734980..bdd232da 100644 --- a/src/openjarvis/optimize/store.py +++ b/src/openjarvis/optimize/store.py @@ -1,3 +1,4 @@ """Backward-compatibility shim -- optimize.store moved to learning.optimize.store.""" + from openjarvis.learning.optimize.store import * # noqa: F401,F403 from openjarvis.learning.optimize.store import __all__ # noqa: F401 diff --git a/src/openjarvis/optimize/trial_runner.py b/src/openjarvis/optimize/trial_runner.py index f1bd5abc..4dab8ed3 100644 --- a/src/openjarvis/optimize/trial_runner.py +++ b/src/openjarvis/optimize/trial_runner.py @@ -1,3 +1,4 @@ """Backward-compat shim: moved to learning.optimize.""" + from openjarvis.learning.optimize.trial_runner import * # noqa: F401,F403 from openjarvis.learning.optimize.trial_runner import __all__ # noqa: F401 diff --git a/src/openjarvis/optimize/types.py b/src/openjarvis/optimize/types.py index 6b96442a..fafadafb 100644 --- a/src/openjarvis/optimize/types.py +++ b/src/openjarvis/optimize/types.py @@ -1,4 +1,5 @@ """Backward-compatibility shim -- optimize.types moved to learning.optimize.types.""" + from openjarvis.learning.optimize.types import * # noqa: F401,F403 from openjarvis.learning.optimize.types import ( _PARAM_TO_RECIPE, # noqa: F401 diff --git a/src/openjarvis/prompt/builder.py b/src/openjarvis/prompt/builder.py index dd844668..cfd38f5a 100644 --- a/src/openjarvis/prompt/builder.py +++ b/src/openjarvis/prompt/builder.py @@ -40,7 +40,8 @@ class SystemPromptBuilder: sections: list[str] = [] sections.append(self._agent_template) soul = self._load_file( - self._mf_config.soul_path, self._sp_config.soul_max_chars, + self._mf_config.soul_path, + self._sp_config.soul_max_chars, ) if soul: sections.append(f"## Agent Persona\n\n{soul}") @@ -51,7 +52,8 @@ class SystemPromptBuilder: if memory: sections.append(f"## Agent Memory\n\n{memory}") user = self._load_file( - self._mf_config.user_path, self._sp_config.user_max_chars, + self._mf_config.user_path, + self._sp_config.user_max_chars, ) if user: sections.append(f"## User Profile\n\n{user}") diff --git a/src/openjarvis/recipes/composer.py b/src/openjarvis/recipes/composer.py index 75b23b76..2983afa4 100644 --- a/src/openjarvis/recipes/composer.py +++ b/src/openjarvis/recipes/composer.py @@ -75,14 +75,16 @@ def recipe_to_eval_suite( bench_cfgs: list[BenchmarkConfig] = [] for bname in bench_names: - bench_cfgs.append(BenchmarkConfig( - name=bname, - backend=backend, - max_samples=max_samples or recipe.eval_max_samples, - agent=recipe.agent_type if has_agent else None, - tools=list(recipe.tools) if has_agent else [], - judge_model=judge_model or recipe.eval_judge_model, - )) + bench_cfgs.append( + BenchmarkConfig( + name=bname, + backend=backend, + max_samples=max_samples or recipe.eval_max_samples, + agent=recipe.agent_type if has_agent else None, + tools=list(recipe.tools) if has_agent else [], + judge_model=judge_model or recipe.eval_judge_model, + ) + ) return EvalSuiteConfig( meta=MetaConfig( diff --git a/src/openjarvis/recipes/loader.py b/src/openjarvis/recipes/loader.py index 446592f0..9506b40a 100644 --- a/src/openjarvis/recipes/loader.py +++ b/src/openjarvis/recipes/loader.py @@ -242,7 +242,8 @@ def _load_operator_as_recipe(path: Path, data: Dict[str, Any]) -> Recipe: system_prompt = agent_data.get("system_prompt", op.get("system_prompt", "")) system_prompt_path = agent_data.get( - "system_prompt_path", op.get("system_prompt_path", ""), + "system_prompt_path", + op.get("system_prompt_path", ""), ) if not system_prompt and system_prompt_path: prompt_p = Path(system_prompt_path) diff --git a/src/openjarvis/sandbox/mount_security.py b/src/openjarvis/sandbox/mount_security.py index db968fbe..a5b703ba 100644 --- a/src/openjarvis/sandbox/mount_security.py +++ b/src/openjarvis/sandbox/mount_security.py @@ -177,15 +177,11 @@ def validate_mounts( valid: List[str] = [] for mount in mounts: if _is_blocked(mount, allowlist.blocked_patterns): - raise ValueError( - f"Mount path blocked by security policy: {mount}" - ) + raise ValueError(f"Mount path blocked by security policy: {mount}") if _is_under_allowed_root(mount, allowlist.roots): valid.append(mount) else: - raise ValueError( - f"Mount path not under any allowed root: {mount}" - ) + raise ValueError(f"Mount path not under any allowed root: {mount}") return valid diff --git a/src/openjarvis/sandbox/runner.py b/src/openjarvis/sandbox/runner.py index 997b6e20..e7a990ea 100644 --- a/src/openjarvis/sandbox/runner.py +++ b/src/openjarvis/sandbox/runner.py @@ -114,9 +114,12 @@ class ContainerRunner: runtime, "run", "--rm", - "--name", container_name, - "--label", "openjarvis-sandbox=true", - "--network", "none", + "--name", + container_name, + "--label", + "openjarvis-sandbox=true", + "--network", + "none", "-i", ] @@ -171,7 +174,9 @@ class ContainerRunner: payload["_workspace"] = workspace args = self._build_docker_args( - container_name, validated_mounts, env, + container_name, + validated_mounts, + env, ) try: @@ -186,9 +191,7 @@ class ContainerRunner: # Kill the container on timeout self.stop(container_name) return { - "content": ( - f"Container timed out after {self._timeout}s." - ), + "content": (f"Container timed out after {self._timeout}s."), "error": True, "error_type": "timeout", } @@ -197,7 +200,9 @@ class ContainerRunner: stderr = proc.stderr.strip() if proc.stderr else "" logger.error( "Container %s exited %d: %s", - container_name, proc.returncode, stderr, + container_name, + proc.returncode, + stderr, ) return { "content": f"Container failed: {stderr}", @@ -216,7 +221,7 @@ class ContainerRunner: if start == -1 or end == -1: return {"content": stdout.strip()} - json_str = stdout[start + len(_OUTPUT_START):end].strip() + json_str = stdout[start + len(_OUTPUT_START) : end].strip() try: return json.loads(json_str) except json.JSONDecodeError: @@ -233,7 +238,8 @@ class ContainerRunner: ) except Exception: logger.debug( - "Failed to stop container %s", container_name, + "Failed to stop container %s", + container_name, exc_info=True, ) @@ -243,8 +249,11 @@ class ContainerRunner: runtime = shutil.which(self._runtime) or self._runtime result = subprocess.run( [ - runtime, "ps", "-aq", - "--filter", "label=openjarvis-sandbox=true", + runtime, + "ps", + "-aq", + "--filter", + "label=openjarvis-sandbox=true", ], capture_output=True, text=True, @@ -263,7 +272,8 @@ class ContainerRunner: ) except Exception: logger.debug( - "Orphan cleanup failed", exc_info=True, + "Orphan cleanup failed", + exc_info=True, ) diff --git a/src/openjarvis/sandbox/wasm_runner.py b/src/openjarvis/sandbox/wasm_runner.py index ff763a7d..223f5819 100644 --- a/src/openjarvis/sandbox/wasm_runner.py +++ b/src/openjarvis/sandbox/wasm_runner.py @@ -10,6 +10,7 @@ from typing import Any, Dict, Optional @dataclass(slots=True) class WasmResult: """Result from a WASM execution.""" + success: bool = True output: str = "" duration_seconds: float = 0.0 @@ -42,6 +43,7 @@ class WasmRunner: """Check if wasmtime is available.""" try: import wasmtime # noqa: F401 + return True except ImportError: return False @@ -63,8 +65,7 @@ class WasmRunner: return WasmResult( success=False, output=( - "wasmtime not installed. Install with: " - "uv sync --extra sandbox-wasm" + "wasmtime not installed. Install with: uv sync --extra sandbox-wasm" ), ) @@ -123,6 +124,7 @@ class WasmRunner: """Validate that bytes represent a valid WASM module.""" try: import wasmtime + config = wasmtime.Config() engine = wasmtime.Engine(config) wasmtime.Module.validate(engine, wasm_bytes) @@ -145,6 +147,7 @@ def create_sandbox_runner(config: Any = None) -> Any: # Fall back to Docker ContainerRunner try: from openjarvis.sandbox.runner import ContainerRunner + return ContainerRunner( image=getattr(config, "image", "openjarvis-sandbox:latest"), timeout=getattr(config, "timeout", 300), diff --git a/src/openjarvis/scheduler/scheduler.py b/src/openjarvis/scheduler/scheduler.py index 8702ee21..ebbfe4ed 100644 --- a/src/openjarvis/scheduler/scheduler.py +++ b/src/openjarvis/scheduler/scheduler.py @@ -224,9 +224,7 @@ class TaskScheduler: else task.tools.split(",") ) tools_list = ( - [t.strip() for t in raw_tools if t.strip()] - if task.tools - else [] + [t.strip() for t in raw_tools if t.strip()] if task.tools else [] ) ask_kwargs: Dict[str, Any] = { "agent": task.agent, @@ -307,9 +305,7 @@ class TaskScheduler: return None @staticmethod - def _compute_next_cron( - cron_expr: str, now: datetime - ) -> Optional[str]: + def _compute_next_cron(cron_expr: str, now: datetime) -> Optional[str]: """Compute the next run time from a cron expression. Uses ``croniter`` if available, otherwise falls back to a basic diff --git a/src/openjarvis/scheduler/store.py b/src/openjarvis/scheduler/store.py index 262207a1..ef297526 100644 --- a/src/openjarvis/scheduler/store.py +++ b/src/openjarvis/scheduler/store.py @@ -116,9 +116,7 @@ class SchedulerStore: def delete_task(self, task_id: str) -> None: """Delete a task by ID.""" - self._conn.execute( - "DELETE FROM scheduled_tasks WHERE id = ?", (task_id,) - ) + self._conn.execute("DELETE FROM scheduled_tasks WHERE id = ?", (task_id,)) self._conn.commit() # -- Run logs ------------------------------------------------------------ @@ -139,13 +137,10 @@ class SchedulerStore: ) self._conn.commit() - def get_run_logs( - self, task_id: str, limit: int = 10 - ) -> List[Dict[str, Any]]: + def get_run_logs(self, task_id: str, limit: int = 10) -> List[Dict[str, Any]]: """Return the most recent run logs for *task_id*.""" rows = self._conn.execute( - "SELECT * FROM task_run_logs WHERE task_id = ? " - "ORDER BY id DESC LIMIT ?", + "SELECT * FROM task_run_logs WHERE task_id = ? ORDER BY id DESC LIMIT ?", (task_id, limit), ).fetchall() return [dict(r) for r in rows] diff --git a/src/openjarvis/scheduler/tools.py b/src/openjarvis/scheduler/tools.py index 950d1221..f846a585 100644 --- a/src/openjarvis/scheduler/tools.py +++ b/src/openjarvis/scheduler/tools.py @@ -93,11 +93,13 @@ class ScheduleTaskTool(BaseTool): ) return ToolResult( tool_name="schedule_task", - content=json.dumps({ - "task_id": task.id, - "next_run": task.next_run, - "status": task.status, - }), + content=json.dumps( + { + "task_id": task.id, + "next_run": task.next_run, + "status": task.status, + } + ), success=True, ) except Exception as exc: diff --git a/src/openjarvis/sdk.py b/src/openjarvis/sdk.py index 1e56b796..835f9e29 100644 --- a/src/openjarvis/sdk.py +++ b/src/openjarvis/sdk.py @@ -46,7 +46,8 @@ class MemoryHandle: if key == "sqlite": self._backend = MemoryRegistry.create( - key, db_path=self._config.memory.db_path, + key, + db_path=self._config.memory.db_path, ) else: self._backend = MemoryRegistry.create(key) @@ -70,7 +71,8 @@ class MemoryHandle: doc_ids: List[str] = [] for chunk in chunks: doc_id = backend.store( - chunk.content, source=chunk.source, + chunk.content, + source=chunk.source, metadata={"index": chunk.index}, ) doc_ids.append(doc_id) @@ -214,6 +216,7 @@ class Jarvis: # Apply security guardrails from openjarvis.security import setup_security + sec = setup_security(self._config, engine, self._bus) engine = sec.engine self._audit_logger = sec.audit_logger @@ -232,7 +235,9 @@ class Jarvis: logger.debug("Failed to create energy monitor: %s", exc) self._energy_monitor = energy_monitor self._engine = InstrumentedEngine( - engine, self._bus, energy_monitor=energy_monitor, + engine, + self._bus, + energy_monitor=energy_monitor, ) def ask( @@ -292,7 +297,9 @@ class Jarvis: # Agent mode if agent is not None: return self._run_agent( - agent, query, model_name, + agent, + query, + model_name, tools=tools or [], temperature=temperature, max_tokens=max_tokens, @@ -457,7 +464,10 @@ class Jarvis: from openjarvis.cli.ask import _build_tools tool_objects = _build_tools( - tools, self._config, self._engine, model_name, + tools, + self._config, + self._engine, + model_name, ) agent_kwargs: Dict[str, Any] = { @@ -492,7 +502,10 @@ class Jarvis: max_context_tokens=self._config.memory.context_max_tokens, ) context_messages = inject_context( - query, [], backend, config=ctx_cfg, + query, + [], + backend, + config=ctx_cfg, ) for msg in context_messages: ctx.conversation.add(msg) @@ -517,7 +530,9 @@ class Jarvis: } def _inject_context( - self, query: str, messages: List[Message], + self, + query: str, + messages: List[Message], ) -> List[Message]: """Inject memory context into messages.""" try: diff --git a/src/openjarvis/security/audit.py b/src/openjarvis/security/audit.py index 878803a7..dcb842ad 100644 --- a/src/openjarvis/security/audit.py +++ b/src/openjarvis/security/audit.py @@ -82,17 +82,19 @@ class AuditLogger: def log(self, event: SecurityEvent) -> None: """Insert a security event into the audit log with Merkle hash chain.""" - findings_json = json.dumps([ - { - "pattern_name": f.pattern_name, - "matched_text": f.matched_text, - "threat_level": f.threat_level.value, - "start": f.start, - "end": f.end, - "description": f.description, - } - for f in event.findings - ]) + findings_json = json.dumps( + [ + { + "pattern_name": f.pattern_name, + "matched_text": f.matched_text, + "threat_level": f.threat_level.value, + "start": f.start, + "end": f.end, + "description": f.description, + } + for f in event.findings + ] + ) # Compute hash chain prev_hash = self.tail_hash() @@ -205,10 +207,7 @@ class AuditLogger: if stored_prev != expected_prev: return False, rid # Verify row_hash - hash_input = ( - f"{stored_prev}|{ts}|{etype}" - f"|{fj}|{preview}|{action}" - ) + hash_input = f"{stored_prev}|{ts}|{etype}|{fj}|{preview}|{action}" computed = hashlib.sha256(hash_input.encode()).hexdigest() if computed != stored_hash: return False, rid @@ -218,9 +217,7 @@ class AuditLogger: def count(self) -> int: """Return the total number of logged security events.""" - row = self._conn.execute( - "SELECT COUNT(*) FROM security_events" - ).fetchone() + row = self._conn.execute("SELECT COUNT(*) FROM security_events").fetchone() return row[0] if row else 0 def close(self) -> None: diff --git a/src/openjarvis/security/capabilities.py b/src/openjarvis/security/capabilities.py index cac6a3eb..95fdaaaf 100644 --- a/src/openjarvis/security/capabilities.py +++ b/src/openjarvis/security/capabilities.py @@ -15,6 +15,7 @@ logger = logging.getLogger(__name__) class Capability(str, Enum): """Fine-grained capability labels.""" + FILE_READ = "file:read" FILE_WRITE = "file:write" NETWORK_FETCH = "network:fetch" @@ -30,13 +31,15 @@ class Capability(str, Enum): @dataclass(slots=True) class CapabilityGrant: """A single capability grant for an agent.""" - capability: str # Capability value or glob pattern - pattern: str = "*" # resource glob pattern + + capability: str # Capability value or glob pattern + pattern: str = "*" # resource glob pattern @dataclass(slots=True) class AgentPolicy: """Policy for a specific agent.""" + agent_id: str grants: List[CapabilityGrant] = field(default_factory=list) deny: List[str] = field(default_factory=list) # explicit denials @@ -63,6 +66,7 @@ class CapabilityPolicy: self._default_deny = default_deny from openjarvis._rust_bridge import get_rust_module + _rust = get_rust_module() self._rust_impl = _rust.CapabilityPolicy(default_deny=default_deny) @@ -72,7 +76,8 @@ class CapabilityPolicy: def grant(self, agent_id: str, capability: str, pattern: str = "*") -> None: """Grant a capability to an agent.""" policy = self._policies.setdefault( - agent_id, AgentPolicy(agent_id=agent_id), + agent_id, + AgentPolicy(agent_id=agent_id), ) policy.grants.append(CapabilityGrant(capability=capability, pattern=pattern)) self._rust_impl.grant(agent_id, capability, pattern) @@ -80,7 +85,8 @@ class CapabilityPolicy: def deny(self, agent_id: str, capability: str) -> None: """Explicitly deny a capability to an agent.""" policy = self._policies.setdefault( - agent_id, AgentPolicy(agent_id=agent_id), + agent_id, + AgentPolicy(agent_id=agent_id), ) policy.deny.append(capability) self._rust_impl.deny(agent_id, capability) @@ -148,14 +154,16 @@ class CapabilityPolicy: """Save policy to a JSON file.""" agents = [] for agent_id, policy in self._policies.items(): - agents.append({ - "agent_id": agent_id, - "grants": [ - {"capability": g.capability, "pattern": g.pattern} - for g in policy.grants - ], - "deny": policy.deny, - }) + agents.append( + { + "agent_id": agent_id, + "grants": [ + {"capability": g.capability, "pattern": g.pattern} + for g in policy.grants + ], + "deny": policy.deny, + } + ) path.write_text(json.dumps({"agents": agents}, indent=2)) diff --git a/src/openjarvis/security/file_policy.py b/src/openjarvis/security/file_policy.py index 36482440..350f499a 100644 --- a/src/openjarvis/security/file_policy.py +++ b/src/openjarvis/security/file_policy.py @@ -5,24 +5,26 @@ from __future__ import annotations from pathlib import Path from typing import Iterable, List, Union -DEFAULT_SENSITIVE_PATTERNS: frozenset[str] = frozenset({ - ".env", - ".env.*", - "*.env", - ".secret", - "*.secrets", - "credentials.*", - "*.pem", - "*.key", - "*.p12", - "*.pfx", - "*.jks", - "id_rsa", - "id_ed25519", - ".htpasswd", - ".pgpass", - ".netrc", -}) +DEFAULT_SENSITIVE_PATTERNS: frozenset[str] = frozenset( + { + ".env", + ".env.*", + "*.env", + ".secret", + "*.secrets", + "credentials.*", + "*.pem", + "*.key", + "*.p12", + "*.pfx", + "*.jks", + "id_rsa", + "id_ed25519", + ".htpasswd", + ".pgpass", + ".netrc", + } +) def is_sensitive_file(path: Union[str, Path]) -> bool: diff --git a/src/openjarvis/security/injection_scanner.py b/src/openjarvis/security/injection_scanner.py index 726930f2..4f83819d 100644 --- a/src/openjarvis/security/injection_scanner.py +++ b/src/openjarvis/security/injection_scanner.py @@ -103,6 +103,7 @@ _INJECTION_PATTERNS = [ @dataclass(slots=True) class InjectionScanResult: """Result of an injection scan.""" + is_clean: bool findings: List[ScanFinding] threat_level: ThreatLevel # highest threat found @@ -125,12 +126,14 @@ class InjectionScanner: for pat, name, level, desc in _INJECTION_PATTERNS ] from openjarvis._rust_bridge import get_rust_module + _rust = get_rust_module() self._rust_impl = _rust.InjectionScanner() def scan(self, text: str) -> InjectionScanResult: """Scan text for injection patterns — always via Rust backend.""" from openjarvis._rust_bridge import injection_result_from_json + return injection_result_from_json(self._rust_impl.scan(text)) diff --git a/src/openjarvis/security/rate_limiter.py b/src/openjarvis/security/rate_limiter.py index 179eebe8..ed258997 100644 --- a/src/openjarvis/security/rate_limiter.py +++ b/src/openjarvis/security/rate_limiter.py @@ -13,6 +13,7 @@ __all__ = ["RateLimitConfig", "RateLimiter", "TokenBucket"] @dataclass(slots=True) class RateLimitConfig: """Configuration for rate limiting.""" + requests_per_minute: int = 60 burst_size: int = 10 # max tokens in bucket enabled: bool = True @@ -67,6 +68,7 @@ class RateLimiter: self._lock = threading.Lock() from openjarvis._rust_bridge import get_rust_module + _rust = get_rust_module() self._rust_impl = _rust.RateLimiter( requests_per_minute=self._config.requests_per_minute, diff --git a/src/openjarvis/security/signing.py b/src/openjarvis/security/signing.py index d02aa848..bbe2ffca 100644 --- a/src/openjarvis/security/signing.py +++ b/src/openjarvis/security/signing.py @@ -12,6 +12,7 @@ logger = logging.getLogger(__name__) @dataclass(slots=True) class KeyPair: """Ed25519 key pair.""" + private_key: bytes public_key: bytes @@ -100,6 +101,7 @@ def verify_b64(data: bytes, signature_b64: str, public_key: bytes) -> bool: def load_public_key(path: str) -> bytes: """Load a raw 32-byte Ed25519 public key from a file.""" from pathlib import Path + raw = Path(path).read_bytes() # If base64-encoded (common), decode if len(raw) > 32: @@ -113,6 +115,7 @@ def load_public_key(path: str) -> bytes: def save_keypair(keypair: KeyPair, private_path: str, public_path: str) -> None: """Save keypair to files (base64-encoded).""" from pathlib import Path + Path(private_path).write_text(base64.b64encode(keypair.private_key).decode()) Path(public_path).write_text(base64.b64encode(keypair.public_key).decode()) diff --git a/src/openjarvis/security/ssrf.py b/src/openjarvis/security/ssrf.py index 55f2f71a..91f82462 100644 --- a/src/openjarvis/security/ssrf.py +++ b/src/openjarvis/security/ssrf.py @@ -7,12 +7,14 @@ import socket from typing import Optional # Cloud metadata endpoints to block -_BLOCKED_HOSTS = frozenset({ - "169.254.169.254", # AWS/GCP/Azure metadata - "metadata.google.internal", - "metadata.google.com", - "100.100.100.200", # Alibaba Cloud metadata -}) +_BLOCKED_HOSTS = frozenset( + { + "169.254.169.254", # AWS/GCP/Azure metadata + "metadata.google.internal", + "metadata.google.com", + "100.100.100.200", # Alibaba Cloud metadata + } +) _BLOCKED_CIDR = [ ipaddress.ip_network("10.0.0.0/8"), @@ -21,8 +23,8 @@ _BLOCKED_CIDR = [ ipaddress.ip_network("127.0.0.0/8"), ipaddress.ip_network("169.254.0.0/16"), # link-local ipaddress.ip_network("::1/128"), - ipaddress.ip_network("fc00::/7"), # unique local - ipaddress.ip_network("fe80::/10"), # link-local v6 + ipaddress.ip_network("fc00::/7"), # unique local + ipaddress.ip_network("fe80::/10"), # link-local v6 ] @@ -59,7 +61,10 @@ def _check_ssrf_python(url: str) -> Optional[str]: # DNS resolution check try: resolved = socket.getaddrinfo( - hostname, None, socket.AF_UNSPEC, socket.SOCK_STREAM, + hostname, + None, + socket.AF_UNSPEC, + socket.SOCK_STREAM, ) for family, stype, proto, canonname, sockaddr in resolved: ip = sockaddr[0] diff --git a/src/openjarvis/security/subprocess_sandbox.py b/src/openjarvis/security/subprocess_sandbox.py index c3fcb452..9ed6ff52 100644 --- a/src/openjarvis/security/subprocess_sandbox.py +++ b/src/openjarvis/security/subprocess_sandbox.py @@ -12,15 +12,26 @@ from typing import Dict, List, Optional logger = logging.getLogger(__name__) # Safe environment variables to pass through -_SAFE_ENV_VARS = frozenset({ - "PATH", "HOME", "USER", "LANG", "TERM", "SHELL", - "LC_ALL", "LC_CTYPE", "TMPDIR", "TZ", -}) +_SAFE_ENV_VARS = frozenset( + { + "PATH", + "HOME", + "USER", + "LANG", + "TERM", + "SHELL", + "LC_ALL", + "LC_CTYPE", + "TMPDIR", + "TZ", + } +) @dataclass(slots=True) class SandboxResult: """Result of a sandboxed subprocess execution.""" + stdout: str = "" stderr: str = "" returncode: int = -1 diff --git a/src/openjarvis/security/taint.py b/src/openjarvis/security/taint.py index ffa4793f..f054f72b 100644 --- a/src/openjarvis/security/taint.py +++ b/src/openjarvis/security/taint.py @@ -13,6 +13,7 @@ from typing import Dict, FrozenSet, Optional, Set class TaintLabel(str, Enum): """Labels for tainted data.""" + PII = "pii" SECRET = "secret" USER_PRIVATE = "user_private" @@ -22,6 +23,7 @@ class TaintLabel(str, Enum): @dataclass(frozen=True) class TaintSet: """Immutable set of taint labels attached to data.""" + labels: FrozenSet[TaintLabel] = field(default_factory=frozenset) def union(self, other: TaintSet) -> TaintSet: @@ -52,14 +54,14 @@ SINK_POLICY: Dict[str, Set[TaintLabel]] = { # Patterns for auto-detecting taint in text _PII_PATTERNS = [ - re.compile(r"\b\d{3}-\d{2}-\d{4}\b"), # SSN + re.compile(r"\b\d{3}-\d{2}-\d{4}\b"), # SSN re.compile(r"\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Z|a-z]{2,}\b"), # email re.compile(r"\b\d{4}[\s-]?\d{4}[\s-]?\d{4}[\s-]?\d{4}\b"), # credit card re.compile(r"\b\+?1?\s*\(?[2-9]\d{2}\)?\s*[-.\s]?\d{3}\s*[-.\s]?\d{4}\b"), # phone ] _SECRET_PATTERNS = [ - re.compile(r"(?:sk|pk|api)[_-][a-zA-Z0-9]{20,}"), # API keys + re.compile(r"(?:sk|pk|api)[_-][a-zA-Z0-9]{20,}"), # API keys re.compile(r"(?:ghp|gho|ghu|ghs|ghr)_[A-Za-z0-9_]{36,}"), # GitHub tokens re.compile(r"-----BEGIN (?:RSA |EC |DSA )?PRIVATE KEY-----"), # Private keys re.compile( @@ -80,13 +82,9 @@ def check_taint(tool_name: str, taint: TaintSet) -> Optional[str]: violations = taint.labels & forbidden if violations: labels_str = ", ".join( - v.value - for v in sorted(violations, key=lambda x: x.value) - ) - return ( - f"Data with labels [{labels_str}] " - f"cannot be sent to '{tool_name}'." + v.value for v in sorted(violations, key=lambda x: x.value) ) + return f"Data with labels [{labels_str}] cannot be sent to '{tool_name}'." return None diff --git a/src/openjarvis/security/types.py b/src/openjarvis/security/types.py index 8fcabce0..dd73df2f 100644 --- a/src/openjarvis/security/types.py +++ b/src/openjarvis/security/types.py @@ -71,11 +71,14 @@ class ScanResult: if not self.findings: return None order = [ - ThreatLevel.LOW, ThreatLevel.MEDIUM, - ThreatLevel.HIGH, ThreatLevel.CRITICAL, + ThreatLevel.LOW, + ThreatLevel.MEDIUM, + ThreatLevel.HIGH, + ThreatLevel.CRITICAL, ] best = max( - self.findings, key=lambda f: order.index(f.threat_level), + self.findings, + key=lambda f: order.index(f.threat_level), ) return best.threat_level diff --git a/src/openjarvis/server/agent_manager_routes.py b/src/openjarvis/server/agent_manager_routes.py index 76f3a585..09ffb65d 100644 --- a/src/openjarvis/server/agent_manager_routes.py +++ b/src/openjarvis/server/agent_manager_routes.py @@ -125,10 +125,7 @@ def _pick_recommended_model( model_ids: list[str], ) -> dict[str, str]: """Pick the second-largest local model from a list.""" - local = [ - m for m in model_ids - if not any(m.startswith(p) for p in _CLOUD_PREFIXES) - ] + local = [m for m in model_ids if not any(m.startswith(p) for p in _CLOUD_PREFIXES)] if not local: return { "model": model_ids[0] if model_ids else "", @@ -310,6 +307,7 @@ def _build_deep_research_tools( if not knowledge_db_path: from openjarvis.core.config import DEFAULT_CONFIG_DIR + knowledge_db_path = str(DEFAULT_CONFIG_DIR / "knowledge.db") if not Path(knowledge_db_path).exists(): @@ -502,42 +500,19 @@ async def _stream_managed_agent( if system_prompt: llm_messages.append(Message(role=Role.SYSTEM, content=system_prompt)) - # Build agent constructor kwargs from config - agent_kwargs: Dict[str, Any] = { - "engine": engine, - "model": model, - } - if bus is not None: - agent_kwargs["bus"] = bus - if config.get("system_prompt"): - agent_kwargs["system_prompt"] = config["system_prompt"] - if config.get("temperature") is not None: - agent_kwargs["temperature"] = config["temperature"] - if config.get("max_tokens") is not None: - agent_kwargs["max_tokens"] = config["max_tokens"] - if config.get("max_turns") is not None: - agent_kwargs["max_turns"] = config["max_turns"] - - # Build DeepResearch tools when applicable + # Resolve agent type and class for DeepResearch tool wiring + agent_type = agent_record.get("agent_type", "") if agent_type == "deep_research": - dr_tools = _build_deep_research_tools(engine=engine, model=model) - if dr_tools: - agent_kwargs["tools"] = dr_tools - - try: - agent = agent_cls(**agent_kwargs) - except TypeError as exc: - logger.warning( - "Agent instantiation failed with all kwargs, retrying minimal: %s", - exc, + dr_tools = _build_deep_research_tools( + engine=engine, model=model, ) - agent = agent_cls(engine=engine, model=model) + # Store on app_state so streaming loop can access them + if app_state is not None and dr_tools: + app_state._dr_tools = dr_tools - # Build conversation context from existing messages - ctx = AgentContext() - messages = manager.list_messages(agent_id, limit=50) - # Messages come in DESC order, reverse for chronological - for m in reversed(messages): + # Load prior conversation context (DESC order, reverse for chronological) + history = manager.list_messages(agent_id, limit=50) + for m in reversed(history): if m["id"] == message_id: continue if m["direction"] == "user_to_agent": diff --git a/src/openjarvis/server/api_routes.py b/src/openjarvis/server/api_routes.py index ca6c93c8..d07a9150 100644 --- a/src/openjarvis/server/api_routes.py +++ b/src/openjarvis/server/api_routes.py @@ -14,22 +14,27 @@ logger = logging.getLogger(__name__) # ---- Request/Response models ---- + class AgentCreateRequest(BaseModel): agent_type: str tools: Optional[List[str]] = None agent_id: Optional[str] = None + class AgentMessageRequest(BaseModel): message: str + class MemoryStoreRequest(BaseModel): content: str metadata: Optional[Dict[str, Any]] = None + class MemorySearchRequest(BaseModel): query: str top_k: int = 5 + class BudgetLimitsRequest(BaseModel): max_tokens_per_day: Optional[int] = None max_requests_per_hour: Optional[int] = None @@ -52,6 +57,7 @@ class OptimizeRunRequest(BaseModel): agents_router = APIRouter(prefix="/v1/agents", tags=["agents"]) + @agents_router.get("") async def list_agents(request: Request): """List available agent types and running agents.""" @@ -59,32 +65,36 @@ async def list_agents(request: Request): try: import openjarvis.agents # noqa: F401 — side-effect registration from openjarvis.core.registry import AgentRegistry + for key in sorted(AgentRegistry.keys()): cls = AgentRegistry.get(key) - registered.append({ - "key": key, - "class": cls.__name__, - "accepts_tools": getattr(cls, "accepts_tools", False), - }) + registered.append( + { + "key": key, + "class": cls.__name__, + "accepts_tools": getattr(cls, "accepts_tools", False), + } + ) except Exception as exc: logger.warning("Failed to list registered agents: %s", exc) running = [] try: from openjarvis.tools.agent_tools import _SPAWNED_AGENTS - running = [ - {"id": k, **v} for k, v in _SPAWNED_AGENTS.items() - ] + + running = [{"id": k, **v} for k, v in _SPAWNED_AGENTS.items()] except ImportError: pass return {"registered": registered, "running": running} + @agents_router.post("") async def create_agent(req: AgentCreateRequest, request: Request): """Spawn a new agent.""" try: from openjarvis.tools.agent_tools import AgentSpawnTool + tool = AgentSpawnTool() params = {"agent_type": req.agent_type} if req.tools: @@ -102,11 +112,13 @@ async def create_agent(req: AgentCreateRequest, request: Request): except ImportError: raise HTTPException(status_code=501, detail="Agent tools not available") + @agents_router.delete("/{agent_id}") async def kill_agent(agent_id: str, request: Request): """Kill a running agent.""" try: from openjarvis.tools.agent_tools import AgentKillTool + tool = AgentKillTool() result = tool.execute(agent_id=agent_id) if not result.success: @@ -115,11 +127,13 @@ async def kill_agent(agent_id: str, request: Request): except ImportError: raise HTTPException(status_code=501, detail="Agent tools not available") + @agents_router.post("/{agent_id}/message") async def message_agent(agent_id: str, req: AgentMessageRequest, request: Request): """Send a message to a running agent.""" try: from openjarvis.tools.agent_tools import AgentSendTool + tool = AgentSendTool() result = tool.execute(agent_id=agent_id, message=req.message) if not result.success: @@ -133,22 +147,26 @@ async def message_agent(agent_id: str, req: AgentMessageRequest, request: Reques memory_router = APIRouter(prefix="/v1/memory", tags=["memory"]) + @memory_router.post("/store") async def memory_store(req: MemoryStoreRequest, request: Request): """Store content in memory.""" try: from openjarvis.tools.storage.sqlite import SQLiteMemory + backend = SQLiteMemory() backend.store(req.content, metadata=req.metadata or {}) return {"status": "stored"} except Exception as exc: raise HTTPException(status_code=500, detail=str(exc)) + @memory_router.post("/search") async def memory_search(req: MemorySearchRequest, request: Request): """Search memory for relevant content.""" try: from openjarvis.tools.storage.sqlite import SQLiteMemory + backend = SQLiteMemory() results = backend.search(req.query, top_k=req.top_k) items = [ @@ -159,11 +177,13 @@ async def memory_search(req: MemorySearchRequest, request: Request): except Exception as exc: raise HTTPException(status_code=500, detail=str(exc)) + @memory_router.get("/stats") async def memory_stats(request: Request): """Get memory backend statistics.""" try: from openjarvis.tools.storage.sqlite import SQLiteMemory + backend = SQLiteMemory() stats = backend.stats() return stats @@ -175,6 +195,7 @@ async def memory_stats(request: Request): traces_router = APIRouter(prefix="/v1/traces", tags=["traces"]) + @traces_router.get("") async def list_traces(request: Request, limit: int = 20): """List recent traces.""" @@ -190,6 +211,7 @@ async def list_traces(request: Request, limit: int = 20): except Exception as exc: return {"traces": [], "error": str(exc)} + @traces_router.get("/{trace_id}") async def get_trace(trace_id: str, request: Request): """Get a specific trace by ID.""" @@ -213,6 +235,7 @@ async def get_trace(trace_id: str, request: Request): telemetry_router = APIRouter(prefix="/v1/telemetry", tags=["telemetry"]) + @telemetry_router.get("/stats") async def telemetry_stats(request: Request): """Get aggregated telemetry statistics.""" @@ -240,11 +263,11 @@ async def telemetry_stats(request: Request): except Exception as exc: return {"error": str(exc)} + @telemetry_router.get("/energy") async def telemetry_energy(request: Request): """Get energy monitoring data.""" try: - from openjarvis.core.config import DEFAULT_CONFIG_DIR from openjarvis.telemetry.aggregator import TelemetryAggregator @@ -286,11 +309,13 @@ async def telemetry_energy(request: Request): skills_router = APIRouter(prefix="/v1/skills", tags=["skills"]) + @skills_router.get("") async def list_skills(request: Request): """List installed skills.""" try: from openjarvis.core.registry import SkillRegistry + skills = [] for key in sorted(SkillRegistry.keys()): skills.append({"name": key}) @@ -299,6 +324,7 @@ async def list_skills(request: Request): logger.warning("Failed to list skills: %s", exc) return {"skills": []} + @skills_router.post("") async def install_skill(request: Request): """Install a skill (placeholder).""" @@ -307,6 +333,7 @@ async def install_skill(request: Request): "message": "Use TOML files in ~/.openjarvis/skills/", } + @skills_router.delete("/{skill_name}") async def remove_skill(skill_name: str, request: Request): """Remove a skill (placeholder).""" @@ -320,31 +347,32 @@ async def remove_skill(skill_name: str, request: Request): sessions_router = APIRouter(prefix="/v1/sessions", tags=["sessions"]) + @sessions_router.get("") async def list_sessions(request: Request, limit: int = 20): """List active sessions.""" try: from openjarvis.sessions.store import SessionStore + store = SessionStore() sessions = store.recent(limit=limit) - items = [ - s.to_dict() if hasattr(s, "to_dict") else str(s) - for s in sessions - ] + items = [s.to_dict() if hasattr(s, "to_dict") else str(s) for s in sessions] return {"sessions": items} except Exception as exc: return {"sessions": [], "error": str(exc)} + @sessions_router.get("/{session_id}") async def get_session(session_id: str, request: Request): """Get a specific session.""" try: from openjarvis.sessions.store import SessionStore + store = SessionStore() session = store.get(session_id) if session is None: raise HTTPException(status_code=404, detail="Session not found") - return session.to_dict() if hasattr(session, 'to_dict') else {"id": session_id} + return session.to_dict() if hasattr(session, "to_dict") else {"id": session_id} except HTTPException: raise except Exception as exc: @@ -364,11 +392,13 @@ _budget_usage: Dict[str, int] = { "requests_this_hour": 0, } + @budget_router.get("") async def get_budget(request: Request): """Get current budget usage and limits.""" return {"limits": _budget_limits, "usage": _budget_usage} + @budget_router.put("/limits") async def set_budget_limits(req: BudgetLimitsRequest, request: Request): """Update budget limits.""" @@ -383,6 +413,7 @@ async def set_budget_limits(req: BudgetLimitsRequest, request: Request): metrics_router = APIRouter(tags=["metrics"]) + @metrics_router.get("/metrics") async def prometheus_metrics(request: Request): """Prometheus-compatible metrics endpoint.""" @@ -393,6 +424,7 @@ async def prometheus_metrics(request: Request): db_path = DEFAULT_CONFIG_DIR / "telemetry.db" if not db_path.exists(): from starlette.responses import PlainTextResponse + return PlainTextResponse("# no telemetry data\n", media_type="text/plain") agg = TelemetryAggregator(db_path) @@ -410,13 +442,13 @@ async def prometheus_metrics(request: Request): f"openjarvis_latency_avg_ms {stats.get('avg_latency_ms', 0)}", ] from starlette.responses import PlainTextResponse + return PlainTextResponse("\n".join(lines) + "\n", media_type="text/plain") except Exception as exc: logger.warning("Failed to collect Prometheus metrics: %s", exc) from starlette.responses import PlainTextResponse - return PlainTextResponse( - "# No metrics available\n", media_type="text/plain" - ) + + return PlainTextResponse("# No metrics available\n", media_type="text/plain") # ---- WebSocket streaming routes ---- @@ -458,7 +490,9 @@ async def websocket_chat_stream(websocket: WebSocket): continue model = data.get("model") or getattr( - websocket.app.state, "model", "default", + websocket.app.state, + "model", + "default", ) engine = getattr(websocket.app.state, "engine", None) if engine is None: @@ -473,8 +507,7 @@ async def websocket_chat_stream(websocket: WebSocket): # Prefer streaming if the engine supports it stream_fn = getattr(engine, "stream", None) if stream_fn is not None and ( - inspect.isasyncgenfunction(stream_fn) - or callable(stream_fn) + inspect.isasyncgenfunction(stream_fn) or callable(stream_fn) ): full_content = "" try: @@ -498,9 +531,14 @@ async def websocket_chat_stream(websocket: WebSocket): # stream() didn't return an iterable; fall back to # generate() result = engine.generate(messages, model=model) - content = result.get("content", "") if isinstance( - result, dict, - ) else str(result) + content = ( + result.get("content", "") + if isinstance( + result, + dict, + ) + else str(result) + ) full_content = content await websocket.send_json( {"type": "chunk", "content": content}, @@ -511,9 +549,14 @@ async def websocket_chat_stream(websocket: WebSocket): else: # No stream method — single-shot generate result = engine.generate(messages, model=model) - content = result.get("content", "") if isinstance( - result, dict, - ) else str(result) + content = ( + result.get("content", "") + if isinstance( + result, + dict, + ) + else str(result) + ) await websocket.send_json( {"type": "chunk", "content": content}, ) @@ -534,6 +577,7 @@ async def websocket_chat_stream(websocket: WebSocket): learning_router = APIRouter(prefix="/v1/learning", tags=["learning"]) + @learning_router.get("/stats") async def learning_stats(request: Request): """Return learning system statistics across all sub-policies.""" @@ -542,6 +586,7 @@ async def learning_stats(request: Request): # Skill discovery try: from openjarvis.learning.agents.skill_discovery import SkillDiscovery + discovery = SkillDiscovery() result["skill_discovery"] = { "available": True, @@ -553,6 +598,7 @@ async def learning_stats(request: Request): return result + @learning_router.get("/policy") async def learning_policy(request: Request): """Return current routing policy configuration.""" @@ -561,6 +607,7 @@ async def learning_policy(request: Request): # Load config and extract learning section try: from openjarvis.core.config import load_config + config = load_config() lc = config.learning result["enabled"] = lc.enabled @@ -723,14 +770,10 @@ async def get_optimize_run(run_id: str, request: Request): "status": run.status, "benchmark": run.benchmark, "trials": len(run.trials), - "best_trial_id": ( - run.best_trial.trial_id if run.best_trial else None - ), + "best_trial_id": (run.best_trial.trial_id if run.best_trial else None), } except Exception as exc: - logger.warning( - "Failed to get optimization run %s: %s", run_id, exc - ) + logger.warning("Failed to get optimization run %s: %s", run_id, exc) return {"run_id": run_id, "status": "not_found"} @@ -762,6 +805,7 @@ def include_all_routes(app) -> None: from openjarvis.server.agent_manager_routes import ( # noqa: PLC0415 create_agent_manager_router, ) + agents_r, templates_r, global_r, tools_r = create_agent_manager_router( app.state.agent_manager ) diff --git a/src/openjarvis/server/app.py b/src/openjarvis/server/app.py index 7ee7a20a..781de49e 100644 --- a/src/openjarvis/server/app.py +++ b/src/openjarvis/server/app.py @@ -32,9 +32,7 @@ class _NoCacheStaticFiles(StaticFiles): async def __call__(self, scope, receive, send): async def _send_with_headers(message): if message["type"] == "http.response.start": - extra = [ - (k.encode(), v.encode()) for k, v in _NO_CACHE_HEADERS.items() - ] + extra = [(k.encode(), v.encode()) for k, v in _NO_CACHE_HEADERS.items()] # Remove etag and last-modified existing = [ (k, v) @@ -88,6 +86,7 @@ def create_app( ) from fastapi.middleware.cors import CORSMiddleware + app.add_middleware( CORSMiddleware, allow_origins=["*"], @@ -159,24 +158,14 @@ def create_app( webhook_router = create_webhook_router( bridge=channel_bridge, - twilio_auth_token=webhook_config.get( - "twilio_auth_token", "" - ), - bluebubbles_password=webhook_config.get( - "bluebubbles_password", "" - ), - whatsapp_verify_token=webhook_config.get( - "whatsapp_verify_token", "" - ), - whatsapp_app_secret=webhook_config.get( - "whatsapp_app_secret", "" - ), + twilio_auth_token=webhook_config.get("twilio_auth_token", ""), + bluebubbles_password=webhook_config.get("bluebubbles_password", ""), + whatsapp_verify_token=webhook_config.get("whatsapp_verify_token", ""), + whatsapp_app_secret=webhook_config.get("whatsapp_app_secret", ""), ) app.include_router(webhook_router) except Exception as exc: - logger.debug( - "Webhook routes init skipped: %s", exc - ) + logger.debug("Webhook routes init skipped: %s", exc) # Serve static frontend assets if the static/ directory exists static_dir = pathlib.Path(__file__).parent / "static" diff --git a/src/openjarvis/server/auth_middleware.py b/src/openjarvis/server/auth_middleware.py index d004c293..7489428d 100644 --- a/src/openjarvis/server/auth_middleware.py +++ b/src/openjarvis/server/auth_middleware.py @@ -30,14 +30,10 @@ class AuthMiddleware(BaseHTTPMiddleware): def __init__(self, app, api_key: str = "") -> None: # noqa: ANN001 super().__init__(app) - self._api_key = api_key or os.environ.get( - "OPENJARVIS_API_KEY", "" - ) + self._api_key = api_key or os.environ.get("OPENJARVIS_API_KEY", "") async def dispatch(self, request: Request, call_next): # noqa: ANN001 - if self._api_key and not self._is_exempt( - request.url.path - ): + if self._api_key and not self._is_exempt(request.url.path): auth = request.headers.get("Authorization", "") if not auth: return JSONResponse( @@ -45,10 +41,7 @@ class AuthMiddleware(BaseHTTPMiddleware): status_code=401, ) scheme, _, token = auth.partition(" ") - if ( - scheme.lower() != "bearer" - or token != self._api_key - ): + if scheme.lower() != "bearer" or token != self._api_key: return JSONResponse( {"detail": "Invalid API key"}, status_code=401, diff --git a/src/openjarvis/server/channel_bridge.py b/src/openjarvis/server/channel_bridge.py index c09f0e56..f4d479ae 100644 --- a/src/openjarvis/server/channel_bridge.py +++ b/src/openjarvis/server/channel_bridge.py @@ -126,16 +126,12 @@ class ChannelBridge: # Command routing stripped = content.strip() if stripped.startswith("/"): - result = self._handle_command( - sender_id, stripped, channel_type - ) + result = self._handle_command(sender_id, stripped, channel_type) if result is not None: return result # Regular chat — route to JarvisSystem.ask() - return self._handle_chat( - sender_id, stripped, channel_type, max_length - ) + return self._handle_chat(sender_id, stripped, channel_type, max_length) # -------------------------------------------------------------- # Command parsing @@ -177,17 +173,11 @@ class ChannelBridge: # Unknown command — fall through to chat return None - def _handle_more( - self, sender_id: str, channel_type: str - ) -> str: - session = self._session_store.get_or_create( - sender_id, channel_type - ) + def _handle_more(self, sender_id: str, channel_type: str) -> str: + session = self._session_store.get_or_create(sender_id, channel_type) pending = session.get("pending_response") if pending: - self._session_store.clear_pending_response( - sender_id, channel_type - ) + self._session_store.clear_pending_response(sender_id, channel_type) return pending return "No pending response." @@ -204,9 +194,7 @@ class ChannelBridge: lines.append(f" {name} — {status}") return "Running agents:\n" + "\n".join(lines) - def _handle_agent_command( - self, agent_id: str, action: str - ) -> str: + def _handle_agent_command(self, agent_id: str, action: str) -> str: if not self._agent_manager: return "No agent manager configured." action_lower = action.strip().lower() @@ -225,11 +213,7 @@ class ChannelBridge: return f"Agent '{agent_id}' resumed." # Treat as a message to the agent result = self._agent_manager.send_message(agent_id, action) - return ( - str(result) - if result - else f"Message sent to agent '{agent_id}'." - ) + return str(result) if result else f"Message sent to agent '{agent_id}'." # -------------------------------------------------------------- # Chat handling @@ -237,9 +221,7 @@ class ChannelBridge: def _handle_sessions(self, sender_id: str) -> str: targets = self._session_store.get_notification_targets() - user_sessions = [ - t for t in targets if t["sender_id"] == sender_id - ] + user_sessions = [t for t in targets if t["sender_id"] == sender_id] if not user_sessions: return "No active sessions with notification preferences." lines = [] @@ -257,27 +239,20 @@ class ChannelBridge: channel_type: str, max_length: int, ) -> str: - self._session_store.append_message( - sender_id, channel_type, "user", content - ) + self._session_store.append_message(sender_id, channel_type, "user", content) # Build context from conversation history - session = self._session_store.get_or_create( - sender_id, channel_type - ) + session = self._session_store.get_or_create(sender_id, channel_type) history = session.get("conversation_history", []) context_lines = [] for msg in history[:-1]: # exclude the message we just appended - context_lines.append( - f"{msg['role']}: {msg['content']}" - ) + context_lines.append(f"{msg['role']}: {msg['content']}") context_str = "\n".join(context_lines) query = content if context_str: query = ( - f"Previous conversation:\n{context_str}\n\n" - f"Current message: {content}" + f"Previous conversation:\n{context_str}\n\nCurrent message: {content}" ) # Try DeepResearchAgent first @@ -295,8 +270,7 @@ class ChannelBridge: except Exception: logger.exception("Error in JarvisSystem.ask()") error_msg = ( - "Sorry, I couldn't process that right now. " - "Try again in a moment." + "Sorry, I couldn't process that right now. Try again in a moment." ) self._session_store.append_message( sender_id, channel_type, "assistant", error_msg @@ -304,8 +278,7 @@ class ChannelBridge: return error_msg else: error_msg = ( - "Sorry, I couldn't process that right now. " - "Try again in a moment." + "Sorry, I couldn't process that right now. Try again in a moment." ) self._session_store.append_message( sender_id, channel_type, "assistant", error_msg @@ -331,14 +304,10 @@ class ChannelBridge: if len(response) <= max_length: return response # Truncate and store full response for /more retrieval - truncation_notice = ( - "\n\n... (reply /more for full response)" - ) + truncation_notice = "\n\n... (reply /more for full response)" cut_at = max_length - len(truncation_notice) truncated = response[:cut_at] + truncation_notice - self._session_store.set_pending_response( - sender_id, channel_type, response - ) + self._session_store.set_pending_response(sender_id, channel_type, response) return truncated # -------------------------------------------------------------- @@ -347,9 +316,7 @@ class ChannelBridge: def _subscribe_notifications(self) -> None: for event_type in _NOTIFICATION_EVENTS: - self._bus.subscribe( - event_type, self._on_notification_event - ) + self._bus.subscribe(event_type, self._on_notification_event) def _on_notification_event(self, event) -> None: # noqa: ANN001 event_key = str(event.event_type) @@ -369,23 +336,18 @@ class ChannelBridge: for target in targets: pref_channel = target["preferred_notification_channel"] sender_id = target["sender_id"] - self._send_notification( - pref_channel, sender_id, message - ) + self._send_notification(pref_channel, sender_id, message) def _format_notification( # noqa: ANN201 - self, event # noqa: ANN001 + self, + event, # noqa: ANN001 ) -> Optional[str]: data = event.data or {} name = data.get("agent_name", data.get("name", "unknown")) if event.event_type == EventType.AGENT_TICK_END: summary = data.get("summary", data.get("result", "")) - return ( - f"Agent '{name}' finished: {summary}" - if summary - else None - ) + return f"Agent '{name}' finished: {summary}" if summary else None if event.event_type == EventType.AGENT_TICK_ERROR: error = data.get("error", "unknown error") return f"Agent '{name}' error: {error}" @@ -414,6 +376,4 @@ class ChannelBridge: try: ch.send(sender_id, message) except Exception: - logger.exception( - "Failed to send notification to %s", channel_type - ) + logger.exception("Failed to send notification to %s", channel_type) diff --git a/src/openjarvis/server/connectors_router.py b/src/openjarvis/server/connectors_router.py index 58eefd96..e28cec40 100644 --- a/src/openjarvis/server/connectors_router.py +++ b/src/openjarvis/server/connectors_router.py @@ -261,7 +261,11 @@ def create_connectors_router(): engine = SyncEngine(pipeline=pipeline) chunks = engine.sync(inst) - return {"connector_id": connector_id, "chunks_indexed": chunks, "status": "complete"} + return { + "connector_id": connector_id, + "chunks_indexed": chunks, + "status": "complete", + } @router.get("/{connector_id}/sync") async def sync_status(connector_id: str): diff --git a/src/openjarvis/server/cost_calculator.py b/src/openjarvis/server/cost_calculator.py index a79459cc..11c5bc4f 100644 --- a/src/openjarvis/server/cost_calculator.py +++ b/src/openjarvis/server/cost_calculator.py @@ -11,6 +11,7 @@ from openjarvis.server.savings import CLOUD_PRICING @dataclass(slots=True) class CostEstimate: """Estimated cost for a provider given a usage scenario.""" + provider: str label: str monthly_cost: float @@ -23,6 +24,7 @@ class CostEstimate: @dataclass(slots=True) class Scenario: """A prebuilt usage scenario.""" + name: str label: str description: str diff --git a/src/openjarvis/server/savings.py b/src/openjarvis/server/savings.py index ae807139..c8e6e539 100644 --- a/src/openjarvis/server/savings.py +++ b/src/openjarvis/server/savings.py @@ -130,8 +130,7 @@ def compute_savings( params_b = pricing.get("params_b", 200.0) params = params_b * 1e9 flops = ( - 2.0 * params * total_tokens_evaluated - if total_tokens_evaluated > 0 else 0.0 + 2.0 * params * total_tokens_evaluated if total_tokens_evaluated > 0 else 0.0 ) # Derive Wh-per-FLOP from the provider's per-token constants: # energy_wh_per_1k_tokens / (1000 * flops_per_token) = Wh per FLOP diff --git a/src/openjarvis/server/session_store.py b/src/openjarvis/server/session_store.py index 627e2345..fcea2601 100644 --- a/src/openjarvis/server/session_store.py +++ b/src/openjarvis/server/session_store.py @@ -55,18 +55,14 @@ class SessionStore: # Public API # ------------------------------------------------------------------ - def get_or_create( - self, sender_id: str, channel_type: str - ) -> Dict[str, Any]: + def get_or_create(self, sender_id: str, channel_type: str) -> Dict[str, Any]: row = self._db.execute( - "SELECT * FROM channel_sessions " - "WHERE sender_id = ? AND channel_type = ?", + "SELECT * FROM channel_sessions WHERE sender_id = ? AND channel_type = ?", (sender_id, channel_type), ).fetchone() if row is None: self._db.execute( - "INSERT INTO channel_sessions " - "(sender_id, channel_type) VALUES (?, ?)", + "INSERT INTO channel_sessions (sender_id, channel_type) VALUES (?, ?)", (sender_id, channel_type), ) self._db.commit() @@ -80,12 +76,8 @@ class SessionStore: return { "sender_id": row["sender_id"], "channel_type": row["channel_type"], - "conversation_history": json.loads( - row["conversation_history"] - ), - "preferred_notification_channel": row[ - "preferred_notification_channel" - ], + "conversation_history": json.loads(row["conversation_history"]), + "preferred_notification_channel": row["preferred_notification_channel"], "pending_response": row["pending_response"], } @@ -103,9 +95,7 @@ class SessionStore: ).fetchone() if row is None: return - history: List[Dict[str, str]] = json.loads( - row["conversation_history"] - ) + history: List[Dict[str, str]] = json.loads(row["conversation_history"]) history.append({"role": role, "content": content}) if len(history) > _MAX_HISTORY_TURNS: history = history[-_MAX_HISTORY_TURNS:] @@ -148,9 +138,7 @@ class SessionStore: ) self._db.commit() - def clear_pending_response( - self, sender_id: str, channel_type: str - ) -> None: + def clear_pending_response(self, sender_id: str, channel_type: str) -> None: self.set_pending_response(sender_id, channel_type, None) def expire_sessions(self, max_age_hours: int = 24) -> int: @@ -164,9 +152,7 @@ class SessionStore: self._db.commit() return cur.rowcount - def get_last_active_channel( - self, sender_id: str - ) -> Optional[str]: + def get_last_active_channel(self, sender_id: str) -> Optional[str]: row = self._db.execute( "SELECT channel_type FROM channel_sessions " "WHERE sender_id = ? " diff --git a/src/openjarvis/server/webhook_routes.py b/src/openjarvis/server/webhook_routes.py index ba869d1a..6af3a15b 100644 --- a/src/openjarvis/server/webhook_routes.py +++ b/src/openjarvis/server/webhook_routes.py @@ -41,9 +41,7 @@ def _validate_twilio_signature( validator = RequestValidator(auth_token) return validator.validate(url, params, signature) except ImportError: - logger.warning( - "twilio SDK not installed — skipping signature validation" - ) + logger.warning("twilio SDK not installed — skipping signature validation") return True @@ -63,9 +61,7 @@ def create_webhook_router( whatsapp_verify_token: WhatsApp verification token. whatsapp_app_secret: WhatsApp app secret for HMAC. """ - router = APIRouter( - prefix="/webhooks", tags=["webhooks"] - ) + router = APIRouter(prefix="/webhooks", tags=["webhooks"]) # ---------------------------------------------------------- # Twilio SMS @@ -75,9 +71,7 @@ def create_webhook_router( async def twilio_incoming(request: Request) -> Response: form = await request.form() params = dict(form) - signature = request.headers.get( - "X-Twilio-Signature", "" - ) + signature = request.headers.get("X-Twilio-Signature", "") url = str(request.url) if twilio_auth_token and not _validate_twilio_signature( @@ -115,9 +109,7 @@ def create_webhook_router( ) -> Response: auth = request.headers.get("Authorization", "") if bluebubbles_password and auth != bluebubbles_password: - return Response( - "Invalid password", status_code=403 - ) + return Response("Invalid password", status_code=403) payload = await request.json() msg_type = payload.get("type", "") @@ -148,12 +140,8 @@ def create_webhook_router( @router.get("/whatsapp") async def whatsapp_verify(request: Request) -> Response: mode = request.query_params.get("hub.mode", "") - token = request.query_params.get( - "hub.verify_token", "" - ) - challenge = request.query_params.get( - "hub.challenge", "" - ) + token = request.query_params.get("hub.verify_token", "") + challenge = request.query_params.get("hub.challenge", "") if mode == "subscribe" and token == whatsapp_verify_token: return PlainTextResponse(challenge) @@ -167,9 +155,7 @@ def create_webhook_router( # Verify signature if whatsapp_app_secret: - signature = request.headers.get( - "X-Hub-Signature-256", "" - ) + signature = request.headers.get("X-Hub-Signature-256", "") expected = ( "sha256=" + hmac.new( @@ -179,9 +165,7 @@ def create_webhook_router( ).hexdigest() ) if not hmac.compare_digest(signature, expected): - return Response( - "Invalid signature", status_code=403 - ) + return Response("Invalid signature", status_code=403) payload = json.loads(body_bytes) for entry in payload.get("entry", []): @@ -191,9 +175,7 @@ def create_webhook_router( if message.get("type") != "text": continue sender = message.get("from", "") - text = message.get("text", {}).get( - "body", "" - ) + text = message.get("text", {}).get("body", "") task = asyncio.create_task( asyncio.to_thread( @@ -203,9 +185,7 @@ def create_webhook_router( "whatsapp", ) ) - task.add_done_callback( - _log_task_exception - ) + task.add_done_callback(_log_task_exception) return Response("OK", status_code=200) diff --git a/src/openjarvis/sessions/__init__.py b/src/openjarvis/sessions/__init__.py index e55c3eb3..f9ba8e84 100644 --- a/src/openjarvis/sessions/__init__.py +++ b/src/openjarvis/sessions/__init__.py @@ -1,4 +1,5 @@ """Cross-channel session management.""" + from openjarvis.sessions.session import Session, SessionIdentity, SessionStore __all__ = ["Session", "SessionIdentity", "SessionStore"] diff --git a/src/openjarvis/sessions/compression.py b/src/openjarvis/sessions/compression.py index c326c7b5..29ffd1fa 100644 --- a/src/openjarvis/sessions/compression.py +++ b/src/openjarvis/sessions/compression.py @@ -13,8 +13,7 @@ class BaseCompressor(ABC): """Abstract base for context compression strategies.""" @abstractmethod - def compress(self, messages: List[Message], threshold: float) -> List[Message]: - ... + def compress(self, messages: List[Message], threshold: float) -> List[Message]: ... @CompressionRegistry.register("session_consolidation") @@ -50,15 +49,10 @@ class RuleBasedPrecompression(BaseCompressor): try: parsed = json.loads(msg.content) truncated = ( - json.dumps(parsed, indent=None)[ - : self.TOOL_OUTPUT_MAX - ] - + suffix + json.dumps(parsed, indent=None)[: self.TOOL_OUTPUT_MAX] + suffix ) except (json.JSONDecodeError, TypeError): - truncated = ( - msg.content[: self.TOOL_OUTPUT_MAX] + suffix - ) + truncated = msg.content[: self.TOOL_OUTPUT_MAX] + suffix result.append(replace(msg, content=truncated)) else: result.append(msg) @@ -89,20 +83,20 @@ class TieredSummaries(BaseCompressor): l0_msgs = messages[l1_end:] result: list[Message] = [] if l2_msgs: - one_liners = "; ".join( - f"{m.role}: {m.content[:50]}" for m in l2_msgs + one_liners = "; ".join(f"{m.role}: {m.content[:50]}" for m in l2_msgs) + result.append( + Message( + role=Role.SYSTEM, + content=f"[Oldest context] {one_liners}", + ) ) - result.append(Message( - role=Role.SYSTEM, - content=f"[Oldest context] {one_liners}", - )) if l1_msgs: - paragraphs = "\n".join( - f"- {m.role}: {m.content[:200]}" for m in l1_msgs + paragraphs = "\n".join(f"- {m.role}: {m.content[:200]}" for m in l1_msgs) + result.append( + Message( + role=Role.SYSTEM, + content=f"[Earlier context]\n{paragraphs}", + ) ) - result.append(Message( - role=Role.SYSTEM, - content=f"[Earlier context]\n{paragraphs}", - )) result.extend(l0_msgs) return result diff --git a/src/openjarvis/sessions/session.py b/src/openjarvis/sessions/session.py index 2809113d..a5eabaf7 100644 --- a/src/openjarvis/sessions/session.py +++ b/src/openjarvis/sessions/session.py @@ -19,6 +19,7 @@ from openjarvis.core.config import DEFAULT_CONFIG_DIR @dataclass(slots=True) class SessionIdentity: """Canonical user identity across channels.""" + user_id: str display_name: str = "" # channel_type -> channel_user_id @@ -30,7 +31,8 @@ class SessionIdentity: @dataclass(slots=True) class SessionMessage: """A single message within a session.""" - role: str # "user" | "assistant" | "system" + + role: str # "user" | "assistant" | "system" content: str channel: str = "" timestamp: float = 0.0 @@ -40,6 +42,7 @@ class SessionMessage: @dataclass class Session: """A conversation session with cross-channel message history.""" + session_id: str = "" identity: Optional[SessionIdentity] = None messages: List[SessionMessage] = field(default_factory=list) @@ -48,9 +51,14 @@ class Session: metadata: Dict[str, Any] = field(default_factory=dict) def add_message(self, role: str, content: str, *, channel: str = "") -> None: - self.messages.append(SessionMessage( - role=role, content=content, channel=channel, timestamp=time.time(), - )) + self.messages.append( + SessionMessage( + role=role, + content=content, + channel=channel, + timestamp=time.time(), + ) + ) self.last_activity = time.time() @@ -124,8 +132,10 @@ class SessionStore: if age_hours > self._max_age_hours: # Session expired, create new return self._create_session( - user_id, channel, - channel_user_id, display_name, + user_id, + channel, + channel_user_id, + display_name, ) channel_ids = json.loads(row[3]) if row[3] else {} @@ -145,7 +155,8 @@ class SessionStore: return Session( session_id=session_id, identity=SessionIdentity( - user_id=row[1], display_name=row[2] or display_name, + user_id=row[1], + display_name=row[2] or display_name, channel_ids=channel_ids, ), messages=messages, @@ -157,7 +168,11 @@ class SessionStore: return self._create_session(user_id, channel, channel_user_id, display_name) def _create_session( - self, user_id: str, channel: str, channel_user_id: str, display_name: str, + self, + user_id: str, + channel: str, + channel_user_id: str, + display_name: str, ) -> Session: session_id = uuid.uuid4().hex[:16] now = time.time() @@ -173,7 +188,8 @@ class SessionStore: return Session( session_id=session_id, identity=SessionIdentity( - user_id=user_id, display_name=display_name, + user_id=user_id, + display_name=display_name, channel_ids=channel_ids, ), created_at=now, @@ -181,8 +197,13 @@ class SessionStore: ) def save_message( - self, session_id: str, role: str, content: str, - *, channel: str = "", metadata: Optional[Dict[str, Any]] = None, + self, + session_id: str, + role: str, + content: str, + *, + channel: str = "", + metadata: Optional[Dict[str, Any]] = None, ) -> None: """Persist a message to a session.""" self._conn.execute( @@ -190,8 +211,14 @@ class SessionStore: " (session_id, role, content," " channel, timestamp, metadata) " "VALUES (?, ?, ?, ?, ?, ?)", - (session_id, role, content, channel, time.time(), - json.dumps(metadata or {})), + ( + session_id, + role, + content, + channel, + time.time(), + json.dumps(metadata or {}), + ), ) self._conn.execute( "UPDATE sessions SET last_activity = ? WHERE session_id = ?", @@ -243,17 +270,18 @@ class SessionStore: age = max_age_hours or self._max_age_hours cutoff = time.time() - (age * 3600) cur = self._conn.execute( - "SELECT session_id FROM sessions WHERE last_activity < ?", (cutoff,), + "SELECT session_id FROM sessions WHERE last_activity < ?", + (cutoff,), ) session_ids = [row[0] for row in cur.fetchall()] for sid in session_ids: self._conn.execute( - "DELETE FROM session_messages" - " WHERE session_id = ?", (sid,), + "DELETE FROM session_messages WHERE session_id = ?", + (sid,), ) self._conn.execute( - "DELETE FROM sessions" - " WHERE session_id = ?", (sid,), + "DELETE FROM sessions WHERE session_id = ?", + (sid,), ) self._conn.commit() return len(session_ids) @@ -261,7 +289,8 @@ class SessionStore: def link_channel(self, session_id: str, channel: str, channel_user_id: str) -> None: """Link a channel identity to an existing session.""" row = self._conn.execute( - "SELECT channel_ids FROM sessions WHERE session_id = ?", (session_id,), + "SELECT channel_ids FROM sessions WHERE session_id = ?", + (session_id,), ).fetchone() if row: channel_ids = json.loads(row[0]) if row[0] else {} @@ -273,7 +302,10 @@ class SessionStore: self._conn.commit() def list_sessions( - self, *, active_only: bool = True, limit: int = 50, + self, + *, + active_only: bool = True, + limit: int = 50, ) -> List[Session]: """List sessions, optionally filtering to active only.""" sql = ( @@ -292,16 +324,19 @@ class SessionStore: rows = self._conn.execute(sql, params).fetchall() sessions = [] for row in rows: - sessions.append(Session( - session_id=row[0], - identity=SessionIdentity( - user_id=row[1], display_name=row[2] or "", - channel_ids=json.loads(row[3]) if row[3] else {}, - ), - created_at=row[4] or 0.0, - last_activity=row[5] or 0.0, - metadata=json.loads(row[6]) if row[6] else {}, - )) + sessions.append( + Session( + session_id=row[0], + identity=SessionIdentity( + user_id=row[1], + display_name=row[2] or "", + channel_ids=json.loads(row[3]) if row[3] else {}, + ), + created_at=row[4] or 0.0, + last_activity=row[5] or 0.0, + metadata=json.loads(row[6]) if row[6] else {}, + ) + ) return sessions def _load_messages(self, session_id: str) -> List[SessionMessage]: @@ -312,7 +347,9 @@ class SessionStore: ).fetchall() return [ SessionMessage( - role=row[0], content=row[1], channel=row[2] or "", + role=row[0], + content=row[1], + channel=row[2] or "", timestamp=row[3] or 0.0, metadata=json.loads(row[4]) if row[4] else {}, ) diff --git a/src/openjarvis/skills/__init__.py b/src/openjarvis/skills/__init__.py index 7405351b..02a46be9 100644 --- a/src/openjarvis/skills/__init__.py +++ b/src/openjarvis/skills/__init__.py @@ -1,4 +1,5 @@ """Skill system — reusable multi-tool compositions.""" + from openjarvis.skills.executor import SkillExecutor from openjarvis.skills.loader import load_skill from openjarvis.skills.tool_adapter import SkillTool diff --git a/src/openjarvis/skills/executor.py b/src/openjarvis/skills/executor.py index 267d5dce..fbecf08e 100644 --- a/src/openjarvis/skills/executor.py +++ b/src/openjarvis/skills/executor.py @@ -100,6 +100,7 @@ class SkillExecutor: @staticmethod def _render_template(template: str, ctx: Dict[str, Any]) -> str: """Simple {key} placeholder rendering.""" + def _replace(match: re.Match) -> str: key = match.group(1) val = ctx.get(key, match.group(0)) diff --git a/src/openjarvis/skills/loader.py b/src/openjarvis/skills/loader.py index 5786ea4a..20119052 100644 --- a/src/openjarvis/skills/loader.py +++ b/src/openjarvis/skills/loader.py @@ -51,11 +51,13 @@ def load_skill( steps = [] for step_data in skill_data.get("steps", []): - steps.append(SkillStep( - tool_name=step_data["tool_name"], - arguments_template=step_data.get("arguments_template", "{}"), - output_key=step_data.get("output_key", ""), - )) + steps.append( + SkillStep( + tool_name=step_data["tool_name"], + arguments_template=step_data.get("arguments_template", "{}"), + output_key=step_data.get("output_key", ""), + ) + ) manifest = SkillManifest( name=skill_data.get("name", path.stem), @@ -72,6 +74,7 @@ def load_skill( if verify_signature and public_key and manifest.signature: try: from openjarvis.security.signing import verify_b64 + valid = verify_b64( manifest.manifest_bytes(), manifest.signature, @@ -89,6 +92,7 @@ def load_skill( if scan_for_injection: try: from openjarvis.security.scanner import SecretScanner + scanner = SecretScanner() for step in manifest.steps: scan_result = scanner.scan(step.arguments_template) diff --git a/src/openjarvis/skills/tool_adapter.py b/src/openjarvis/skills/tool_adapter.py index fde896b8..a615102d 100644 --- a/src/openjarvis/skills/tool_adapter.py +++ b/src/openjarvis/skills/tool_adapter.py @@ -61,7 +61,9 @@ class SkillTool(BaseTool): content=result.context.get( result.step_results[-1].tool_name if result.step_results else "", result.step_results[-1].content if result.step_results else "", - ) if result.step_results else "", + ) + if result.step_results + else "", success=result.success, metadata={"skill": self._manifest.name, "steps": len(result.step_results)}, ) diff --git a/src/openjarvis/skills/types.py b/src/openjarvis/skills/types.py index 9844c41a..c0d82ab8 100644 --- a/src/openjarvis/skills/types.py +++ b/src/openjarvis/skills/types.py @@ -9,14 +9,16 @@ from typing import Any, Dict, List @dataclass(slots=True) class SkillStep: """A single step in a skill pipeline.""" + tool_name: str arguments_template: str = "{}" # Jinja2-style template - output_key: str = "" # Key to store result in context + output_key: str = "" # Key to store result in context @dataclass(slots=True) class SkillManifest: """Manifest describing a reusable skill.""" + name: str version: str = "0.1.0" description: str = "" @@ -29,6 +31,7 @@ class SkillManifest: def manifest_bytes(self) -> bytes: """Serialize the manifest (excluding signature) for signing/verification.""" import json + data = { "name": self.name, "version": self.version, diff --git a/src/openjarvis/speech/deepgram.py b/src/openjarvis/speech/deepgram.py index 124e1cd8..9f20f367 100644 --- a/src/openjarvis/speech/deepgram.py +++ b/src/openjarvis/speech/deepgram.py @@ -62,7 +62,8 @@ class DeepgramSpeechBackend(SpeechBackend): options = options_kwargs response = self._client.listen.rest.v("1").transcribe_file( - payload, options, + payload, + options, ) # Extract transcript from response diff --git a/src/openjarvis/telemetry/aggregator.py b/src/openjarvis/telemetry/aggregator.py index 31754888..f4beee43 100644 --- a/src/openjarvis/telemetry/aggregator.py +++ b/src/openjarvis/telemetry/aggregator.py @@ -136,14 +136,9 @@ class TelemetryAggregator: has_itl = self._safe_col("mean_itl_ms") if has_pte: - extra_cols += ( - ", SUM(prompt_tokens_evaluated)" - " AS prompt_tokens_evaluated" - ) + extra_cols += ", SUM(prompt_tokens_evaluated) AS prompt_tokens_evaluated" if has_tpj: - extra_cols += ( - ", AVG(tokens_per_joule) AS avg_tokens_per_joule" - ) + extra_cols += ", AVG(tokens_per_joule) AS avg_tokens_per_joule" if has_derived: extra_cols += ( ", AVG(energy_per_output_token_joules)" @@ -197,9 +192,7 @@ class TelemetryAggregator: avg_throughput_tok_per_sec=r["avg_throughput_tok_per_sec"] or 0.0, ) if has_pte: - ms.prompt_tokens_evaluated = ( - r["prompt_tokens_evaluated"] or 0 - ) + ms.prompt_tokens_evaluated = r["prompt_tokens_evaluated"] or 0 if has_tpj: ms.avg_tokens_per_joule = r["avg_tokens_per_joule"] or 0.0 if has_derived: @@ -208,12 +201,8 @@ class TelemetryAggregator: ) ms.avg_throughput_per_watt = r["avg_throughput_per_watt"] or 0.0 if has_phase: - ms.total_prefill_energy_joules = ( - r["total_prefill_energy_joules"] or 0.0 - ) - ms.total_decode_energy_joules = ( - r["total_decode_energy_joules"] or 0.0 - ) + ms.total_prefill_energy_joules = r["total_prefill_energy_joules"] or 0.0 + ms.total_decode_energy_joules = r["total_decode_energy_joules"] or 0.0 if has_itl: ms.avg_mean_itl_ms = r["avg_mean_itl_ms"] or 0.0 ms.avg_median_itl_ms = r["avg_median_itl_ms"] or 0.0 @@ -236,9 +225,7 @@ class TelemetryAggregator: has_itl = self._safe_col("mean_itl_ms") if has_tpj: - extra_cols += ( - ", AVG(tokens_per_joule) AS avg_tokens_per_joule" - ) + extra_cols += ", AVG(tokens_per_joule) AS avg_tokens_per_joule" if has_derived: extra_cols += ( ", AVG(energy_per_output_token_joules)" @@ -295,12 +282,8 @@ class TelemetryAggregator: ) es.avg_throughput_per_watt = r["avg_throughput_per_watt"] or 0.0 if has_phase: - es.total_prefill_energy_joules = ( - r["total_prefill_energy_joules"] or 0.0 - ) - es.total_decode_energy_joules = ( - r["total_decode_energy_joules"] or 0.0 - ) + es.total_prefill_energy_joules = r["total_prefill_energy_joules"] or 0.0 + es.total_decode_energy_joules = r["total_decode_energy_joules"] or 0.0 if has_itl: es.avg_mean_itl_ms = r["avg_mean_itl_ms"] or 0.0 es.avg_median_itl_ms = r["avg_median_itl_ms"] or 0.0 @@ -330,9 +313,9 @@ class TelemetryAggregator: def _weighted_avg(attr: str) -> float: if total_calls == 0: return 0.0 - return sum( - getattr(m, attr) * m.call_count for m in model_stats - ) / total_calls + return ( + sum(getattr(m, attr) * m.call_count for m in model_stats) / total_calls + ) return AggregatedStats( total_calls=total_calls, diff --git a/src/openjarvis/telemetry/batch.py b/src/openjarvis/telemetry/batch.py index c00517c6..fb71e49d 100644 --- a/src/openjarvis/telemetry/batch.py +++ b/src/openjarvis/telemetry/batch.py @@ -69,15 +69,11 @@ class EnergyBatch: total_requests = ctx._total_requests per_request_energy = list(ctx._per_request_energy) - energy_per_token = ( - total_energy / total_tokens if total_tokens > 0 else 0.0 - ) + energy_per_token = total_energy / total_tokens if total_tokens > 0 else 0.0 energy_per_request = ( total_energy / total_requests if total_requests > 0 else 0.0 ) - mean_throughput = ( - total_tokens / elapsed if elapsed > 0 else 0.0 - ) + mean_throughput = total_tokens / elapsed if elapsed > 0 else 0.0 self.metrics = BatchMetrics( batch_id=self.batch_id, diff --git a/src/openjarvis/telemetry/energy_apple.py b/src/openjarvis/telemetry/energy_apple.py index 1be8c4ca..4cffef1d 100644 --- a/src/openjarvis/telemetry/energy_apple.py +++ b/src/openjarvis/telemetry/energy_apple.py @@ -50,7 +50,9 @@ def _detect_chip() -> tuple[str, float]: try: r = subprocess.run( ["sysctl", "-n", "machdep.cpu.brand_string"], - capture_output=True, text=True, timeout=3, + capture_output=True, + text=True, + timeout=3, ) brand = r.stdout.strip() except Exception as exc: @@ -113,7 +115,8 @@ class AppleEnergyMonitor(EnergyMonitor): yield from self._sample_cputime(result) def _sample_zeus( - self, result: EnergySample, + self, + result: EnergySample, ) -> Generator[EnergySample, None, None]: assert self._monitor is not None window_name = f"openjarvis_{time.monotonic_ns()}" @@ -140,7 +143,8 @@ class AppleEnergyMonitor(EnergyMonitor): result.mean_power_watts = result.energy_joules / wall def _sample_cputime( - self, result: EnergySample, + self, + result: EnergySample, ) -> Generator[EnergySample, None, None]: """Estimate energy from user+system CPU time and chip TDP. diff --git a/src/openjarvis/telemetry/energy_monitor.py b/src/openjarvis/telemetry/energy_monitor.py index 94dee6d8..9b869b69 100644 --- a/src/openjarvis/telemetry/energy_monitor.py +++ b/src/openjarvis/telemetry/energy_monitor.py @@ -119,6 +119,7 @@ def create_energy_monitor( try: from openjarvis.telemetry.energy_nvidia import NvidiaEnergyMonitor + vendor_map["nvidia"] = NvidiaEnergyMonitor default_order.append(NvidiaEnergyMonitor) except Exception as exc: @@ -126,6 +127,7 @@ def create_energy_monitor( try: from openjarvis.telemetry.energy_amd import AmdEnergyMonitor + vendor_map["amd"] = AmdEnergyMonitor default_order.append(AmdEnergyMonitor) except Exception as exc: @@ -133,6 +135,7 @@ def create_energy_monitor( try: from openjarvis.telemetry.energy_apple import AppleEnergyMonitor + vendor_map["apple"] = AppleEnergyMonitor default_order.append(AppleEnergyMonitor) except Exception as exc: @@ -140,6 +143,7 @@ def create_energy_monitor( try: from openjarvis.telemetry.energy_rapl import RaplEnergyMonitor + vendor_map["cpu_rapl"] = RaplEnergyMonitor default_order.append(RaplEnergyMonitor) except Exception as exc: diff --git a/src/openjarvis/telemetry/energy_nvidia.py b/src/openjarvis/telemetry/energy_nvidia.py index 61235be8..d5093c9b 100644 --- a/src/openjarvis/telemetry/energy_nvidia.py +++ b/src/openjarvis/telemetry/energy_nvidia.py @@ -144,13 +144,9 @@ class NvidiaEnergyMonitor(EnergyMonitor): now = time.monotonic() with lock: power_ticks.append(powers) - util_ticks.append( - sum(utils) / len(utils) if utils else 0.0 - ) + util_ticks.append(sum(utils) / len(utils) if utils else 0.0) mem_ticks.append(sum(mems)) - temp_ticks.append( - sum(temps) / len(temps) if temps else 0.0 - ) + temp_ticks.append(sum(temps) / len(temps) if temps else 0.0) timestamps.append(now) stop_event.wait(self._poll_interval_s) @@ -185,8 +181,15 @@ class NvidiaEnergyMonitor(EnergyMonitor): thread = threading.Thread( target=self._polling_loop, - args=(power_ticks, util_ticks, mem_ticks, temp_ticks, - timestamps, lock, stop_event), + args=( + power_ticks, + util_ticks, + mem_ticks, + temp_ticks, + timestamps, + lock, + stop_event, + ), daemon=True, ) @@ -203,8 +206,7 @@ class NvidiaEnergyMonitor(EnergyMonitor): if self._hw_counter_available and energy_start is not None: energy_end = self._read_energy_counters() total_mj = sum( - end - start - for start, end in zip(energy_start, energy_end) + end - start for start, end in zip(energy_start, energy_end) ) result.energy_joules = total_mj / 1000.0 result.gpu_energy_joules = result.energy_joules diff --git a/src/openjarvis/telemetry/gpu_monitor.py b/src/openjarvis/telemetry/gpu_monitor.py index f5faa659..6690b941 100644 --- a/src/openjarvis/telemetry/gpu_monitor.py +++ b/src/openjarvis/telemetry/gpu_monitor.py @@ -216,13 +216,9 @@ class GpuMonitor: for tick_snaps in all_snapshots: total_power = sum(s.power_watts for s in tick_snaps) - mean_util = ( - sum(s.utilization_pct for s in tick_snaps) / len(tick_snaps) - ) + mean_util = sum(s.utilization_pct for s in tick_snaps) / len(tick_snaps) total_mem = sum(s.memory_used_gb for s in tick_snaps) - mean_temp = ( - sum(s.temperature_c for s in tick_snaps) / len(tick_snaps) - ) + mean_temp = sum(s.temperature_c for s in tick_snaps) / len(tick_snaps) tick_powers.append(total_power) tick_utils.append(mean_util) tick_mems.append(total_mem) diff --git a/src/openjarvis/telemetry/steady_state.py b/src/openjarvis/telemetry/steady_state.py index eb1ab26d..d3fc79b8 100644 --- a/src/openjarvis/telemetry/steady_state.py +++ b/src/openjarvis/telemetry/steady_state.py @@ -69,12 +69,12 @@ class SteadyStateDetector: return True # Not enough post-warmup samples for a full window yet - post_warmup = self._throughputs[cfg.warmup_samples:] + post_warmup = self._throughputs[cfg.warmup_samples :] if len(post_warmup) < cfg.window_size: return False # Compute CV over the last window_size values - window = post_warmup[-cfg.window_size:] + window = post_warmup[-cfg.window_size :] mean = statistics.mean(window) if mean == 0: self._consecutive_stable = 0 diff --git a/src/openjarvis/telemetry/vllm_metrics.py b/src/openjarvis/telemetry/vllm_metrics.py index de4e0148..cf53fade 100644 --- a/src/openjarvis/telemetry/vllm_metrics.py +++ b/src/openjarvis/telemetry/vllm_metrics.py @@ -134,7 +134,9 @@ class VLLMMetricsScraper: resp = httpx.get(f"{self._host}/metrics", timeout=5.0) resp.raise_for_status() except ( - httpx.ConnectError, httpx.TimeoutException, httpx.HTTPStatusError, + httpx.ConnectError, + httpx.TimeoutException, + httpx.HTTPStatusError, ) as exc: logger.debug("Failed to fetch vLLM metrics: %s", exc) return VLLMMetrics() diff --git a/src/openjarvis/tools/_stubs.py b/src/openjarvis/tools/_stubs.py index 143ddaa9..98a73d1f 100644 --- a/src/openjarvis/tools/_stubs.py +++ b/src/openjarvis/tools/_stubs.py @@ -133,7 +133,9 @@ class ToolExecutor: if self._capability_policy and tool.spec.required_capabilities: for cap in tool.spec.required_capabilities: if not self._capability_policy.check( - self._agent_id, cap, tool_call.name, + self._agent_id, + cap, + tool_call.name, ): if self._bus: self._bus.publish( @@ -194,10 +196,7 @@ class ToolExecutor: ), success=False, ) - prompt = ( - f"Allow execution of tool" - f" '{tool_call.name}' with args {params}?" - ) + prompt = f"Allow execution of tool '{tool_call.name}' with args {params}?" if not self._confirm_callback(prompt): return ToolResult( tool_name=tool_call.name, @@ -227,10 +226,7 @@ class ToolExecutor: ) result = ToolResult( tool_name=tool_call.name, - content=( - f"Tool '{tool_call.name}' timed out" - f" after {timeout:.0f}s." - ), + content=(f"Tool '{tool_call.name}' timed out after {timeout:.0f}s."), success=False, ) except Exception as exc: diff --git a/src/openjarvis/tools/agent_tools.py b/src/openjarvis/tools/agent_tools.py index 1b7d6386..7966dc01 100644 --- a/src/openjarvis/tools/agent_tools.py +++ b/src/openjarvis/tools/agent_tools.py @@ -58,23 +58,18 @@ class AgentSpawnTool(BaseTool): }, "query": { "type": "string", - "description": ( - "Optional initial query to send to" - " the agent." - ), + "description": ("Optional initial query to send to the agent."), }, "tools": { "type": "string", "description": ( - "Comma-separated tool names to" - " attach to the agent." + "Comma-separated tool names to attach to the agent." ), }, "agent_id": { "type": "string", "description": ( - "Custom agent ID. Auto-generated" - " if not provided." + "Custom agent ID. Auto-generated if not provided." ), }, }, @@ -140,9 +135,7 @@ class AgentSendTool(BaseTool): def spec(self) -> ToolSpec: return ToolSpec( name="agent_send", - description=( - "Send a message to a running agent by its ID." - ), + description=("Send a message to a running agent by its ID."), parameters={ "type": "object", "properties": { @@ -203,11 +196,13 @@ class AgentSendTool(BaseTool): return ToolResult( tool_name="agent_send", - content=json.dumps({ - "agent_id": agent_id, - "delivered": True, - "message": message, - }), + content=json.dumps( + { + "agent_id": agent_id, + "delivered": True, + "message": message, + } + ), success=True, ) @@ -228,8 +223,7 @@ class AgentListTool(BaseTool): return ToolSpec( name="agent_list", description=( - "List all spawned agents with their status," - " type, and creation time." + "List all spawned agents with their status, type, and creation time." ), parameters={ "type": "object", @@ -249,12 +243,14 @@ class AgentListTool(BaseTool): agents = [] for agent_id, info in _SPAWNED_AGENTS.items(): - agents.append({ - "agent_id": agent_id, - "agent_type": info["agent_type"], - "status": info["status"], - "created_at": info["created_at"], - }) + agents.append( + { + "agent_id": agent_id, + "agent_type": info["agent_type"], + "status": info["status"], + "created_at": info["created_at"], + } + ) return ToolResult( tool_name="agent_list", @@ -278,10 +274,7 @@ class AgentKillTool(BaseTool): def spec(self) -> ToolSpec: return ToolSpec( name="agent_kill", - description=( - "Stop a running agent by its ID. Requires" - " confirmation." - ), + description=("Stop a running agent by its ID. Requires confirmation."), parameters={ "type": "object", "properties": { @@ -318,10 +311,12 @@ class AgentKillTool(BaseTool): return ToolResult( tool_name="agent_kill", - content=json.dumps({ - "agent_id": agent_id, - "status": "stopped", - }), + content=json.dumps( + { + "agent_id": agent_id, + "status": "stopped", + } + ), success=True, ) diff --git a/src/openjarvis/tools/apply_patch.py b/src/openjarvis/tools/apply_patch.py index 5924c452..6438177a 100644 --- a/src/openjarvis/tools/apply_patch.py +++ b/src/openjarvis/tools/apply_patch.py @@ -16,9 +16,7 @@ from openjarvis.tools._stubs import BaseTool, ToolSpec # Hunk / patch parsing helpers # --------------------------------------------------------------------------- -_HUNK_HEADER_RE = re.compile( - r"^@@ -(\d+)(?:,(\d+))? \+(\d+)(?:,(\d+))? @@" -) +_HUNK_HEADER_RE = re.compile(r"^@@ -(\d+)(?:,(\d+))? \+(\d+)(?:,(\d+))? @@") @dataclass @@ -168,7 +166,7 @@ def _apply_hunks(original: str, hunks: List[_Hunk]) -> str: # Splice the new lines into orig_lines consumed = check_pos - pos - orig_lines[pos:pos + consumed] = new_lines + orig_lines[pos : pos + consumed] = new_lines offset += len(new_lines) - consumed return "".join(orig_lines) @@ -199,9 +197,7 @@ class ApplyPatchTool(BaseTool): "properties": { "patch": { "type": "string", - "description": ( - "The unified diff patch text to apply." - ), + "description": ("The unified diff patch text to apply."), }, "path": { "type": "string", @@ -213,8 +209,7 @@ class ApplyPatchTool(BaseTool): "backup": { "type": "boolean", "description": ( - "Create a .bak backup before applying" - " (default: true)." + "Create a .bak backup before applying (default: true)." ), }, }, diff --git a/src/openjarvis/tools/audio_tool.py b/src/openjarvis/tools/audio_tool.py index 62198ed2..4fb77928 100644 --- a/src/openjarvis/tools/audio_tool.py +++ b/src/openjarvis/tools/audio_tool.py @@ -117,8 +117,7 @@ class AudioTranscribeTool(BaseTool): return ToolResult( tool_name="audio_transcribe", content=( - f"Unsupported provider '{provider}'." - " Supported: 'openai', 'local'." + f"Unsupported provider '{provider}'. Supported: 'openai', 'local'." ), success=False, ) @@ -130,8 +129,7 @@ class AudioTranscribeTool(BaseTool): return ToolResult( tool_name="audio_transcribe", content=( - "openai package not installed." - " Install with: pip install openai" + "openai package not installed. Install with: pip install openai" ), success=False, ) diff --git a/src/openjarvis/tools/browser.py b/src/openjarvis/tools/browser.py index d142491f..c026d587 100644 --- a/src/openjarvis/tools/browser.py +++ b/src/openjarvis/tools/browser.py @@ -25,8 +25,7 @@ class _BrowserSession: from playwright.sync_api import sync_playwright except ImportError: raise ImportError( - "playwright not installed. Install with: " - "uv sync --extra browser" + "playwright not installed. Install with: uv sync --extra browser" ) self._playwright = sync_playwright().start() self._browser = self._playwright.chromium.launch(headless=True) @@ -134,8 +133,7 @@ class BrowserNavigateTool(BaseTool): return ToolResult( tool_name="browser_navigate", content=( - "playwright not installed. Install with: " - "uv sync --extra browser" + "playwright not installed. Install with: uv sync --extra browser" ), success=False, ) @@ -214,8 +212,7 @@ class BrowserClickTool(BaseTool): return ToolResult( tool_name="browser_click", content=( - "playwright not installed. Install with: " - "uv sync --extra browser" + "playwright not installed. Install with: uv sync --extra browser" ), success=False, ) @@ -260,8 +257,7 @@ class BrowserTypeTool(BaseTool): "clear": { "type": "boolean", "description": ( - "If true, clear the field before typing." - " Default: true." + "If true, clear the field before typing. Default: true." ), }, }, @@ -306,8 +302,7 @@ class BrowserTypeTool(BaseTool): return ToolResult( tool_name="browser_type", content=( - "playwright not installed. Install with: " - "uv sync --extra browser" + "playwright not installed. Install with: uv sync --extra browser" ), success=False, ) @@ -348,8 +343,7 @@ class BrowserScreenshotTool(BaseTool): "full_page": { "type": "boolean", "description": ( - "If true, capture the full scrollable page." - " Default: false." + "If true, capture the full scrollable page. Default: false." ), }, }, @@ -387,8 +381,7 @@ class BrowserScreenshotTool(BaseTool): return ToolResult( tool_name="browser_screenshot", content=( - "playwright not installed. Install with: " - "uv sync --extra browser" + "playwright not installed. Install with: uv sync --extra browser" ), success=False, ) @@ -425,8 +418,7 @@ class BrowserExtractTool(BaseTool): "selector": { "type": "string", "description": ( - "CSS selector to extract from." - " Default: 'body'." + "CSS selector to extract from. Default: 'body'." ), }, "extract_type": { @@ -522,8 +514,7 @@ class BrowserExtractTool(BaseTool): return ToolResult( tool_name="browser_extract", content=( - "playwright not installed. Install with: " - "uv sync --extra browser" + "playwright not installed. Install with: uv sync --extra browser" ), success=False, ) diff --git a/src/openjarvis/tools/calculator.py b/src/openjarvis/tools/calculator.py index 2ec101eb..a3e02803 100644 --- a/src/openjarvis/tools/calculator.py +++ b/src/openjarvis/tools/calculator.py @@ -92,10 +92,12 @@ def safe_eval(expression: str) -> float: """Evaluate a math expression safely — Rust backend with Python fallback.""" try: from openjarvis._rust_bridge import get_rust_module + _rust = get_rust_module() return float(_rust.CalculatorTool().execute(expression)) except ImportError: import ast as _ast + tree = _ast.parse(expression, mode="eval") return float(_safe_eval_node(tree.body)) @@ -121,8 +123,7 @@ class CalculatorTool(BaseTool): "expression": { "type": "string", "description": ( - "Math expression to evaluate" - " (e.g. '2+3*4', 'sqrt(16)')" + "Math expression to evaluate (e.g. '2+3*4', 'sqrt(16)')" ), }, }, diff --git a/src/openjarvis/tools/code_interpreter_docker.py b/src/openjarvis/tools/code_interpreter_docker.py index 97fe5fe8..c2147b65 100644 --- a/src/openjarvis/tools/code_interpreter_docker.py +++ b/src/openjarvis/tools/code_interpreter_docker.py @@ -100,10 +100,12 @@ class DockerCodeInterpreterTool(BaseTool): result = container.wait(timeout=self._timeout) exit_code = result.get("StatusCode", -1) stdout = container.logs( - stdout=True, stderr=False, + stdout=True, + stderr=False, ).decode("utf-8", errors="replace") stderr = container.logs( - stdout=False, stderr=True, + stdout=False, + stderr=True, ).decode("utf-8", errors="replace") finally: container.remove(force=True) @@ -112,9 +114,7 @@ class DockerCodeInterpreterTool(BaseTool): if stderr: output += ("\n" if output else "") + stderr if len(output) > self._max_output: - output = ( - output[: self._max_output] + "\n... (output truncated)" - ) + output = output[: self._max_output] + "\n... (output truncated)" return ToolResult( tool_name="code_interpreter_docker", @@ -125,16 +125,10 @@ class DockerCodeInterpreterTool(BaseTool): except Exception as exc: error_type = type(exc).__name__ - if ( - "timeout" in str(exc).lower() - or "read timed out" in str(exc).lower() - ): + if "timeout" in str(exc).lower() or "read timed out" in str(exc).lower(): return ToolResult( tool_name="code_interpreter_docker", - content=( - f"Execution timed out after" - f" {self._timeout} seconds." - ), + content=(f"Execution timed out after {self._timeout} seconds."), success=False, ) return ToolResult( diff --git a/src/openjarvis/tools/db_query.py b/src/openjarvis/tools/db_query.py index 28ee1c5c..3907a559 100644 --- a/src/openjarvis/tools/db_query.py +++ b/src/openjarvis/tools/db_query.py @@ -137,8 +137,7 @@ class DatabaseQueryTool(BaseTool): "max_rows": { "type": "integer", "description": ( - "Maximum number of result rows to return." - " Default: 100." + "Maximum number of result rows to return. Default: 100." ), }, }, @@ -178,7 +177,10 @@ class DatabaseQueryTool(BaseTool): # Route to the appropriate backend if db_url: return self._execute_postgresql( - query, db_url, read_only, max_rows, + query, + db_url, + read_only, + max_rows, ) return self._execute_sqlite(query, db_path, read_only, max_rows) @@ -249,8 +251,7 @@ class DatabaseQueryTool(BaseTool): content = _format_table(column_names, rows) else: content = ( - f"Query executed successfully." - f" Rows affected: {cursor.rowcount}" + f"Query executed successfully. Rows affected: {cursor.rowcount}" ) return ToolResult( @@ -330,8 +331,7 @@ class DatabaseQueryTool(BaseTool): content = _format_table(column_names, rows) else: content = ( - f"Query executed successfully." - f" Rows affected: {cursor.rowcount}" + f"Query executed successfully. Rows affected: {cursor.rowcount}" ) return ToolResult( diff --git a/src/openjarvis/tools/file_read.py b/src/openjarvis/tools/file_read.py index 6e5b7e38..c96195d6 100644 --- a/src/openjarvis/tools/file_read.py +++ b/src/openjarvis/tools/file_read.py @@ -29,10 +29,7 @@ class FileReadTool(BaseTool): def spec(self) -> ToolSpec: return ToolSpec( name="file_read", - description=( - "Read the contents of a file." - " Returns the text content." - ), + description=("Read the contents of a file. Returns the text content."), parameters={ "type": "object", "properties": { @@ -42,10 +39,7 @@ class FileReadTool(BaseTool): }, "max_lines": { "type": "integer", - "description": ( - "Max lines to return" - " (default: all)." - ), + "description": ("Max lines to return (default: all)."), }, }, "required": ["path"], @@ -116,6 +110,7 @@ class FileReadTool(BaseTool): ) try: from openjarvis._rust_bridge import get_rust_module + _rust = get_rust_module() text = _rust.FileReadTool().execute(str(path)) except ImportError: diff --git a/src/openjarvis/tools/file_write.py b/src/openjarvis/tools/file_write.py index 71be96df..dfd789fc 100644 --- a/src/openjarvis/tools/file_write.py +++ b/src/openjarvis/tools/file_write.py @@ -29,10 +29,7 @@ class FileWriteTool(BaseTool): def spec(self) -> ToolSpec: return ToolSpec( name="file_write", - description=( - "Write content to a file." - " Supports write and append modes." - ), + description=("Write content to a file. Supports write and append modes."), parameters={ "type": "object", "properties": { @@ -158,6 +155,7 @@ class FileWriteTool(BaseTool): if mode == "write": try: from openjarvis._rust_bridge import get_rust_module + _rust = get_rust_module() _rust.FileWriteTool().execute(str(path), content) except ImportError: diff --git a/src/openjarvis/tools/git_tool.py b/src/openjarvis/tools/git_tool.py index e23b5609..b660f974 100644 --- a/src/openjarvis/tools/git_tool.py +++ b/src/openjarvis/tools/git_tool.py @@ -121,8 +121,7 @@ class GitStatusTool(BaseTool): "repo_path": { "type": "string", "description": ( - "Path to the git repository." - " Default: current directory." + "Path to the git repository. Default: current directory." ), }, }, @@ -176,22 +175,19 @@ class GitDiffTool(BaseTool): "repo_path": { "type": "string", "description": ( - "Path to the git repository." - " Default: current directory." + "Path to the git repository. Default: current directory." ), }, "staged": { "type": "boolean", "description": ( - "Show staged changes instead of" - " unstaged. Default: false." + "Show staged changes instead of unstaged. Default: false." ), }, "path": { "type": "string", "description": ( - "Specific file path to diff." - " Default: all files." + "Specific file path to diff. Default: all files." ), }, }, @@ -262,8 +258,7 @@ class GitCommitTool(BaseTool): "repo_path": { "type": "string", "description": ( - "Path to the git repository." - " Default: current directory." + "Path to the git repository. Default: current directory." ), }, "files": { @@ -304,7 +299,8 @@ class GitCommitTool(BaseTool): success=False, ) add_result = _run_git( - ["git", "add"] + file_list, cwd=repo_path, + ["git", "add"] + file_list, + cwd=repo_path, ) if not add_result.success: return ToolResult( @@ -316,7 +312,8 @@ class GitCommitTool(BaseTool): # Commit return _run_git( - ["git", "commit", "-m", message], cwd=repo_path, + ["git", "commit", "-m", message], + cwd=repo_path, ) @@ -345,23 +342,16 @@ class GitLogTool(BaseTool): "repo_path": { "type": "string", "description": ( - "Path to the git repository." - " Default: current directory." + "Path to the git repository. Default: current directory." ), }, "count": { "type": "integer", - "description": ( - "Number of commits to show." - " Default: 10." - ), + "description": ("Number of commits to show. Default: 10."), }, "oneline": { "type": "boolean", - "description": ( - "Use --oneline format." - " Default: true." - ), + "description": ("Use --oneline format. Default: true."), }, }, "required": [], diff --git a/src/openjarvis/tools/http_request.py b/src/openjarvis/tools/http_request.py index d519815a..fe61adbb 100644 --- a/src/openjarvis/tools/http_request.py +++ b/src/openjarvis/tools/http_request.py @@ -106,6 +106,7 @@ class HttpRequestTool(BaseTool): _rust = None try: from openjarvis._rust_bridge import get_rust_module + _rust = get_rust_module() except ImportError: pass diff --git a/src/openjarvis/tools/image_tool.py b/src/openjarvis/tools/image_tool.py index 960665a8..5b18b5ee 100644 --- a/src/openjarvis/tools/image_tool.py +++ b/src/openjarvis/tools/image_tool.py @@ -23,8 +23,7 @@ class ImageGenerateTool(BaseTool): return ToolSpec( name="image_generate", description=( - "Generate an image from a text description." - " Returns the image URL." + "Generate an image from a text description. Returns the image URL." ), parameters={ "type": "object", @@ -82,8 +81,7 @@ class ImageGenerateTool(BaseTool): return ToolResult( tool_name="image_generate", content=( - f"Unsupported provider '{provider}'." - " Only 'openai' is supported." + f"Unsupported provider '{provider}'. Only 'openai' is supported." ), success=False, ) @@ -94,8 +92,7 @@ class ImageGenerateTool(BaseTool): return ToolResult( tool_name="image_generate", content=( - "openai package not installed." - " Install with: pip install openai" + "openai package not installed. Install with: pip install openai" ), success=False, ) @@ -137,10 +134,7 @@ class ImageGenerateTool(BaseTool): except Exception as exc: return ToolResult( tool_name="image_generate", - content=( - f"Image generated but failed to save: {exc}." - f" URL: {url}" - ), + content=(f"Image generated but failed to save: {exc}. URL: {url}"), success=False, metadata={"url": url, "size": size, "provider": provider}, ) diff --git a/src/openjarvis/tools/knowledge_tools.py b/src/openjarvis/tools/knowledge_tools.py index 309357db..aab22c49 100644 --- a/src/openjarvis/tools/knowledge_tools.py +++ b/src/openjarvis/tools/knowledge_tools.py @@ -34,8 +34,7 @@ class KGAddEntityTool(BaseTool): "entity_type": { "type": "string", "description": ( - "Entity type (e.g., 'concept'," - " 'tool', 'user')." + "Entity type (e.g., 'concept', 'tool', 'user')." ), }, "name": { @@ -44,9 +43,7 @@ class KGAddEntityTool(BaseTool): }, "properties": { "type": "object", - "description": ( - "Additional properties." - ), + "description": ("Additional properties."), }, }, "required": ["entity_id", "entity_type", "name"], @@ -57,15 +54,16 @@ class KGAddEntityTool(BaseTool): def execute(self, **params: Any) -> ToolResult: if not self._backend or not hasattr( - self._backend, "add_entity", + self._backend, + "add_entity", ): return ToolResult( tool_name="kg_add_entity", - content="No knowledge graph backend" - " available.", + content="No knowledge graph backend available.", success=False, ) from openjarvis.tools.storage.knowledge_graph import Entity + entity = Entity( entity_id=params["entity_id"], entity_type=params["entity_type"], @@ -73,7 +71,7 @@ class KGAddEntityTool(BaseTool): properties=params.get("properties", {}), ) self._backend.add_entity(entity) - name = params['name'] + name = params["name"] return ToolResult( tool_name="kg_add_entity", content=f"Entity '{name}' added.", @@ -108,17 +106,11 @@ class KGAddRelationTool(BaseTool): }, "relation_type": { "type": "string", - "description": ( - "Relation type (e.g.," - " 'used', 'depends_on')." - ), + "description": ("Relation type (e.g., 'used', 'depends_on')."), }, "weight": { "type": "number", - "description": ( - "Relation weight" - " (default 1.0)." - ), + "description": ("Relation weight (default 1.0)."), }, }, "required": ["source_id", "target_id", "relation_type"], @@ -129,15 +121,16 @@ class KGAddRelationTool(BaseTool): def execute(self, **params: Any) -> ToolResult: if not self._backend or not hasattr( - self._backend, "add_relation", + self._backend, + "add_relation", ): return ToolResult( tool_name="kg_add_relation", - content="No knowledge graph backend" - " available.", + content="No knowledge graph backend available.", success=False, ) from openjarvis.tools.storage.knowledge_graph import Relation + relation = Relation( source_id=params["source_id"], target_id=params["target_id"], @@ -145,7 +138,7 @@ class KGAddRelationTool(BaseTool): weight=params.get("weight", 1.0), ) self._backend.add_relation(relation) - rtype = params['relation_type'] + rtype = params["relation_type"] return ToolResult( tool_name="kg_add_relation", content=f"Relation '{rtype}' added.", @@ -172,21 +165,15 @@ class KGQueryTool(BaseTool): "properties": { "entity_type": { "type": "string", - "description": ( - "Filter by entity type." - ), + "description": ("Filter by entity type."), }, "relation_type": { "type": "string", - "description": ( - "Filter by relation type." - ), + "description": ("Filter by relation type."), }, "limit": { "type": "integer", - "description": ( - "Max results (default 50)." - ), + "description": ("Max results (default 50)."), }, }, }, @@ -196,12 +183,12 @@ class KGQueryTool(BaseTool): def execute(self, **params: Any) -> ToolResult: if not self._backend or not hasattr( - self._backend, "query_pattern", + self._backend, + "query_pattern", ): return ToolResult( tool_name="kg_query", - content="No knowledge graph backend" - " available.", + content="No knowledge graph backend available.", success=False, ) result = self._backend.query_pattern( @@ -250,31 +237,21 @@ class KGNeighborsTool(BaseTool): "properties": { "entity_id": { "type": "string", - "description": ( - "Entity ID to find" - " neighbors for." - ), + "description": ("Entity ID to find neighbors for."), }, "relation_type": { "type": "string", - "description": ( - "Filter by relation type." - ), + "description": ("Filter by relation type."), }, "direction": { "type": "string", "description": ( - "Direction: 'in', 'out'," - " or 'both'" - " (default 'both')." + "Direction: 'in', 'out', or 'both' (default 'both')." ), }, "limit": { "type": "integer", - "description": ( - "Max results" - " (default 50)." - ), + "description": ("Max results (default 50)."), }, }, "required": ["entity_id"], @@ -285,12 +262,12 @@ class KGNeighborsTool(BaseTool): def execute(self, **params: Any) -> ToolResult: if not self._backend or not hasattr( - self._backend, "neighbors", + self._backend, + "neighbors", ): return ToolResult( tool_name="kg_neighbors", - content="No knowledge graph backend" - " available.", + content="No knowledge graph backend available.", success=False, ) neighbors = self._backend.neighbors( diff --git a/src/openjarvis/tools/mcp_adapter.py b/src/openjarvis/tools/mcp_adapter.py index c276aa9a..50a710c6 100644 --- a/src/openjarvis/tools/mcp_adapter.py +++ b/src/openjarvis/tools/mcp_adapter.py @@ -39,9 +39,7 @@ class MCPToolAdapter(BaseTool): result = self._client.call_tool(self._spec.name, params) content_parts = result.get("content", []) text = "\n".join( - p.get("text", "") - for p in content_parts - if isinstance(p, dict) + p.get("text", "") for p in content_parts if isinstance(p, dict) ) return ToolResult( tool_name=self._spec.name, diff --git a/src/openjarvis/tools/memory_manage.py b/src/openjarvis/tools/memory_manage.py index ff2731fb..8088e50a 100644 --- a/src/openjarvis/tools/memory_manage.py +++ b/src/openjarvis/tools/memory_manage.py @@ -86,9 +86,7 @@ class MemoryManageTool(BaseTool): content="Entry cannot be empty.", ) self._memory_path.parent.mkdir(parents=True, exist_ok=True) - existing = ( - self._memory_path.read_text() if self._memory_path.exists() else "" - ) + existing = self._memory_path.read_text() if self._memory_path.exists() else "" self._memory_path.write_text(existing.rstrip() + f"\n- {entry}\n") return ToolResult( tool_name=self.spec.name, diff --git a/src/openjarvis/tools/pdf_tool.py b/src/openjarvis/tools/pdf_tool.py index c47ad0fb..12e61aa0 100644 --- a/src/openjarvis/tools/pdf_tool.py +++ b/src/openjarvis/tools/pdf_tool.py @@ -58,8 +58,7 @@ class PDFExtractTool(BaseTool): return ToolSpec( name="pdf_extract", description=( - "Extract text from a PDF file." - " Returns the extracted text content." + "Extract text from a PDF file. Returns the extracted text content." ), parameters={ "type": "object", @@ -77,10 +76,7 @@ class PDFExtractTool(BaseTool): }, "max_chars": { "type": "integer", - "description": ( - "Maximum characters to return." - " Default 50000." - ), + "description": ("Maximum characters to return. Default 50000."), }, }, "required": ["file_path"], diff --git a/src/openjarvis/tools/repl.py b/src/openjarvis/tools/repl.py index f891c56b..1394c046 100644 --- a/src/openjarvis/tools/repl.py +++ b/src/openjarvis/tools/repl.py @@ -39,18 +39,50 @@ _BLOCKED_PATTERNS = [ # Layer 2: Restricted builtins — remove dangerous ones _REMOVED_BUILTINS = { - "open", "exec", "eval", "compile", "__import__", - "breakpoint", "exit", "quit", "input", + "open", + "exec", + "eval", + "compile", + "__import__", + "breakpoint", + "exit", + "quit", + "input", } # Layer 3: Safe import allowlist -_SAFE_IMPORT_MODULES = frozenset({ - "math", "cmath", "decimal", "fractions", "random", "statistics", - "itertools", "functools", "operator", "collections", "string", - "re", "textwrap", "datetime", "time", "calendar", - "json", "csv", "copy", "dataclasses", "enum", "typing", - "heapq", "bisect", "array", "pprint", "abc", "numbers", -}) +_SAFE_IMPORT_MODULES = frozenset( + { + "math", + "cmath", + "decimal", + "fractions", + "random", + "statistics", + "itertools", + "functools", + "operator", + "collections", + "string", + "re", + "textwrap", + "datetime", + "time", + "calendar", + "json", + "csv", + "copy", + "dataclasses", + "enum", + "typing", + "heapq", + "bisect", + "array", + "pprint", + "abc", + "numbers", + } +) def _make_safe_import(allowed: frozenset = _SAFE_IMPORT_MODULES): diff --git a/src/openjarvis/tools/scan_chunks.py b/src/openjarvis/tools/scan_chunks.py index f1e13086..5076b7e8 100644 --- a/src/openjarvis/tools/scan_chunks.py +++ b/src/openjarvis/tools/scan_chunks.py @@ -164,9 +164,7 @@ class ScanChunksTool(BaseTool): ), ] - result = self._engine.generate( - messages, model=self._model, max_tokens=1024 - ) + result = self._engine.generate(messages, model=self._model, max_tokens=1024) content = result.get("content", "").strip() if content and "NOTHING_RELEVANT" not in content: findings.append(content) diff --git a/src/openjarvis/tools/shell_exec.py b/src/openjarvis/tools/shell_exec.py index 6611df7a..e6f0d360 100644 --- a/src/openjarvis/tools/shell_exec.py +++ b/src/openjarvis/tools/shell_exec.py @@ -47,9 +47,7 @@ class ShellExecTool(BaseTool): }, "timeout": { "type": "integer", - "description": ( - "Timeout in seconds (default 30, max 300)." - ), + "description": ("Timeout in seconds (default 30, max 300)."), }, "working_dir": { "type": "string", @@ -127,6 +125,7 @@ class ShellExecTool(BaseTool): try: from openjarvis._rust_bridge import get_rust_module + _rust = get_rust_module() output = _rust.ShellExecTool().execute(command, working_dir) return ToolResult( diff --git a/src/openjarvis/tools/skill_manage.py b/src/openjarvis/tools/skill_manage.py index adc1b894..c9767315 100644 --- a/src/openjarvis/tools/skill_manage.py +++ b/src/openjarvis/tools/skill_manage.py @@ -70,9 +70,7 @@ class SkillManageTool(BaseTool): content=f"Unknown action: {action}", ) - def _create( - self, name: str, description: str, steps: List[dict] - ) -> ToolResult: + def _create(self, name: str, description: str, steps: List[dict]) -> ToolResult: if not name: return ToolResult( tool_name=self.spec.name, @@ -91,9 +89,7 @@ class SkillManageTool(BaseTool): lines.append("[[skill.steps]]") lines.append(f'tool_name = "{step.get("tool_name", "")}"') if "arguments_template" in step: - lines.append( - f"arguments_template = '{step['arguments_template']}'" - ) + lines.append(f"arguments_template = '{step['arguments_template']}'") if "output_key" in step: lines.append(f'output_key = "{step["output_key"]}"') lines.append("") @@ -123,8 +119,7 @@ class SkillManageTool(BaseTool): return ToolResult( tool_name=self.spec.name, success=True, - content="Available skills:\n" - + "\n".join(f"- {s}" for s in skills), + content="Available skills:\n" + "\n".join(f"- {s}" for s in skills), ) def _load(self, name: str) -> ToolResult: diff --git a/src/openjarvis/tools/storage/bm25.py b/src/openjarvis/tools/storage/bm25.py index 23536d99..ac739a68 100644 --- a/src/openjarvis/tools/storage/bm25.py +++ b/src/openjarvis/tools/storage/bm25.py @@ -46,11 +46,14 @@ class BM25Memory(MemoryBackend): meta_json = json.dumps(metadata) if metadata else None doc_id = self._rust_impl.store(content, source, meta_json) bus = get_event_bus() - bus.publish(EventType.MEMORY_STORE, { - "backend": self.backend_id, - "doc_id": doc_id, - "source": source, - }) + bus.publish( + EventType.MEMORY_STORE, + { + "backend": self.backend_id, + "doc_id": doc_id, + "source": source, + }, + ) return doc_id def retrieve( @@ -64,15 +67,19 @@ class BM25Memory(MemoryBackend): if not query.strip(): return [] from openjarvis._rust_bridge import retrieval_results_from_json + results = retrieval_results_from_json( self._rust_impl.retrieve(query, top_k), ) bus = get_event_bus() - bus.publish(EventType.MEMORY_RETRIEVE, { - "backend": self.backend_id, - "query": query, - "num_results": len(results), - }) + bus.publish( + EventType.MEMORY_RETRIEVE, + { + "backend": self.backend_id, + "query": query, + "num_results": len(results), + }, + ) return results def delete(self, doc_id: str) -> bool: diff --git a/src/openjarvis/tools/storage/chunking.py b/src/openjarvis/tools/storage/chunking.py index 96af345c..70e7ada5 100644 --- a/src/openjarvis/tools/storage/chunking.py +++ b/src/openjarvis/tools/storage/chunking.py @@ -75,22 +75,21 @@ def chunk_text( # If adding this paragraph would exceed chunk_size and we already # have content, flush the current chunk first. - if ( - current_tokens - and len(current_tokens) + len(para_tokens) > cfg.chunk_size - ): + if current_tokens and len(current_tokens) + len(para_tokens) > cfg.chunk_size: chunk_content = " ".join(current_tokens) if _count_tokens(chunk_content) >= cfg.min_chunk_size: - chunks.append(Chunk( - content=chunk_content, - source=source, - offset=chunk_start_offset, - index=len(chunks), - )) + chunks.append( + Chunk( + content=chunk_content, + source=source, + offset=chunk_start_offset, + index=len(chunks), + ) + ) # Keep the overlap tail for the next chunk if cfg.chunk_overlap > 0 and len(current_tokens) > cfg.chunk_overlap: - overlap = current_tokens[-cfg.chunk_overlap:] + overlap = current_tokens[-cfg.chunk_overlap :] current_tokens = list(overlap) else: current_tokens = [] @@ -102,26 +101,30 @@ def chunk_text( if current_tokens: chunk_content = " ".join(current_tokens) if _count_tokens(chunk_content) >= cfg.min_chunk_size: - chunks.append(Chunk( - content=chunk_content, - source=source, - offset=chunk_start_offset, - index=len(chunks), - )) + chunks.append( + Chunk( + content=chunk_content, + source=source, + offset=chunk_start_offset, + index=len(chunks), + ) + ) current_tokens = [] # Split the oversized paragraph into fixed windows idx = 0 while idx < len(para_tokens): - window = para_tokens[idx:idx + cfg.chunk_size] + window = para_tokens[idx : idx + cfg.chunk_size] chunk_content = " ".join(window) if _count_tokens(chunk_content) >= cfg.min_chunk_size: - chunks.append(Chunk( - content=chunk_content, - source=source, - offset=current_offset + idx, - index=len(chunks), - )) + chunks.append( + Chunk( + content=chunk_content, + source=source, + offset=current_offset + idx, + index=len(chunks), + ) + ) step = max(1, cfg.chunk_size - cfg.chunk_overlap) idx += step @@ -136,12 +139,14 @@ def chunk_text( if current_tokens: chunk_content = " ".join(current_tokens) if _count_tokens(chunk_content) >= cfg.min_chunk_size: - chunks.append(Chunk( - content=chunk_content, - source=source, - offset=chunk_start_offset, - index=len(chunks), - )) + chunks.append( + Chunk( + content=chunk_content, + source=source, + offset=chunk_start_offset, + index=len(chunks), + ) + ) return chunks diff --git a/src/openjarvis/tools/storage/colbert_backend.py b/src/openjarvis/tools/storage/colbert_backend.py index 46d59720..dc1becc0 100644 --- a/src/openjarvis/tools/storage/colbert_backend.py +++ b/src/openjarvis/tools/storage/colbert_backend.py @@ -62,9 +62,7 @@ class ColBERTMemory(MemoryBackend): self._device = device # id -> (content, source, metadata) - self._documents: Dict[ - str, Tuple[str, str, Dict[str, Any]] - ] = {} + self._documents: Dict[str, Tuple[str, str, Dict[str, Any]]] = {} # id -> token-level embedding tensor self._embeddings: Dict[str, Any] = {} @@ -135,10 +133,12 @@ class ColBERTMemory(MemoryBackend): # Normalize to unit vectors for cosine similarity q_norm = _torch.nn.functional.normalize( - query_embs.float(), dim=-1, + query_embs.float(), + dim=-1, ) d_norm = _torch.nn.functional.normalize( - doc_embs.float(), dim=-1, + doc_embs.float(), + dim=-1, ) # (Q, D) x (D, N) -> (Q, N) cosine similarity matrix @@ -166,11 +166,14 @@ class ColBERTMemory(MemoryBackend): self._embeddings[doc_id] = self._encode(content) bus = get_event_bus() - bus.publish(EventType.MEMORY_STORE, { - "backend": self.backend_id, - "doc_id": doc_id, - "source": source, - }) + bus.publish( + EventType.MEMORY_STORE, + { + "backend": self.backend_id, + "doc_id": doc_id, + "source": source, + }, + ) return doc_id def retrieve( @@ -183,11 +186,14 @@ class ColBERTMemory(MemoryBackend): """Search for *query* and return the top-k results.""" if not query.strip() or not self._documents: bus = get_event_bus() - bus.publish(EventType.MEMORY_RETRIEVE, { - "backend": self.backend_id, - "query": query, - "num_results": 0, - }) + bus.publish( + EventType.MEMORY_RETRIEVE, + { + "backend": self.backend_id, + "query": query, + "num_results": 0, + }, + ) return [] query_embs = self._encode(query) @@ -202,19 +208,24 @@ class ColBERTMemory(MemoryBackend): results: List[RetrievalResult] = [] for doc_id, score in scored[:top_k]: content, source, metadata = self._documents[doc_id] - results.append(RetrievalResult( - content=content, - score=score, - source=source, - metadata=dict(metadata), - )) + results.append( + RetrievalResult( + content=content, + score=score, + source=source, + metadata=dict(metadata), + ) + ) bus = get_event_bus() - bus.publish(EventType.MEMORY_RETRIEVE, { - "backend": self.backend_id, - "query": query, - "num_results": len(results), - }) + bus.publish( + EventType.MEMORY_RETRIEVE, + { + "backend": self.backend_id, + "query": query, + "num_results": len(results), + }, + ) return results def delete(self, doc_id: str) -> bool: diff --git a/src/openjarvis/tools/storage/context.py b/src/openjarvis/tools/storage/context.py index 4770f71e..07f4d0be 100644 --- a/src/openjarvis/tools/storage/context.py +++ b/src/openjarvis/tools/storage/context.py @@ -52,8 +52,7 @@ def build_context_message( content = ( "The following context was retrieved from the knowledge" " base. Use it to inform your response, citing sources" - " where applicable:\n\n" - + context_text + " where applicable:\n\n" + context_text ) return Message(role=Role.SYSTEM, content=content) @@ -109,12 +108,15 @@ def inject_context( # Publish event bus = get_event_bus() - bus.publish(EventType.MEMORY_RETRIEVE, { - "context_injection": True, - "query": query, - "num_results": len(truncated), - "total_tokens": total_tokens, - }) + bus.publish( + EventType.MEMORY_RETRIEVE, + { + "context_injection": True, + "query": query, + "num_results": len(truncated), + "total_tokens": total_tokens, + }, + ) # Build context message and prepend ctx_msg = build_context_message(truncated) diff --git a/src/openjarvis/tools/storage/embeddings.py b/src/openjarvis/tools/storage/embeddings.py index ad410161..8bcd74cc 100644 --- a/src/openjarvis/tools/storage/embeddings.py +++ b/src/openjarvis/tools/storage/embeddings.py @@ -35,9 +35,7 @@ class SentenceTransformerEmbedder(Embedder): ``all-MiniLM-L6-v2`` (384-dim, ~22 MB). """ - def __init__( - self, model_name: str = "all-MiniLM-L6-v2" - ) -> None: + def __init__(self, model_name: str = "all-MiniLM-L6-v2") -> None: try: from sentence_transformers import ( SentenceTransformer, @@ -50,15 +48,11 @@ class SentenceTransformerEmbedder(Embedder): ) from exc self._model = SentenceTransformer(model_name) - self._dim: int = ( - self._model.get_sentence_embedding_dimension() - ) + self._dim: int = self._model.get_sentence_embedding_dimension() def embed(self, texts: list[str]) -> Any: """Return a numpy array of shape ``(len(texts), dim)``.""" - return self._model.encode( - texts, convert_to_numpy=True - ) + return self._model.encode(texts, convert_to_numpy=True) def dim(self) -> int: """Return the embedding dimensionality.""" diff --git a/src/openjarvis/tools/storage/faiss_backend.py b/src/openjarvis/tools/storage/faiss_backend.py index fc5db8ce..06d0614f 100644 --- a/src/openjarvis/tools/storage/faiss_backend.py +++ b/src/openjarvis/tools/storage/faiss_backend.py @@ -46,9 +46,7 @@ class FAISSMemory(MemoryBackend): embedder = SentenceTransformerEmbedder() self._embedder = embedder self._index = faiss.IndexFlatIP(self._embedder.dim()) - self._documents: Dict[ - str, Tuple[str, str, Dict[str, Any]] - ] = {} + self._documents: Dict[str, Tuple[str, str, Dict[str, Any]]] = {} self._id_map: List[str] = [] self._deleted: Set[str] = set() @@ -116,9 +114,7 @@ class FAISSMemory(MemoryBackend): scores, indices = self._index.search(vec, k) results: List[RetrievalResult] = [] - for score, idx in zip( - scores[0].tolist(), indices[0].tolist() - ): + for score, idx in zip(scores[0].tolist(), indices[0].tolist()): if idx < 0: continue doc_id = self._id_map[idx] @@ -149,10 +145,7 @@ class FAISSMemory(MemoryBackend): def delete(self, doc_id: str) -> bool: """Soft-delete *doc_id*. Return True if it existed.""" - if ( - doc_id not in self._documents - or doc_id in self._deleted - ): + if doc_id not in self._documents or doc_id in self._deleted: return False self._deleted.add(doc_id) return True diff --git a/src/openjarvis/tools/storage/hybrid.py b/src/openjarvis/tools/storage/hybrid.py index 8c0aa5e2..1f9cf703 100644 --- a/src/openjarvis/tools/storage/hybrid.py +++ b/src/openjarvis/tools/storage/hybrid.py @@ -55,12 +55,14 @@ def reciprocal_rank_fusion( scores.items(), key=lambda x: x[1], reverse=True ): original = best_result[content_key] - fused.append(RetrievalResult( - content=original.content, - score=fused_score, - source=original.source, - metadata=original.metadata, - )) + fused.append( + RetrievalResult( + content=original.content, + score=fused_score, + source=original.source, + metadata=original.metadata, + ) + ) return fused @@ -101,21 +103,28 @@ class HybridMemory(MemoryBackend): """Store in both sub-backends with the same doc id.""" # Store in sparse first to get the id sparse_id = self._sparse.store( - content, source=source, metadata=metadata, + content, + source=source, + metadata=metadata, ) # Store in dense — it generates its own id dense_id = self._dense.store( - content, source=source, metadata=metadata, + content, + source=source, + metadata=metadata, ) # Map sparse_id -> dense_id so we can delete from both self._id_map[sparse_id] = dense_id bus = get_event_bus() - bus.publish(EventType.MEMORY_STORE, { - "backend": self.backend_id, - "doc_id": sparse_id, - "source": source, - }) + bus.publish( + EventType.MEMORY_STORE, + { + "backend": self.backend_id, + "doc_id": sparse_id, + "source": source, + }, + ) return sparse_id def retrieve( @@ -130,10 +139,12 @@ class HybridMemory(MemoryBackend): fetch_k = top_k * 3 sparse_results = self._sparse.retrieve( - query, top_k=fetch_k, + query, + top_k=fetch_k, ) dense_results = self._dense.retrieve( - query, top_k=fetch_k, + query, + top_k=fetch_k, ) fused = reciprocal_rank_fusion( @@ -143,11 +154,14 @@ class HybridMemory(MemoryBackend): ) bus = get_event_bus() - bus.publish(EventType.MEMORY_RETRIEVE, { - "backend": self.backend_id, - "query": query, - "num_results": min(len(fused), top_k), - }) + bus.publish( + EventType.MEMORY_RETRIEVE, + { + "backend": self.backend_id, + "query": query, + "num_results": min(len(fused), top_k), + }, + ) return fused[:top_k] diff --git a/src/openjarvis/tools/storage/ingest.py b/src/openjarvis/tools/storage/ingest.py index 45670682..4ab5b532 100644 --- a/src/openjarvis/tools/storage/ingest.py +++ b/src/openjarvis/tools/storage/ingest.py @@ -9,19 +9,60 @@ from typing import List, Optional, Tuple from openjarvis.tools.storage.chunking import Chunk, ChunkConfig, chunk_text # Directories to skip when walking a tree -_SKIP_DIRS = frozenset({ - "__pycache__", ".git", ".hg", ".svn", "node_modules", - ".venv", "venv", ".tox", ".mypy_cache", ".ruff_cache", - ".pytest_cache", "__pypackages__", ".eggs", "*.egg-info", -}) +_SKIP_DIRS = frozenset( + { + "__pycache__", + ".git", + ".hg", + ".svn", + "node_modules", + ".venv", + "venv", + ".tox", + ".mypy_cache", + ".ruff_cache", + ".pytest_cache", + "__pypackages__", + ".eggs", + "*.egg-info", + } +) # Extension -> file-type mapping -_CODE_EXTS = frozenset({ - ".py", ".js", ".ts", ".tsx", ".jsx", ".rs", ".go", ".java", - ".c", ".cpp", ".h", ".hpp", ".rb", ".sh", ".bash", ".zsh", - ".lua", ".swift", ".kt", ".scala", ".cs", ".r", ".sql", - ".yaml", ".yml", ".toml", ".json", ".xml", ".html", ".css", -}) +_CODE_EXTS = frozenset( + { + ".py", + ".js", + ".ts", + ".tsx", + ".jsx", + ".rs", + ".go", + ".java", + ".c", + ".cpp", + ".h", + ".hpp", + ".rb", + ".sh", + ".bash", + ".zsh", + ".lua", + ".swift", + ".kt", + ".scala", + ".cs", + ".r", + ".sql", + ".yaml", + ".yml", + ".toml", + ".json", + ".xml", + ".html", + ".css", + } +) @dataclass(slots=True) @@ -163,11 +204,31 @@ def ingest_path( # Skip binary-looking files if child.suffix.lower() in { - ".png", ".jpg", ".jpeg", ".gif", ".bmp", ".ico", - ".mp3", ".mp4", ".wav", ".avi", ".mov", - ".zip", ".tar", ".gz", ".bz2", ".7z", - ".exe", ".dll", ".so", ".dylib", ".o", - ".pyc", ".pyo", ".class", ".wasm", + ".png", + ".jpg", + ".jpeg", + ".gif", + ".bmp", + ".ico", + ".mp3", + ".mp4", + ".wav", + ".avi", + ".mov", + ".zip", + ".tar", + ".gz", + ".bz2", + ".7z", + ".exe", + ".dll", + ".so", + ".dylib", + ".o", + ".pyc", + ".pyo", + ".class", + ".wasm", }: continue diff --git a/src/openjarvis/tools/storage/knowledge_graph.py b/src/openjarvis/tools/storage/knowledge_graph.py index 6c06d34b..4890de61 100644 --- a/src/openjarvis/tools/storage/knowledge_graph.py +++ b/src/openjarvis/tools/storage/knowledge_graph.py @@ -16,8 +16,9 @@ from openjarvis.core.registry import MemoryRegistry @dataclass(slots=True) class Entity: """A node in the knowledge graph.""" + entity_id: str - entity_type: str # "agent", "tool", "model", "user", "concept", etc. + entity_type: str # "agent", "tool", "model", "user", "concept", etc. name: str properties: Dict[str, Any] = field(default_factory=dict) created_at: float = 0.0 @@ -26,6 +27,7 @@ class Entity: @dataclass(slots=True) class Relation: """An edge between two entities.""" + source_id: str target_id: str relation_type: str # "used", "produced", "depends_on", "similar_to", etc. @@ -37,6 +39,7 @@ class Relation: @dataclass(slots=True) class GraphQueryResult: """Result from a graph pattern query.""" + entities: List[Entity] = field(default_factory=list) relations: List[Relation] = field(default_factory=list) @@ -90,17 +93,21 @@ class KnowledgeGraphMemory: # -- MemoryBackend ABC -- def store( - self, key: str, content: str, + self, + key: str, + content: str, metadata: Optional[Dict[str, Any]] = None, ) -> None: """Store content as an entity (MemoryBackend interface).""" meta = metadata or {} - self.add_entity(Entity( - entity_id=key, - entity_type=meta.get("entity_type", "document"), - name=meta.get("name", key), - properties={"content": content, **(meta.get("properties", {}))}, - )) + self.add_entity( + Entity( + entity_id=key, + entity_type=meta.get("entity_type", "document"), + name=meta.get("name", key), + properties={"content": content, **(meta.get("properties", {}))}, + ) + ) def retrieve(self, key: str) -> Optional[str]: """Retrieve content by entity_id (MemoryBackend interface).""" @@ -122,19 +129,20 @@ class KnowledgeGraphMemory: for row in rows: eid, etype, name, props_json, ts = row props = json.loads(props_json) if props_json else {} - results.append({ - "key": eid, - "content": props.get("content", ""), - "score": 1.0, - "metadata": {"entity_type": etype, "name": name}, - }) + results.append( + { + "key": eid, + "content": props.get("content", ""), + "score": 1.0, + "metadata": {"entity_type": etype, "name": name}, + } + ) return results def delete(self, key: str) -> bool: """Delete an entity and its relations.""" self._conn.execute( - "DELETE FROM relations" - " WHERE source_id = ? OR target_id = ?", + "DELETE FROM relations WHERE source_id = ? OR target_id = ?", (key, key), ) cur = self._conn.execute("DELETE FROM entities WHERE entity_id = ?", (key,)) @@ -157,8 +165,13 @@ class KnowledgeGraphMemory: " (entity_id, entity_type, name," " properties, created_at) " "VALUES (?, ?, ?, ?, ?)", - (entity.entity_id, entity.entity_type, entity.name, - json.dumps(entity.properties), ts), + ( + entity.entity_id, + entity.entity_type, + entity.name, + json.dumps(entity.properties), + ts, + ), ) self._conn.commit() @@ -166,12 +179,15 @@ class KnowledgeGraphMemory: """Get entity by ID.""" row = self._conn.execute( "SELECT entity_id, entity_type, name, properties, created_at " - "FROM entities WHERE entity_id = ?", (entity_id,), + "FROM entities WHERE entity_id = ?", + (entity_id,), ).fetchone() if not row: return None return Entity( - entity_id=row[0], entity_type=row[1], name=row[2], + entity_id=row[0], + entity_type=row[1], + name=row[2], properties=json.loads(row[3]) if row[3] else {}, created_at=row[4] or 0.0, ) @@ -184,8 +200,14 @@ class KnowledgeGraphMemory: " (source_id, target_id, relation_type," " weight, properties, created_at) " "VALUES (?, ?, ?, ?, ?, ?)", - (relation.source_id, relation.target_id, relation.relation_type, - relation.weight, json.dumps(relation.properties), ts), + ( + relation.source_id, + relation.target_id, + relation.relation_type, + relation.weight, + json.dumps(relation.properties), + ts, + ), ) self._conn.commit() @@ -246,11 +268,15 @@ class KnowledgeGraphMemory: (entity_type, limit), ).fetchall() for row in rows: - entities.append(Entity( - entity_id=row[0], entity_type=row[1], name=row[2], - properties=json.loads(row[3]) if row[3] else {}, - created_at=row[4] or 0.0, - )) + entities.append( + Entity( + entity_id=row[0], + entity_type=row[1], + name=row[2], + properties=json.loads(row[3]) if row[3] else {}, + created_at=row[4] or 0.0, + ) + ) if relation_type: rows = self._conn.execute( @@ -262,11 +288,16 @@ class KnowledgeGraphMemory: (relation_type, limit), ).fetchall() for row in rows: - relations.append(Relation( - source_id=row[0], target_id=row[1], relation_type=row[2], - weight=row[3], properties=json.loads(row[4]) if row[4] else {}, - created_at=row[5] or 0.0, - )) + relations.append( + Relation( + source_id=row[0], + target_id=row[1], + relation_type=row[2], + weight=row[3], + properties=json.loads(row[4]) if row[4] else {}, + created_at=row[5] or 0.0, + ) + ) return GraphQueryResult(entities=entities, relations=relations) diff --git a/src/openjarvis/tools/storage/sqlite.py b/src/openjarvis/tools/storage/sqlite.py index 708ca5a4..e9d77172 100644 --- a/src/openjarvis/tools/storage/sqlite.py +++ b/src/openjarvis/tools/storage/sqlite.py @@ -33,11 +33,13 @@ class SQLiteMemory(MemoryBackend): def __init__(self, db_path: str | Path = "") -> None: if not db_path: from openjarvis.core.config import DEFAULT_CONFIG_DIR + db_path = str(DEFAULT_CONFIG_DIR / "memory.db") self._db_path = str(db_path) from openjarvis._rust_bridge import get_rust_module + _rust = get_rust_module() self._rust_impl = _rust.SQLiteMemory(self._db_path) self._conn = None # type: ignore[assignment] @@ -71,11 +73,14 @@ class SQLiteMemory(MemoryBackend): meta_json = json.dumps(metadata) if metadata else None doc_id = self._rust_impl.store(content, source, meta_json) bus = get_event_bus() - bus.publish(EventType.MEMORY_STORE, { - "backend": self.backend_id, - "doc_id": doc_id, - "source": source, - }) + bus.publish( + EventType.MEMORY_STORE, + { + "backend": self.backend_id, + "doc_id": doc_id, + "source": source, + }, + ) return doc_id def retrieve( @@ -90,15 +95,19 @@ class SQLiteMemory(MemoryBackend): return [] from openjarvis._rust_bridge import retrieval_results_from_json + results = retrieval_results_from_json( self._rust_impl.retrieve(query, top_k), ) bus = get_event_bus() - bus.publish(EventType.MEMORY_RETRIEVE, { - "backend": self.backend_id, - "query": query, - "num_results": len(results), - }) + bus.publish( + EventType.MEMORY_RETRIEVE, + { + "backend": self.backend_id, + "query": query, + "num_results": len(results), + }, + ) return results def delete(self, doc_id: str) -> bool: diff --git a/src/openjarvis/tools/storage_tools.py b/src/openjarvis/tools/storage_tools.py index 01ceb477..fd588fef 100644 --- a/src/openjarvis/tools/storage_tools.py +++ b/src/openjarvis/tools/storage_tools.py @@ -62,7 +62,8 @@ class MemoryStoreTool(BaseTool): ) try: doc_id = self._backend.store( - content, source=params.get("source", ""), + content, + source=params.get("source", ""), ) return ToolResult( tool_name="memory_store", @@ -131,9 +132,7 @@ class MemoryRetrieveTool(BaseTool): content="No results found.", success=True, ) - formatted = "\n---\n".join( - f"[{r.score:.2f}] {r.content}" for r in results - ) + formatted = "\n---\n".join(f"[{r.score:.2f}] {r.content}" for r in results) return ToolResult( tool_name="memory_retrieve", content=formatted, diff --git a/src/openjarvis/tools/templates/__init__.py b/src/openjarvis/tools/templates/__init__.py index bcdff251..92b9f707 100644 --- a/src/openjarvis/tools/templates/__init__.py +++ b/src/openjarvis/tools/templates/__init__.py @@ -1,4 +1,5 @@ """MCP tool templates — reusable tool definitions from TOML.""" + from openjarvis.tools.templates.loader import ToolTemplate, discover_templates __all__ = ["ToolTemplate", "discover_templates"] diff --git a/src/openjarvis/tools/templates/loader.py b/src/openjarvis/tools/templates/loader.py index 94d92efe..014c736a 100644 --- a/src/openjarvis/tools/templates/loader.py +++ b/src/openjarvis/tools/templates/loader.py @@ -76,9 +76,14 @@ class ToolTemplate(BaseTool): ) # Safe evaluation with params available safe_builtins = { - "str": str, "int": int, "float": float, - "len": len, "sorted": sorted, - "list": list, "dict": dict, "json": json, + "str": str, + "int": int, + "float": float, + "len": len, + "sorted": sorted, + "list": list, + "dict": dict, + "json": json, } result = eval( # noqa: S307 expr, @@ -104,8 +109,11 @@ class ToolTemplate(BaseTool): for key, val in params.items(): cmd = cmd.replace(f"{{{key}}}", str(val)) result = subprocess.run( # noqa: S602, S603 - cmd, shell=True, capture_output=True, - text=True, timeout=30, + cmd, + shell=True, + capture_output=True, + text=True, + timeout=30, ) output = result.stdout or result.stderr return ToolResult( @@ -148,7 +156,8 @@ class ToolTemplate(BaseTool): return ToolResult( tool_name=self._name, content=json.dumps( - parsed, indent=2, + parsed, + indent=2, ), success=True, ) diff --git a/src/openjarvis/tools/think.py b/src/openjarvis/tools/think.py index 3c353ab0..8c7b4805 100644 --- a/src/openjarvis/tools/think.py +++ b/src/openjarvis/tools/think.py @@ -42,6 +42,7 @@ class ThinkTool(BaseTool): thought = params.get("thought", "") try: from openjarvis._rust_bridge import get_rust_module + _rust = get_rust_module() content = _rust.ThinkTool().execute(thought) except (ImportError, ModuleNotFoundError): diff --git a/src/openjarvis/tools/user_profile_manage.py b/src/openjarvis/tools/user_profile_manage.py index 67199a23..2e718349 100644 --- a/src/openjarvis/tools/user_profile_manage.py +++ b/src/openjarvis/tools/user_profile_manage.py @@ -21,9 +21,7 @@ class UserProfileManageTool(BaseTool): def spec(self) -> ToolSpec: return ToolSpec( name="user_profile_manage", - description=( - "Read, add, update, or remove entries in user profile." - ), + description=("Read, add, update, or remove entries in user profile."), parameters={ "type": "object", "properties": { @@ -86,9 +84,7 @@ class UserProfileManageTool(BaseTool): content="Entry cannot be empty.", ) self._user_path.parent.mkdir(parents=True, exist_ok=True) - existing = ( - self._user_path.read_text() if self._user_path.exists() else "" - ) + existing = self._user_path.read_text() if self._user_path.exists() else "" self._user_path.write_text(existing.rstrip() + f"\n- {entry}\n") return ToolResult( tool_name=self.spec.name, diff --git a/src/openjarvis/workflow/__init__.py b/src/openjarvis/workflow/__init__.py index ab243111..e82c5ffb 100644 --- a/src/openjarvis/workflow/__init__.py +++ b/src/openjarvis/workflow/__init__.py @@ -1,4 +1,5 @@ """Workflow engine — DAG-based multi-agent pipelines.""" + from openjarvis.workflow.builder import WorkflowBuilder from openjarvis.workflow.engine import WorkflowEngine from openjarvis.workflow.graph import WorkflowGraph diff --git a/src/openjarvis/workflow/builder.py b/src/openjarvis/workflow/builder.py index 43063c19..56df7123 100644 --- a/src/openjarvis/workflow/builder.py +++ b/src/openjarvis/workflow/builder.py @@ -32,13 +32,15 @@ class WorkflowBuilder: tools: Optional[List[str]] = None, config: Optional[Dict[str, Any]] = None, ) -> WorkflowBuilder: - self._nodes.append(WorkflowNode( - id=node_id, - node_type=NodeType.AGENT, - agent=agent, - tools=tools or [], - config=config or {}, - )) + self._nodes.append( + WorkflowNode( + id=node_id, + node_type=NodeType.AGENT, + agent=agent, + tools=tools or [], + config=config or {}, + ) + ) return self def add_tool( @@ -48,11 +50,13 @@ class WorkflowBuilder: tool_name: str, tool_args: str = "{}", ) -> WorkflowBuilder: - self._nodes.append(WorkflowNode( - id=node_id, - node_type=NodeType.TOOL, - config={"tool_name": tool_name, "tool_args": tool_args}, - )) + self._nodes.append( + WorkflowNode( + id=node_id, + node_type=NodeType.TOOL, + config={"tool_name": tool_name, "tool_args": tool_args}, + ) + ) return self def add_condition( @@ -61,11 +65,13 @@ class WorkflowBuilder: *, expr: str, ) -> WorkflowBuilder: - self._nodes.append(WorkflowNode( - id=node_id, - node_type=NodeType.CONDITION, - condition_expr=expr, - )) + self._nodes.append( + WorkflowNode( + id=node_id, + node_type=NodeType.CONDITION, + condition_expr=expr, + ) + ) return self def add_loop( @@ -76,13 +82,15 @@ class WorkflowBuilder: max_iterations: int = 10, exit_condition: str = "", ) -> WorkflowBuilder: - self._nodes.append(WorkflowNode( - id=node_id, - node_type=NodeType.LOOP, - agent=agent, - max_iterations=max_iterations, - condition_expr=exit_condition, - )) + self._nodes.append( + WorkflowNode( + id=node_id, + node_type=NodeType.LOOP, + agent=agent, + max_iterations=max_iterations, + condition_expr=exit_condition, + ) + ) return self def add_transform( @@ -91,11 +99,13 @@ class WorkflowBuilder: *, transform: str = "concatenate", ) -> WorkflowBuilder: - self._nodes.append(WorkflowNode( - id=node_id, - node_type=NodeType.TRANSFORM, - transform_expr=transform, - )) + self._nodes.append( + WorkflowNode( + id=node_id, + node_type=NodeType.TRANSFORM, + transform_expr=transform, + ) + ) return self def connect( @@ -105,17 +115,24 @@ class WorkflowBuilder: *, condition: str = "", ) -> WorkflowBuilder: - self._edges.append(WorkflowEdge( - source=source, target=target, condition=condition, - )) + self._edges.append( + WorkflowEdge( + source=source, + target=target, + condition=condition, + ) + ) return self def sequential(self, *node_ids: str) -> WorkflowBuilder: """Connect nodes in sequential order.""" for i in range(len(node_ids) - 1): - self._edges.append(WorkflowEdge( - source=node_ids[i], target=node_ids[i + 1], - )) + self._edges.append( + WorkflowEdge( + source=node_ids[i], + target=node_ids[i + 1], + ) + ) return self def build(self) -> WorkflowGraph: diff --git a/src/openjarvis/workflow/engine.py b/src/openjarvis/workflow/engine.py index 3cf44de5..58f49827 100644 --- a/src/openjarvis/workflow/engine.py +++ b/src/openjarvis/workflow/engine.py @@ -191,7 +191,10 @@ class WorkflowEngine: return result def _get_node_input( - self, node: WorkflowNode, outputs: Dict[str, str], graph: WorkflowGraph, + self, + node: WorkflowNode, + outputs: Dict[str, str], + graph: WorkflowGraph, ) -> str: """Get input for a node from predecessor outputs.""" preds = graph.predecessors(node.id) @@ -201,8 +204,11 @@ class WorkflowEngine: return outputs.get("_input", "") def _run_agent_node( - self, node: WorkflowNode, outputs: Dict[str, str], - system: Any, graph: WorkflowGraph, + self, + node: WorkflowNode, + outputs: Dict[str, str], + system: Any, + graph: WorkflowGraph, ) -> WorkflowStepResult: """Execute an agent node.""" input_text = self._get_node_input(node, outputs, graph) @@ -231,13 +237,17 @@ class WorkflowEngine: ) def _run_tool_node( - self, node: WorkflowNode, outputs: Dict[str, str], system: Any, + self, + node: WorkflowNode, + outputs: Dict[str, str], + system: Any, ) -> WorkflowStepResult: """Execute a tool node.""" tool_name = node.config.get("tool_name", "") tool_args = node.config.get("tool_args", "{}") if system and system.tool_executor: from openjarvis.core.types import ToolCall + tc = ToolCall(id=f"wf_{node.id}", name=tool_name, arguments=tool_args) tr = system.tool_executor.execute(tc) return WorkflowStepResult( @@ -252,13 +262,17 @@ class WorkflowEngine: ) def _run_condition_node( - self, node: WorkflowNode, outputs: Dict[str, str], + self, + node: WorkflowNode, + outputs: Dict[str, str], ) -> WorkflowStepResult: """Evaluate a condition expression against outputs.""" expr = node.condition_expr if not expr: return WorkflowStepResult( - node_id=node.id, success=True, output="true", + node_id=node.id, + success=True, + output="true", ) # Simple expression evaluation — check if key exists and is truthy # Supports: "node_id.success", "node_id.output contains 'text'" @@ -273,7 +287,9 @@ class WorkflowEngine: ) def _run_transform_node( - self, node: WorkflowNode, outputs: Dict[str, str], + self, + node: WorkflowNode, + outputs: Dict[str, str], ) -> WorkflowStepResult: """Apply a text transformation.""" expr = node.transform_expr @@ -289,8 +305,11 @@ class WorkflowEngine: return WorkflowStepResult(node_id=node.id, output=combined) def _run_loop_node( - self, node: WorkflowNode, outputs: Dict[str, str], - system: Any, graph: WorkflowGraph, + self, + node: WorkflowNode, + outputs: Dict[str, str], + system: Any, + graph: WorkflowGraph, ) -> WorkflowStepResult: """Execute a loop node (re-runs agent until condition or max iterations).""" input_text = self._get_node_input(node, outputs, graph) @@ -303,8 +322,7 @@ class WorkflowEngine: # Check if loop should terminate if ( node.condition_expr - and node.condition_expr.lower() - in last_output.lower() + and node.condition_expr.lower() in last_output.lower() ): break else: diff --git a/tests/agents/conftest.py b/tests/agents/conftest.py index 41277334..996ae028 100644 --- a/tests/agents/conftest.py +++ b/tests/agents/conftest.py @@ -33,7 +33,9 @@ def scenario_harness(tmp_path): executor.set_system(system) scheduler = AgentScheduler( - manager=manager, executor=executor, event_bus=bus, + manager=manager, + executor=executor, + event_bus=bus, ) return ScenarioHarness( diff --git a/tests/agents/fake_engine.py b/tests/agents/fake_engine.py index 805b05a4..d99e6254 100644 --- a/tests/agents/fake_engine.py +++ b/tests/agents/fake_engine.py @@ -45,11 +45,14 @@ class FakeEngine(InferenceEngine): result: Dict[str, Any] = { "content": resp.get("content", ""), - "usage": resp.get("usage", { - "prompt_tokens": 10, - "completion_tokens": 5, - "total_tokens": 15, - }), + "usage": resp.get( + "usage", + { + "prompt_tokens": 10, + "completion_tokens": 5, + "total_tokens": 15, + }, + ), "model": model, "finish_reason": "tool_calls" if resp.get("tool_calls") else "stop", } @@ -58,7 +61,11 @@ class FakeEngine(InferenceEngine): return result async def stream( - self, messages: list, *, model: str, **kw: Any, + self, + messages: list, + *, + model: str, + **kw: Any, ) -> AsyncIterator[str]: result = self.generate(messages, model=model, **kw) yield result["content"] diff --git a/tests/agents/test_channel_agent.py b/tests/agents/test_channel_agent.py index c75c879f..d2631dea 100644 --- a/tests/agents/test_channel_agent.py +++ b/tests/agents/test_channel_agent.py @@ -3,11 +3,9 @@ from __future__ import annotations import time -from typing import Any, Dict, List, Optional +from typing import Any, Dict, List from unittest.mock import MagicMock -import pytest - from openjarvis.agents._stubs import AgentResult from openjarvis.agents.channel_agent import ChannelAgent, classify_query from openjarvis.channels._stubs import ( @@ -17,7 +15,6 @@ from openjarvis.channels._stubs import ( ChannelStatus, ) - # --------------------------------------------------------------------------- # FakeChannel test helper # --------------------------------------------------------------------------- @@ -115,7 +112,10 @@ class TestClassifyQuery: assert classify_query("What's Sarah's email?") == "quick" def test_compare_is_deep(self): - assert classify_query("Compare what Sarah and Mike said about the budget") == "deep" + assert ( + classify_query("Compare what Sarah and Mike said about the budget") + == "deep" + ) # --------------------------------------------------------------------------- diff --git a/tests/agents/test_channel_agent_integration.py b/tests/agents/test_channel_agent_integration.py index 8892fa28..2226dfc7 100644 --- a/tests/agents/test_channel_agent_integration.py +++ b/tests/agents/test_channel_agent_integration.py @@ -195,18 +195,14 @@ def test_quick_query_inline_response(tmp_path: Path) -> None: # Response sent inline (no escalation link) assert "openjarvis://" not in sent_content, ( - "Quick query must NOT produce an escalation link, " - f"but got:\n{sent_content}" + f"Quick query must NOT produce an escalation link, but got:\n{sent_content}" ) # Response contains meeting information assert any( keyword in sent_content.lower() for keyword in ("meeting", "monday", "10am", "team sync", "sync") - ), ( - "Response should contain meeting info, " - f"but got:\n{sent_content}" - ) + ), f"Response should contain meeting info, but got:\n{sent_content}" # --------------------------------------------------------------------------- @@ -233,9 +229,7 @@ def test_deep_query_escalation_link(tmp_path: Path) -> None: "type": "function", "function": { "name": "knowledge_search", - "arguments": json.dumps( - {"query": "budget API redesign"} - ), + "arguments": json.dumps({"query": "budget API redesign"}), }, } ], @@ -286,6 +280,5 @@ def test_deep_query_escalation_link(tmp_path: Path) -> None: # Response contains the escalation link assert "openjarvis://" in sent_content, ( - "Deep query must produce an escalation link, " - f"but got:\n{sent_content}" + f"Deep query must produce an escalation link, but got:\n{sent_content}" ) diff --git a/tests/agents/test_fake_engine.py b/tests/agents/test_fake_engine.py index e0815612..a9cd7c93 100644 --- a/tests/agents/test_fake_engine.py +++ b/tests/agents/test_fake_engine.py @@ -8,10 +8,12 @@ from tests.agents.fake_engine import FakeEngine def test_fake_engine_returns_responses_in_order(): - engine = FakeEngine([ - {"content": "first"}, - {"content": "second"}, - ]) + engine = FakeEngine( + [ + {"content": "first"}, + {"content": "second"}, + ] + ) r1 = engine.generate([], model="m") r2 = engine.generate([], model="m") assert r1["content"] == "first" @@ -33,10 +35,16 @@ def test_fake_engine_raises_on_request(): def test_fake_engine_tool_calls(): - engine = FakeEngine([{ - "content": "", - "tool_calls": [{"id": "1", "function": {"name": "think", "arguments": "{}"}}], - }]) + engine = FakeEngine( + [ + { + "content": "", + "tool_calls": [ + {"id": "1", "function": {"name": "think", "arguments": "{}"}} + ], + } + ] + ) r = engine.generate([], model="m") assert r["finish_reason"] == "tool_calls" assert len(r["tool_calls"]) == 1 diff --git a/tests/agents/test_scenario_harness_smoke.py b/tests/agents/test_scenario_harness_smoke.py index 32776740..3970f0bb 100644 --- a/tests/agents/test_scenario_harness_smoke.py +++ b/tests/agents/test_scenario_harness_smoke.py @@ -8,10 +8,13 @@ from tests.agents.scenario_harness import ScenarioHarness def test_harness_creates_and_runs_agent(scenario_harness: ScenarioHarness): """Verify the harness wires up correctly — create agent, run tick.""" h = scenario_harness - agent = h.manager.create_agent("Smoke Test", config={ - "schedule_type": "manual", - "instruction": "Say hello.", - }) + agent = h.manager.create_agent( + "Smoke Test", + config={ + "schedule_type": "manual", + "instruction": "Say hello.", + }, + ) h.executor.execute_tick(agent["id"]) updated = h.manager.get_agent(agent["id"]) assert updated["status"] == "idle" diff --git a/tests/channels/conftest.py b/tests/channels/conftest.py index 1ed8dcd8..66c6c45a 100644 --- a/tests/channels/conftest.py +++ b/tests/channels/conftest.py @@ -34,8 +34,13 @@ def scenario_harness(tmp_path): scheduler = AgentScheduler(manager=manager, executor=executor, event_bus=bus) return ScenarioHarness( - manager=manager, executor=executor, scheduler=scheduler, - bus=bus, engine=engine, system=system, db_path=db_path, + manager=manager, + executor=executor, + scheduler=scheduler, + bus=bus, + engine=engine, + system=system, + db_path=db_path, ) diff --git a/tests/channels/test_agent_channel_e2e.py b/tests/channels/test_agent_channel_e2e.py index 1eaefa97..5bf4519b 100644 --- a/tests/channels/test_agent_channel_e2e.py +++ b/tests/channels/test_agent_channel_e2e.py @@ -13,10 +13,13 @@ def test_agent_sends_to_webchat(scenario_harness: ScenarioHarness): webchat = WebChatChannel() webchat.connect() - agent = h.manager.create_agent("Channel Agent", config={ - "schedule_type": "manual", - "instruction": "Send a report to the general channel.", - }) + agent = h.manager.create_agent( + "Channel Agent", + config={ + "schedule_type": "manual", + "instruction": "Send a report to the general channel.", + }, + ) h.executor.execute_tick(agent["id"]) data = h.manager.get_agent(agent["id"]) @@ -28,10 +31,13 @@ def test_agent_sends_to_webchat(scenario_harness: ScenarioHarness): def test_channel_failure_does_not_crash_agent(scenario_harness: ScenarioHarness): """Agent continues if channel send fails.""" h = scenario_harness - agent = h.manager.create_agent("Resilient Agent", config={ - "schedule_type": "manual", - "instruction": "Try to send a message.", - }) + agent = h.manager.create_agent( + "Resilient Agent", + config={ + "schedule_type": "manual", + "instruction": "Try to send a message.", + }, + ) h.executor.execute_tick(agent["id"]) data = h.manager.get_agent(agent["id"]) @@ -47,10 +53,13 @@ def test_agent_with_channel_binding(scenario_harness: ScenarioHarness): webchat = WebChatChannel() webchat.connect() - agent = h.manager.create_agent("Bound Agent", config={ - "schedule_type": "manual", - "instruction": "Monitor and report.", - }) + agent = h.manager.create_agent( + "Bound Agent", + config={ + "schedule_type": "manual", + "instruction": "Monitor and report.", + }, + ) aid = agent["id"] # Bind a channel diff --git a/tests/channels/test_channel_contract.py b/tests/channels/test_channel_contract.py index 44f04252..95bf5b9c 100644 --- a/tests/channels/test_channel_contract.py +++ b/tests/channels/test_channel_contract.py @@ -19,9 +19,7 @@ from openjarvis.core.registry import ChannelRegistry # We store the actual class objects, not registry keys, so they survive # the autouse _clean_registries fixture. importlib.reload(openjarvis.channels) -_ALL_CHANNELS = [ - (key, ChannelRegistry.get(key)) for key in ChannelRegistry.keys() -] +_ALL_CHANNELS = [(key, ChannelRegistry.get(key)) for key in ChannelRegistry.keys()] @pytest.fixture(params=_ALL_CHANNELS, ids=lambda x: x[0]) diff --git a/tests/channels/test_imessage_daemon.py b/tests/channels/test_imessage_daemon.py index 6184267e..e70c77de 100644 --- a/tests/channels/test_imessage_daemon.py +++ b/tests/channels/test_imessage_daemon.py @@ -5,8 +5,6 @@ from __future__ import annotations import sqlite3 from pathlib import Path -import pytest - def _create_fake_chat_db(db_path: Path) -> None: conn = sqlite3.connect(str(db_path)) @@ -25,13 +23,10 @@ def _create_fake_chat_db(db_path: Path) -> None: ); """) conn.execute("INSERT INTO handle VALUES (1, '+15551234567')") - conn.execute( - "INSERT INTO chat VALUES (1, '+15551234567', 'Test Chat')" - ) + conn.execute("INSERT INTO chat VALUES (1, '+15551234567', 'Test Chat')") conn.execute("INSERT INTO chat_message_join VALUES (1, 1)") conn.execute( - "INSERT INTO message VALUES " - "(1, 'Hello agent', 1, 700000000000000000, 0)" + "INSERT INTO message VALUES (1, 'Hello agent', 1, 700000000000000000, 0)" ) conn.commit() conn.close() @@ -39,10 +34,12 @@ def _create_fake_chat_db(db_path: Path) -> None: def test_poll_new_messages(tmp_path: Path) -> None: from openjarvis.channels.imessage_daemon import poll_new_messages + db_path = tmp_path / "chat.db" _create_fake_chat_db(db_path) messages = poll_new_messages( - db_path=str(db_path), last_rowid=0, + db_path=str(db_path), + last_rowid=0, chat_identifier="+15551234567", ) assert len(messages) == 1 @@ -52,10 +49,12 @@ def test_poll_new_messages(tmp_path: Path) -> None: def test_poll_skips_old_messages(tmp_path: Path) -> None: from openjarvis.channels.imessage_daemon import poll_new_messages + db_path = tmp_path / "chat.db" _create_fake_chat_db(db_path) messages = poll_new_messages( - db_path=str(db_path), last_rowid=1, + db_path=str(db_path), + last_rowid=1, chat_identifier="+15551234567", ) assert len(messages) == 0 @@ -63,10 +62,12 @@ def test_poll_skips_old_messages(tmp_path: Path) -> None: def test_poll_filters_by_chat(tmp_path: Path) -> None: from openjarvis.channels.imessage_daemon import poll_new_messages + db_path = tmp_path / "chat.db" _create_fake_chat_db(db_path) messages = poll_new_messages( - db_path=str(db_path), last_rowid=0, + db_path=str(db_path), + last_rowid=0, chat_identifier="+15559999999", ) assert len(messages) == 0 @@ -74,6 +75,7 @@ def test_poll_filters_by_chat(tmp_path: Path) -> None: def test_poll_skips_own_messages(tmp_path: Path) -> None: from openjarvis.channels.imessage_daemon import poll_new_messages + db_path = tmp_path / "chat.db" conn = sqlite3.connect(str(db_path)) conn.executescript(""" @@ -91,18 +93,16 @@ def test_poll_skips_own_messages(tmp_path: Path) -> None: ); """) conn.execute("INSERT INTO handle VALUES (1, '+15551234567')") - conn.execute( - "INSERT INTO chat VALUES (1, '+15551234567', 'Test')" - ) + conn.execute("INSERT INTO chat VALUES (1, '+15551234567', 'Test')") conn.execute("INSERT INTO chat_message_join VALUES (1, 1)") conn.execute( - "INSERT INTO message VALUES " - "(1, 'My own msg', 1, 700000000000000000, 1)" + "INSERT INTO message VALUES (1, 'My own msg', 1, 700000000000000000, 1)" ) conn.commit() conn.close() messages = poll_new_messages( - db_path=str(db_path), last_rowid=0, + db_path=str(db_path), + last_rowid=0, chat_identifier="+15551234567", ) assert len(messages) == 0 diff --git a/tests/channels/test_tier1_gmail.py b/tests/channels/test_tier1_gmail.py index 75292689..c344eb0f 100644 --- a/tests/channels/test_tier1_gmail.py +++ b/tests/channels/test_tier1_gmail.py @@ -103,8 +103,8 @@ class TestSend: def test_gmail_send_exception_returns_false(self): ch = GmailChannel() mock_service = MagicMock() - mock_service.users().messages().send().execute.side_effect = ( - RuntimeError("API error") + mock_service.users().messages().send().execute.side_effect = RuntimeError( + "API error" ) ch._service = mock_service ch._status = ChannelStatus.CONNECTED diff --git a/tests/channels/test_tier2_twitter.py b/tests/channels/test_tier2_twitter.py index fcc15924..182ac7a3 100644 --- a/tests/channels/test_tier2_twitter.py +++ b/tests/channels/test_tier2_twitter.py @@ -76,7 +76,9 @@ def test_twitter_send_reply(): ch._status = ChannelStatus.CONNECTED result = ch.send( - "timeline", "This is a reply", conversation_id="999888777", + "timeline", + "This is a reply", + conversation_id="999888777", ) assert result is True mock_client.create_tweet.assert_called_once_with( @@ -105,8 +107,7 @@ def test_twitter_event_bus_integration(): assert EventType.CHANNEL_MESSAGE_SENT in event_types sent_event = next( - e for e in bus.history - if e.event_type == EventType.CHANNEL_MESSAGE_SENT + e for e in bus.history if e.event_type == EventType.CHANNEL_MESSAGE_SENT ) assert sent_event.data["content"] == "Event test!" assert sent_event.data["channel"] == "timeline" diff --git a/tests/channels/test_twilio_sms.py b/tests/channels/test_twilio_sms.py index e5b58ccb..563f2da5 100644 --- a/tests/channels/test_twilio_sms.py +++ b/tests/channels/test_twilio_sms.py @@ -18,9 +18,7 @@ def _register_twilio(): TwilioSMSChannel, ) - ChannelRegistry.register_value( - "twilio", TwilioSMSChannel - ) + ChannelRegistry.register_value("twilio", TwilioSMSChannel) class TestRegistration: @@ -49,9 +47,7 @@ class TestInit: monkeypatch.setenv("TWILIO_ACCOUNT_SID", "AC_env") monkeypatch.setenv("TWILIO_AUTH_TOKEN", "token_env") - monkeypatch.setenv( - "TWILIO_PHONE_NUMBER", "+15559876543" - ) + monkeypatch.setenv("TWILIO_PHONE_NUMBER", "+15559876543") ch = TwilioSMSChannel() assert ch._account_sid == "AC_env" @@ -70,9 +66,7 @@ class TestSend: ch.connect() mock_client = MagicMock() - mock_client.messages.create.return_value = MagicMock( - sid="SM_test" - ) + mock_client.messages.create.return_value = MagicMock(sid="SM_test") ch._client = mock_client result = ch.send("+15559999999", "Hello via SMS!") @@ -96,9 +90,7 @@ class TestSend: ch.connect() mock_client = MagicMock() - mock_client.messages.create.side_effect = Exception( - "API error" - ) + mock_client.messages.create.side_effect = Exception("API error") ch._client = mock_client result = ch.send("+15559999999", "Hello!") @@ -119,9 +111,7 @@ class TestSend: ch.connect() mock_client = MagicMock() - mock_client.messages.create.return_value = MagicMock( - sid="SM_test" - ) + mock_client.messages.create.return_value = MagicMock(sid="SM_test") ch._client = mock_client ch.send("+15559999999", "Hello!") @@ -140,10 +130,7 @@ class TestStatus: auth_token="token_test", phone_number="+15551234567", ) - with patch( - "openjarvis.channels.twilio_sms" - "._create_twilio_client" - ): + with patch("openjarvis.channels.twilio_sms._create_twilio_client"): ch.connect() assert ch.status() == ChannelStatus.CONNECTED @@ -157,10 +144,7 @@ class TestStatus: auth_token="token_test", phone_number="+15551234567", ) - with patch( - "openjarvis.channels.twilio_sms" - "._create_twilio_client" - ): + with patch("openjarvis.channels.twilio_sms._create_twilio_client"): ch.connect() ch.disconnect() assert ch.status() == ChannelStatus.DISCONNECTED diff --git a/tests/cli/test_deep_research_setup.py b/tests/cli/test_deep_research_setup.py index 7fb9f990..eed748b7 100644 --- a/tests/cli/test_deep_research_setup.py +++ b/tests/cli/test_deep_research_setup.py @@ -5,17 +5,12 @@ from __future__ import annotations import gzip import sqlite3 from pathlib import Path -from typing import List - -import pytest - -from openjarvis.connectors._stubs import Document - # --------------------------------------------------------------------------- # Helpers # --------------------------------------------------------------------------- + def _create_fake_notes_db(db_path: Path) -> None: """Create a minimal Apple Notes SQLite database.""" conn = sqlite3.connect(str(db_path)) @@ -36,7 +31,8 @@ def _create_fake_notes_db(db_path: Path) -> None: """) content = gzip.compress(b"Test note about meetings") conn.execute( - "INSERT INTO ZICCLOUDSYNCINGOBJECT VALUES (1, NULL, 'Test Note', 694310400.0, 'n1', 1)" + "INSERT INTO ZICCLOUDSYNCINGOBJECT VALUES " + "(1, NULL, 'Test Note', 694310400.0, 'n1', 1)" ) conn.execute("INSERT INTO ZICNOTEDATA VALUES (1, ?, 1)", (content,)) conn.commit() @@ -48,7 +44,10 @@ def _create_fake_imessage_db(db_path: Path) -> None: conn = sqlite3.connect(str(db_path)) conn.executescript(""" CREATE TABLE handle (ROWID INTEGER PRIMARY KEY, id TEXT); - CREATE TABLE chat (ROWID INTEGER PRIMARY KEY, chat_identifier TEXT, display_name TEXT); + CREATE TABLE chat ( + ROWID INTEGER PRIMARY KEY, + chat_identifier TEXT, display_name TEXT + ); CREATE TABLE chat_message_join (chat_id INTEGER, message_id INTEGER); CREATE TABLE message ( ROWID INTEGER PRIMARY KEY, text TEXT, handle_id INTEGER, @@ -58,7 +57,9 @@ def _create_fake_imessage_db(db_path: Path) -> None: conn.execute("INSERT INTO handle VALUES (1, '+15551234567')") conn.execute("INSERT INTO chat VALUES (1, '+15551234567', 'Test Chat')") conn.execute("INSERT INTO chat_message_join VALUES (1, 1)") - conn.execute("INSERT INTO message VALUES (1, 'Hello from test', 1, 694310400000000000, 0)") + conn.execute( + "INSERT INTO message VALUES (1, 'Hello from test', 1, 694310400000000000, 0)" + ) conn.commit() conn.close() @@ -118,7 +119,10 @@ def test_detect_includes_obsidian_when_vault_exists(tmp_path: Path) -> None: def test_ingest_sources(tmp_path: Path) -> None: """ingest_sources connects and ingests documents into KnowledgeStore.""" - from openjarvis.cli.deep_research_setup_cmd import detect_local_sources, ingest_sources + from openjarvis.cli.deep_research_setup_cmd import ( + detect_local_sources, + ingest_sources, + ) from openjarvis.connectors.store import KnowledgeStore notes_db = tmp_path / "NoteStore.sqlite" diff --git a/tests/connectors/test_dropbox.py b/tests/connectors/test_dropbox.py index 5c8fc636..edcdb507 100644 --- a/tests/connectors/test_dropbox.py +++ b/tests/connectors/test_dropbox.py @@ -95,9 +95,7 @@ def test_sync_yields_documents( """sync() yields one Document per file with correct metadata.""" # Set up fake credentials so is_connected() returns True creds_path = Path(connector._credentials_path) - creds_path.write_text( - json.dumps({"token": "fake-access-token"}), encoding="utf-8" - ) + creds_path.write_text(json.dumps({"token": "fake-access-token"}), encoding="utf-8") # Configure mocks mock_list.return_value = _LIST_RESPONSE @@ -132,9 +130,7 @@ def test_sync_yields_documents( def test_disconnect(connector, tmp_path: Path) -> None: """disconnect() deletes the credentials file.""" creds_path = Path(connector._credentials_path) - creds_path.write_text( - json.dumps({"token": "fake-access-token"}), encoding="utf-8" - ) + creds_path.write_text(json.dumps({"token": "fake-access-token"}), encoding="utf-8") assert connector.is_connected() is True connector.disconnect() diff --git a/tests/connectors/test_granola.py b/tests/connectors/test_granola.py index d1934959..8bb33118 100644 --- a/tests/connectors/test_granola.py +++ b/tests/connectors/test_granola.py @@ -148,9 +148,7 @@ def test_sync_yields_documents( # Write fake credentials so is_connected() returns True creds_path = Path(connector._credentials_path) creds_path.parent.mkdir(parents=True, exist_ok=True) - creds_path.write_text( - json.dumps({"token": "grl_fake_key"}), encoding="utf-8" - ) + creds_path.write_text(json.dumps({"token": "grl_fake_key"}), encoding="utf-8") mock_list.return_value = _LIST_RESPONSE mock_get.side_effect = [_NOTE_1, _NOTE_2] @@ -211,9 +209,7 @@ def test_disconnect(connector, tmp_path: Path) -> None: """disconnect() deletes the credentials file.""" creds_path = Path(connector._credentials_path) creds_path.parent.mkdir(parents=True, exist_ok=True) - creds_path.write_text( - json.dumps({"token": "grl_fake_key"}), encoding="utf-8" - ) + creds_path.write_text(json.dumps({"token": "grl_fake_key"}), encoding="utf-8") assert connector.is_connected() is True connector.disconnect() diff --git a/tests/connectors/test_incremental_sync.py b/tests/connectors/test_incremental_sync.py index 20cda4ba..f54b7719 100644 --- a/tests/connectors/test_incremental_sync.py +++ b/tests/connectors/test_incremental_sync.py @@ -109,7 +109,7 @@ def test_first_sync_passes_no_since(engine: SyncEngine) -> None: def test_second_sync_passes_since(engine: SyncEngine) -> None: - """After the first sync a checkpoint is written; the second sync receives since != None.""" + """After first sync, checkpoint is written; second sync gets since.""" docs = [_make_doc("second:doc:0", content="Document for incremental sync")] connector = TimestampConnector(docs) diff --git a/tests/connectors/test_live_smoke.py b/tests/connectors/test_live_smoke.py index a627bac4..a1c290aa 100644 --- a/tests/connectors/test_live_smoke.py +++ b/tests/connectors/test_live_smoke.py @@ -61,10 +61,7 @@ def test_live_obsidian_full_pipeline() -> None: # Search for engine/inference result = tool.execute(query="inference engine") assert result.success - print( - f" 'inference engine' query: " - f"{result.metadata['num_results']} results" - ) + print(f" 'inference engine' query: {result.metadata['num_results']} results") # Search with source filter result = tool.execute(query="agent", source="obsidian") @@ -81,7 +78,8 @@ def test_live_obsidian_full_pipeline() -> None: if result.metadata["num_results"] > 0: # Results should have source attribution assert "[obsidian]" in result.content - print(f" 'registry pattern': found with attribution") + print(" 'registry pattern': found with attribution") - print(f"\n SMOKE TEST PASSED — {items} chunks indexed, " - f"search working end-to-end") + print( + f"\n SMOKE TEST PASSED — {items} chunks indexed, search working end-to-end" + ) diff --git a/tests/connectors/test_notion.py b/tests/connectors/test_notion.py index aec24694..e42bd525 100644 --- a/tests/connectors/test_notion.py +++ b/tests/connectors/test_notion.py @@ -108,9 +108,7 @@ def test_sync_yields_documents( """sync() yields one Document per page with correct metadata.""" # Write fake credentials so is_connected() returns True creds_path = Path(connector._credentials_path) - creds_path.write_text( - json.dumps({"token": "ntn_fake"}), encoding="utf-8" - ) + creds_path.write_text(json.dumps({"token": "ntn_fake"}), encoding="utf-8") mock_search.return_value = _SEARCH_RESPONSE mock_blocks.return_value = _BLOCKS_RESPONSE @@ -215,9 +213,7 @@ def test_render_blocks_to_markdown() -> None: def test_disconnect(connector, tmp_path: Path) -> None: """disconnect() deletes the credentials file.""" creds_path = Path(connector._credentials_path) - creds_path.write_text( - json.dumps({"token": "ntn_fake"}), encoding="utf-8" - ) + creds_path.write_text(json.dumps({"token": "ntn_fake"}), encoding="utf-8") assert connector.is_connected() is True connector.disconnect() diff --git a/tests/connectors/test_outlook.py b/tests/connectors/test_outlook.py index 57a6a910..c6799c97 100644 --- a/tests/connectors/test_outlook.py +++ b/tests/connectors/test_outlook.py @@ -5,8 +5,6 @@ from __future__ import annotations from pathlib import Path from unittest.mock import MagicMock, patch -import pytest - from openjarvis.connectors.oauth import load_tokens from openjarvis.connectors.outlook import OutlookConnector from openjarvis.core.registry import ConnectorRegistry diff --git a/tests/connectors/test_slack_connector.py b/tests/connectors/test_slack_connector.py index b47a341a..3735ffbf 100644 --- a/tests/connectors/test_slack_connector.py +++ b/tests/connectors/test_slack_connector.py @@ -120,9 +120,7 @@ def test_sync_yields_documents( """ # Set up fake credentials so is_connected() returns True creds_path = Path(connector._credentials_path) - creds_path.write_text( - json.dumps({"token": "fake-access-token"}), encoding="utf-8" - ) + creds_path.write_text(json.dumps({"token": "fake-access-token"}), encoding="utf-8") # Configure mocks mock_users.return_value = _USERS_RESPONSE @@ -176,9 +174,7 @@ def test_sync_yields_documents( def test_disconnect(connector, tmp_path: Path) -> None: """disconnect() deletes the credentials file.""" creds_path = Path(connector._credentials_path) - creds_path.write_text( - json.dumps({"token": "fake-access-token"}), encoding="utf-8" - ) + creds_path.write_text(json.dumps({"token": "fake-access-token"}), encoding="utf-8") assert connector.is_connected() is True connector.disconnect() diff --git a/tests/core/test_config.py b/tests/core/test_config.py index bf4ec66f..9e0556ec 100644 --- a/tests/core/test_config.py +++ b/tests/core/test_config.py @@ -464,9 +464,12 @@ class TestApplyTomlSectionListNormalization: from openjarvis.core.config import _apply_toml_section target = LearningConfig() - _apply_toml_section(target, { - "reward_weights": ["accuracy=0.8", "latency=0.2"], - }) + _apply_toml_section( + target, + { + "reward_weights": ["accuracy=0.8", "latency=0.2"], + }, + ) assert target.metrics.accuracy_weight == 0.8 assert target.metrics.latency_weight == 0.2 @@ -475,9 +478,12 @@ class TestApplyTomlSectionListNormalization: from openjarvis.core.config import _apply_toml_section target = AgentConfig() - _apply_toml_section(target, { - "tools": ["web_search", "http_request", "file_read"], - }) + _apply_toml_section( + target, + { + "tools": ["web_search", "http_request", "file_read"], + }, + ) assert isinstance(target.tools, str) assert target.tools == "web_search,http_request,file_read" diff --git a/tests/engine/test_cloud_minimax.py b/tests/engine/test_cloud_minimax.py index 5d1ef5d9..a58dc354 100644 --- a/tests/engine/test_cloud_minimax.py +++ b/tests/engine/test_cloud_minimax.py @@ -193,9 +193,7 @@ class TestMiniMaxGenerate: ) assert actual_temp >= 0.01, "Temperature should be clamped above zero" - def test_temperature_clamped_at_max( - self, monkeypatch: pytest.MonkeyPatch - ) -> None: + def test_temperature_clamped_at_max(self, monkeypatch: pytest.MonkeyPatch) -> None: """MiniMax requires temperature <= 1.0; verify high values are clamped.""" engine = _make_cloud_engine(monkeypatch) fake_client = mock.MagicMock() @@ -236,9 +234,7 @@ class TestMiniMaxGenerate: assert tc["id"] == "call_minimax_123" assert tc["name"] == "search" - def test_no_tool_calls_when_absent( - self, monkeypatch: pytest.MonkeyPatch - ) -> None: + def test_no_tool_calls_when_absent(self, monkeypatch: pytest.MonkeyPatch) -> None: engine = _make_cloud_engine(monkeypatch) fake_client = mock.MagicMock() fake_client.chat.completions.create.return_value = _fake_minimax_response( @@ -255,9 +251,7 @@ class TestMiniMaxGenerate: engine = _make_cloud_engine(monkeypatch) assert engine._minimax_client is None - with pytest.raises( - EngineConnectionError, match="MiniMax client not available" - ): + with pytest.raises(EngineConnectionError, match="MiniMax client not available"): engine.generate( [Message(role=Role.USER, content="Hi")], model="MiniMax-M2.5" ) diff --git a/tests/evals/test_pinchbench_grading.py b/tests/evals/test_pinchbench_grading.py index d2396bfc..3bc6e0bc 100644 --- a/tests/evals/test_pinchbench_grading.py +++ b/tests/evals/test_pinchbench_grading.py @@ -29,12 +29,12 @@ def _make_record(**meta_overrides) -> EvalRecord: class TestGradeAutomated: def test_simple_pass(self, tmp_path): - code = ''' + code = """ def grade(transcript, workspace_path): from pathlib import Path f = Path(workspace_path) / "output.txt" return {"file_exists": 1.0 if f.exists() else 0.0} -''' +""" (tmp_path / "output.txt").write_text("hello") record = _make_record(automated_checks=code) result = _grade_automated(record, [], str(tmp_path)) @@ -42,18 +42,18 @@ def grade(transcript, workspace_path): assert result["breakdown"]["file_exists"] == 1.0 def test_simple_fail(self, tmp_path): - code = ''' + code = """ def grade(transcript, workspace_path): from pathlib import Path f = Path(workspace_path) / "output.txt" return {"file_exists": 1.0 if f.exists() else 0.0} -''' +""" record = _make_record(automated_checks=code) result = _grade_automated(record, [], str(tmp_path)) assert result["score"] == 0.0 def test_transcript_inspection(self, tmp_path): - code = ''' + code = """ def grade(transcript, workspace_path): used_read = False for entry in transcript: @@ -65,20 +65,22 @@ def grade(transcript, workspace_path): if item.get("type") == "toolCall" and name == "read_file": used_read = True return {"used_read_file": 1.0 if used_read else 0.0} -''' - transcript = [{ - "type": "message", - "message": { - "role": "assistant", - "content": [ - { - "type": "toolCall", - "name": "read_file", - "params": {"path": "a.txt"}, - } - ], - }, - }] +""" + transcript = [ + { + "type": "message", + "message": { + "role": "assistant", + "content": [ + { + "type": "toolCall", + "name": "read_file", + "params": {"path": "a.txt"}, + } + ], + }, + } + ] record = _make_record(automated_checks=code) result = _grade_automated(record, transcript, str(tmp_path)) assert result["score"] == 1.0 diff --git a/tests/evals/test_pinchbench_integration.py b/tests/evals/test_pinchbench_integration.py index 4edf9334..b14733d4 100644 --- a/tests/evals/test_pinchbench_integration.py +++ b/tests/evals/test_pinchbench_integration.py @@ -20,7 +20,8 @@ def _create_test_repo(tmp_path: Path) -> Path: assets_dir.mkdir() # Write a simple automated task - (tasks_dir / "task_00_sanity.md").write_text(textwrap.dedent("""\ + (tasks_dir / "task_00_sanity.md").write_text( + textwrap.dedent("""\ --- id: task_00_sanity name: Sanity Check @@ -56,7 +57,8 @@ def _create_test_repo(tmp_path: Path) -> Path: "has_hello": 1.0 if has_hello else 0.0, } ``` - """)) + """) + ) return repo diff --git a/tests/evals/test_pinchbench_transcript.py b/tests/evals/test_pinchbench_transcript.py index d7850c6b..7fa930b2 100644 --- a/tests/evals/test_pinchbench_transcript.py +++ b/tests/evals/test_pinchbench_transcript.py @@ -78,6 +78,7 @@ def test_events_to_transcript_tool_name_mapping(): def test_events_to_transcript_empty(): """Empty events produce empty transcript.""" from openjarvis.evals.scorers.pinchbench import events_to_transcript + assert events_to_transcript([]) == [] diff --git a/tests/server/test_auth_middleware.py b/tests/server/test_auth_middleware.py index 38bec7b3..c6b4cb00 100644 --- a/tests/server/test_auth_middleware.py +++ b/tests/server/test_auth_middleware.py @@ -4,9 +4,7 @@ from __future__ import annotations import pytest -pytest.importorskip( - "fastapi", reason="openjarvis[server] not installed" -) +pytest.importorskip("fastapi", reason="openjarvis[server] not installed") from fastapi import FastAPI from fastapi.testclient import TestClient @@ -55,9 +53,7 @@ class TestAuthMiddleware: def test_accepts_valid_key(self, client): resp = client.get( "/v1/models", - headers={ - "Authorization": "Bearer oj_sk_test123" - }, + headers={"Authorization": "Bearer oj_sk_test123"}, ) assert resp.status_code == 200 diff --git a/tests/server/test_channel_bridge.py b/tests/server/test_channel_bridge.py index ba06b10e..bbaf35da 100644 --- a/tests/server/test_channel_bridge.py +++ b/tests/server/test_channel_bridge.py @@ -9,9 +9,7 @@ from unittest.mock import MagicMock import pytest -pytest.importorskip( - "fastapi", reason="openjarvis[server] not installed" -) +pytest.importorskip("fastapi", reason="openjarvis[server] not installed") from openjarvis.channels._stubs import ( BaseChannel, @@ -47,9 +45,7 @@ class FakeChannel(BaseChannel): conversation_id: str = "", metadata: Dict[str, Any] | None = None, ) -> bool: - self.sent.append( - {"channel": channel, "content": content} - ) + self.sent.append({"channel": channel, "content": content}) return True def status(self) -> ChannelStatus: @@ -65,9 +61,7 @@ class FakeChannel(BaseChannel): @pytest.fixture def store(): with tempfile.TemporaryDirectory() as tmpdir: - s = SessionStore( - db_path=str(Path(tmpdir) / "sessions.db") - ) + s = SessionStore(db_path=str(Path(tmpdir) / "sessions.db")) yield s s.close() @@ -106,9 +100,7 @@ class TestBackwardCompatible: def test_status_connected(self, bridge): assert bridge.status() == ChannelStatus.CONNECTED - def test_status_disconnected_when_none_connected( - self, store, bus, mock_system - ): + def test_status_disconnected_when_none_connected(self, store, bus, mock_system): fake = FakeChannel() # not connected b = ChannelBridge( channels={"fake": fake}, @@ -132,9 +124,7 @@ class TestCommandParsing: assert "notify" in reply.lower() def test_notify_command(self, bridge, store): - reply = bridge.handle_incoming( - "user1", "/notify slack", "fake" - ) + reply = bridge.handle_incoming("user1", "/notify slack", "fake") assert "slack" in reply.lower() session = store.get_or_create("user1", "fake") assert session["preferred_notification_channel"] == "slack" @@ -142,71 +132,45 @@ class TestCommandParsing: def test_agents_command(self, bridge): bridge._agent_manager = MagicMock() bridge._agent_manager.list_agents.return_value = [] - reply = bridge.handle_incoming( - "user1", "/agents", "fake" - ) - assert ( - "no" in reply.lower() or "agent" in reply.lower() - ) + reply = bridge.handle_incoming("user1", "/agents", "fake") + assert "no" in reply.lower() or "agent" in reply.lower() - def test_unknown_command_falls_through_to_chat( - self, bridge, mock_system - ): + def test_unknown_command_falls_through_to_chat(self, bridge, mock_system): bridge.handle_incoming("user1", "/unknown_cmd", "fake") # Should treat as regular chat mock_system.ask.assert_called_once() def test_more_command_returns_pending(self, bridge, store): store.get_or_create("user1", "fake") - store.set_pending_response( - "user1", "fake", "the rest of the long response" - ) + store.set_pending_response("user1", "fake", "the rest of the long response") reply = bridge.handle_incoming("user1", "/more", "fake") assert "the rest of the long response" in reply class TestChatRouting: def test_routes_to_system_ask(self, bridge, mock_system): - reply = bridge.handle_incoming( - "user1", "what is 2+2?", "fake" - ) + reply = bridge.handle_incoming("user1", "what is 2+2?", "fake") mock_system.ask.assert_called_once() call_kwargs = mock_system.ask.call_args assert "2+2" in str(call_kwargs) assert reply == "Hello from Jarvis!" - def test_stores_conversation_history( - self, bridge, store, mock_system - ): + def test_stores_conversation_history(self, bridge, store, mock_system): bridge.handle_incoming("user1", "hello", "fake") session = store.get_or_create("user1", "fake") # user + assistant assert len(session["conversation_history"]) == 2 - assert ( - session["conversation_history"][0]["role"] == "user" - ) - assert ( - session["conversation_history"][1]["role"] - == "assistant" - ) + assert session["conversation_history"][0]["role"] == "user" + assert session["conversation_history"][1]["role"] == "assistant" - def test_error_returns_friendly_message( - self, bridge, mock_system - ): + def test_error_returns_friendly_message(self, bridge, mock_system): mock_system.ask.side_effect = RuntimeError("engine down") - reply = bridge.handle_incoming( - "user1", "hello", "fake" - ) - assert ( - "sorry" in reply.lower() - or "couldn't" in reply.lower() - ) + reply = bridge.handle_incoming("user1", "hello", "fake") + assert "sorry" in reply.lower() or "couldn't" in reply.lower() class TestResponseFormatting: - def test_truncates_long_sms_response( - self, bridge, mock_system - ): + def test_truncates_long_sms_response(self, bridge, mock_system): mock_system.ask.return_value = {"content": "x" * 2000} reply = bridge.handle_incoming( "user1", @@ -217,12 +181,8 @@ class TestResponseFormatting: assert len(reply) <= 1600 assert "/more" in reply - def test_short_response_not_truncated( - self, bridge, mock_system - ): - mock_system.ask.return_value = { - "content": "short answer" - } + def test_short_response_not_truncated(self, bridge, mock_system): + mock_system.ask.return_value = {"content": "short answer"} reply = bridge.handle_incoming("user1", "hi", "fake") assert reply == "short answer" assert "/more" not in reply diff --git a/tests/server/test_channel_bridge_deep_research.py b/tests/server/test_channel_bridge_deep_research.py index 97360472..e70348d6 100644 --- a/tests/server/test_channel_bridge_deep_research.py +++ b/tests/server/test_channel_bridge_deep_research.py @@ -4,8 +4,6 @@ from __future__ import annotations from unittest.mock import MagicMock -import pytest - def test_handle_chat_uses_deep_research_agent() -> None: """When a DeepResearch agent is configured, route through it.""" diff --git a/tests/server/test_deep_research_tools_wiring.py b/tests/server/test_deep_research_tools_wiring.py index 2f7c6756..f566a4a7 100644 --- a/tests/server/test_deep_research_tools_wiring.py +++ b/tests/server/test_deep_research_tools_wiring.py @@ -5,8 +5,6 @@ from __future__ import annotations from pathlib import Path from unittest.mock import MagicMock -import pytest - from openjarvis.connectors.store import KnowledgeStore diff --git a/tests/server/test_session_store.py b/tests/server/test_session_store.py index bff077a1..c078b25e 100644 --- a/tests/server/test_session_store.py +++ b/tests/server/test_session_store.py @@ -13,9 +13,7 @@ from openjarvis.server.session_store import SessionStore @pytest.fixture def store(): with tempfile.TemporaryDirectory() as tmpdir: - s = SessionStore( - db_path=str(Path(tmpdir) / "sessions.db") - ) + s = SessionStore(db_path=str(Path(tmpdir) / "sessions.db")) yield s s.close() @@ -38,9 +36,7 @@ class TestGetOrCreate: def test_separate_sessions_per_channel(self, store): store.get_or_create("user123", "twilio") store.get_or_create("user123", "slack") - store.append_message( - "user123", "twilio", "user", "hello via sms" - ) + store.append_message("user123", "twilio", "user", "hello via sms") twilio_session = store.get_or_create("user123", "twilio") slack_session = store.get_or_create("user123", "slack") assert len(twilio_session["conversation_history"]) == 1 @@ -51,9 +47,7 @@ class TestAppendMessage: def test_appends_message(self, store): store.get_or_create("user1", "slack") store.append_message("user1", "slack", "user", "hi") - store.append_message( - "user1", "slack", "assistant", "hello!" - ) + store.append_message("user1", "slack", "assistant", "hello!") session = store.get_or_create("user1", "slack") assert len(session["conversation_history"]) == 2 assert session["conversation_history"][0] == { @@ -68,46 +62,30 @@ class TestAppendMessage: def test_caps_history_at_max_turns(self, store): store.get_or_create("user1", "slack") for i in range(25): - store.append_message( - "user1", "slack", "user", f"msg {i}" - ) + store.append_message("user1", "slack", "user", f"msg {i}") session = store.get_or_create("user1", "slack") assert len(session["conversation_history"]) == 20 - assert ( - session["conversation_history"][0]["content"] - == "msg 5" - ) + assert session["conversation_history"][0]["content"] == "msg 5" class TestNotificationPreference: def test_set_and_get(self, store): store.get_or_create("user1", "twilio") - store.set_notification_preference( - "user1", "twilio", "slack" - ) + store.set_notification_preference("user1", "twilio", "slack") session = store.get_or_create("user1", "twilio") - assert ( - session["preferred_notification_channel"] == "slack" - ) + assert session["preferred_notification_channel"] == "slack" class TestPendingResponse: def test_set_and_get(self, store): store.get_or_create("user1", "twilio") - store.set_pending_response( - "user1", "twilio", "full long response text here" - ) + store.set_pending_response("user1", "twilio", "full long response text here") session = store.get_or_create("user1", "twilio") - assert ( - session["pending_response"] - == "full long response text here" - ) + assert session["pending_response"] == "full long response text here" def test_clear_pending(self, store): store.get_or_create("user1", "twilio") - store.set_pending_response( - "user1", "twilio", "some text" - ) + store.set_pending_response("user1", "twilio", "some text") store.clear_pending_response("user1", "twilio") session = store.get_or_create("user1", "twilio") assert session["pending_response"] is None @@ -116,13 +94,10 @@ class TestPendingResponse: class TestExpireSessions: def test_expire_clears_old_history(self, store): store.get_or_create("user1", "twilio") - store.append_message( - "user1", "twilio", "user", "old message" - ) + store.append_message("user1", "twilio", "user", "old message") # Force the updated_at to be old store._db.execute( - "UPDATE channel_sessions " - "SET updated_at = datetime('now', '-25 hours')" + "UPDATE channel_sessions SET updated_at = datetime('now', '-25 hours')" ) store._db.commit() store.expire_sessions(max_age_hours=24) @@ -134,9 +109,7 @@ class TestLastActiveChannel: def test_returns_most_recent(self, store): store.get_or_create("user1", "twilio") store.get_or_create("user1", "slack") - store.append_message( - "user1", "slack", "user", "latest" - ) + store.append_message("user1", "slack", "user", "latest") result = store.get_last_active_channel("user1") assert result == "slack" diff --git a/tests/server/test_webhook_routes.py b/tests/server/test_webhook_routes.py index 856e7980..6493bda2 100644 --- a/tests/server/test_webhook_routes.py +++ b/tests/server/test_webhook_routes.py @@ -9,9 +9,7 @@ from unittest.mock import MagicMock, patch import pytest -pytest.importorskip( - "fastapi", reason="openjarvis[server] not installed" -) +pytest.importorskip("fastapi", reason="openjarvis[server] not installed") from fastapi import FastAPI from fastapi.testclient import TestClient @@ -41,12 +39,9 @@ class TestTwilioWebhook: def twilio_client(self, twilio_app): return TestClient(twilio_app) - def test_valid_twilio_webhook( - self, twilio_client, mock_bridge - ): + def test_valid_twilio_webhook(self, twilio_client, mock_bridge): with patch( - "openjarvis.server.webhook_routes" - "._validate_twilio_signature", + "openjarvis.server.webhook_routes._validate_twilio_signature", return_value=True, ): resp = twilio_client.post( @@ -62,8 +57,7 @@ class TestTwilioWebhook: def test_invalid_signature_rejected(self, twilio_client): with patch( - "openjarvis.server.webhook_routes" - "._validate_twilio_signature", + "openjarvis.server.webhook_routes._validate_twilio_signature", return_value=False, ): resp = twilio_client.post( @@ -92,17 +86,13 @@ class TestBlueBubblesWebhook: def bb_client(self, bb_app): return TestClient(bb_app) - def test_valid_bluebubbles_webhook( - self, bb_client, mock_bridge - ): + def test_valid_bluebubbles_webhook(self, bb_client, mock_bridge): resp = bb_client.post( "/webhooks/bluebubbles", json={ "type": "new-message", "data": { - "handle": { - "address": "user@icloud.com" - }, + "handle": {"address": "user@icloud.com"}, "text": "hello from imessage", "guid": "msg-123", }, @@ -191,9 +181,7 @@ class TestWhatsAppWebhook: ) assert resp.status_code == 403 - def test_valid_message_webhook( - self, wa_client, mock_bridge - ): + def test_valid_message_webhook(self, wa_client, mock_bridge): payload = { "entry": [ { @@ -203,9 +191,7 @@ class TestWhatsAppWebhook: "messages": [ { "from": "15551234567", - "text": { - "body": "hello wa" - }, + "text": {"body": "hello wa"}, "id": "wamid.abc123", "type": "text", } @@ -217,9 +203,7 @@ class TestWhatsAppWebhook: ] } body_bytes = json.dumps(payload).encode() - sig = hmac.new( - b"wa_secret", body_bytes, hashlib.sha256 - ).hexdigest() + sig = hmac.new(b"wa_secret", body_bytes, hashlib.sha256).hexdigest() resp = wa_client.post( "/webhooks/whatsapp", content=body_bytes, diff --git a/tests/tools/test_knowledge_search.py b/tests/tools/test_knowledge_search.py index 8d41e13c..9fced764 100644 --- a/tests/tools/test_knowledge_search.py +++ b/tests/tools/test_knowledge_search.py @@ -145,7 +145,9 @@ def test_tool_uses_two_stage_retriever(tmp_path: Path) -> None: from openjarvis.connectors.retriever import TwoStageRetriever store = KnowledgeStore(db_path=str(tmp_path / "ts_test.db")) - store.store(content="Deep learning research paper", source="gdrive", doc_type="document") + store.store( + content="Deep learning research paper", source="gdrive", doc_type="document" + ) retriever = TwoStageRetriever(store=store) tool = KnowledgeSearchTool(store=store, retriever=retriever) result = tool.execute(query="deep learning") diff --git a/tests/tools/test_knowledge_sql.py b/tests/tools/test_knowledge_sql.py index c2be20b9..7f434e7b 100644 --- a/tests/tools/test_knowledge_sql.py +++ b/tests/tools/test_knowledge_sql.py @@ -14,7 +14,9 @@ from openjarvis.core.registry import ToolRegistry def store(tmp_path: Path) -> KnowledgeStore: ks = KnowledgeStore(str(tmp_path / "test.db")) ks.store("Hello from Alice", source="imessage", author="Alice", doc_type="message") - ks.store("Hello from Alice again", source="imessage", author="Alice", doc_type="message") + ks.store( + "Hello from Alice again", source="imessage", author="Alice", doc_type="message" + ) ks.store("Meeting notes Q1", source="granola", author="Bob", doc_type="document") ks.store("Email about Spain trip", source="gmail", author="Carol", doc_type="email") return ks @@ -22,6 +24,7 @@ def store(tmp_path: Path) -> KnowledgeStore: def test_select_count(store: KnowledgeStore) -> None: from openjarvis.tools.knowledge_sql import KnowledgeSQLTool + tool = KnowledgeSQLTool(store=store) result = tool.execute(query="SELECT COUNT(*) as total FROM knowledge_chunks") assert result.success @@ -30,9 +33,14 @@ def test_select_count(store: KnowledgeStore) -> None: def test_group_by_author(store: KnowledgeStore) -> None: from openjarvis.tools.knowledge_sql import KnowledgeSQLTool + tool = KnowledgeSQLTool(store=store) result = tool.execute( - query="SELECT author, COUNT(*) as n FROM knowledge_chunks GROUP BY author ORDER BY n DESC" + query=( + "SELECT author, COUNT(*) as n " + "FROM knowledge_chunks " + "GROUP BY author ORDER BY n DESC" + ) ) assert result.success assert "Alice" in result.content @@ -41,6 +49,7 @@ def test_group_by_author(store: KnowledgeStore) -> None: def test_rejects_non_select(store: KnowledgeStore) -> None: from openjarvis.tools.knowledge_sql import KnowledgeSQLTool + tool = KnowledgeSQLTool(store=store) result = tool.execute(query="DELETE FROM knowledge_chunks") assert not result.success @@ -49,6 +58,7 @@ def test_rejects_non_select(store: KnowledgeStore) -> None: def test_rejects_drop(store: KnowledgeStore) -> None: from openjarvis.tools.knowledge_sql import KnowledgeSQLTool + tool = KnowledgeSQLTool(store=store) result = tool.execute(query="DROP TABLE knowledge_chunks") assert not result.success @@ -56,6 +66,7 @@ def test_rejects_drop(store: KnowledgeStore) -> None: def test_handles_bad_sql(store: KnowledgeStore) -> None: from openjarvis.tools.knowledge_sql import KnowledgeSQLTool + tool = KnowledgeSQLTool(store=store) result = tool.execute(query="SELECT * FROM nonexistent_table") assert not result.success @@ -63,6 +74,7 @@ def test_handles_bad_sql(store: KnowledgeStore) -> None: def test_filter_by_source(store: KnowledgeStore) -> None: from openjarvis.tools.knowledge_sql import KnowledgeSQLTool + tool = KnowledgeSQLTool(store=store) result = tool.execute( query="SELECT title, author FROM knowledge_chunks WHERE source = 'gmail'" @@ -73,5 +85,6 @@ def test_filter_by_source(store: KnowledgeStore) -> None: def test_registered() -> None: from openjarvis.tools.knowledge_sql import KnowledgeSQLTool + ToolRegistry.register_value("knowledge_sql", KnowledgeSQLTool) assert ToolRegistry.contains("knowledge_sql") diff --git a/tests/tools/test_scan_chunks.py b/tests/tools/test_scan_chunks.py index 58a8162b..2696c5a6 100644 --- a/tests/tools/test_scan_chunks.py +++ b/tests/tools/test_scan_chunks.py @@ -32,6 +32,7 @@ def _fake_engine() -> MagicMock: def test_scan_finds_semantic_matches(store: KnowledgeStore) -> None: from openjarvis.tools.scan_chunks import ScanChunksTool + engine = _fake_engine() tool = ScanChunksTool(store=store, engine=engine, model="test") result = tool.execute(question="Which VCs have I spoken with?") @@ -42,6 +43,7 @@ def test_scan_finds_semantic_matches(store: KnowledgeStore) -> None: def test_scan_respects_source_filter(store: KnowledgeStore) -> None: from openjarvis.tools.scan_chunks import ScanChunksTool + engine = _fake_engine() tool = ScanChunksTool(store=store, engine=engine, model="test") result = tool.execute(question="What trips?", source="imessage") @@ -54,6 +56,7 @@ def test_scan_respects_source_filter(store: KnowledgeStore) -> None: def test_scan_empty_store(tmp_path: Path) -> None: from openjarvis.tools.scan_chunks import ScanChunksTool + ks = KnowledgeStore(str(tmp_path / "empty.db")) engine = _fake_engine() tool = ScanChunksTool(store=ks, engine=engine, model="test") @@ -64,5 +67,6 @@ def test_scan_empty_store(tmp_path: Path) -> None: def test_registered() -> None: from openjarvis.tools.scan_chunks import ScanChunksTool + ToolRegistry.register_value("scan_chunks", ScanChunksTool) assert ToolRegistry.contains("scan_chunks")