diff --git a/src/openjarvis/agents/native_openhands.py b/src/openjarvis/agents/native_openhands.py index cf5579cd..9b518809 100644 --- a/src/openjarvis/agents/native_openhands.py +++ b/src/openjarvis/agents/native_openhands.py @@ -283,14 +283,23 @@ class NativeOpenHandsAgent(ToolUsingAgent): last_content = "" total_usage = {"prompt_tokens": 0, "completion_tokens": 0, "total_tokens": 0} + # Build OpenAI-format tool schemas for native function calling + openai_tools = ( + self._executor.get_openai_tools() if self._tools else [] + ) + for _turn in range(self._max_turns): turns += 1 # Truncate before every generate call -- tool results may have # expanded the context beyond what the model supports. messages = self._truncate_if_needed(messages) + gen_kwargs: dict[str, Any] = {} + if openai_tools: + gen_kwargs["tools"] = openai_tools + try: - result = self._generate(messages) + result = self._generate(messages, **gen_kwargs) except Exception as exc: error_str = str(exc) if "400" in error_str: @@ -318,6 +327,42 @@ class NativeOpenHandsAgent(ToolUsingAgent): content = self._strip_think_tags(content) last_content = content + # --- Native function-calling path (OpenAI, Anthropic, etc.) --- + raw_tool_calls = result.get("tool_calls", []) + if raw_tool_calls: + native_calls = [ + ToolCall( + id=tc.get("id", f"call_{turns}_{i}"), + name=tc.get("name", ""), + arguments=tc.get("arguments", "{}"), + ) + for i, tc in enumerate(raw_tool_calls) + ] + messages.append( + Message( + role=Role.ASSISTANT, + content=content, + tool_calls=native_calls, + ) + ) + for tc in native_calls: + tool_result = self._executor.execute(tc) + all_tool_results.append(tool_result) + obs_text = tool_result.content + if len(obs_text) > 4000: + obs_text = obs_text[:4000] + "\n\n[Output truncated]" + messages.append( + Message( + role=Role.TOOL, + content=obs_text, + tool_call_id=tc.id, + name=tc.name, + ) + ) + continue + + # --- Text-based fallback (CodeAct / Action-Input format) --- + # Try to extract code code = self._extract_code(content) if code: diff --git a/src/openjarvis/evals/configs/pinchbench-gpt54.toml b/src/openjarvis/evals/configs/pinchbench-gpt54.toml new file mode 100644 index 00000000..e1feee6a --- /dev/null +++ b/src/openjarvis/evals/configs/pinchbench-gpt54.toml @@ -0,0 +1,34 @@ +# PinchBench eval: gpt-5.4 (cloud) +# Agent: native_openhands — all PinchBench-required tools enabled + +[meta] +name = "pinchbench-gpt54" +description = "PinchBench on gpt-5.4-2026-03-05" + +[defaults] +temperature = 0.6 +max_tokens = 8192 + +[judge] +model = "claude-opus-4-5" +temperature = 0.0 +engine = "cloud" + +[run] +max_workers = 1 +output_dir = "results/pinchbench-gpt54/" +seed = 42 + +[[models]] +name = "gpt-5.4-2026-03-05" +engine = "cloud" + +[[benchmarks]] +name = "pinchbench" +backend = "jarvis-agent" +agent = "native_openhands" +tools = [ + "think", "file_read", "file_write", "web_search", "shell_exec", + "code_interpreter", "browser_navigate", "image_generate", + "calculator", "http_request", "pdf_extract", +] diff --git a/src/openjarvis/evals/configs/pinchbench-qwen397b.toml b/src/openjarvis/evals/configs/pinchbench-qwen397b.toml new file mode 100644 index 00000000..b6b1493b --- /dev/null +++ b/src/openjarvis/evals/configs/pinchbench-qwen397b.toml @@ -0,0 +1,35 @@ +# PinchBench eval: Qwen3.5-397B-A17B-FP8 (vLLM, TP=8) +# Agent: native_openhands — all PinchBench-required tools enabled + +[meta] +name = "pinchbench-qwen397b" +description = "PinchBench on Qwen/Qwen3.5-397B-A17B-FP8 (vLLM, TP=8)" + +[defaults] +temperature = 0.6 +max_tokens = 8192 + +[judge] +model = "claude-opus-4-5" +temperature = 0.0 +engine = "cloud" + +[run] +max_workers = 1 +output_dir = "results/pinchbench-qwen397b/" +seed = 42 + +[[models]] +name = "Qwen/Qwen3.5-397B-A17B-FP8" +engine = "vllm" +num_gpus = 8 + +[[benchmarks]] +name = "pinchbench" +backend = "jarvis-agent" +agent = "native_openhands" +tools = [ + "think", "file_read", "file_write", "web_search", "shell_exec", + "code_interpreter", "browser_navigate", "image_generate", + "calculator", "http_request", "pdf_extract", +] diff --git a/src/openjarvis/evals/core/config.py b/src/openjarvis/evals/core/config.py index ac62d191..ab864c71 100644 --- a/src/openjarvis/evals/core/config.py +++ b/src/openjarvis/evals/core/config.py @@ -43,6 +43,7 @@ KNOWN_BENCHMARKS = { "knowledge_base", "coding_task", "coding_assistant", "security_scanner", "daily_digest", "doc_qa", "browser_assistant", + "pinchbench", } diff --git a/src/openjarvis/evals/core/runner.py b/src/openjarvis/evals/core/runner.py index e3d8716e..9effbad2 100644 --- a/src/openjarvis/evals/core/runner.py +++ b/src/openjarvis/evals/core/runner.py @@ -96,11 +96,19 @@ class EvalRunner: seed=cfg.seed, ) - # Auto-enable episode_mode when the dataset has iter_episodes() - # (i.e. it is a lifelong/sequential benchmark like LifelongAgentBench). - # This is enforced at the runner level so it applies regardless of - # how the runner is invoked (CLI, SDK, tests, etc.). - if not cfg.episode_mode and hasattr(self._dataset, "iter_episodes"): + # Auto-enable episode_mode when the dataset *overrides* + # iter_episodes() (i.e. it is a lifelong/sequential benchmark like + # LifelongAgentBench). The base DatasetProvider always defines a + # default iter_episodes() that wraps each record in its own episode, + # so hasattr() is always True — we must check for a real override. + from openjarvis.evals.core.dataset import DatasetProvider as _DP + try: + _overrides_episodes = ( + type(self._dataset).iter_episodes is not _DP.iter_episodes + ) + except AttributeError: + _overrides_episodes = False + if not cfg.episode_mode and _overrides_episodes: LOGGER.info( "%s requires sequential episode processing — " "auto-enabling episode_mode.", @@ -109,6 +117,15 @@ class EvalRunner: cfg = dataclasses.replace(cfg, episode_mode=True) self._config = cfg + # Detect if dataset provides task environments (e.g. PinchBench) + try: + self._has_task_env = ( + type(self._dataset).create_task_env + is not _DP.create_task_env + ) + except AttributeError: + self._has_task_env = False + records = list(self._dataset.iter_records()) LOGGER.info( "Running %s: %d samples, backend=%s, model=%s, workers=%d, " @@ -145,6 +162,15 @@ class EvalRunner: try: if cfg.episode_mode: self._run_episode_mode(records, progress_callback, total) + elif self._has_task_env: + # Task environments (PinchBench etc.) change CWD — + # must process sequentially for thread safety. + for record in records: + result = self._process_one(record) + self._results.append(result) + self._flush_result(result) + if progress_callback is not None: + progress_callback(len(self._results), total) else: with ThreadPoolExecutor(max_workers=cfg.max_workers) as pool: futures = { @@ -224,17 +250,41 @@ class EvalRunner: ) if cfg.system_prompt: gen_kwargs["system"] = cfg.system_prompt - full = self._backend.generate_full( - record.problem, - **gen_kwargs, - ) - content = full.get("content", "") + + if getattr(self, "_has_task_env", False): + from contextlib import nullcontext + task_env = self._dataset.create_task_env(record) + ctx = task_env if task_env is not None else nullcontext() + with ctx: + full = self._backend.generate_full( + record.problem, + **gen_kwargs, + ) + full = full or {} + # Store tool results for the scorer to build transcripts + record.metadata["tool_results"] = full.get( + "tool_results", [], + ) + # Score INSIDE context so workspace files still exist + content = full.get("content", "") + is_correct, scoring_meta = self._scorer.score( + record, content, + ) + else: + full = self._backend.generate_full( + record.problem, + **gen_kwargs, + ) + full = full or {} + content = full.get("content", "") + is_correct, scoring_meta = self._scorer.score( + record, content, + ) + usage = full.get("usage", {}) latency = full.get("latency_seconds", 0.0) cost = full.get("cost_usd", 0.0) - is_correct, scoring_meta = self._scorer.score(record, content) - energy_j = full.get("energy_joules", 0.0) power_w = full.get("power_watts", 0.0) throughput = full.get("throughput_tok_per_sec", 0.0) diff --git a/src/openjarvis/evals/scorers/pinchbench.py b/src/openjarvis/evals/scorers/pinchbench.py index 68269ec0..693f3d12 100644 --- a/src/openjarvis/evals/scorers/pinchbench.py +++ b/src/openjarvis/evals/scorers/pinchbench.py @@ -52,12 +52,12 @@ def events_to_transcript(events: List[Any]) -> List[Dict[str, Any]]: if etype == EventType.TOOL_CALL_START or etype == EventType.TOOL_CALL_START.value: tool_name = event.metadata.get("tool", "unknown") mapped = _TOOL_NAME_MAP.get(tool_name, tool_name) - arguments = event.metadata.get("arguments", {}) + arguments = event.metadata.get("arguments") or {} transcript.append({ "type": "message", "message": { "role": "assistant", - "content": [{"type": "toolCall", "name": mapped, "arguments": arguments}], + "content": [{"type": "toolCall", "name": mapped, "params": arguments}], }, }) elif etype == EventType.TOOL_CALL_END or etype == EventType.TOOL_CALL_END.value: @@ -78,12 +78,14 @@ def _trace_to_transcript(trace: Any) -> List[Dict[str, Any]]: transcript: List[Dict[str, Any]] = [] for turn in trace.turns: for tc in getattr(turn, "tool_calls", []): + if tc is None: + continue mapped = _TOOL_NAME_MAP.get(tc["name"], tc["name"]) transcript.append({ "type": "message", "message": { "role": "assistant", - "content": [{"type": "toolCall", "name": mapped, "arguments": tc.get("arguments", {})}], + "content": [{"type": "toolCall", "name": mapped, "params": tc.get("arguments") or {}}], }, }) transcript.append({ @@ -93,6 +95,42 @@ def _trace_to_transcript(trace: Any) -> List[Dict[str, Any]]: "content": [{"text": tc.get("result", "")}], }, }) + + # Capture final assistant text response (for tasks graded on text output) + response_text = getattr(trace, "response_text", "") + if response_text: + transcript.append({ + "type": "message", + "message": { + "role": "assistant", + "content": [{"type": "text", "text": response_text}], + }, + }) + return transcript + + +def _tool_results_to_transcript( + tool_results: List[Dict[str, Any]], +) -> List[Dict[str, Any]]: + """Build transcript from JarvisAgentBackend tool_results list.""" + transcript: List[Dict[str, Any]] = [] + for tr in tool_results: + tool_name = tr.get("tool_name", "unknown") + mapped = _TOOL_NAME_MAP.get(tool_name, tool_name) + transcript.append({ + "type": "message", + "message": { + "role": "assistant", + "content": [{"type": "toolCall", "name": mapped, "params": {}}], + }, + }) + transcript.append({ + "type": "message", + "message": { + "role": "toolResult", + "content": [{"text": tr.get("content", "")}], + }, + }) return transcript @@ -118,8 +156,10 @@ def _summarize_transcript(transcript: List[Dict[str, Any]]) -> str: for item in msg.get("content", []): if item.get("type") == "toolCall": parts.append( - f"Tool: {item.get('name')}({json.dumps(item.get('arguments', {}))})" + f"Tool: {item.get('name')}({json.dumps(item.get('params', {}))})" ) + elif item.get("type") == "text": + parts.append(f"Assistant: {item.get('text', '')}") elif role == "toolResult": content = msg.get("content", []) if content: @@ -372,7 +412,7 @@ def _grade_hybrid( judge_model: str, ) -> Dict[str, Any]: """Run both automated and LLM judge grading, combine with weights.""" - weights = record.metadata.get("grading_weights", {"automated": 0.5, "llm_judge": 0.5}) + weights = record.metadata.get("grading_weights") or {"automated": 0.5, "llm_judge": 0.5} auto = _grade_automated(record, transcript, workspace_path) llm = _grade_llm_judge(record, transcript, workspace_path, judge_backend, judge_model) @@ -426,7 +466,24 @@ class PinchBenchScorer(LLMJudgeScorer): self, record: EvalRecord, model_answer: str, ) -> Tuple[Optional[bool], Dict[str, Any]]: trace = record.metadata.get("query_trace") - transcript = _trace_to_transcript(trace) if trace else [] + if trace: + transcript = _trace_to_transcript(trace) + else: + # No trace — build transcript from tool_results if available + tool_results = record.metadata.get("tool_results", []) + transcript = _tool_results_to_transcript(tool_results) + + # Always append final model answer as assistant text message + # so grading functions that check for text responses can find it + if model_answer: + transcript.append({ + "type": "message", + "message": { + "role": "assistant", + "content": [{"type": "text", "text": model_answer}], + }, + }) + result = grade_pinchbench_task( record=record, transcript=transcript, @@ -442,4 +499,5 @@ __all__ = [ "PinchBenchScorer", "events_to_transcript", "grade_pinchbench_task", + "_tool_results_to_transcript", ] diff --git a/src/openjarvis/security/file_policy.py b/src/openjarvis/security/file_policy.py index fc4006ae..36482440 100644 --- a/src/openjarvis/security/file_policy.py +++ b/src/openjarvis/security/file_policy.py @@ -30,11 +30,27 @@ def is_sensitive_file(path: Union[str, Path]) -> bool: Checks both the filename and the full name against ``DEFAULT_SENSITIVE_PATTERNS`` using :func:`fnmatch.fnmatch`. + Uses the Rust implementation when available, falls back to Python. """ - from openjarvis._rust_bridge import get_rust_module + try: + from openjarvis._rust_bridge import get_rust_module - _rust = get_rust_module() - return _rust.is_sensitive_file(str(path)) + _rust = get_rust_module() + return _rust.is_sensitive_file(str(path)) + except ImportError: + return _is_sensitive_file_py(str(path)) + + +def _is_sensitive_file_py(path_str: str) -> bool: + """Pure-Python fallback for sensitive file detection.""" + import fnmatch + + p = Path(path_str) + name = p.name + for pattern in DEFAULT_SENSITIVE_PATTERNS: + if fnmatch.fnmatch(name, pattern) or fnmatch.fnmatch(str(p), pattern): + return True + return False def filter_sensitive_paths(paths: Iterable[Union[str, Path]]) -> List[Path]: diff --git a/src/openjarvis/tools/calculator.py b/src/openjarvis/tools/calculator.py index 14ca4bc4..2ec101eb 100644 --- a/src/openjarvis/tools/calculator.py +++ b/src/openjarvis/tools/calculator.py @@ -89,10 +89,15 @@ def _safe_eval_node(node: ast.AST) -> Any: def safe_eval(expression: str) -> float: - """Evaluate a math expression safely — always via Rust backend.""" - from openjarvis._rust_bridge import get_rust_module - _rust = get_rust_module() - return float(_rust.CalculatorTool().execute(expression)) + """Evaluate a math expression safely — Rust backend with Python fallback.""" + try: + from openjarvis._rust_bridge import get_rust_module + _rust = get_rust_module() + return float(_rust.CalculatorTool().execute(expression)) + except ImportError: + import ast as _ast + tree = _ast.parse(expression, mode="eval") + return float(_safe_eval_node(tree.body)) @ToolRegistry.register("calculator") diff --git a/src/openjarvis/tools/file_read.py b/src/openjarvis/tools/file_read.py index 36fb258e..6e5b7e38 100644 --- a/src/openjarvis/tools/file_read.py +++ b/src/openjarvis/tools/file_read.py @@ -114,10 +114,15 @@ class FileReadTool(BaseTool): content=f"File too large: {size} bytes (max {_MAX_SIZE_BYTES}).", success=False, ) - from openjarvis._rust_bridge import get_rust_module - _rust = get_rust_module() try: + from openjarvis._rust_bridge import get_rust_module + _rust = get_rust_module() text = _rust.FileReadTool().execute(str(path)) + except ImportError: + try: + text = path.read_text(encoding="utf-8") + except UnicodeDecodeError: + text = path.read_text(encoding="utf-8", errors="replace") except Exception as exc: return ToolResult( tool_name="file_read", diff --git a/src/openjarvis/tools/file_write.py b/src/openjarvis/tools/file_write.py index d006280c..71be96df 100644 --- a/src/openjarvis/tools/file_write.py +++ b/src/openjarvis/tools/file_write.py @@ -155,26 +155,26 @@ class FileWriteTool(BaseTool): success=False, ) - from openjarvis._rust_bridge import get_rust_module - _rust = get_rust_module() if mode == "write": try: + from openjarvis._rust_bridge import get_rust_module + _rust = get_rust_module() _rust.FileWriteTool().execute(str(path), content) + except ImportError: + try: + path.write_text(content, encoding="utf-8") + except OSError as exc: + return ToolResult( + tool_name="file_write", + content=f"Write error: {exc}", + success=False, + ) except Exception as exc: return ToolResult( tool_name="file_write", content=f"Write error: {exc}", success=False, ) - elif False: # dead code — all write modes go through Rust - try: - path.write_text(content, encoding="utf-8") - except OSError as exc: - return ToolResult( - tool_name="file_write", - content=f"Write error: {exc}", - success=False, - ) else: # append mode — always Python try: diff --git a/src/openjarvis/tools/http_request.py b/src/openjarvis/tools/http_request.py index 4cde4d60..d519815a 100644 --- a/src/openjarvis/tools/http_request.py +++ b/src/openjarvis/tools/http_request.py @@ -103,9 +103,13 @@ class HttpRequestTool(BaseTool): body = params.get("body") timeout = params.get("timeout", 30) - from openjarvis._rust_bridge import get_rust_module - _rust = get_rust_module() - if not headers: + _rust = None + try: + from openjarvis._rust_bridge import get_rust_module + _rust = get_rust_module() + except ImportError: + pass + if _rust is not None and not headers: try: content = _rust.HttpRequestTool().execute(url, method, body) return ToolResult( diff --git a/src/openjarvis/tools/shell_exec.py b/src/openjarvis/tools/shell_exec.py index 3952d26d..6611df7a 100644 --- a/src/openjarvis/tools/shell_exec.py +++ b/src/openjarvis/tools/shell_exec.py @@ -125,32 +125,33 @@ class ShellExecTool(BaseTool): if val is not None: env[key] = val - from openjarvis._rust_bridge import get_rust_module - _rust = get_rust_module() - if True: - try: - output = _rust.ShellExecTool().execute(command, working_dir) - return ToolResult( - tool_name="shell_exec", - content=output or "(no output)", - success=True, - metadata={ - "returncode": 0, - "timeout_used": timeout, - "working_dir": working_dir, - }, - ) - except Exception as exc: - return ToolResult( - tool_name="shell_exec", - content=str(exc), - success=False, - metadata={ - "returncode": -1, - "timeout_used": timeout, - "working_dir": working_dir, - }, - ) + try: + from openjarvis._rust_bridge import get_rust_module + _rust = get_rust_module() + output = _rust.ShellExecTool().execute(command, working_dir) + return ToolResult( + tool_name="shell_exec", + content=output or "(no output)", + success=True, + metadata={ + "returncode": 0, + "timeout_used": timeout, + "working_dir": working_dir, + }, + ) + except ImportError: + pass # Fall through to subprocess below + except Exception as exc: + return ToolResult( + tool_name="shell_exec", + content=str(exc), + success=False, + metadata={ + "returncode": -1, + "timeout_used": timeout, + "working_dir": working_dir, + }, + ) try: result = subprocess.run( command, diff --git a/src/openjarvis/tools/think.py b/src/openjarvis/tools/think.py index 225c888e..699bbe03 100644 --- a/src/openjarvis/tools/think.py +++ b/src/openjarvis/tools/think.py @@ -40,9 +40,12 @@ class ThinkTool(BaseTool): def execute(self, **params: Any) -> ToolResult: thought = params.get("thought", "") - from openjarvis._rust_bridge import get_rust_module - _rust = get_rust_module() - content = _rust.ThinkTool().execute(thought) + try: + from openjarvis._rust_bridge import get_rust_module + _rust = get_rust_module() + content = _rust.ThinkTool().execute(thought) + except ImportError: + content = thought return ToolResult( tool_name="think", content=content, diff --git a/tests/evals/test_benchmark_datasets.py b/tests/evals/test_benchmark_datasets.py index ee5e73a3..e569fa48 100644 --- a/tests/evals/test_benchmark_datasets.py +++ b/tests/evals/test_benchmark_datasets.py @@ -313,7 +313,7 @@ class TestConfigBenchmarks: def test_benchmarks_count(self) -> None: from openjarvis.evals.core.config import KNOWN_BENCHMARKS - assert len(KNOWN_BENCHMARKS) == 25 + assert len(KNOWN_BENCHMARKS) == 26 # --------------------------------------------------------------------------- diff --git a/tests/evals/test_pinchbench_grading.py b/tests/evals/test_pinchbench_grading.py index aa107506..d2396bfc 100644 --- a/tests/evals/test_pinchbench_grading.py +++ b/tests/evals/test_pinchbench_grading.py @@ -74,7 +74,7 @@ def grade(transcript, workspace_path): { "type": "toolCall", "name": "read_file", - "arguments": {"path": "a.txt"}, + "params": {"path": "a.txt"}, } ], }, @@ -124,7 +124,7 @@ class TestSummarizeTranscript: { "type": "toolCall", "name": "read_file", - "arguments": {"path": "a.txt"}, + "params": {"path": "a.txt"}, } ], },