admin/OpenJarvis
1
0
Fork 0
mirror of https://github.com/open-jarvis/OpenJarvis.git synced 2026-07-31 03:12:16 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(evals): PinchBench harness fixes — scores from 26% to 84% (#124)

Browse Source 
* fix(evals): PinchBench harness fixes — scores from 26% to 84%

Multiple infrastructure bugs prevented PinchBench from producing
accurate scores. This commit fixes the eval harness so model scores
match the official leaderboard (Qwen3.5-397B: 26% → 84%, GPT-5.4:
5% → 58%).

Key fixes:

- EvalRunner: wrap generation in PinchBenchTaskEnv context so workspace
  files persist through grading (was deleting before scorer ran)
- EvalRunner: detect task_env datasets and force sequential processing
  (CWD changes aren't thread-safe)
- EvalRunner: fix episode_mode auto-detection to check for real
  iter_episodes() override instead of hasattr() (always True)
- Scorer: use "params" field in transcripts to match PinchBench grade()
  functions (was "arguments")
- Scorer: add None guard in _trace_to_transcript for tool_calls
- Scorer: capture final assistant text response in transcript so
  text-only tasks (like sanity check) can be graded
- Scorer: add _tool_results_to_transcript() helper for EvalRunner path
- native_openhands: add native function-calling support (tools=
  parameter) with text-based fallback, matching monitor_operative
- Tools: add Python fallbacks for file_read, file_write, shell_exec,
  calculator, think, http_request when openjarvis_rust unavailable
- Security: add Python fallback for is_sensitive_file()
- Config: add "pinchbench" to KNOWN_BENCHMARKS
- Add PinchBench eval configs for Qwen3.5-397B and GPT-5.4

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

* fix(evals): fix hybrid grading crash when grading_weights is None

The 4 tasks with grading_type=hybrid (task_10, task_13, task_16_market,
task_22) crashed because grading_weights was explicitly None in task
metadata. dict.get("key", default) returns None (not the default) when
the key exists with value None. Use `or` to coalesce None to defaults.

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

* fix(evals): handle MagicMock datasets in runner type checks

The iter_episodes and create_task_env type identity checks fail with
AttributeError when the dataset is a MagicMock (used in tracker tests).
Wrap in try/except to default to False for non-DatasetProvider objects.

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

---------

Co-authored-by: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
Jon Saad-Falcon
2026-03-25 21:16:57 -07:00
committed by GitHub
co-authored by Claude Opus 4.6
parent 43b1240a44
commit 1d24c64f11
15 changed files with 331 additions and 74 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/openjarvis/agents/native_openhands.py
+46 -1
View File
@@ -283,14 +283,23 @@ class NativeOpenHandsAgent(ToolUsingAgent):
last_content = ""
total_usage = {"prompt_tokens": 0, "completion_tokens": 0, "total_tokens": 0}
# Build OpenAI-format tool schemas for native function calling
openai_tools = (
self._executor.get_openai_tools() if self._tools else []
)
for _turn in range(self._max_turns):
turns += 1
# Truncate before every generate call -- tool results may have
# expanded the context beyond what the model supports.
messages = self._truncate_if_needed(messages)
gen_kwargs: dict[str, Any] = {}
if openai_tools:
gen_kwargs["tools"] = openai_tools
try:
result = self._generate(messages)
result = self._generate(messages, **gen_kwargs)
except Exception as exc:
error_str = str(exc)
if "400" in error_str:
@@ -318,6 +327,42 @@ class NativeOpenHandsAgent(ToolUsingAgent):
content = self._strip_think_tags(content)
last_content = content
# --- Native function-calling path (OpenAI, Anthropic, etc.) ---
raw_tool_calls = result.get("tool_calls", [])
if raw_tool_calls:
native_calls = [
ToolCall(
id=tc.get("id", f"call_{turns}_{i}"),
name=tc.get("name", ""),
arguments=tc.get("arguments", "{}"),
)
for i, tc in enumerate(raw_tool_calls)
]
messages.append(
Message(
role=Role.ASSISTANT,
content=content,
tool_calls=native_calls,
)
)
for tc in native_calls:
tool_result = self._executor.execute(tc)
all_tool_results.append(tool_result)
obs_text = tool_result.content
if len(obs_text) > 4000:
obs_text = obs_text[:4000] + "\n\n[Output truncated]"
messages.append(
Message(
role=Role.TOOL,
content=obs_text,
tool_call_id=tc.id,
name=tc.name,
)
)
continue
# --- Text-based fallback (CodeAct / Action-Input format) ---
# Try to extract code
code = self._extract_code(content)
if code:
src/openjarvis/evals/configs/pinchbench-gpt54.toml
+34
View File
@@ -0,0 +1,34 @@
# PinchBench eval: gpt-5.4 (cloud)
# Agent: native_openhands — all PinchBench-required tools enabled
[meta]
name = "pinchbench-gpt54"
description = "PinchBench on gpt-5.4-2026-03-05"
[defaults]
temperature = 0.6
max_tokens = 8192
[judge]
model = "claude-opus-4-5"
temperature = 0.0
engine = "cloud"
[run]
max_workers = 1
output_dir = "results/pinchbench-gpt54/"
seed = 42
[[models]]
name = "gpt-5.4-2026-03-05"
engine = "cloud"
[[benchmarks]]
name = "pinchbench"
backend = "jarvis-agent"
agent = "native_openhands"
tools = [
"think", "file_read", "file_write", "web_search", "shell_exec",
"code_interpreter", "browser_navigate", "image_generate",
"calculator", "http_request", "pdf_extract",
]
src/openjarvis/evals/configs/pinchbench-qwen397b.toml
+35
View File
@@ -0,0 +1,35 @@
# PinchBench eval: Qwen3.5-397B-A17B-FP8 (vLLM, TP=8)
# Agent: native_openhands — all PinchBench-required tools enabled
[meta]
name = "pinchbench-qwen397b"
description = "PinchBench on Qwen/Qwen3.5-397B-A17B-FP8 (vLLM, TP=8)"
[defaults]
temperature = 0.6
max_tokens = 8192
[judge]
model = "claude-opus-4-5"
temperature = 0.0
engine = "cloud"
[run]
max_workers = 1
output_dir = "results/pinchbench-qwen397b/"
seed = 42
[[models]]
name = "Qwen/Qwen3.5-397B-A17B-FP8"
engine = "vllm"
num_gpus = 8
[[benchmarks]]
name = "pinchbench"
backend = "jarvis-agent"
agent = "native_openhands"
tools = [
"think", "file_read", "file_write", "web_search", "shell_exec",
"code_interpreter", "browser_navigate", "image_generate",
"calculator", "http_request", "pdf_extract",
]
src/openjarvis/evals/core/config.py
+1
View File
@@ -43,6 +43,7 @@ KNOWN_BENCHMARKS = {
"knowledge_base", "coding_task",
"coding_assistant", "security_scanner", "daily_digest",
"doc_qa", "browser_assistant",
"pinchbench",
}
src/openjarvis/evals/core/runner.py
+62 -12
View File
@@ -96,11 +96,19 @@ class EvalRunner:
seed=cfg.seed,
)
# Auto-enable episode_mode when the dataset has iter_episodes()
# (i.e. it is a lifelong/sequential benchmark like LifelongAgentBench).
# This is enforced at the runner level so it applies regardless of
# how the runner is invoked (CLI, SDK, tests, etc.).
if not cfg.episode_mode and hasattr(self._dataset, "iter_episodes"):
# Auto-enable episode_mode when the dataset *overrides*
# iter_episodes() (i.e. it is a lifelong/sequential benchmark like
# LifelongAgentBench). The base DatasetProvider always defines a
# default iter_episodes() that wraps each record in its own episode,
# so hasattr() is always True — we must check for a real override.
from openjarvis.evals.core.dataset import DatasetProvider as _DP
try:
_overrides_episodes = (
type(self._dataset).iter_episodes is not _DP.iter_episodes
)
except AttributeError:
_overrides_episodes = False
if not cfg.episode_mode and _overrides_episodes:
LOGGER.info(
"%s requires sequential episode processing — "
"auto-enabling episode_mode.",
@@ -109,6 +117,15 @@ class EvalRunner:
cfg = dataclasses.replace(cfg, episode_mode=True)
self._config = cfg
# Detect if dataset provides task environments (e.g. PinchBench)
try:
self._has_task_env = (
type(self._dataset).create_task_env
is not _DP.create_task_env
)
except AttributeError:
self._has_task_env = False
records = list(self._dataset.iter_records())
LOGGER.info(
"Running %s: %d samples, backend=%s, model=%s, workers=%d, "
@@ -145,6 +162,15 @@ class EvalRunner:
try:
if cfg.episode_mode:
self._run_episode_mode(records, progress_callback, total)
elif self._has_task_env:
# Task environments (PinchBench etc.) change CWD —
# must process sequentially for thread safety.
for record in records:
result = self._process_one(record)
self._results.append(result)
self._flush_result(result)
if progress_callback is not None:
progress_callback(len(self._results), total)
else:
with ThreadPoolExecutor(max_workers=cfg.max_workers) as pool:
futures = {
@@ -224,17 +250,41 @@ class EvalRunner:
)
if cfg.system_prompt:
gen_kwargs["system"] = cfg.system_prompt
full = self._backend.generate_full(
record.problem,
**gen_kwargs,
)
content = full.get("content", "")
if getattr(self, "_has_task_env", False):
from contextlib import nullcontext
task_env = self._dataset.create_task_env(record)
ctx = task_env if task_env is not None else nullcontext()
with ctx:
full = self._backend.generate_full(
record.problem,
**gen_kwargs,
)
full = full or {}
# Store tool results for the scorer to build transcripts
record.metadata["tool_results"] = full.get(
"tool_results", [],
)
# Score INSIDE context so workspace files still exist
content = full.get("content", "")
is_correct, scoring_meta = self._scorer.score(
record, content,
)
else:
full = self._backend.generate_full(
record.problem,
**gen_kwargs,
)
full = full or {}
content = full.get("content", "")
is_correct, scoring_meta = self._scorer.score(
record, content,
)
usage = full.get("usage", {})
latency = full.get("latency_seconds", 0.0)
cost = full.get("cost_usd", 0.0)
is_correct, scoring_meta = self._scorer.score(record, content)
energy_j = full.get("energy_joules", 0.0)
power_w = full.get("power_watts", 0.0)
throughput = full.get("throughput_tok_per_sec", 0.0)
src/openjarvis/evals/scorers/pinchbench.py
+64 -6
View File
@@ -52,12 +52,12 @@ def events_to_transcript(events: List[Any]) -> List[Dict[str, Any]]:
if etype == EventType.TOOL_CALL_START or etype == EventType.TOOL_CALL_START.value:
tool_name = event.metadata.get("tool", "unknown")
mapped = _TOOL_NAME_MAP.get(tool_name, tool_name)
arguments = event.metadata.get("arguments", {})
arguments = event.metadata.get("arguments") or {}
transcript.append({
"type": "message",
"message": {
"role": "assistant",
"content": [{"type": "toolCall", "name": mapped, "arguments": arguments}],
"content": [{"type": "toolCall", "name": mapped, "params": arguments}],
},
})
elif etype == EventType.TOOL_CALL_END or etype == EventType.TOOL_CALL_END.value:
@@ -78,12 +78,14 @@ def _trace_to_transcript(trace: Any) -> List[Dict[str, Any]]:
transcript: List[Dict[str, Any]] = []
for turn in trace.turns:
for tc in getattr(turn, "tool_calls", []):
if tc is None:
continue
mapped = _TOOL_NAME_MAP.get(tc["name"], tc["name"])
transcript.append({
"type": "message",
"message": {
"role": "assistant",
"content": [{"type": "toolCall", "name": mapped, "arguments": tc.get("arguments", {})}],
"content": [{"type": "toolCall", "name": mapped, "params": tc.get("arguments") or {}}],
},
})
transcript.append({
@@ -93,6 +95,42 @@ def _trace_to_transcript(trace: Any) -> List[Dict[str, Any]]:
"content": [{"text": tc.get("result", "")}],
},
})
# Capture final assistant text response (for tasks graded on text output)
response_text = getattr(trace, "response_text", "")
if response_text:
transcript.append({
"type": "message",
"message": {
"role": "assistant",
"content": [{"type": "text", "text": response_text}],
},
})
return transcript
def _tool_results_to_transcript(
tool_results: List[Dict[str, Any]],
) -> List[Dict[str, Any]]:
"""Build transcript from JarvisAgentBackend tool_results list."""
transcript: List[Dict[str, Any]] = []
for tr in tool_results:
tool_name = tr.get("tool_name", "unknown")
mapped = _TOOL_NAME_MAP.get(tool_name, tool_name)
transcript.append({
"type": "message",
"message": {
"role": "assistant",
"content": [{"type": "toolCall", "name": mapped, "params": {}}],
},
})
transcript.append({
"type": "message",
"message": {
"role": "toolResult",
"content": [{"text": tr.get("content", "")}],
},
})
return transcript
@@ -118,8 +156,10 @@ def _summarize_transcript(transcript: List[Dict[str, Any]]) -> str:
for item in msg.get("content", []):
if item.get("type") == "toolCall":
parts.append(
f"Tool: {item.get('name')}({json.dumps(item.get('arguments', {}))})"
f"Tool: {item.get('name')}({json.dumps(item.get('params', {}))})"
)
elif item.get("type") == "text":
parts.append(f"Assistant: {item.get('text', '')}")
elif role == "toolResult":
content = msg.get("content", [])
if content:
@@ -372,7 +412,7 @@ def _grade_hybrid(
judge_model: str,
) -> Dict[str, Any]:
"""Run both automated and LLM judge grading, combine with weights."""
weights = record.metadata.get("grading_weights", {"automated": 0.5, "llm_judge": 0.5})
weights = record.metadata.get("grading_weights") or {"automated": 0.5, "llm_judge": 0.5}
auto = _grade_automated(record, transcript, workspace_path)
llm = _grade_llm_judge(record, transcript, workspace_path, judge_backend, judge_model)
@@ -426,7 +466,24 @@ class PinchBenchScorer(LLMJudgeScorer):
self, record: EvalRecord, model_answer: str,
) -> Tuple[Optional[bool], Dict[str, Any]]:
trace = record.metadata.get("query_trace")
transcript = _trace_to_transcript(trace) if trace else []
if trace:
transcript = _trace_to_transcript(trace)
else:
# No trace — build transcript from tool_results if available
tool_results = record.metadata.get("tool_results", [])
transcript = _tool_results_to_transcript(tool_results)
# Always append final model answer as assistant text message
# so grading functions that check for text responses can find it
if model_answer:
transcript.append({
"type": "message",
"message": {
"role": "assistant",
"content": [{"type": "text", "text": model_answer}],
},
})
result = grade_pinchbench_task(
record=record,
transcript=transcript,
@@ -442,4 +499,5 @@ __all__ = [
"PinchBenchScorer",
"events_to_transcript",
"grade_pinchbench_task",
"_tool_results_to_transcript",
]
src/openjarvis/security/file_policy.py
+19 -3
View File
@@ -30,11 +30,27 @@ def is_sensitive_file(path: Union[str, Path]) -> bool:
Checks both the filename and the full name against
``DEFAULT_SENSITIVE_PATTERNS`` using :func:`fnmatch.fnmatch`.
Uses the Rust implementation when available, falls back to Python.
"""
from openjarvis._rust_bridge import get_rust_module
try:
from openjarvis._rust_bridge import get_rust_module
_rust = get_rust_module()
return _rust.is_sensitive_file(str(path))
_rust = get_rust_module()
return _rust.is_sensitive_file(str(path))
except ImportError:
return _is_sensitive_file_py(str(path))
def _is_sensitive_file_py(path_str: str) -> bool:
"""Pure-Python fallback for sensitive file detection."""
import fnmatch
p = Path(path_str)
name = p.name
for pattern in DEFAULT_SENSITIVE_PATTERNS:
if fnmatch.fnmatch(name, pattern) or fnmatch.fnmatch(str(p), pattern):
return True
return False
def filter_sensitive_paths(paths: Iterable[Union[str, Path]]) -> List[Path]:
src/openjarvis/tools/calculator.py
+9 -4
View File
@@ -89,10 +89,15 @@ def _safe_eval_node(node: ast.AST) -> Any:
def safe_eval(expression: str) -> float:
"""Evaluate a math expression safely — always via Rust backend."""
from openjarvis._rust_bridge import get_rust_module
_rust = get_rust_module()
return float(_rust.CalculatorTool().execute(expression))
"""Evaluate a math expression safely — Rust backend with Python fallback."""
try:
from openjarvis._rust_bridge import get_rust_module
_rust = get_rust_module()
return float(_rust.CalculatorTool().execute(expression))
except ImportError:
import ast as _ast
tree = _ast.parse(expression, mode="eval")
return float(_safe_eval_node(tree.body))
@ToolRegistry.register("calculator")
src/openjarvis/tools/file_read.py
+7 -2
View File
@@ -114,10 +114,15 @@ class FileReadTool(BaseTool):
content=f"File too large: {size} bytes (max {_MAX_SIZE_BYTES}).",
success=False,
)
from openjarvis._rust_bridge import get_rust_module
_rust = get_rust_module()
try:
from openjarvis._rust_bridge import get_rust_module
_rust = get_rust_module()
text = _rust.FileReadTool().execute(str(path))
except ImportError:
try:
text = path.read_text(encoding="utf-8")
except UnicodeDecodeError:
text = path.read_text(encoding="utf-8", errors="replace")
except Exception as exc:
return ToolResult(
tool_name="file_read",
src/openjarvis/tools/file_write.py
+11 -11
View File
@@ -155,26 +155,26 @@ class FileWriteTool(BaseTool):
success=False,
)
from openjarvis._rust_bridge import get_rust_module
_rust = get_rust_module()
if mode == "write":
try:
from openjarvis._rust_bridge import get_rust_module
_rust = get_rust_module()
_rust.FileWriteTool().execute(str(path), content)
except ImportError:
try:
path.write_text(content, encoding="utf-8")
except OSError as exc:
return ToolResult(
tool_name="file_write",
content=f"Write error: {exc}",
success=False,
)
except Exception as exc:
return ToolResult(
tool_name="file_write",
content=f"Write error: {exc}",
success=False,
)
elif False: # dead code — all write modes go through Rust
try:
path.write_text(content, encoding="utf-8")
except OSError as exc:
return ToolResult(
tool_name="file_write",
content=f"Write error: {exc}",
success=False,
)
else:
# append mode — always Python
try:
src/openjarvis/tools/http_request.py
+7 -3
View File
@@ -103,9 +103,13 @@ class HttpRequestTool(BaseTool):
body = params.get("body")
timeout = params.get("timeout", 30)
from openjarvis._rust_bridge import get_rust_module
_rust = get_rust_module()
if not headers:
_rust = None
try:
from openjarvis._rust_bridge import get_rust_module
_rust = get_rust_module()
except ImportError:
pass
if _rust is not None and not headers:
try:
content = _rust.HttpRequestTool().execute(url, method, body)
return ToolResult(
src/openjarvis/tools/shell_exec.py
+27 -26
View File
@@ -125,32 +125,33 @@ class ShellExecTool(BaseTool):
if val is not None:
env[key] = val
from openjarvis._rust_bridge import get_rust_module
_rust = get_rust_module()
if True:
try:
output = _rust.ShellExecTool().execute(command, working_dir)
return ToolResult(
tool_name="shell_exec",
content=output or "(no output)",
success=True,
metadata={
"returncode": 0,
"timeout_used": timeout,
"working_dir": working_dir,
},
)
except Exception as exc:
return ToolResult(
tool_name="shell_exec",
content=str(exc),
success=False,
metadata={
"returncode": -1,
"timeout_used": timeout,
"working_dir": working_dir,
},
)
try:
from openjarvis._rust_bridge import get_rust_module
_rust = get_rust_module()
output = _rust.ShellExecTool().execute(command, working_dir)
return ToolResult(
tool_name="shell_exec",
content=output or "(no output)",
success=True,
metadata={
"returncode": 0,
"timeout_used": timeout,
"working_dir": working_dir,
},
)
except ImportError:
pass # Fall through to subprocess below
except Exception as exc:
return ToolResult(
tool_name="shell_exec",
content=str(exc),
success=False,
metadata={
"returncode": -1,
"timeout_used": timeout,
"working_dir": working_dir,
},
)
try:
result = subprocess.run(
command,
src/openjarvis/tools/think.py
+6 -3
View File
@@ -40,9 +40,12 @@ class ThinkTool(BaseTool):
def execute(self, **params: Any) -> ToolResult:
thought = params.get("thought", "")
from openjarvis._rust_bridge import get_rust_module
_rust = get_rust_module()
content = _rust.ThinkTool().execute(thought)
try:
from openjarvis._rust_bridge import get_rust_module
_rust = get_rust_module()
content = _rust.ThinkTool().execute(thought)
except ImportError:
content = thought
return ToolResult(
tool_name="think",
content=content,
tests/evals/test_benchmark_datasets.py
+1 -1
View File
@@ -313,7 +313,7 @@ class TestConfigBenchmarks:
def test_benchmarks_count(self) -> None:
from openjarvis.evals.core.config import KNOWN_BENCHMARKS
assert len(KNOWN_BENCHMARKS) == 25
assert len(KNOWN_BENCHMARKS) == 26
# ---------------------------------------------------------------------------
tests/evals/test_pinchbench_grading.py
+2 -2
View File
@@ -74,7 +74,7 @@ def grade(transcript, workspace_path):
{
"type": "toolCall",
"name": "read_file",
"arguments": {"path": "a.txt"},
"params": {"path": "a.txt"},
}
],
},
@@ -124,7 +124,7 @@ class TestSummarizeTranscript:
{
"type": "toolCall",
"name": "read_file",
"arguments": {"path": "a.txt"},
"params": {"path": "a.txt"},
}
],
},