diff --git a/src/openjarvis/evals/cli.py b/src/openjarvis/evals/cli.py index b3674be1..e291e73f 100644 --- a/src/openjarvis/evals/cli.py +++ b/src/openjarvis/evals/cli.py @@ -599,6 +599,7 @@ def _run_terminalbench_native(config, console: Console) -> object: ) import re + # Docker compose project names must be lowercase alphanumeric + hyphens/underscores model_slug = re.sub(r"[^a-z0-9_-]", "-", model.lower().replace("/", "-")) run_id = f"tb2-{model_slug}" diff --git a/src/openjarvis/evals/datasets/toolcall15.py b/src/openjarvis/evals/datasets/toolcall15.py index f27395df..f14e6897 100644 --- a/src/openjarvis/evals/datasets/toolcall15.py +++ b/src/openjarvis/evals/datasets/toolcall15.py @@ -672,8 +672,7 @@ class ToolCall15Dataset(DatasetProvider): # instructions, available tools, and user message so the # model can respond with tool calls via any backend. tool_descriptions = "\n".join( - f"- {t['function']['name']}: " - f"{t['function']['description']}" + f"- {t['function']['name']}: {t['function']['description']}" for t in TOOLS ) prompt = ( diff --git a/src/openjarvis/evals/scorers/toolcall15.py b/src/openjarvis/evals/scorers/toolcall15.py index 66eb66a2..edbb2b63 100644 --- a/src/openjarvis/evals/scorers/toolcall15.py +++ b/src/openjarvis/evals/scorers/toolcall15.py @@ -41,20 +41,24 @@ def _extract_tool_calls( for tc in getattr(turn, "tool_calls", []): if tc is None: continue - tool_calls.append({ - "name": tc.get("name", ""), - "arguments": tc.get("arguments") or {}, - }) + tool_calls.append( + { + "name": tc.get("name", ""), + "arguments": tc.get("arguments") or {}, + } + ) if tool_calls: return tool_calls # Try tool_results list (from JarvisAgentBackend) tool_results = record.metadata.get("tool_results", []) for tr in tool_results: - tool_calls.append({ - "name": tr.get("tool_name", ""), - "arguments": tr.get("arguments") or {}, - }) + tool_calls.append( + { + "name": tr.get("tool_name", ""), + "arguments": tr.get("arguments") or {}, + } + ) if tool_calls: return tool_calls