| {html_lib.escape(h.strip())} | " for h in head) + + "
|---|
| {html_lib.escape(cell.strip())} | " for cell in row) + "
\1", s)
+ return s
+
+ for raw in lines:
+ line = raw.rstrip()
+ if line.startswith("|") and "|" in line[1:]:
+ if not in_table:
+ in_table = True
+ table_rows = []
+ cells = [c for c in line.strip().strip("|").split("|")]
+ table_rows.append(cells)
+ continue
+ if in_table:
+ flush_table()
+ in_table = False
+ if line.startswith("### "):
+ out.append(f"{inline(line[4:])}
") + elif line.startswith("## "): + out.append(f"{inline(line[3:])}
") + elif line.startswith("# "): + out.append(f"{inline(line[2:])}
") + elif line.startswith("> "): + out.append(f"{inline(line[2:])}") + elif line.startswith("- "): + out.append(f"
") + elif line.strip() == "": + out.append("") + else: + out.append(f"
{inline(line)}
") + if in_table: + flush_table() + return "\n".join(out) + + +# ---------- HTML output ---------- + +CSS = """ +body { font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', sans-serif; + background: #fafbfc; color: #1a1a1a; max-width: 1280px; margin: 0 auto; + padding: 24px 16px 60px; line-height: 1.55; } +h1 { border-bottom: 2px solid #1a1a1a; padding-bottom: 8px; } +h2 { margin-top: 40px; border-bottom: 1px solid #d1d5db; padding-bottom: 6px; } +.card { background: white; border: 1px solid #e5e7eb; border-radius: 8px; + box-shadow: 0 1px 3px rgba(0,0,0,0.05); padding: 16px 20px; margin: 18px 0; } +.hero { display: grid; grid-template-columns: repeat(auto-fit, minmax(220px, 1fr)); + gap: 12px; margin: 16px 0 24px; } +.hero .item { background: white; border: 1px solid #e5e7eb; border-left: 3px solid #3b82f6; + border-radius: 6px; padding: 12px 14px; } +.hero .item.cost { border-left-color: #10b981; } +.hero .label { font-size: 0.78em; color: #6b7280; text-transform: uppercase; letter-spacing: 0.5px; } +.hero .value { font-size: 1.15em; font-weight: 600; margin-top: 4px; } +.hero .sub { font-size: 0.85em; color: #4b5563; margin-top: 2px; } +table.summary { width: 100%; border-collapse: collapse; font-size: 0.85em; margin-top: 12px; } +table.summary th, table.summary td { border: 1px solid #e5e7eb; padding: 6px 8px; text-align: left; } +table.summary th { background: #f3f4f6; cursor: pointer; user-select: none; } +table.summary tr:nth-child(even) { background: #fafafa; } +table.summary .best { background: #dcfce7 !important; } +section { margin-top: 28px; } +section img { max-width: 100%; height: auto; display: block; margin: 8px 0; + border: 1px solid #e5e7eb; border-radius: 6px; background: white; } +.legend { font-size: 0.85em; color: #6b7280; } +.legend .sw { display: inline-block; width: 11px; height: 11px; border-radius: 50%; + vertical-align: middle; margin-right: 4px; } +.desc { color: #4b5563; } +table.md { width: 100%; border-collapse: collapse; font-size: 0.85em; margin: 12px 0; } +table.md th, table.md td { border: 1px solid #e5e7eb; padding: 4px 8px; text-align: left; } +table.md th { background: #f3f4f6; } +table.md tr:nth-child(even) { background: #fafafa; } +code { background: #f3f4f6; padding: 1px 4px; border-radius: 3px; font-size: 0.92em; } +hr { border: none; border-top: 1px solid #d1d5db; margin: 20px 0; } +blockquote { border-left: 3px solid #d1d5db; padding-left: 12px; color: #4b5563; margin: 8px 0; } +""" + +SORT_JS = """ +document.querySelectorAll('table.summary th').forEach((th, idx) => { + th.addEventListener('click', () => { + const tbl = th.closest('table'); + const tbody = tbl.querySelector('tbody'); + const rows = Array.from(tbody.querySelectorAll('tr')); + const asc = th.dataset.dir !== 'asc'; + rows.sort((a, b) => { + const av = a.cells[idx].dataset.sort ?? a.cells[idx].innerText; + const bv = b.cells[idx].dataset.sort ?? b.cells[idx].innerText; + const af = parseFloat(av), bf = parseFloat(bv); + const an = isNaN(af) ? av : af, bn = isNaN(bf) ? bv : bf; + return (an < bn ? -1 : an > bn ? 1 : 0) * (asc ? 1 : -1); + }); + rows.forEach(r => tbody.appendChild(r)); + tbl.querySelectorAll('th').forEach(t => delete t.dataset.dir); + th.dataset.dir = asc ? 'asc' : 'desc'; + }); +}); +""" + + +def build_html(cells: list[Cell], plots: list[tuple[Axis, Path]]) -> str: + ts = datetime.now().strftime("%Y-%m-%d %H:%M") + # headline cards + by_bench = {"gaia": [c for c in cells if c.bench == "gaia"], + "swe": [c for c in cells if c.bench == "swe"]} + def best_acc(lst): + return max(lst, key=lambda c: c.accuracy) if lst else None + def best_acc_per_dollar(lst): + cand = [c for c in lst if c.cost_usd > 0] + return max(cand, key=lambda c: c.accuracy / c.cost_usd) if cand else None + + hero = ['']
+ for bench_key, label in (("gaia", "GAIA"), ("swe", "SWE-bench")):
+ b = best_acc(by_bench[bench_key])
+ if b:
+ hero.append(f"""""")
+ for bench_key, label in (("gaia", "GAIA"), ("swe", "SWE-bench")):
+ b = best_acc_per_dollar(by_bench[bench_key])
+ if b:
+ ratio = b.accuracy / b.cost_usd
+ hero.append(f"""""")
+ hero.append("
")
+
+ # summary table
+ legend_html = 'Best acc — {label}
+ {b.accuracy:.3f} · {b.paradigm} × {CLOUD_TOKENS.get(b.cloud, b.cloud)}
+ cost: ${b.cost_usd:.2f}
Best acc / $ — {label}
+ {b.accuracy:.3f} for ${b.cost_usd:.2f}
+ {b.paradigm} × {CLOUD_TOKENS.get(b.cloud, b.cloud)} ({ratio:.3f} acc/$)
'
+ for p in PARADIGM_ORDER:
+ col = PARADIGM_COLOR.get(p, "#000")
+ legend_html += f'{p} '
+ legend_html += '
'
+
+ rows_html = []
+ sorted_cells = sorted(cells, key=lambda c: (c.bench, -c.accuracy))
+ # mark best per bench
+ best_per_bench = {b: best_acc(by_bench[b]) for b in by_bench}
+ for c in sorted_cells:
+ is_best = best_per_bench.get(c.bench) is c
+ cls = "best" if is_best else ""
+ rows_html.append(f"""| paradigm | local | cloud | bench | +accuracy ↕ | cost ↕ | latency_med ↕ | tool_calls ↕ | +web_searches ↕ | +tokens_local ↕ | tokens_cloud ↕ | +
|---|
{html_lib.escape(axis.title)}
+{html_lib.escape(axis.description)}
+OpenJarvis Hybrid n=100 Ablation
+Comparing local-cloud paradigms across 6 cloud models on GAIA + SWE-bench-Verified · {len(cells)} cells · generated {ts}
+ +
+
+
+Headline findings
+{''.join(hero)} +
+
+
+{''.join(axis_sections)}
+
+All cells (sortable)
+{legend_html} +{summary_table} +
+
+
+
+
+"""
+
+
+def main():
+ cells = load_cells()
+ print(f"loaded {len(cells)} cells")
+ plots = []
+ for axis in axis_definitions():
+ png = render_axis_png(axis, cells)
+ if png is not None:
+ plots.append((axis, png))
+ print(f" rendered {png.name}")
+ html = build_html(cells, plots)
+ OUT_HTML.write_text(html)
+ print(f"wrote {OUT_HTML} ({len(html):,} bytes)")
+
+
+if __name__ == "__main__":
+ main()
diff --git a/src/openjarvis/agents/hybrid/_base.py b/src/openjarvis/agents/hybrid/_base.py
index 15ffd114..8cff722a 100644
--- a/src/openjarvis/agents/hybrid/_base.py
+++ b/src/openjarvis/agents/hybrid/_base.py
@@ -64,6 +64,35 @@ ANTHROPIC_WEB_SEARCH_TOOL = {
}
+def build_web_search_tool(max_uses: int = 8) -> Dict[str, Any]:
+ """Build an Anthropic server-side web_search tool block with a custom cap.
+
+ Web_search is server-side: Anthropic runs the searches internally before
+ returning, so ``max_uses`` is the only knob the caller has to bound cost
+ per task. Defaults to 8 (matches ``ANTHROPIC_WEB_SEARCH_TOOL``).
+ """
+ return {
+ "type": "web_search_20250305",
+ "name": "web_search",
+ "max_uses": int(max_uses),
+ }
+
+
+def web_search_cfg(method_cfg: Optional[Dict[str, Any]]) -> Tuple[bool, int]:
+ """Parse ``method_cfg.web_search = { enabled, max_uses }``.
+
+ Defaults: enabled=False, max_uses=8. ``enabled`` defaults to False so
+ existing cells stay one-shot (backwards compat). Cells opting in flip
+ ``enabled=true`` in the registry. Returns ``(enabled, max_uses)``.
+ """
+ if not method_cfg:
+ return False, 8
+ ws = method_cfg.get("web_search")
+ if not isinstance(ws, dict):
+ return False, 8
+ return bool(ws.get("enabled", False)), int(ws.get("max_uses", 8))
+
+
# ---------- Thread-local trace buffer ----------
#
# Every call through ``_call_anthropic`` / ``_call_openai`` / ``_call_vllm``
@@ -352,6 +381,75 @@ class LocalCloudAgent(BaseAgent):
})
return text, p, c
+ @staticmethod
+ def _call_gemini(
+ model: str,
+ *,
+ user: str,
+ system: Optional[str] = None,
+ max_tokens: int = 4096,
+ temperature: float = 0.0,
+ timeout: float = 600.0,
+ trace_role: str = "cloud",
+ ) -> Tuple[str, int, int]:
+ """Single Gemini Developer-API call. Returns (text, p_tok, c_tok).
+
+ Tool-call parity with Anthropic is intentionally NOT implemented —
+ skillorchestra / baseline-cloud only need text generation, and the
+ google-genai tool-config plumbing diverges enough from the
+ Anthropic/OpenAI shape that wiring it would double the surface
+ area of this file. If a future paradigm needs Gemini tool use,
+ extend this helper rather than hand-rolling it in the agent.
+
+ Captures the call into the active per-task trace via the
+ ``"gemini"`` kind so the dashboard's trace renderer picks it up.
+ """
+ from google import genai
+ from google.genai import types
+
+ client = genai.Client(http_options=types.HttpOptions(timeout=int(timeout * 1000)))
+ cfg = types.GenerateContentConfig(
+ temperature=temperature,
+ max_output_tokens=max_tokens,
+ )
+ if system:
+ cfg.system_instruction = system
+ t0 = time.time()
+ resp = client.models.generate_content(
+ model=model,
+ contents=user,
+ config=cfg,
+ )
+ latency = time.time() - t0
+ # `resp.text` is the convenience accessor that concatenates every
+ # text part in the first candidate. Empty if the model emitted
+ # only non-text parts (which we don't request).
+ text = (resp.text or "") if hasattr(resp, "text") else ""
+ um = getattr(resp, "usage_metadata", None)
+ p = int(getattr(um, "prompt_token_count", 0) or 0) if um else 0
+ c = int(getattr(um, "candidates_token_count", 0) or 0) if um else 0
+ finish_reason = None
+ try:
+ finish_reason = str(resp.candidates[0].finish_reason)
+ except Exception:
+ pass
+ _record_event({
+ "kind": "gemini",
+ "role": trace_role,
+ "model": model,
+ "system": system,
+ "user": user,
+ "response": text,
+ "tokens_in": p,
+ "tokens_out": c,
+ "temperature": temperature,
+ "max_tokens": max_tokens,
+ "finish_reason": finish_reason,
+ "latency_s": latency,
+ "ts": time.time(),
+ })
+ return text, p, c
+
@staticmethod
def _call_vllm(
model: str,
@@ -426,6 +524,122 @@ class LocalCloudAgent(BaseAgent):
})
return text, p, c
+ @staticmethod
+ def _call_anthropic_agent(
+ model: str,
+ *,
+ user: str,
+ system: Optional[str] = None,
+ max_tokens: int = 4096,
+ temperature: float = 0.0,
+ tools: Optional[list] = None,
+ max_turns: int = 8,
+ timeout: float = 600.0,
+ max_retries: int = 5,
+ trace_role: str = "cloud",
+ ) -> Tuple[str, int, int, int, int]:
+ """Multi-turn Anthropic loop with optional tools.
+
+ Returns ``(final_text, prompt_tokens_sum, completion_tokens_sum,
+ n_web_searches_sum, turns)``. The loop appends the assistant
+ response (preserving server-tool blocks so Anthropic's continuation
+ is valid) and re-prompts until the model stops with ``end_turn``
+ or we hit ``max_turns``. Server-side tools (web_search) execute
+ inside a single message and don't require a tool_result echo —
+ the model just continues thinking with the search results
+ already in its context. Client-side tools aren't executed here;
+ if the model emits a client-side ``tool_use`` block we stop
+ (paradigms that want client tools should call ``_call_anthropic``
+ directly and handle their own dispatch).
+ """
+ import anthropic
+
+ client = anthropic.Anthropic(timeout=timeout, max_retries=max_retries)
+ # Build the conversation. We grow ``messages`` across turns so the
+ # model sees its own prior assistant content (text + server tool
+ # use + web_search_tool_result). For server tools Anthropic is
+ # happy as long as we feed the raw assistant content back; no
+ # synthetic user tool_result block is needed.
+ messages: List[Dict[str, Any]] = [{"role": "user", "content": user}]
+ p_total = 0
+ c_total = 0
+ n_searches_total = 0
+ last_text = ""
+ turns = 0
+ for turn in range(max(1, max_turns)):
+ turns = turn + 1
+ kwargs: Dict[str, Any] = {
+ "model": model,
+ "max_tokens": max_tokens,
+ "messages": messages,
+ }
+ if system:
+ kwargs["system"] = system
+ if supports_temperature(model):
+ kwargs["temperature"] = temperature
+ if tools:
+ kwargs["tools"] = tools
+ t0 = time.time()
+ msg = client.messages.create(**kwargs)
+ latency = time.time() - t0
+ text = "".join(b.text for b in msg.content if hasattr(b, "text"))
+ srv = getattr(msg.usage, "server_tool_use", None)
+ n_searches = getattr(srv, "web_search_requests", 0) if srv else 0
+ content_blocks = [_serialize_block(b) for b in msg.content]
+ tool_use_blocks = [
+ b for b in content_blocks
+ if b.get("type") in ("tool_use", "server_tool_use")
+ ]
+ tool_result_blocks = [
+ b for b in content_blocks
+ if b.get("type") in ("web_search_tool_result", "tool_result")
+ ]
+ stop_reason = getattr(msg, "stop_reason", None)
+ _record_event({
+ "kind": "anthropic",
+ "role": trace_role,
+ "model": model,
+ "system": system if turn == 0 else None,
+ "user": user if turn == 0 else None,
+ "turn": turn,
+ "response": text,
+ "content_blocks": content_blocks,
+ "tool_calls": tool_use_blocks,
+ "tool_results": tool_result_blocks,
+ "tokens_in": msg.usage.input_tokens,
+ "tokens_out": msg.usage.output_tokens,
+ "n_web_searches": n_searches,
+ "tools_declared": tools,
+ "stop_reason": stop_reason,
+ "latency_s": latency,
+ "ts": time.time(),
+ })
+ p_total += msg.usage.input_tokens
+ c_total += msg.usage.output_tokens
+ n_searches_total += n_searches
+ if text:
+ last_text = text
+ # If the model wants a client-side tool we don't dispatch
+ # here — break and let the caller (or future loop variant)
+ # handle it. Only ``server_tool_use`` blocks (web_search)
+ # are auto-continued by Anthropic itself.
+ client_tool_use = any(
+ b.get("type") == "tool_use" for b in content_blocks
+ )
+ if client_tool_use:
+ break
+ if stop_reason == "end_turn" or stop_reason is None:
+ break
+ # Otherwise: ``stop_reason`` like "max_tokens" or "tool_use"
+ # (server side) — Anthropic returned mid-thought. Append the
+ # assistant turn and ask it to continue.
+ messages.append({"role": "assistant", "content": msg.content})
+ messages.append({
+ "role": "user",
+ "content": "Continue.",
+ })
+ return last_text, p_total, c_total, n_searches_total, turns
+
def _call_cloud(
self,
*,
@@ -460,6 +674,23 @@ class LocalCloudAgent(BaseAgent):
temperature=temperature,
**kwargs,
)
+ if self._cloud_endpoint == "gemini":
+ # Gemini helper doesn't accept tools / response_format kwargs.
+ # Drop them rather than letting them surface as a TypeError so
+ # paradigms that opportunistically pass these for OpenAI /
+ # Anthropic still work when routed to Gemini.
+ kwargs.pop("tools", None)
+ kwargs.pop("tool_choice", None)
+ kwargs.pop("response_format", None)
+ kwargs.pop("output_config", None)
+ return self._call_gemini(
+ self._cloud_model,
+ user=user,
+ system=system,
+ max_tokens=max_tokens,
+ temperature=temperature,
+ **kwargs,
+ )
raise ValueError(f"unsupported cloud endpoint: {self._cloud_endpoint!r}")
# ------------------------------------------------------------------
@@ -609,7 +840,9 @@ __all__ = [
"LocalCloudAgent",
"NO_TEMP_PREFIXES",
"WEB_SEARCH_COST_PER_CALL",
+ "build_web_search_tool",
"estimate_cost",
"is_gpt5_family",
"supports_temperature",
+ "web_search_cfg",
]
diff --git a/src/openjarvis/agents/hybrid/advisors.py b/src/openjarvis/agents/hybrid/advisors.py
index 35797759..9ec75fd9 100644
--- a/src/openjarvis/agents/hybrid/advisors.py
+++ b/src/openjarvis/agents/hybrid/advisors.py
@@ -28,7 +28,12 @@ import urllib.request
from typing import Any, Dict, Optional, Tuple
from openjarvis.agents._stubs import AgentContext
-from openjarvis.agents.hybrid._base import LocalCloudAgent
+from openjarvis.agents.hybrid._base import (
+ WEB_SEARCH_COST_PER_CALL,
+ LocalCloudAgent,
+ build_web_search_tool,
+ web_search_cfg,
+)
from openjarvis.agents.hybrid.mini_swe_agent import (
run_swe_agent_loop,
)
@@ -112,13 +117,32 @@ class AdvisorsAgent(LocalCloudAgent):
advisor_max_tokens = int(cfg.get("advisor_max_tokens", 1024))
advisor_temperature = float(cfg.get("advisor_temperature", 0.2))
- # 1. Initial executor pass
- initial_resp, e1_in, e1_out = self._call_cloud(
- user=f"Question:\n{question}",
- system=EXECUTOR_INITIAL_SYS,
- max_tokens=executor_max_tokens,
- temperature=0.0,
- )
+ ws_enabled, ws_max_uses = web_search_cfg(cfg)
+ use_ws = ws_enabled and self._cloud_endpoint == "anthropic"
+ gaia_max_turns = int(cfg.get("gaia_max_turns", 8))
+ ws_tool = [build_web_search_tool(ws_max_uses)] if use_ws else None
+ n_searches_total = 0
+
+ # 1. Initial executor pass — advisor (Qwen) doesn't get tools;
+ # only the cloud executor passes do.
+ if use_ws:
+ initial_resp, e1_in, e1_out, n_s1, _ = self._call_anthropic_agent(
+ self._cloud_model,
+ user=f"Question:\n{question}",
+ system=EXECUTOR_INITIAL_SYS,
+ max_tokens=executor_max_tokens,
+ temperature=0.0,
+ tools=ws_tool,
+ max_turns=gaia_max_turns,
+ )
+ n_searches_total += n_s1
+ else:
+ initial_resp, e1_in, e1_out = self._call_cloud(
+ user=f"Question:\n{question}",
+ system=EXECUTOR_INITIAL_SYS,
+ max_tokens=executor_max_tokens,
+ temperature=0.0,
+ )
# 2. Advisor pass (local)
if not self._local_endpoint or not self._local_model:
@@ -147,25 +171,41 @@ class AdvisorsAgent(LocalCloudAgent):
f"Produce your best final answer now, respecting the question's "
f"answer-format rules."
)
- final_answer, e2_in, e2_out = self._call_cloud(
- user=final_user,
- system=EXECUTOR_FINAL_SYS,
- max_tokens=executor_max_tokens,
- temperature=0.0,
- )
+ if use_ws:
+ final_answer, e2_in, e2_out, n_s2, _ = self._call_anthropic_agent(
+ self._cloud_model,
+ user=final_user,
+ system=EXECUTOR_FINAL_SYS,
+ max_tokens=executor_max_tokens,
+ temperature=0.0,
+ tools=ws_tool,
+ max_turns=gaia_max_turns,
+ )
+ n_searches_total += n_s2
+ else:
+ final_answer, e2_in, e2_out = self._call_cloud(
+ user=final_user,
+ system=EXECUTOR_FINAL_SYS,
+ max_tokens=executor_max_tokens,
+ temperature=0.0,
+ )
tokens_local = adv_in + adv_out
tokens_cloud = e1_in + e1_out + e2_in + e2_out
cost = self.cost_usd(self._cloud_model, e1_in + e2_in, e1_out + e2_out)
+ cost += n_searches_total * WEB_SEARCH_COST_PER_CALL
meta: Dict[str, Any] = {
"tokens_local": tokens_local,
"tokens_cloud": tokens_cloud,
"cost_usd": cost,
"turns": 3,
+ "web_search_uses": n_searches_total,
"traces": {
"initial_response": initial_resp,
"advisor_feedback": advisor_text,
+ "web_search_enabled": use_ws,
+ "n_web_searches": n_searches_total,
"note": "inference-only advisor (untrained); lower bound on the technique.",
},
}
diff --git a/src/openjarvis/agents/hybrid/archon.py b/src/openjarvis/agents/hybrid/archon.py
index 9c0482cf..8ab35f75 100644
--- a/src/openjarvis/agents/hybrid/archon.py
+++ b/src/openjarvis/agents/hybrid/archon.py
@@ -42,7 +42,13 @@ import types
from typing import Any, Dict, List, Optional, Tuple
from openjarvis.agents._stubs import AgentContext
-from openjarvis.agents.hybrid._base import LocalCloudAgent, _record_event
+from openjarvis.agents.hybrid._base import (
+ WEB_SEARCH_COST_PER_CALL,
+ LocalCloudAgent,
+ _record_event,
+ build_web_search_tool,
+ web_search_cfg,
+)
from openjarvis.agents.hybrid._prices import (
NO_TEMP_PREFIXES,
)
@@ -125,8 +131,11 @@ def _tally() -> Dict[str, int]:
counts = {
"cloud_prompt": 0, "cloud_completion": 0,
"local_prompt": 0, "local_completion": 0,
+ "n_web_searches": 0,
}
_TALLY_LOCAL.counts = counts
+ # Backfill for older threads that started before we added the key.
+ counts.setdefault("n_web_searches", 0)
return counts
@@ -134,9 +143,24 @@ def _reset_tally() -> None:
_TALLY_LOCAL.counts = {
"cloud_prompt": 0, "cloud_completion": 0,
"local_prompt": 0, "local_completion": 0,
+ "n_web_searches": 0,
}
+# Per-thread web_search tool: when set, the Anthropic generator declares
+# it on every call. Set inside ``_run_paradigm`` before invoking Archon
+# so the ranker/fuser passes pick it up; cleared in ``finally``.
+_WS_LOCAL = threading.local()
+
+
+def _set_anthropic_web_search(tool: Optional[Dict[str, Any]]) -> None:
+ _WS_LOCAL.tool = tool
+
+
+def _get_anthropic_web_search() -> Optional[Dict[str, Any]]:
+ return getattr(_WS_LOCAL, "tool", None)
+
+
def _make_local_generator(local_endpoint: str, local_model: str):
"""Archon custom-generator signature: (model, messages, max_tokens, temperature) -> str."""
from openai import OpenAI
@@ -237,6 +261,9 @@ def _wrap_archon_cloud_generators() -> None:
)
if not model.startswith(NO_TEMP_PREFIXES):
kwargs["temperature"] = temperature
+ ws_tool = _get_anthropic_web_search()
+ if ws_tool is not None:
+ kwargs["tools"] = [ws_tool]
t0 = _time.time()
resp = client.messages.create(**kwargs)
text = "".join(b.text for b in resp.content if hasattr(b, "text"))
@@ -244,6 +271,9 @@ def _wrap_archon_cloud_generators() -> None:
if u:
_tally()["cloud_prompt"] += getattr(u, "input_tokens", 0) or 0
_tally()["cloud_completion"] += getattr(u, "output_tokens", 0) or 0
+ srv = getattr(u, "server_tool_use", None) if u else None
+ n_searches = getattr(srv, "web_search_requests", 0) if srv else 0
+ _tally()["n_web_searches"] += int(n_searches)
_record_event({
"kind": "archon_cloud_anthropic",
"model": model,
@@ -252,6 +282,8 @@ def _wrap_archon_cloud_generators() -> None:
"response": text.strip(),
"tokens_in": getattr(u, "input_tokens", 0) if u else 0,
"tokens_out": getattr(u, "output_tokens", 0) if u else 0,
+ "n_web_searches": int(n_searches),
+ "tools_declared": kwargs.get("tools"),
"latency_s": _time.time() - t0,
"ts": _time.time(),
})
@@ -430,6 +462,14 @@ class ArchonAgent(LocalCloudAgent):
archon_cfg = {"name": f"hybrid-archon-{arch}", "layers": layers}
_reset_tally()
+ # Web_search opt-in: when enabled, declare the native server-side
+ # tool on Anthropic ranker/fuser passes (via thread-local). The
+ # local proposers run on vLLM and don't see it.
+ ws_enabled, ws_max_uses = web_search_cfg(cfg)
+ if ws_enabled:
+ _set_anthropic_web_search(build_web_search_tool(ws_max_uses))
+ else:
+ _set_anthropic_web_search(None)
archon = Archon(archon_cfg)
try:
@@ -439,6 +479,8 @@ class ArchonAgent(LocalCloudAgent):
])
except Exception as e:
answer = f"[archon error: {e!r}]"
+ finally:
+ _set_anthropic_web_search(None)
if isinstance(answer, list):
answer = answer[-1] if answer else ""
@@ -446,16 +488,19 @@ class ArchonAgent(LocalCloudAgent):
cp = _tally()["cloud_prompt"]
cc = _tally()["cloud_completion"]
+ n_searches = _tally().get("n_web_searches", 0)
cost = _cost_cloud(ranker_model, cp, cc)
if fuser_model != ranker_model:
# Conservative: charge both at the more expensive of the two.
cost = max(cost, _cost_cloud(fuser_model, cp, cc))
+ cost += n_searches * WEB_SEARCH_COST_PER_CALL
meta = {
"tokens_local": _tally()["local_prompt"] + _tally()["local_completion"],
"tokens_cloud": cp + cc,
"cost_usd": cost,
"turns": (K + 2) if arch == "ensemble_rank_fuse" else 1,
+ "web_search_uses": n_searches,
"traces": {
"architecture": arch,
"n_samples": K,
@@ -463,6 +508,8 @@ class ArchonAgent(LocalCloudAgent):
"fuser_model": fuser_model,
"local_model": self._local_model,
"tokens_breakdown": dict(_tally()),
+ "web_search_enabled": ws_enabled,
+ "n_web_searches": n_searches,
},
}
return answer, meta
diff --git a/src/openjarvis/agents/hybrid/baseline_cloud.py b/src/openjarvis/agents/hybrid/baseline_cloud.py
new file mode 100644
index 00000000..71fca7f4
--- /dev/null
+++ b/src/openjarvis/agents/hybrid/baseline_cloud.py
@@ -0,0 +1,167 @@
+"""BaselineCloudAgent — cloud-only reference for the hybrid ablation.
+
+Used as the "what does the cloud do alone?" row in the n=100 ablation
+matrix (see ``.openjarvis/experiments/hybrid/docs/results-table.md``).
+No local model is involved — ``local_*`` settings are ignored.
+
+On GAIA the agent makes one cloud call with the formatted prompt (which
+already carries the ``FINAL ANSWER:`` format reminder from
+``_prompts.format_gaia``) and returns the text. On SWE-bench-Verified
+the agent delegates to :func:`run_swe_agent_loop` with ``backbone="cloud"``
+so the model gets to run bash and read the repo — same wiring as the
+``mini-swe-agent-swebenchverified-opus-*`` cells. As of 2026-05-15
+``_loop_cloud`` dispatches to per-endpoint loops for Anthropic, OpenAI,
+and Gemini so all three cloud backbones get the proper bash-agent loop
+on SWE (previously OpenAI / Gemini SWE cells silently fell back to a
+one-shot blind patch — fixed).
+
+Construction args mirror :class:`LocalCloudAgent`. The ``cloud`` block
+in the cell registry determines the cloud model + endpoint; ``local``
+is accepted for schema compatibility but unused.
+"""
+
+from __future__ import annotations
+
+from typing import Any, Dict, Optional, Tuple
+
+from openjarvis.agents._stubs import AgentContext
+from openjarvis.agents.hybrid._base import (
+ WEB_SEARCH_COST_PER_CALL,
+ LocalCloudAgent,
+ build_web_search_tool,
+ web_search_cfg,
+)
+from openjarvis.agents.hybrid._prices import cost as estimate_cost
+from openjarvis.agents.hybrid.mini_swe_agent import run_swe_agent_loop
+from openjarvis.core.registry import AgentRegistry
+
+
+@AgentRegistry.register("baseline_cloud")
+class BaselineCloudAgent(LocalCloudAgent):
+ """Cloud-only baseline used as a reference in the n=100 ablation.
+
+ Configurable knobs via ``cfg``:
+
+ - ``cloud_max_tokens`` (int, default 4096): max_tokens per GAIA call
+ and per turn of the SWE agent loop.
+ - ``swe_max_turns`` (int, default 30): SWE-bench loop turn cap.
+ - ``swe_bash_timeout_s`` (int, default 120): SWE-bench bash timeout.
+ """
+
+ agent_id = "baseline_cloud"
+
+ def _run_paradigm(
+ self,
+ input: str,
+ context: Optional[AgentContext],
+ **kwargs: Any,
+ ) -> Tuple[str, Dict[str, Any]]:
+ cfg = self._cfg
+ task: Dict[str, Any] = {}
+ if context is not None:
+ task = context.metadata.get("task") or {}
+
+ is_swe = bool(
+ task.get("problem_statement")
+ and task.get("repo")
+ and task.get("base_commit")
+ )
+
+ if is_swe:
+ out = run_swe_agent_loop(
+ task,
+ backbone="cloud",
+ backbone_model=self._cloud_model,
+ cloud_endpoint=self._cloud_endpoint,
+ initial_prompt=input,
+ max_turns=int(cfg.get("swe_max_turns", 30)),
+ bash_timeout=int(cfg.get("swe_bash_timeout_s", 120)),
+ output_cap=int(cfg.get("swe_output_cap", 10_000)),
+ turn_max_tokens=int(cfg.get("cloud_max_tokens", 4096)),
+ trace_prefix="baseline_cloud",
+ )
+ meta = {
+ "tokens_local": 0,
+ "tokens_cloud": out["tokens_in"] + out["tokens_out"],
+ "cost_usd": out["cost_usd"],
+ "turns": out["turns"],
+ "traces": {
+ "backbone": "cloud",
+ "max_turns_hit": out["max_turns_hit"],
+ "patch_chars": len(out["patch"]),
+ "final_summary": out["final_summary"],
+ },
+ }
+ return out["answer"], meta
+
+ # GAIA branch. If `web_search.enabled` is true AND we're on
+ # Anthropic, run the multi-turn agent loop with the native
+ # server-side web_search tool. Otherwise fall back to the
+ # legacy one-shot call (preserves behavior of every existing
+ # non-opted-in cell).
+ ws_enabled, ws_max_uses = web_search_cfg(cfg)
+ gaia_max_turns = int(cfg.get("gaia_max_turns", 8))
+ if ws_enabled and self._cloud_endpoint == "anthropic":
+ text, p_tok, c_tok, n_searches, turns = self._call_anthropic_agent(
+ self._cloud_model,
+ user=input,
+ max_tokens=int(cfg.get("cloud_max_tokens", 4096)),
+ temperature=0.0,
+ tools=[build_web_search_tool(ws_max_uses)],
+ max_turns=gaia_max_turns,
+ )
+ cost = (
+ estimate_cost(self._cloud_model, p_tok, c_tok)
+ + n_searches * WEB_SEARCH_COST_PER_CALL
+ )
+ meta = {
+ "tokens_local": 0,
+ "tokens_cloud": p_tok + c_tok,
+ "cost_usd": cost,
+ "turns": turns,
+ "web_search_uses": n_searches,
+ "traces": {
+ "mode": "anthropic_agent_loop",
+ "is_swe": is_swe,
+ "cloud_endpoint": self._cloud_endpoint,
+ "web_search_enabled": True,
+ "web_search_max_uses": ws_max_uses,
+ "n_web_searches": n_searches,
+ },
+ }
+ return text, meta
+
+ if ws_enabled and self._cloud_endpoint != "anthropic":
+ # OpenAI / Gemini don't have a parity native web_search tool
+ # wired here. Skip cleanly rather than fake one. Cells that
+ # want web_search must run on Anthropic until those backends
+ # are wired.
+ self.record_trace_event({
+ "kind": "web_search_skipped",
+ "reason": "non_anthropic_endpoint",
+ "endpoint": self._cloud_endpoint,
+ })
+
+ # One-shot direct cloud call. GAIA only — SWE goes through the
+ # mini-SWE-agent loop above (now supports anthropic/openai/gemini).
+ text, p_tok, c_tok = self._call_cloud(
+ user=input,
+ max_tokens=int(cfg.get("cloud_max_tokens", 4096)),
+ temperature=0.0,
+ )
+ meta = {
+ "tokens_local": 0,
+ "tokens_cloud": p_tok + c_tok,
+ "cost_usd": estimate_cost(self._cloud_model, p_tok, c_tok),
+ "turns": 1,
+ "web_search_uses": 0,
+ "traces": {
+ "mode": "one_shot",
+ "is_swe": is_swe,
+ "cloud_endpoint": self._cloud_endpoint,
+ },
+ }
+ return text, meta
+
+
+__all__ = ["BaselineCloudAgent"]
diff --git a/src/openjarvis/agents/hybrid/conductor.py b/src/openjarvis/agents/hybrid/conductor.py
index 1cac4993..4c1f7bc3 100644
--- a/src/openjarvis/agents/hybrid/conductor.py
+++ b/src/openjarvis/agents/hybrid/conductor.py
@@ -39,7 +39,12 @@ from pathlib import Path
from typing import Any, Dict, List, Optional, Tuple
from openjarvis.agents._stubs import AgentContext
-from openjarvis.agents.hybrid._base import LocalCloudAgent
+from openjarvis.agents.hybrid._base import (
+ WEB_SEARCH_COST_PER_CALL,
+ LocalCloudAgent,
+ build_web_search_tool,
+ web_search_cfg,
+)
from openjarvis.agents.hybrid._prices import (
PRICES,
is_gpt5_family,
@@ -340,6 +345,130 @@ def _resolve_worker_pool(
return resolved
+# Endpoints conductor's `_call_worker` actually knows how to dispatch to.
+# Web-search and gemini are NOT supported here — toolorchestra has the
+# web-search dispatcher; gemini isn't wired into _call_worker.
+_CONDUCTOR_VALID_ENDPOINTS = ("vllm", "openai", "anthropic")
+
+
+def _resolve_worker_pool(
+ cfg: Dict[str, Any],
+ local_model: Optional[str],
+ local_endpoint: Optional[str],
+ cloud_model: str,
+) -> List[Dict[str, Any]]:
+ """Return the worker pool for this run.
+
+ Strict replace, not merge: if ``cfg["worker_pool"]`` is set, the
+ default pool is ignored entirely. Falls back to ``_default_pool`` when
+ the override is absent.
+
+ Each user-supplied entry must be a dict with keys ``id``, ``name``,
+ ``endpoint``, and ``model``. ``endpoint`` must be one of
+ ``vllm`` / ``openai`` / ``anthropic`` — conductor does not wire
+ web-search or gemini workers.
+
+ Substitution: ``model = "$local"`` (or ``"