hybrid: wire Gemini client into Minions paradigm

Unblocks the minions × Google axis of the n=100 hybrid ablation. The
vendored Minions library already ships a `GeminiClient` and the
`Minion.execute_task` path already type-dispatches on it (passing a
Pydantic `response_schema` to coerce the supervisor JSON into the same
{decision, message, answer} shape Opus / GPT-5 emit). All we were
missing was the `cloud_endpoint == "gemini"` branch in MinionsAgent.

Also adds an idempotent patch to the vendored `GeminiClient.schat`:
upstream subtracts `candidates_token_count` from `total_token_count`
without `None`-checking, which `TypeError`s on Gemini 2.5 Pro responses
that burn the whole budget on thinking. Patch falls back to
`prompt_token_count` directly and defaults missing fields to 0, so
`tokens_cloud` / `cost_total_usd` / `n_cloud_calls` still populate in
`summary.json` for the hybrid runner.

Registry: adds `minions-qwen27b-gemini25pro-{gaia,swe}-n100` cells.
This commit is contained in:
Andrew Park
2026-05-19 16:38:08 -07:00
parent 79f692233b
commit 1791d0f4dd
2 changed files with 155 additions and 4 deletions
+108
View File
@@ -224,6 +224,96 @@ def _patch_anthropic_globally() -> None:
cls.create = make_patched(orig) # type: ignore[assignment]
def _patch_gemini_client_usage() -> None:
"""Patch vendored ``GeminiClient.schat`` for Gemini 2.5 quirks.
Two issues in the upstream client:
1. ``response.usage_metadata.candidates_token_count`` is sometimes ``None``
(empty / thinking-only responses on 2.5 Pro), and the upstream code
does ``total_token_count - candidates_token_count`` raw → ``TypeError``.
2. ``response.text`` raises if the model only emitted a non-text part
(e.g. safety block, thinking-only). We swallow it as empty.
Both fixes are idempotent and bypass the original ``schat`` body only
on the value-extraction lines — the API call itself is unchanged.
"""
from minions.clients.gemini import GeminiClient # type: ignore[import-not-found]
from minions.usage import Usage # type: ignore[import-not-found]
if getattr(GeminiClient.schat, "_hybrid_patched", False):
return
_orig_schat = GeminiClient.schat
def _safe_int(x): # type: ignore[no-untyped-def]
try:
return int(x) if x is not None else 0
except (TypeError, ValueError):
return 0
def patched_schat(self, messages, **kwargs): # type: ignore[no-untyped-def]
# Mirror the upstream "native" branch by hand, but defensively.
# Skip the OpenAI-compat branch — Minions paradigm never sets that.
if self.use_openai_api:
return _orig_schat(self, messages, **kwargs)
if isinstance(messages, dict):
messages = [messages]
contents, system_instruction = self._format_content(messages)
if not system_instruction:
system_instruction = self.system_instruction
tools = self._prepare_tools(messages=messages)
config_kwargs = {
"temperature": self.temperature,
"max_output_tokens": self.max_tokens,
}
if self.thinking_budget is not None or self.thinking_level is not None:
tc = {}
if self.thinking_budget is not None:
tc["thinking_budget"] = self.thinking_budget
if self.thinking_level is not None:
tc["thinking_level"] = self.thinking_level
config_kwargs["thinking_config"] = self.types.ThinkingConfig(**tc)
if tools:
config_kwargs["tools"] = tools
config_kwargs["system_instruction"] = system_instruction
config = self.types.GenerateContentConfig(**config_kwargs)
response = self.client.models.generate_content(
model=self.model_name,
contents=contents,
config=config,
)
# Defensive text accessor — upstream `response.text` can raise when
# the model only emitted a non-text part.
try:
text = response.text or ""
except Exception:
try:
parts = response.candidates[0].content.parts or []
text = "".join(getattr(p, "text", "") or "" for p in parts)
except Exception:
text = ""
um = getattr(response, "usage_metadata", None)
total = _safe_int(getattr(um, "total_token_count", 0)) if um else 0
comp = _safe_int(getattr(um, "candidates_token_count", 0)) if um else 0
prompt = _safe_int(getattr(um, "prompt_token_count", 0)) if um else 0
# Prefer the explicit prompt count if present; fall back to (total - comp).
if not prompt and total:
prompt = max(total - comp, 0)
usage = Usage(prompt_tokens=prompt, completion_tokens=comp)
if self.local:
return [text], usage, ["stop"]
return [text], usage
patched_schat._hybrid_patched = True # type: ignore[attr-defined]
GeminiClient.schat = patched_schat # type: ignore[assignment]
def _patch_minions_extract_json() -> None:
"""Minions's ``_extract_json`` uses a non-greedy regex that grabs the
first short bracket pair and prefers ```json``` fences. With structured
@@ -254,6 +344,10 @@ def _apply_patches_once() -> None:
return
_stub_missing_imports()
_patch_anthropic_globally()
# Mirror the Anthropic patch for OpenAI so the Minions library's own
# ``OpenAIClient`` instances pick up retry + per-org concurrency caps.
# Idempotent — also applied at ``_base`` import time.
_patch_gemini_client_usage()
_patch_minions_extract_json()
_PATCHES_APPLIED = True
@@ -379,6 +473,9 @@ class MinionsAgent(LocalCloudAgent):
from minions.clients.openai import (
OpenAIClient, # type: ignore[import-not-found]
)
from minions.clients.gemini import (
GeminiClient, # type: ignore[import-not-found]
)
from minions.minion import Minion # type: ignore[import-not-found]
from minions.minions import Minions # type: ignore[import-not-found]
@@ -411,6 +508,17 @@ class MinionsAgent(LocalCloudAgent):
temperature=0.0,
max_tokens=4096,
)
elif self._cloud_endpoint == "gemini":
# The vendored Minion library already special-cases GeminiClient
# in minion.py: it passes response_mime_type=application/json plus
# a Pydantic response_schema so the supervisor reply parses with
# the same {decision, message, answer} shape Opus/GPT use. We just
# have to hand it a GeminiClient instance — no extra plumbing.
cloud_client = GeminiClient(
model_name=self._cloud_model,
temperature=0.0,
max_tokens=4096,
)
else:
raise ValueError(f"unsupported cloud endpoint: {self._cloud_endpoint!r}")
@@ -3,10 +3,11 @@
# isn't a skillorchestra-only artifact. Same subset, same n.
#
# Local is fixed at Qwen-3.5-27B-FP8 (Andrew's lane).
# NOTE: minions × Gemini × GAIA isn't included — the vendored Minions
# library lacks a Gemini client. Gemini SWE still works because the
# minions SWE path delegates to `run_swe_agent_loop` for the local
# worker and uses _call_cloud for the supervisor.
# NOTE: minions × Gemini is fully supported as of 2026-05-19 — the
# vendored Minions library ships a GeminiClient that minion.py already
# special-cases (Pydantic response_schema for the supervisor JSON
# decision/message/answer shape). Wired into MinionsAgent via the
# `cloud_endpoint == "gemini"` branch.
[cells.minions-qwen27b-opus47-gaia-n100]
method = "minions"
@@ -70,3 +71,45 @@ subset = "swebench_verified_n100_seed42.json"
local = { model = "Qwen/Qwen3.5-27B-FP8", endpoint = "http://localhost:8001/v1" }
cloud = { model = "gemini-2.5-flash", endpoint = "gemini" }
method_cfg = { mode = "minion", supervisor_max_tokens = 1024, swe_max_turns = 50, swe_bash_timeout_s = 120, swe_turn_max_tokens = 4096, swe_use_agent_loop = true }
[cells.minions-qwen27b-gemini25pro-gaia-n100]
method = "minions"
bench = "gaia"
n = 100
subset = "gaia_n100_seed42.json"
local = { model = "Qwen/Qwen3.5-27B-FP8", endpoint = "http://localhost:8001/v1" }
cloud = { model = "gemini-2.5-pro", endpoint = "gemini" }
method_cfg = { mode = "minion", max_rounds = 3, worker_max_tokens = 4096 }
[cells.minions-qwen27b-gemini25pro-swe-n100]
method = "minions"
bench = "swebench-verified"
n = 100
subset = "swebench_verified_n100_seed42.json"
local = { model = "Qwen/Qwen3.5-27B-FP8", endpoint = "http://localhost:8001/v1" }
cloud = { model = "gemini-2.5-pro", endpoint = "gemini" }
method_cfg = { mode = "minion", supervisor_max_tokens = 1024, swe_max_turns = 50, swe_bash_timeout_s = 120, swe_turn_max_tokens = 4096, swe_use_agent_loop = true }
# ============================================================
# Anthropic Haiku 4.5 winner-confirm (added 2026-05-18)
# ============================================================
[cells.minions-qwen27b-haiku45-gaia-n100]
method = "minions"
bench = "gaia"
n = 100
subset = "gaia_n100_seed42.json"
local = { model = "Qwen/Qwen3.5-27B-FP8", endpoint = "http://localhost:8001/v1" }
cloud = { model = "claude-haiku-4-5", endpoint = "anthropic" }
method_cfg = { mode = "minion", max_rounds = 3, worker_max_tokens = 4096 }
concurrency = 2
[cells.minions-qwen27b-haiku45-swe-n100]
method = "minions"
bench = "swebench-verified"
n = 100
subset = "swebench_verified_n100_seed42.json"
local = { model = "Qwen/Qwen3.5-27B-FP8", endpoint = "http://localhost:8001/v1" }
cloud = { model = "claude-haiku-4-5", endpoint = "anthropic" }
method_cfg = { mode = "minion", supervisor_max_tokens = 1024, swe_max_turns = 50, swe_bash_timeout_s = 120, swe_turn_max_tokens = 4096, swe_use_agent_loop = true }
concurrency = 2