orchestrator: track the SFT training pipeline and fix ToolOrchestra tracing

The orchestrator SFT pipeline was living entirely in the working tree and
never got committed. One script in that lineage (clean_sft_data.py) has
already been lost with no way to recover it, so track the rest before the
same thing happens again:

  run_sft_fsdp.py      full-parameter FSDP trainer (launched via accelerate)
  sft_tokenize.py      conversation -> tokens + assistant-only masking
  render_sft_data.py   record -> chat-template rendering
  make_splits.py       train / holdout / overfit splits
  format_eval_sample.py, upload_to_braintrust.py
  scripts/train/       fsdp4.yaml, fsdp8.yaml, eval sbatch

toolorchestra/tracing.py was also untracked despite being imported by
unified.py and rollout.py, which left the branch broken for a fresh
checkout. Add it along with the rest of the ToolOrchestra package split.

Also here: mmlu_pro / supergpqa scorers, expert pricing for the OpenRouter
Qwen builds (estimates, flagged in-file), an expanded calculator/web_search
tool surface, and the reject-sampling clean gate that run_sft_fsdp reads
via --require-clean.

Paths in the sbatch and litellm helpers are now env-driven rather than
hardcoded to one cluster home.

.gitignore

@@ -133,3 +133,7 @@ oj-debug.*.json
 
 # Generated orchestrator SFT/eval data artifacts
 data/
+
+# Training run artifacts (model checkpoints + W&B run dirs — regenerated, large)
+checkpoints/
+wandb/

litellm/config.yaml

@@ -0,0 +1,74 @@
+# LiteLLM proxy config for OpenJarvis ToolOrchestra expert spend tracking.
+# Launch:  litellm --config litellm/config.yaml --port 4000 --host 127.0.0.1
+# Keys are read from process env (source OpenJarvis/.env first).
+# No Postgres on this box (docker socket denied) -> admin UI/virtual keys are
+# disabled; spend is written to litellm/logs/spend.jsonl by spend_logger.py.
+
+model_list:
+  # ---- OpenAI ----
+  - model_name: gpt-5.5
+    litellm_params:
+      model: openai/gpt-5.5
+      api_key: os.environ/OPENAI_API_KEY
+  - model_name: gpt-5-mini
+    litellm_params:
+      model: openai/gpt-5-mini
+      api_key: os.environ/OPENAI_API_KEY
+  - model_name: gpt-4o
+    litellm_params:
+      model: openai/gpt-4o
+      api_key: os.environ/OPENAI_API_KEY
+
+  # ---- Anthropic ----
+  - model_name: claude-opus-4-8
+    litellm_params:
+      model: anthropic/claude-opus-4-8
+      api_key: os.environ/ANTHROPIC_API_KEY
+  - model_name: claude-sonnet-4-6
+    litellm_params:
+      model: anthropic/claude-sonnet-4-6
+      api_key: os.environ/ANTHROPIC_API_KEY
+
+  # ---- Gemini ----
+  - model_name: gemini-2.5-pro
+    litellm_params:
+      model: gemini/gemini-2.5-pro
+      api_key: os.environ/GEMINI_API_KEY
+  - model_name: gemini-2.5-flash
+    litellm_params:
+      model: gemini/gemini-2.5-flash
+      api_key: os.environ/GEMINI_API_KEY
+
+  # ---- OpenRouter (Qwen coder etc.) ----
+  - model_name: qwen-coder
+    litellm_params:
+      model: openrouter/qwen/qwen-2.5-coder-32b-instruct
+      api_key: os.environ/OPENROUTER_API_KEY
+  - model_name: openrouter/*
+    litellm_params:
+      model: openrouter/*
+      api_key: os.environ/OPENROUTER_API_KEY
+
+  # ---- Local vLLM (self-hosted students / experts) ----
+  - model_name: qwen3.5-9b-local
+    litellm_params:
+      model: openai/Qwen/Qwen3.5-9B
+      api_base: http://localhost:8011/v1
+      api_key: EMPTY
+      input_cost_per_token: 0.0
+      output_cost_per_token: 0.0
+  - model_name: qwen3.6-27b-local
+    litellm_params:
+      model: openai/Qwen/Qwen3.6-27B-FP8
+      api_base: http://localhost:8002/v1
+      api_key: EMPTY
+      input_cost_per_token: 0.0
+      output_cost_per_token: 0.0
+
+litellm_settings:
+  # JSONL spend logger (Postgres-free spend tracking).
+  callbacks: spend_logger.proxy_handler_instance
+  drop_params: true
+
+general_settings:
+  master_key: os.environ/LITELLM_MASTER_KEY

litellm/spend_logger.py

@@ -0,0 +1,87 @@
+"""Custom LiteLLM callback that appends per-request cost/usage to a JSONL file.
+
+This is the spend-tracking fallback used when no Postgres is available for the
+LiteLLM admin UI. Every successful (and failed) proxy request writes one line to
+$OJ_LITELLM_SPEND_LOG. Aggregate with `litellm/spend_report.py`.
+"""
+
+from __future__ import annotations
+
+import datetime
+import json
+import os
+import threading
+
+from litellm.integrations.custom_logger import CustomLogger
+
+LOG_PATH = os.environ.get(
+    "OJ_LITELLM_SPEND_LOG",
+    os.path.join(os.path.dirname(os.path.abspath(__file__)), "logs", "spend.jsonl"),
+)
+os.makedirs(os.path.dirname(LOG_PATH), exist_ok=True)
+
+_lock = threading.Lock()
+
+
+def _write(kwargs, response_obj, start_time, end_time, status: str) -> None:
+    try:
+        cost = kwargs.get("response_cost")
+        litellm_params = kwargs.get("litellm_params") or {}
+        metadata = litellm_params.get("metadata") or {}
+        usage = None
+        try:
+            usage = getattr(response_obj, "usage", None) or (
+                response_obj.get("usage") if isinstance(response_obj, dict) else None
+            )
+        except Exception:
+            usage = None
+
+        def _u(field):
+            if usage is None:
+                return None
+            return getattr(usage, field, None) if not isinstance(usage, dict) else usage.get(field)
+
+        dur = None
+        try:
+            if start_time and end_time:
+                dur = (end_time - start_time).total_seconds()
+        except Exception:
+            dur = None
+
+        rec = {
+            "ts": datetime.datetime.utcnow().isoformat() + "Z",
+            "status": status,
+            "model": kwargs.get("model"),
+            "model_group": metadata.get("model_group"),
+            "call_type": kwargs.get("call_type"),
+            "cost_usd": cost,
+            "prompt_tokens": _u("prompt_tokens"),
+            "completion_tokens": _u("completion_tokens"),
+            "total_tokens": _u("total_tokens"),
+            "duration_s": dur,
+            "key_alias": metadata.get("user_api_key_alias"),
+            "exception": str(kwargs.get("exception")) if status == "failure" else None,
+        }
+        line = json.dumps(rec)
+        with _lock:
+            with open(LOG_PATH, "a") as f:
+                f.write(line + "\n")
+    except Exception as exc:  # never let logging break a request
+        print(f"[spend_logger] error: {exc}")
+
+
+class SpendLogger(CustomLogger):
+    def log_success_event(self, kwargs, response_obj, start_time, end_time):
+        _write(kwargs, response_obj, start_time, end_time, "success")
+
+    async def async_log_success_event(self, kwargs, response_obj, start_time, end_time):
+        _write(kwargs, response_obj, start_time, end_time, "success")
+
+    def log_failure_event(self, kwargs, response_obj, start_time, end_time):
+        _write(kwargs, response_obj, start_time, end_time, "failure")
+
+    async def async_log_failure_event(self, kwargs, response_obj, start_time, end_time):
+        _write(kwargs, response_obj, start_time, end_time, "failure")
+
+
+proxy_handler_instance = SpendLogger()

litellm/spend_report.py

@@ -0,0 +1,43 @@
+#!/usr/bin/env python3
+"""Aggregate litellm/logs/spend.jsonl into a per-model spend summary.
+
+Postgres-free stand-in for the LiteLLM admin UI. Usage:
+    .venv/bin/python litellm/spend_report.py [path-to-spend.jsonl]
+"""
+import json
+import os
+import sys
+from collections import defaultdict
+
+path = sys.argv[1] if len(sys.argv) > 1 else os.environ.get(
+    "OJ_LITELLM_SPEND_LOG",
+    os.path.join(os.path.dirname(os.path.abspath(__file__)), "logs", "spend.jsonl"),
+)
+
+agg = defaultdict(lambda: {"calls": 0, "ok": 0, "fail": 0, "cost": 0.0,
+                           "ptok": 0, "ctok": 0})
+total_cost = 0.0
+with open(path) as f:
+    for line in f:
+        line = line.strip()
+        if not line:
+            continue
+        r = json.loads(line)
+        g = r.get("model_group") or r.get("model") or "?"
+        a = agg[g]
+        a["calls"] += 1
+        a["ok"] += 1 if r.get("status") == "success" else 0
+        a["fail"] += 1 if r.get("status") == "failure" else 0
+        a["cost"] += float(r.get("cost_usd") or 0)
+        a["ptok"] += int(r.get("prompt_tokens") or 0)
+        a["ctok"] += int(r.get("completion_tokens") or 0)
+        total_cost += float(r.get("cost_usd") or 0)
+
+hdr = f"{'model':<24}{'calls':>7}{'ok':>5}{'fail':>6}{'p_tok':>9}{'c_tok':>9}{'cost_usd':>12}"
+print(hdr)
+print("-" * len(hdr))
+for g, a in sorted(agg.items(), key=lambda kv: -kv[1]["cost"]):
+    print(f"{g:<24}{a['calls']:>7}{a['ok']:>5}{a['fail']:>6}"
+          f"{a['ptok']:>9}{a['ctok']:>9}{a['cost']:>12.6f}")
+print("-" * len(hdr))
+print(f"{'TOTAL':<24}{'':>7}{'':>5}{'':>6}{'':>9}{'':>9}{total_cost:>12.6f}")

pyproject.toml

@@ -28,6 +28,7 @@ classifiers = [
     "Topic :: Scientific/Engineering :: Artificial Intelligence",
 ]
 dependencies = [
+    "braintrust>=0.0.150",
     "click>=8",
     "datasets>=4.5.0",
     "ddgs>=9.11.4",

scripts/orchestrator/build_orchestrator_sft.py

@@ -42,6 +42,10 @@ from __future__ import annotations
 import argparse
 import json
 import logging
+import os
+import sys
+import time
+from pathlib import Path
 from typing import Optional
 
 from openjarvis.agents.hybrid.expert_registry import orchestrator_catalog
@@ -63,12 +67,25 @@ def main(argv: Optional[list[str]] = None) -> int:
     p = argparse.ArgumentParser(
         description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter
     )
-    p.add_argument("--out", default="data/orchestrator_sft_v1.jsonl")
+    # Default output is timestamped (e.g. data/orchestrator_sft_06-22-1230am.jsonl)
+    # so every run lands in its own file and concurrent runs never collide. Pass
+    # --out explicitly to override (sharded runs do this, one file per shard).
+    p.add_argument("--out", default=None,
+                   help="Output JSONL. Default: data/orchestrator_sft_<MM-DD-HHMMam>.jsonl")
     # The orchestrator == the local Qwen3-8B self-sampling over its own rollouts.
     p.add_argument("--orchestrator-endpoint", default="http://localhost:8001/v1",
                    help="OpenAI-compatible vLLM base URL serving the orchestrator.")
     p.add_argument("--orchestrator-model", default="qwen3-8b")
     p.add_argument("--orchestrator-api-key", default="EMPTY")
+    # Provenance stamps: when set, every written record carries these fields so
+    # the JSONL is self-identifying once pooled across model families (gemma vs
+    # qwen). Default None -> no stamping (existing qwen runs unaffected).
+    p.add_argument("--gen-model", default=None,
+                   help="Full HF id of the generating orchestrator, stamped as "
+                        "record['gen_model'] (e.g. google/gemma-4-26B-A4B-it).")
+    p.add_argument("--orchestrator-label", default=None,
+                   help="Short label stamped as record['orchestrator_model'] "
+                        "(e.g. gemma-4-26b).")
     # Local OSS model endpoints; repeatable "model_id=base_url" (e.g.
     # "Qwen/Qwen3.5-9B=http://localhost:8001/v1"). Unmapped local models are
     # still listed but served unconfigured (base_url=None).
@@ -77,13 +94,112 @@ def main(argv: Optional[list[str]] = None) -> int:
                    help="Local model id -> vLLM base URL (repeatable).")
     p.add_argument("--max-tasks", type=int, default=None,
                    help="Cap on tasks (default: all of load_sft_tasks()).")
+    p.add_argument("--skip-task-ids-from", action="append", default=[],
+                   metavar="GLOB",
+                   help="Glob of prior data.jsonl files; skip task_ids already "
+                        "generated there so this run only does unseen prompts "
+                        "(resume). Repeatable; applied before sharding.")
     p.add_argument("--samples-per-task", type=int, default=8)
     p.add_argument("--max-keep-per-task", type=int, default=1)
     p.add_argument("--max-turns", type=int, default=8)
     p.add_argument("--temperature", type=float, default=0.7)
+    # Orchestrator completion cap per turn. The library default (4096) intermittently
+    # truncates the final answer mid-sentence on longer reasoning turns; bump it so
+    # the trace ends cleanly. Raise further (e.g. 16384) if traces still cut off.
+    p.add_argument("--max-tokens", type=int, default=8192,
+                   help="Per-turn completion cap for the orchestrator (default 8192).")
+    # Number of tasks rolled out concurrently against vLLM. Each in-flight task
+    # issues its samples sequentially, so ~concurrency requests hit the server at
+    # once. ~30-50 is comfortable on 1xL40S (prefix caching + continuous batching).
+    p.add_argument("--concurrency", type=int, default=32,
+                   help="Tasks rolled out in parallel (default 32).")
+    # Balanced-smoke pull: cap//4 from each of GeneralThought + OpenThoughts
+    # code/math/science instead of the GeneralThought-only fast cap path. Use for a
+    # representative smoke; the real run omits --max-tasks for the full 8K balanced set.
+    p.add_argument("--balanced", action=argparse.BooleanOptionalAction, default=True,
+                   help="Draw an EVEN cross-domain sample (GeneralThought + "
+                        "OpenThoughts code/math/science). Default ON — pass "
+                        "--no-balanced for the old GeneralThought-only skew.")
+    # Data-parallel sharding: split the (deterministic, seed-42) task list across
+    # N independent driver processes, each pointed at its own orchestrator vLLM
+    # replica. Shard i takes tasks[i::N] (a strided slice, so every shard stays
+    # domain-balanced). Each writes its own --out file; concatenate the shard
+    # JSONLs afterward. Lets the GPU-bound orchestrator scale across idle GPUs.
+    p.add_argument("--shard-index", type=int, default=0,
+                   help="This shard's index in [0, shard-count).")
+    p.add_argument("--shard-count", type=int, default=1,
+                   help="Total number of shards (default 1 = no sharding).")
+    # Throughput knob: stop sampling a task as soon as max-keep-per-task passing
+    # trajectories are found, instead of always running all --samples-per-task.
+    # ~3-4x faster when most tasks solve early, but keeps the *first* passers
+    # rather than the *cheapest* of N (drops the cost-optimisation signal).
+    p.add_argument("--stop-at-keep", action="store_true",
+                   help="Short-circuit a task once max-keep passing samples found.")
+    # Anonymize model experts (opaque random labels, uniform description, no cost
+    # line, shuffled order) so the policy can't route on a model's name/position/
+    # cost. The anon->real map is saved per record (metrics.anon_map) for analysis.
+    p.add_argument("--anonymize-experts", action="store_true",
+                   help="Hide expert identity (random labels + shuffle) when routing.")
+    # By default we keep EVERY rolled-out trajectory (correct + incorrect), each
+    # tagged with ``correct`` (verifier verdict) and ``kept`` (the cheapest-correct
+    # sample the rejection sampler would pick). Lets you compute accuracy / inspect
+    # failures from the JSONL directly. --rejection-only restores the old
+    # drop-the-failures behaviour (only cheapest-correct written).
+    p.add_argument("--rejection-only", action="store_true",
+                   help="Drop incorrect rollouts; write only the cheapest-correct "
+                        "sample per task (the original behaviour).")
+    # Rollouts call shell_exec / file_write with model-chosen relative paths
+    # (e.g. ``solution.py``), which otherwise land in the repo root. Run them
+    # from a throwaway scratch dir so generated files never dirty the tree.
+    # gitignored via the existing ``scratch/`` rule.
+    p.add_argument("--scratch-dir", default="scratch/sft-rollouts",
+                   help="CWD for rollouts; stray tool-written files go here.")
     args = p.parse_args(argv)
 
     logging.basicConfig(level=logging.INFO, format="%(message)s")
+    # Quiet the per-request HTTP / dataset-stream spam so the run log stays
+    # readable (rollout calls and dataset shards otherwise flood it).
+    for _noisy in ("httpx", "httpcore", "urllib3", "datasets", "fsspec",
+                   "huggingface_hub", "openai"):
+        logging.getLogger(_noisy).setLevel(logging.WARNING)
+
+    # Every run lands in its OWN datetimed folder, all collected flat under
+    # data/runs/ — so runs sort by time, never collide, and never litter data/.
+    # --out is treated as an optional LABEL for the folder, not a path; the data
+    # file inside is always data.jsonl (+ .stats.json/.pretty.json/.txt/.html).
+    # Folder name: <MM-DD-HHMMam/pm>_<label> (local PT) e.g. 06-24-0349pm_gemma_20_anon
+    stamp = time.strftime("%m-%d-%I%M%p").lower()
+    label = Path(args.out).stem if args.out else "orchestrator_sft"
+    run_dir = (Path("data/runs") / f"{stamp}_{label}").resolve()
+    if run_dir.exists():  # parallel runs, same label+minute -> disambiguate
+        run_dir = run_dir.with_name(f"{run_dir.name}-{os.getpid()}")
+    out_p = run_dir / "data.jsonl"
+    lock_p = out_p.with_suffix(out_p.suffix + ".lock")
+    out_p.parent.mkdir(parents=True, exist_ok=True)
+    lock_p.write_text(f"{stamp} pid={os.getpid()}")
+    args.out = str(out_p)
+    import atexit
+    atexit.register(lambda: lock_p.exists() and lock_p.unlink())
+    logging.info("Run dir: %s", run_dir)
+
+    # Enrich Braintrust rollout traces with run-level provenance (no-op-safe;
+    # tracing.run_context() reads these). setdefault so an explicit env wins.
+    os.environ.setdefault("OJ_RUN_LABEL", label)
+    os.environ.setdefault("OJ_GEN_MODEL", args.gen_model or args.orchestrator_model)
+    os.environ.setdefault("OJ_RUN_STAGE",
+                          "smoke" if args.max_tasks else "prod")
+    os.environ["OJ_CFG_TEMPERATURE"] = str(args.temperature)
+    os.environ["OJ_CFG_MAX_TURNS"] = str(args.max_turns)
+    os.environ["OJ_CFG_ANONYMIZE"] = str(bool(args.anonymize_experts))
+    os.environ["OJ_CFG_REJECTION_ONLY"] = str(bool(args.rejection_only))
+
+    # Sandbox the rollouts: chdir into a scratch dir so any file a rollout writes
+    # (shell_exec redirects, file_write with a relative path) lands there instead
+    # of the repo root. (--out is already absolute via run_dir.resolve() above.)
+    scratch = Path(args.scratch_dir).resolve()
+    scratch.mkdir(parents=True, exist_ok=True)
+    os.chdir(scratch)
+    logging.info("Rollout scratch dir (cwd): %s", scratch)
 
     local_endpoints = {}
     for item in args.local_endpoint:
@@ -91,6 +207,11 @@ def main(argv: Optional[list[str]] = None) -> int:
         if model_id and url:
             local_endpoints[model_id] = url
     tools = orchestrator_catalog(local_endpoints=local_endpoints or None)
+    # Drop the no-op ``think`` scratchpad tool: it adds turns/cost and is a
+    # harness-leakage vector (the model narrates the routing rule inside a think
+    # call, which the reasoning-scrub can't reach). The model still reasons in
+    # its own <think> blocks — it just can't emit reasoning AS a tool call.
+    tools = [t for t in tools if t.name != "think"]
     logging.info("Tool catalog (%d): %s", len(tools), [t.name for t in tools])
 
     call_orch = make_call_orchestrator(
@@ -98,24 +219,71 @@ def main(argv: Optional[list[str]] = None) -> int:
         base_url=args.orchestrator_endpoint,
         api_key=args.orchestrator_api_key,
         temperature=args.temperature,
+        max_tokens=args.max_tokens,
     )
     dispatch = make_dispatch({})
 
+    # Build the provenance stamp once (None if neither flag given).
+    record_extra = {}
+    if args.gen_model:
+        record_extra["gen_model"] = args.gen_model
+    if args.orchestrator_label:
+        record_extra["orchestrator_model"] = args.orchestrator_label
+    record_extra = record_extra or None
+
     def rollout_fn(task):
         try:
             return run_unified_rollout(
                 task.instruction, tools,
                 call_orchestrator=call_orch, dispatch=dispatch,
                 max_turns=args.max_turns,
+                anonymize=args.anonymize_experts,
             )
         except Exception as exc:  # network/key failures shouldn't kill the run
             logging.warning("rollout failed for %s: %s", task.task_id, exc)
             return None
 
-    tasks = load_sft_tasks()
-    if args.max_tasks is not None:
-        tasks = tasks[: args.max_tasks]
-    logging.info("Loaded %d SFT tasks", len(tasks))
+    # cap= caps the task count for smoke runs; --balanced makes that small sample
+    # cross-domain (GeneralThought + OpenThoughts code/math/science) instead of the
+    # GeneralThought-only fast path. The real run omits --max-tasks (full 8K balanced).
+    tasks = load_sft_tasks(cap=args.max_tasks, balanced=args.balanced)
+    # Resume on unseen prompts: drop task_ids already generated in prior runs
+    # (per-track seen set via --skip-task-ids-from). Filter BEFORE sharding so the
+    # remaining unseen tasks stride evenly and never re-cover finished prompts.
+    if args.skip_task_ids_from:
+        import glob as _glob
+        skip_ids = set()
+        for pattern in args.skip_task_ids_from:
+            for fp in _glob.glob(pattern):
+                try:
+                    with open(fp) as fh:
+                        for line in fh:
+                            line = line.strip()
+                            if not line:
+                                continue
+                            try:
+                                tid = json.loads(line).get("task_id")
+                            except Exception:
+                                continue
+                            if tid:
+                                skip_ids.add(tid)
+                except OSError:
+                    continue
+        if skip_ids:
+            n_before = len(tasks)
+            tasks = [t for t in tasks if t.task_id not in skip_ids]
+            logging.info("Resume: %d done task_ids -> skipping; %d of %d tasks remain",
+                         len(skip_ids), len(tasks), n_before)
+    if args.shard_count > 1:
+        n_all = len(tasks)
+        tasks = tasks[args.shard_index::args.shard_count]
+        logging.info("Shard %d/%d -> %d of %d tasks",
+                     args.shard_index, args.shard_count, len(tasks), n_all)
+    from collections import Counter as _Counter
+    _dom = _Counter(getattr(t, "domain", "unknown") for t in tasks)
+    logging.info("Loaded %d SFT tasks | balanced=%s | domain split: %s",
+                 len(tasks), args.balanced,
+                 ", ".join(f"{d}={n}" for d, n in sorted(_dom.items())))
 
     stats = generate_sft_dataset(
         args.out,
@@ -126,7 +294,23 @@ def main(argv: Optional[list[str]] = None) -> int:
         samples_per_task=args.samples_per_task,
         max_keep_per_task=args.max_keep_per_task,
         reward_fn=lambda r: -r.cost_usd,  # cheapest-correct gets highest reward
+        concurrency=args.concurrency,
+        stop_at_keep=args.stop_at_keep,
+        keep_all=not args.rejection_only,
+        record_extra=record_extra,
     )
+    # Emit human-viewable companions (<out>.pretty.json + <out>.txt) so the
+    # traces / full message turns can be inspected without un-escaping the JSONL.
+    try:
+        from render_sft_data import render  # same scripts/orchestrator dir
+    except ImportError:
+        sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
+        from render_sft_data import render
+    if stats.get("records_written", 0) > 0:
+        rinfo = render(args.out)
+        logging.info("Rendered viewable traces: %s + %s",
+                     rinfo["pretty"], rinfo["txt"])
+
     print(json.dumps(stats, indent=2))
     return 0
 

scripts/orchestrator/build_unified_sft.py

@@ -1,103 +0,0 @@
-#!/usr/bin/env python
-"""Build ToolOrchestra SFT data by rejection sampling over ToolScale.
-
-For each ToolScale task, a teacher orchestrator is rolled out N times over the
-faithful unified tool catalog (one tool per model); passing trajectories are
-serialized into the conversations JSONL the SFT trainer consumes.
-
-Needs API access for the teacher + expert models (set the usual env keys).
-
-Example:
-    uv run python scripts/orchestrator/build_unified_sft.py \
-        --out data/orchestrator_unified_sft.jsonl \
-        --teacher-model gpt-5 --max-tasks 200 --samples-per-task 4 \
-        --local-model qwen3:8b --local-endpoint http://localhost:8001/v1
-"""
-
-from __future__ import annotations
-
-import argparse
-import json
-import logging
-import os
-from typing import Optional
-
-from openjarvis.agents.hybrid.expert_registry import default_catalog
-from openjarvis.agents.hybrid.toolorchestra.unified import (
-    make_call_orchestrator,
-    make_dispatch,
-)
-from openjarvis.agents.hybrid.toolorchestra.rollout import run_unified_rollout
-from openjarvis.learning.intelligence.orchestrator.sft_data.reject_sample import (
-    generate_sft_dataset,
-    gold_coverage_verify,
-)
-from openjarvis.learning.intelligence.orchestrator.sft_data.toolscale import (
-    load_toolscale,
-)
-
-
-def main(argv: Optional[list[str]] = None) -> int:
-    p = argparse.ArgumentParser(description=__doc__)
-    p.add_argument("--out", default="data/orchestrator_unified_sft.jsonl")
-    p.add_argument("--teacher-model", default="claude-sonnet-4-6")
-    p.add_argument("--teacher-base-url", default="https://api.anthropic.com/v1/",
-                   help="OpenAI-compatible base URL; pass '' for OpenAI cloud.")
-    p.add_argument("--max-tasks", type=int, default=200)
-    p.add_argument("--samples-per-task", type=int, default=4)
-    p.add_argument("--max-keep-per-task", type=int, default=1)
-    p.add_argument("--max-turns", type=int, default=50)
-    p.add_argument("--temperature", type=float, default=1.0)
-    p.add_argument("--local-model", default=None)
-    p.add_argument("--local-endpoint", default=None)
-    args = p.parse_args(argv)
-
-    logging.basicConfig(level=logging.INFO, format="%(message)s")
-
-    tools = default_catalog(
-        local_model=args.local_model, local_endpoint=args.local_endpoint,
-    )
-    logging.info("Tool catalog (%d): %s", len(tools), [t.name for t in tools])
-
-    base_url = args.teacher_base_url or None
-    api_key = (
-        os.environ.get("ANTHROPIC_API_KEY")
-        if base_url and "anthropic" in base_url
-        else os.environ.get("OPENAI_API_KEY")
-    )
-    call_orch = make_call_orchestrator(
-        args.teacher_model,
-        base_url=base_url,
-        api_key=api_key,
-        temperature=args.temperature,
-    )
-    dispatch = make_dispatch({})
-
-    def rollout_fn(task):
-        try:
-            return run_unified_rollout(
-                task.instruction, tools,
-                call_orchestrator=call_orch, dispatch=dispatch,
-                max_turns=args.max_turns,
-            )
-        except Exception as exc:  # network/key failures shouldn't kill the run
-            logging.warning("rollout failed for %s: %s", task.task_id, exc)
-            return None
-
-    tasks = load_toolscale(max_tasks=args.max_tasks)
-    stats = generate_sft_dataset(
-        args.out,
-        tasks=tasks,
-        tools=tools,
-        rollout_fn=rollout_fn,
-        verify_fn=gold_coverage_verify,
-        samples_per_task=args.samples_per_task,
-        max_keep_per_task=args.max_keep_per_task,
-        reward_fn=lambda r: -r.cost_usd,  # cheapest-correct gets highest reward
-    )
-    print(json.dumps(stats, indent=2))
-    return 0
-
-
-if __name__ == "__main__":
-    raise SystemExit(main())

scripts/orchestrator/chat_orchestrator.py

@@ -0,0 +1,109 @@
+#!/usr/bin/env python
+"""Interactive REPL to chat with a trained orchestrator checkpoint.
+
+Reuses the SAME routing loop the eval uses (OrchestratorBackend.generate_full),
+so what you see here is exactly how it behaves on the benchmarks: it reads your
+question, decides whether to answer itself / call a tool / delegate to an expert
+model, runs that rollout, and prints the final answer + a routing summary.
+
+Prereqs:
+  1. Serve the checkpoint you want on a vLLM port (see --endpoint). e.g. 8k:
+       PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True CUDA_VISIBLE_DEVICES=0 \
+       .venv/bin/python -m vllm.entrypoints.openai.api_server \
+           --model "$OJ_WORK/qwen_eval_ckpts/8k_served" \
+           --served-model-name sft-qwen-8k --port 8020 \
+           --gpu-memory-utilization 0.88 --trust-remote-code --max-model-len 32768 \
+           --enforce-eager --enable-auto-tool-choice --tool-call-parser qwen3_xml
+  2. source .env         (cloud experts GPT-5.5 / Opus need the API keys)
+
+Usage:
+  .venv/bin/python scripts/orchestrator/chat_orchestrator.py \
+      --endpoint http://localhost:8020/v1 --model sft-qwen-8k
+
+  # point local expert tiers at their own vLLMs if you have them up (optional):
+  #   --local-endpoint Qwen/Qwen3.6-27B=http://localhost:8002/v1
+"""
+import argparse
+import sys
+import time
+
+
+def _parse_args(argv=None):
+    p = argparse.ArgumentParser(description=__doc__,
+                                formatter_class=argparse.RawDescriptionHelpFormatter)
+    p.add_argument("--endpoint", default="http://localhost:8020/v1",
+                   help="vLLM endpoint serving the orchestrator checkpoint.")
+    p.add_argument("--model", default="sft-qwen-8k",
+                   help="Served-model-name of the checkpoint.")
+    p.add_argument("--api-key", default="EMPTY", help="API key for the endpoint (local vLLM = EMPTY).")
+    p.add_argument("--temperature", type=float, default=0.0)
+    p.add_argument("--max-turns", type=int, default=8, help="Same default as the eval.")
+    p.add_argument("--max-tokens", type=int, default=3000,
+                   help="Keep high: the model reasons a lot before it emits the delegation.")
+    p.add_argument("--local-endpoint", action="append", default=[],
+                   help="MODEL_ID=URL for a local expert tier. Repeatable.")
+    return p.parse_args(argv)
+
+
+def main(argv=None):
+    args = _parse_args(argv)
+
+    local_endpoints = {}
+    for pair in args.local_endpoint:
+        if "=" not in pair:
+            raise SystemExit(f"--local-endpoint expects MODEL_ID=URL, got: {pair!r}")
+        mid, url = pair.split("=", 1)
+        local_endpoints[mid.strip()] = url.strip()
+
+    from openjarvis.learning.intelligence.orchestrator.eval_backend import (
+        OrchestratorBackend,
+    )
+
+    backend = OrchestratorBackend(
+        orchestrator_endpoint=args.endpoint,
+        orchestrator_model=args.model,
+        api_key=args.api_key,
+        local_endpoints=local_endpoints,
+        max_turns=args.max_turns,
+        temperature=args.temperature,
+    )
+
+    print(f"orchestrator: {args.model} @ {args.endpoint}  "
+          f"(temp={args.temperature}, max_turns={args.max_turns})")
+    print("type a question and hit enter. Ctrl-D or 'quit' to exit.\n")
+
+    try:
+        while True:
+            try:
+                prompt = input("you> ").strip()
+            except EOFError:
+                print()
+                break
+            if not prompt:
+                continue
+            if prompt.lower() in {"quit", "exit"}:
+                break
+
+            started = time.time()
+            full = backend.generate_full(
+                prompt, model=args.model, temperature=args.temperature,
+                max_tokens=args.max_tokens,
+            )
+            elapsed = time.time() - started
+
+            if full.get("error"):
+                print(f"\n[ERROR] {full['error']}\n", file=sys.stderr)
+                continue
+
+            print(f"\norch> {full.get('content', '').strip()}\n")
+            print(f"  [turns={full.get('turn_count', '?')} "
+                  f"tool_calls={full.get('tool_calls', '?')} "
+                  f"cost=${full.get('cost_usd', 0.0):.4f} "
+                  f"{elapsed:.1f}s]\n")
+    finally:
+        backend.close()
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

scripts/orchestrator/eval_orchestrator.py

@@ -301,8 +301,29 @@ def main(argv: Optional[List[str]] = None) -> int:
     finally:
         backend.close()
 
+    # Resolve the EXACT model behind the served alias (e.g. "gemma-sft" ->
+    # the real id + checkpoint path vLLM reports), so results are self-identifying
+    # instead of an opaque label. Best-effort: never fail the summary over it.
+    served_id, served_path = args.orchestrator_model, None
+    try:
+        import urllib.request
+        req = urllib.request.Request(
+            args.orchestrator_endpoint.rstrip("/") + "/models",
+            headers={"Authorization": f"Bearer {args.orchestrator_api_key}"},
+        )
+        with urllib.request.urlopen(req, timeout=10) as resp:
+            data = json.load(resp).get("data", [])
+        if data:
+            served_id = data[0].get("id", served_id)
+            # vLLM reports the loaded checkpoint path in `root` (or `parent`).
+            served_path = data[0].get("root") or data[0].get("parent")
+    except Exception:
+        pass
+
     combined = {
         "orchestrator_model": args.orchestrator_model,
+        "orchestrator_served_id": served_id,
+        "orchestrator_model_path": served_path,
         "orchestrator_endpoint": args.orchestrator_endpoint,
         "judge_model": args.judge_model,
         "judge_engine": args.judge_engine,

scripts/orchestrator/format_eval_sample.py

@@ -0,0 +1,239 @@
+#!/usr/bin/env python
+"""Render graded orchestrator-eval JSONL records into clean, readable .txt files.
+
+Companion to ``format_sft_sample.py`` (same banner/indent visual style), but
+for the *scored* eval outputs written by ``eval_orchestrator.py``:
+
+    results/orch-eval-<model>-<tranche>-<suite>/<benchmark>_orchestrator.jsonl
+
+Each record has: record_id, benchmark, model, model_answer (the model's
+FINAL_ANSWER string), is_correct, score, latency_seconds, cost_usd, error, and
+scoring_metadata. The full step-by-step routing trace is NOT saved — only the
+final answer + gold + score — so this renderer cannot show intermediate steps.
+
+The QUESTION text is not in the result JSONL; it's loaded from the benchmark
+dataset by ``record_id`` (via ``openjarvis.evals.cli._build_dataset``). If the
+dataset can't be loaded / the id doesn't match, we render everything else and
+show "(question unavailable)".
+
+Gold answer lives in ``scoring_metadata`` (shape varies by scorer):
+  * MCQ (mmlu-pro, supergpqa): ``reference_letter``.
+  * LLM-judge (gaia): parsed from ``raw_judge_output`` (`gold target "..."`),
+    plus the judge's ``reasoning``.
+  * anything else: dumped verbatim under SCORING (raw).
+
+Usage:
+  .venv/bin/python scripts/orchestrator/format_eval_sample.py \
+      --input results/orch-eval-gemma-2k-full/gaia_orchestrator.jsonl --n 2
+
+  ... --input <file> --lines 1,5,42          # specific 1-indexed lines
+  ... --input <file> --all                    # one .txt per record
+  ... --input <file> --all --only-wrong       # only incorrect samples
+"""
+from __future__ import annotations
+
+import argparse
+import json
+import re
+from pathlib import Path
+
+WIDTH = 80
+
+# Friendly aliases -> the registry keys _build_dataset expects (mirrors
+# eval_orchestrator.BENCHMARK_ALIASES for the ones that write .jsonl here).
+BENCHMARK_ALIASES = {
+    "mmlu_pro": "mmlu-pro",
+    "mmlu-pro": "mmlu-pro",
+    "gaia": "gaia",
+    "taubench": "taubench",
+    "supergpqa": "supergpqa",
+    "terminalbench_v2_1": "terminalbench-v2.1",
+    "terminalbench-v2.1": "terminalbench-v2.1",
+}
+
+
+def _banner(label: str) -> str:
+    """Full-width headline: ``━━━  LABEL  ━━━━━━…``."""
+    prefix = f"━━━  {label}  "
+    fill = "━" * max(3, WIDTH - len(prefix))
+    return f"\n{prefix}{fill}\n"
+
+
+def _indent(text: str, pad: str = "    ") -> str:
+    text = (text or "").rstrip()
+    return "\n".join(pad + ln if ln.strip() else ln for ln in text.splitlines())
+
+
+def _normalize_benchmark(name: str) -> str:
+    key = (name or "").strip()
+    return BENCHMARK_ALIASES.get(key, BENCHMARK_ALIASES.get(key.lower(), key))
+
+
+# ---------------------------------------------------------------------------
+# Question map: record_id -> problem text, loaded from the benchmark dataset.
+# ---------------------------------------------------------------------------
+def build_question_map(benchmark: str, n: int, seed: int = 42) -> dict:
+    """Best-effort {record_id: problem}. Returns {} if the dataset can't load."""
+    try:
+        from openjarvis.evals.cli import _build_dataset
+
+        ds = _build_dataset(_normalize_benchmark(benchmark))
+        ds.load(max_samples=n, seed=seed)
+        return {r.record_id: r.problem for r in ds.iter_records()}
+    except Exception as exc:  # noqa: BLE001 - never fail the render over this
+        print(f"  [warn] could not load dataset for {benchmark!r}: "
+              f"{type(exc).__name__}: {exc}")
+        return {}
+
+
+# ---------------------------------------------------------------------------
+# Gold / judge parsing out of scoring_metadata.
+# ---------------------------------------------------------------------------
+_GOLD_TARGET_RE = re.compile(r'gold target\s*"([^"]*)"', re.IGNORECASE)
+_JUDGE_REASONING_RE = re.compile(
+    r"reasoning:\s*(.*?)(?:\n\s*correct:|\Z)", re.IGNORECASE | re.DOTALL
+)
+
+
+def parse_scoring(meta: dict) -> dict:
+    """Return {gold, judge, kind, clean}. ``clean`` False => dump raw block.
+
+    kind is one of 'mcq' | 'judge' | 'unknown'.
+    """
+    if not isinstance(meta, dict):
+        return {"gold": None, "judge": None, "kind": "unknown", "clean": False}
+
+    # MCQ scorer (mmlu-pro / supergpqa).
+    if "reference_letter" in meta:
+        return {
+            "gold": meta.get("reference_letter"),
+            "judge": None,
+            "kind": "mcq",
+            "clean": True,
+        }
+
+    # LLM-judge scorer (gaia).
+    raw = meta.get("raw_judge_output")
+    if raw:
+        gold_m = _GOLD_TARGET_RE.search(raw)
+        reason_m = _JUDGE_REASONING_RE.search(raw)
+        return {
+            "gold": gold_m.group(1) if gold_m else None,
+            "judge": (reason_m.group(1).strip() if reason_m else raw.strip()),
+            "kind": "judge",
+            "clean": True,
+        }
+
+    # Unknown shape (e.g. {"reason": "no_choice_letter_extracted"}).
+    return {"gold": None, "judge": None, "kind": "unknown", "clean": False}
+
+
+# ---------------------------------------------------------------------------
+# Render one record.
+# ---------------------------------------------------------------------------
+def format_record(rec: dict, question: str | None) -> str:
+    parsed = parse_scoring(rec.get("scoring_metadata"))
+    is_correct = rec.get("is_correct")
+    verdict = "CORRECT" if is_correct else ("INCORRECT" if is_correct is False else "UNSCORED")
+
+    def _fmt(v, fmt):
+        try:
+            return fmt.format(v)
+        except (ValueError, TypeError):
+            return str(v)
+
+    top = "  ·  ".join([
+        str(rec.get("record_id", "?")),
+        str(rec.get("benchmark", "?")),
+        verdict,
+        f"score {_fmt(rec.get('score'), '{:.3f}')}",
+        f"lat {_fmt(rec.get('latency_seconds'), '{:.1f}')}s",
+        f"cost ${_fmt(rec.get('cost_usd'), '{:.4f}')}",
+    ])
+    parts = [top]
+
+    err = rec.get("error")
+    if err:
+        parts.append(f"    error: {err}")
+
+    parts.append(_banner("QUESTION"))
+    parts.append(_indent(question if question else "(question unavailable)"))
+
+    parts.append(_banner("MODEL ANSWER"))
+    parts.append(_indent(str(rec.get("model_answer") or "(empty)")))
+
+    parts.append(_banner("GOLD"))
+    gold = parsed["gold"]
+    parts.append(_indent(str(gold) if gold not in (None, "") else "(gold unavailable — see SCORING below)"))
+
+    if parsed["kind"] == "judge" and parsed["judge"]:
+        parts.append(_banner("JUDGE"))
+        parts.append(_indent(parsed["judge"]))
+
+    if not parsed["clean"]:
+        parts.append(_banner("SCORING (raw)"))
+        parts.append(_indent(json.dumps(rec.get("scoring_metadata"), indent=2, default=str)))
+
+    return "\n".join(parts).rstrip() + "\n"
+
+
+def main(argv=None):
+    p = argparse.ArgumentParser(
+        description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter
+    )
+    p.add_argument("--input", required=True, help="Graded *_orchestrator.jsonl file.")
+    p.add_argument("--out-dir", default="results/formatted", help="Where the .txt files go.")
+    p.add_argument("--seed", type=int, default=42, help="Subset seed used at eval time.")
+    p.add_argument("--only-wrong", action="store_true",
+                   help="Render only incorrect/unscored samples.")
+    g = p.add_mutually_exclusive_group()
+    g.add_argument("--n", type=int, default=1, help="Format the first N records.")
+    g.add_argument("--lines", help="Comma-separated 1-indexed line numbers.")
+    g.add_argument("--all", action="store_true", help="Format every record.")
+    args = p.parse_args(argv)
+
+    src = Path(args.input)
+    out_dir = Path(args.out_dir)
+    out_dir.mkdir(parents=True, exist_ok=True)
+
+    records = [json.loads(l) for l in src.open() if l.strip()]
+    if not records:
+        print("no records in input")
+        return 1
+
+    # Pick indices first (so we only load the dataset once, sized to the file).
+    if args.lines:
+        idxs = [int(x) - 1 for x in args.lines.split(",") if x.strip()]
+    elif args.all:
+        idxs = list(range(len(records)))
+    else:
+        idxs = list(range(min(args.n, len(records))))
+    idxs = [i for i in idxs if 0 <= i < len(records)]
+
+    if args.only_wrong:
+        idxs = [i for i in idxs if not records[i].get("is_correct")]
+
+    benchmark = _normalize_benchmark(records[0].get("benchmark", src.stem.split("_")[0]))
+    # Load the whole subset so any selected id resolves (the eval-time subset is
+    # the first len(records) of seed=42).
+    qmap = build_question_map(benchmark, n=len(records), seed=args.seed)
+
+    written = []
+    for i in idxs:
+        rec = records[i]
+        rid = str(rec.get("record_id", i))
+        bench = rec.get("benchmark", benchmark)
+        fname = re.sub(r"[^A-Za-z0-9_.-]", "_", f"{bench}__{rid}")[:120] + ".txt"
+        out = out_dir / fname
+        out.write_text(format_record(rec, qmap.get(rid)))
+        written.append(out)
+
+    for w in written:
+        print(w)
+    print(f"\nwrote {len(written)} file(s) to {out_dir}/"
+          + ("  (dataset unavailable — questions omitted)" if not qmap else ""))
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

scripts/orchestrator/make_splits.py

@@ -0,0 +1,120 @@
+#!/usr/bin/env python
+"""Carve train / holdout / overfit100 splits from a clean orchestrator-SFT pool.
+
+Replaces make_tranches.py + make_gemma_tranches.py, which carved 1k/2k/4k/8k
+data-scaling tranches from the June pool. That lineage is dead: the tranches it
+produced trained checkpoints that evaluated at base ~= 1k ~= 2k (the pool taught
+the orchestrator to re-derive answers itself instead of routing), and the pool
+was deleted. See data/DELETED_ERA1_MANIFEST.md.
+
+Naming is uniform: ``{name}_orch_{split}_{date}.jsonl`` with name in
+{qwen, gemma, pooled}. Pass several pools to merge-and-restratify into `pooled`.
+
+The holdout is domain-stratified so val-loss is leak-free, and `overfit100` is a
+strict prefix of train (it is a memorisation sanity check — the model must be
+able to fit 100 rows it *has* seen, else the format/masking is broken).
+Deterministic (seed 42).
+
+    # one pool -> qwen_orch_{train,holdout,overfit100}_0711.jsonl
+    python scripts/orchestrator/make_splits.py --name qwen \
+        --pool data/sft_variants/qwen_orch_clean_0711.jsonl
+
+    # merge both -> pooled_orch_{train,holdout,overfit100}_0711.jsonl
+    python scripts/orchestrator/make_splits.py --name pooled \
+        --pool data/sft_variants/qwen_orch_clean_0711.jsonl \
+        --pool data/sft_variants/gemma_orch_clean_0711.jsonl
+"""
+import argparse
+import json
+import os
+import random
+import sys
+from collections import defaultdict
+from datetime import datetime
+from pathlib import Path
+
+OUT = Path("data/sft_variants")
+SEED = 42
+OVERFIT_N = 100
+# Orchestrator that generated each pool — stamped onto every row so provenance
+# survives a merge (the filename encodes it too, but the field is cheap insurance
+# and is what the `pooled` splits rely on to stay attributable).
+ORCH_MODEL = {"qwen": "qwen3.5-9b", "gemma": "gemma-4-26b"}
+
+
+def main() -> int:
+    ap = argparse.ArgumentParser(description=__doc__,
+                                 formatter_class=argparse.RawDescriptionHelpFormatter)
+    ap.add_argument("--name", required=True,
+                    help="split family: qwen | gemma | pooled (drives the filename)")
+    ap.add_argument("--pool", action="append", required=True, metavar="PATH",
+                    help="clean pool jsonl; repeat to merge (use with --name pooled)")
+    ap.add_argument("--holdout-frac", type=float, default=0.15,
+                    help="fraction held out, domain-stratified (default 0.15)")
+    ap.add_argument("--date", default=datetime.now().strftime("%m%d"),
+                    help="date tag in the filename (default: today, MMDD)")
+    ap.add_argument("--out", type=Path, default=OUT)
+    args = ap.parse_args()
+
+    rows = []
+    for p in args.pool:
+        path = Path(p)
+        if not path.exists():
+            print(f"!! pool not found: {path}", file=sys.stderr)
+            return 1
+        # Infer the source orchestrator from the filename so merged pools stay
+        # attributable; don't clobber a stamp the pool already carries.
+        stem = path.name.split("_", 1)[0]
+        loaded = [json.loads(line) for line in path.open() if line.strip()]
+        for r in loaded:
+            r.setdefault("orchestrator_model", ORCH_MODEL.get(stem, stem))
+        rows.extend(loaded)
+        print(f"loaded {len(loaded):>5} rows from {path}")
+    print(f"pool total: {len(rows)}")
+
+    # Stratified holdout: per-domain, proportional to domain size.
+    by_dom = defaultdict(list)
+    for i, r in enumerate(rows):
+        by_dom[r.get("domain", "misc")].append(i)
+    rng = random.Random(SEED)
+    holdout_idx: set = set()
+    for dom, idxs in sorted(by_dom.items()):
+        k = round(args.holdout_frac * len(idxs))
+        pick = rng.sample(idxs, min(k, len(idxs)))
+        holdout_idx.update(pick)
+        print(f"  {dom:<9} pool={len(idxs):>4} holdout={len(pick)}")
+
+    holdout = [rows[i] for i in sorted(holdout_idx)]
+    train = [r for i, r in enumerate(rows) if i not in holdout_idx]  # order preserved
+    overfit = train[:OVERFIT_N]
+    print(f"train={len(train)}  holdout={len(holdout)}  overfit={len(overfit)}")
+
+    args.out.mkdir(parents=True, exist_ok=True)
+    written = {}
+    for split, data in (("train", train), ("holdout", holdout),
+                        (f"overfit{OVERFIT_N}", overfit)):
+        f = args.out / f"{args.name}_orch_{split}_{args.date}.jsonl"
+        f.write_text("".join(json.dumps(r) + "\n" for r in data))
+        written[split] = f
+        print(f"wrote {f} ({len(data)})")
+
+    # ---- auto-upload train + holdout to Braintrust ----
+    # Gated by OJ_BRAINTRUST_AUTOUPLOAD (default ON); a total no-op / never raises
+    # if the key/pkg is missing or the upload errors (see upload_to_braintrust).
+    sys.path.insert(0, str(Path(__file__).resolve().parent))
+    try:
+        from upload_to_braintrust import autoupload
+
+        autoupload(
+            [f"{written['train']}={args.name}_train_{args.date}",
+             f"{written['holdout']}={args.name}_holdout_{args.date}"],
+            run_label=os.getenv("OJ_RUN_LABEL", f"{args.name}_splits_{args.date}"),
+            description=f"{args.name} orchestrator-SFT splits (make_splits.py)",
+        )
+    except Exception as exc:  # telemetry must never break the data pipeline
+        print(f"[braintrust] autoupload hook skipped ({exc})")
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

scripts/orchestrator/render_sft_data.py

@@ -0,0 +1,290 @@
+#!/usr/bin/env python
+"""Render an orchestrator SFT JSONL into human-viewable companions.
+
+The trainer-facing ``*.jsonl`` packs every trajectory onto one line with all
+newlines escaped, which is unreadable by eye. This writes two siblings next to
+it so you can actually inspect what the orchestrator did:
+
+* ``<name>.pretty.json`` — the same records as an indented JSON array.
+* ``<name>.txt``         — a clean plain-text transcript. The shared system
+  prompt + tool catalog (identical across every record) is printed ONCE at the
+  top; each trajectory then shows just its turns: the user question, the
+  assistant's reasoning/answer, tool calls rendered as ``-> name({args})``, and
+  tool results truncated so a single web-search dump doesn't bury the trace.
+
+Usage:
+    .venv/bin/python scripts/orchestrator/render_sft_data.py <path.jsonl>
+"""
+
+from __future__ import annotations
+
+import html as _htmllib
+import json
+import re
+import sys
+from pathlib import Path
+from typing import List
+
+# Tool observations (web_search dumps, code stdout, file reads) can be huge; cap
+# them in the transcript so the actual orchestration stays legible. The full
+# content is always in the .jsonl / .pretty.json.
+_OBS_CAP = 1200
+_RULE = "-" * 80
+_TOOL_CALL_RE = re.compile(r"<tool_call>\s*(\{.*?\})\s*</tool_call>", re.DOTALL)
+
+
+def _parse_tool_calls(content: str):
+    """Parse the model's ``<tool_call>{json}</tool_call>`` tags into a list of
+    parsed ``[(name, arguments_dict), ...]`` calls, de-duplicating the
+    double-emit the template produces (each call shows up twice)."""
+    seen = []
+    for m in _TOOL_CALL_RE.finditer(content):
+        try:
+            call = json.loads(m.group(1))
+            item = (call.get("name"), call.get("arguments", {}))
+        except Exception:
+            item = (None, {"_raw": m.group(1)})
+        key = json.dumps(item, ensure_ascii=False, sort_keys=True, default=str)
+        if not seen or seen[-1][0] != key:  # collapse the adjacent duplicate
+            seen.append((key, item))
+    return [it for _, it in seen]
+
+
+def _call_label(name, anon_map) -> str:
+    """Display a call name; if it's an anonymized expert label, append the real
+    model it maps to (from the record's ``metrics.anon_map``)."""
+    real = (anon_map or {}).get(name)
+    return f"{name}  ->  {real}" if real else f"{name}"
+
+
+def _fmt_tool_calls(content: str, anon_map=None) -> str:
+    """Plain-text tool calls with real (un-escaped) newlines in each arg value."""
+    blocks = []
+    for name, args in _parse_tool_calls(content):
+        lines = [f"-> {_call_label(name, anon_map)}"]
+        if isinstance(args, dict):
+            for k, v in args.items():
+                val = v if isinstance(v, str) else json.dumps(v, ensure_ascii=False, indent=2)
+                indented = "\n".join("       " + ln for ln in val.splitlines())
+                lines.append(f"   {k}:\n{indented}")
+        blocks.append("\n".join(lines))
+    return "\n".join(blocks)
+
+
+def _clip(text: str, cap: int) -> str:
+    text = text or ""
+    if len(text) <= cap:
+        return text
+    return f"{text[:cap]}\n  ... [truncated {len(text) - cap} chars]"
+
+
+def _render_turn(turn: dict, anon_map=None) -> str:
+    role = str(turn.get("role", "?")).lower()
+    content = turn.get("content", "") or ""
+
+    if role == "user":
+        return f"USER:\n{content.strip()}"
+
+    if role in ("tool", "function"):
+        name = turn.get("name", "tool")
+        return f"TOOL [{name}]:\n{_clip(content.strip(), _OBS_CAP)}"
+
+    if role == "assistant":
+        calls = _fmt_tool_calls(content, anon_map)
+        # Strip the raw <tool_call> tags out of the prose so it isn't shown twice.
+        prose = _TOOL_CALL_RE.sub("", content).strip()
+        parts = []
+        if prose:
+            parts.append(f"ASSISTANT:\n{prose}")
+        if calls:
+            parts.append(f"ASSISTANT calls:\n{calls}")
+        return "\n".join(parts) if parts else "ASSISTANT: (empty)"
+
+    # system handled separately (printed once); fall through for anything else
+    return f"{role.upper()}:\n{content.strip()}"
+
+
+def _transcript(record: dict, index: int) -> str:
+    m = record.get("metrics", {}) or {}
+    correct = record.get("correct")
+    kept = record.get("kept")
+    flags = []
+    if correct is not None:
+        flags.append("correct" if correct else "WRONG")
+    if kept is not None and kept:
+        flags.append("kept")
+    head = (
+        f"### [{index}] task={record.get('task_id')}  domain={record.get('domain')}"
+        + (f"  {' '.join(flags)}" if flags else "")
+        + f"\n    cost=${m.get('cost_usd')}  tokens={m.get('tokens')}  "
+        f"tool_calls={m.get('num_tool_calls')}  turns={m.get('num_turns')}  "
+        f"reward={record.get('reward')}"
+    )
+    anon_map = m.get("anon_map")
+    body = [
+        _render_turn(t, anon_map)
+        for t in record.get("conversations", [])
+        if str(t.get("role", "")).lower() != "system"
+    ]
+    return head + "\n\n" + "\n\n".join(body)
+
+
+def _system_header(records: List[dict]) -> str:
+    """The system message is identical across records — render it once."""
+    for r in records:
+        for t in r.get("conversations", []):
+            if str(t.get("role", "")).lower() == "system":
+                return (
+                    "=" * 80 + "\nSHARED SYSTEM PROMPT + TOOL CATALOG "
+                    "(identical for every record below)\n" + "=" * 80
+                    + f"\n{t.get('content', '').strip()}\n"
+                )
+    return ""
+
+
+_HTML_CSS = """
+body{font:14px/1.5 system-ui,sans-serif;margin:0;background:#f5f5f7;color:#1d1d1f}
+header{position:sticky;top:0;background:#1d1d1f;color:#fff;padding:12px 20px;z-index:9}
+header b{font-size:16px}
+.rec{background:#fff;margin:14px 20px;border-radius:10px;box-shadow:0 1px 3px #0002;overflow:hidden}
+.rec>summary{cursor:pointer;padding:10px 16px;font-weight:600;list-style:none;display:flex;gap:14px;align-items:center;flex-wrap:wrap}
+.rec>summary::-webkit-details-marker{display:none}
+.rec>summary:hover{background:#fafafa}
+.body{padding:6px 16px 16px}
+.turn{margin:10px 0;padding:10px 14px;border-radius:8px;white-space:pre-wrap;word-break:break-word}
+.user{background:#e8f0fe}
+.assistant{background:#f1f8e9}
+.call{background:#fff3e0}
+.call .cname{font-family:ui-monospace,monospace;font-weight:700;color:#b25b00;margin-bottom:6px}
+.call .cname .real{background:#b25b00;color:#fff;padding:1px 7px;border-radius:10px;font-size:12px;margin-left:4px}
+.call .arg{margin:6px 0 0 14px}
+.call .arg b{font-family:ui-monospace,monospace;color:#7a4500}
+.call .arg pre{white-space:pre-wrap;word-break:break-word;margin:3px 0 0;padding:8px 10px;background:#fff;border:1px solid #f0d9b8;border-radius:6px;font-size:13px}
+.tool details{background:#fafafa;border:1px solid #eee;border-radius:8px;padding:6px 10px;margin:10px 0}
+.tool summary{cursor:pointer;color:#555;font-weight:600}
+.tool pre{white-space:pre-wrap;word-break:break-word;margin:8px 0 0;font-size:13px}
+.role{font-size:11px;text-transform:uppercase;letter-spacing:.5px;color:#888;margin-bottom:4px}
+.pill{font-size:12px;padding:2px 8px;border-radius:10px;font-weight:600}
+.ok{background:#d7f5dd;color:#1a7f37}.wrong{background:#ffe0e0;color:#c0392b}
+.kept{background:#fff0c2;color:#9a6700}.dom{background:#eee;color:#444}
+.meta{font-weight:400;color:#888;font-size:12px}
+#sys details{margin:14px 20px;background:#fff;border-radius:10px;padding:10px 16px}
+#sys pre{white-space:pre-wrap;word-break:break-word;font-size:12px;color:#444}
+"""
+
+
+def _esc(s: str) -> str:
+    return _htmllib.escape(s or "")
+
+
+def _html_turns(record: dict) -> str:
+    anon_map = (record.get("metrics", {}) or {}).get("anon_map") or {}
+    out = []
+    for t in record.get("conversations", []):
+        role = str(t.get("role", "")).lower()
+        content = t.get("content", "") or ""
+        if role == "system":
+            continue
+        if role == "user":
+            out.append(f'<div class="turn user"><div class="role">user</div>{_esc(content.strip())}</div>')
+        elif role in ("tool", "function"):
+            name = _esc(t.get("name", "tool"))
+            out.append(
+                f'<div class="tool"><details><summary>tool result · {name} '
+                f'({len(content)} chars)</summary><pre>{_esc(content.strip())}</pre></details></div>'
+            )
+        elif role == "assistant":
+            prose = _TOOL_CALL_RE.sub("", content).strip()
+            if prose:
+                out.append(f'<div class="turn assistant"><div class="role">assistant</div>{_esc(prose)}</div>')
+            for name, args in _parse_tool_calls(content):
+                real = anon_map.get(name)
+                label = (f'{_esc(str(name))} <span class="real">&rarr; {_esc(str(real))}</span>'
+                         if real else _esc(str(name)))
+                rows = [f'<div class="cname">&rarr; {label}</div>']
+                if isinstance(args, dict):
+                    for k, v in args.items():
+                        val = v if isinstance(v, str) else json.dumps(v, ensure_ascii=False, indent=2)
+                        rows.append(f'<div class="arg"><b>{_esc(str(k))}</b><pre>{_esc(val)}</pre></div>')
+                out.append(f'<div class="turn call"><div class="role">tool call</div>{"".join(rows)}</div>')
+    return "".join(out)
+
+
+def _html_record(record: dict, index: int) -> str:
+    m = record.get("metrics", {}) or {}
+    pills = [f'<span class="pill dom">{_esc(str(record.get("domain")))}</span>']
+    correct = record.get("correct")
+    if correct is not None:
+        pills.append('<span class="pill ok">correct</span>' if correct
+                     else '<span class="pill wrong">wrong</span>')
+    if record.get("kept"):
+        pills.append('<span class="pill kept">kept</span>')
+    meta = (f'<span class="meta">cost ${m.get("cost_usd")} · {m.get("tokens")} tok · '
+            f'{m.get("num_tool_calls")} calls · {m.get("num_turns")} turns</span>')
+    summary = (f'<span>#{index}</span><span>{_esc(str(record.get("task_id")))}</span>'
+               + "".join(pills) + meta)
+    return (f'<details class="rec"><summary>{summary}</summary>'
+            f'<div class="body">{_html_turns(record)}</div></details>')
+
+
+def _html_doc(records: List[dict], title: str) -> str:
+    sys_txt = ""
+    for r in records:
+        for t in r.get("conversations", []):
+            if str(t.get("role", "")).lower() == "system":
+                sys_txt = t.get("content", "")
+                break
+        if sys_txt:
+            break
+    recs_html = "".join(_html_record(r, i + 1) for i, r in enumerate(records))
+    return (
+        f"<!doctype html><html><head><meta charset='utf-8'><title>{_esc(title)}</title>"
+        f"<style>{_HTML_CSS}</style></head><body>"
+        f"<header><b>{_esc(title)}</b> &nbsp; {len(records)} records "
+        f"&middot; click a row to expand</header>"
+        f"<div id='sys'><details><summary style='cursor:pointer;font-weight:600;padding:4px'>"
+        f"Shared system prompt + tool catalog (same for all)</summary>"
+        f"<pre>{_esc(sys_txt.strip())}</pre></details></div>"
+        f"{recs_html}</body></html>"
+    )
+
+
+def render(jsonl_path: str) -> dict:
+    src = Path(jsonl_path)
+    records: List[dict] = []
+    with src.open() as fh:
+        for line in fh:
+            line = line.strip()
+            if line:
+                records.append(json.loads(line))
+
+    pretty = src.with_suffix(".pretty.json")
+    pretty.write_text(json.dumps(records, indent=2, ensure_ascii=False))
+
+    txt = src.with_suffix(".txt")
+    banner = (
+        f"ORCHESTRATOR SFT TRANSCRIPT — {src.name}\n{len(records)} records\n\n"
+    )
+    sep = f"\n\n{_RULE}\n\n"
+    parts = [banner + _system_header(records)]
+    parts.append(sep.join(_transcript(r, i + 1) for i, r in enumerate(records)))
+    txt.write_text("\n\n".join(parts))
+
+    htmlp = src.with_suffix(".html")
+    htmlp.write_text(_html_doc(records, src.name))
+
+    return {"records": len(records), "pretty": str(pretty),
+            "txt": str(txt), "html": str(htmlp)}
+
+
+def main(argv: List[str]) -> int:
+    if not argv:
+        print(__doc__)
+        return 2
+    info = render(argv[0])
+    print(json.dumps(info, indent=2))
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main(sys.argv[1:]))

scripts/orchestrator/run_sft_fsdp.py

@@ -0,0 +1,338 @@
+#!/usr/bin/env python
+"""Full-parameter SFT for the orchestrator policy via FSDP (multi-GPU).
+
+Launch with accelerate so the model + optimizer shard across GPUs (an 8-9B full
+fine-tune won't fit one L40S otherwise):
+
+    accelerate launch --config_file <fsdp.yaml> \
+        scripts/orchestrator/run_sft_fsdp.py \
+        --data data/runs/<...>/data.jsonl [--data <more> ...] \
+        --variant correct --model Qwen/Qwen3.5-9B \
+        --out checkpoints/sft_correct --epochs 3 --batch-size 8 --grad-accum 8 \
+        --wandb-project orchestrator-sft
+
+--variant selects which trajectories to train on:
+  all            -> every record
+  correct        -> only records the verifier marked correct
+  correct_routed -> only correct records that delegated to a model expert
+
+By default (--require-clean) records explicitly marked clean=False by the clean
+gate (bloated / garbled / unrouted) are also dropped; rows missing the flag
+(legacy data) are kept. Pass --no-require-clean to skip this filter.
+
+Reuses the conversation->tokens + assistant-only masking from sft_tokenize.py.
+On these no-NVLink L40S, the accelerate/FSDP NCCL env (NCCL_P2P_DISABLE=1 etc.)
+must be set by the launcher.
+"""
+from __future__ import annotations
+
+import argparse
+import json
+import math
+import os
+import re
+import sys
+import time
+from pathlib import Path
+
+# reuse the data builder from the LoRA launcher
+sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
+from sft_tokenize import build_example  # noqa: E402
+
+MODEL_EXPERTS = {"gpt_5_5", "claude_opus_4_8", "qwen3_5_9b", "qwen3_6_27b_fp8",
+                 "qwen3_5_122b_a10b_fp8", "qwen3_5_397b_a17b_fp8"}
+
+
+def record_is_correct(r: dict) -> bool:
+    c = r.get("correct")
+    if c is None:
+        c = r.get("metrics", {}).get("correct")
+    return bool(c)
+
+
+def record_routed_to_expert(r: dict) -> bool:
+    """True if the trajectory called a model expert (resolving anon labels)."""
+    amap = r.get("metrics", {}).get("anon_map", {})
+    names = re.findall(r'"name"\s*:\s*"([a-z0-9_]+)"', json.dumps(r.get("conversations", [])))
+    for n in names:
+        real = amap.get(n, n)
+        if real in MODEL_EXPERTS:
+            return True
+    return False
+
+
+def record_clean_flag(r: dict):
+    """The record's `clean` flag: True/False, or None if absent (legacy data).
+
+    None means the clean gate never ran on this row, so we can't judge it -> we
+    keep it (legacy behavior) rather than silently dropping older datasets.
+    """
+    c = r.get("clean")
+    if c is None:
+        c = r.get("metrics", {}).get("clean")
+    return c
+
+
+def select(records, variant: str, require_clean: bool = True):
+    """Filter by --variant, then (default) drop records explicitly marked unclean.
+
+    require_clean drops rows whose `clean` flag is False. Rows missing the flag
+    (legacy) are kept regardless, matching pre-clean-gate behavior.
+    """
+    if variant == "all":
+        base = list(records)
+    elif variant == "correct":
+        base = [r for r in records if record_is_correct(r)]
+    elif variant == "correct_routed":
+        base = [r for r in records if record_is_correct(r) and record_routed_to_expert(r)]
+    else:
+        raise ValueError(f"unknown variant {variant}")
+    if not require_clean:
+        return base
+    return [r for r in base if record_clean_flag(r) is not False]
+
+
+def main() -> int:
+    p = argparse.ArgumentParser()
+    p.add_argument("--data", action="append", required=True, help="JSONL(s) (repeatable)")
+    p.add_argument("--val-data", default=None,
+                   help="Held-out JSONL for val-loss (excluded from --data). Eval'd per epoch.")
+    p.add_argument("--variant", choices=["all", "correct", "correct_routed"], default="correct")
+    p.add_argument("--require-clean", action=argparse.BooleanOptionalAction, default=True,
+                   help="Drop records whose `clean` flag is False (bloated / garbled / "
+                        "unrouted). Rows missing the flag (legacy data) are kept. "
+                        "Use --no-require-clean to disable.")
+    p.add_argument("--model", default="Qwen/Qwen3.5-9B")
+    p.add_argument("--out", required=True)
+    p.add_argument("--epochs", type=float, default=3.0)
+    p.add_argument("--batch-size", type=int, default=8)
+    p.add_argument("--grad-accum", type=int, default=8)
+    p.add_argument("--lr", type=float, default=1e-5)
+    p.add_argument("--max-seq", type=int, default=8192)
+    p.add_argument("--supervise-last-only", action="store_true",
+                   help="Legacy: supervise ONLY the final assistant turn. Default "
+                        "(off) supervises every assistant turn incl. routing.")
+    p.add_argument("--warmup-ratio", type=float, default=0.03)
+    p.add_argument("--seed", type=int, default=42)
+    p.add_argument("--wandb-project", default="orchestrator-sft")
+    p.add_argument("--wandb-name", default="")
+    args = p.parse_args()
+
+    import random
+    from datetime import timedelta
+    import torch
+    from torch.utils.data import DataLoader
+    from accelerate import Accelerator
+    from accelerate.utils import set_seed, InitProcessGroupKwargs
+    from transformers import AutoModelForCausalLM, AutoTokenizer
+
+    set_seed(args.seed)
+    # 30-min collective timeout: a slow first-step shard gather / graph build on a
+    # contended node was tripping the default 10-min NCCL watchdog (job 3892 died
+    # at step 1). This only delays a *real* hang's crash; it fixes spurious ones.
+    pg_timeout_min = int(os.environ.get("SFT_PG_TIMEOUT_MIN", "30"))
+    accelerator = Accelerator(
+        gradient_accumulation_steps=args.grad_accum,
+        kwargs_handlers=[InitProcessGroupKwargs(timeout=timedelta(minutes=pg_timeout_min))])
+    is_main = accelerator.is_main_process
+
+    def log(m):
+        if is_main:
+            print(f"[fsdp-sft] {time.strftime('%H:%M:%S')} {m}", flush=True)
+
+    tok = AutoTokenizer.from_pretrained(args.model, trust_remote_code=True)
+    if tok.pad_token_id is None:
+        tok.pad_token = tok.eos_token
+
+    # ---- data ----
+    records = []
+    for f in args.data:
+        records += [json.loads(l) for l in open(f) if l.strip()]
+    sel = select(records, args.variant, require_clean=args.require_clean)
+    log(f"variant={args.variant}: {len(sel)}/{len(records)} records")
+    if args.require_clean:
+        pre_clean = select(records, args.variant, require_clean=False)
+        n_unclean = len(pre_clean) - len(sel)
+        log(f"require_clean=True: dropped {n_unclean} unclean rows "
+            f"({len(pre_clean)}->{len(sel)}); use --no-require-clean to keep them")
+    else:
+        log("require_clean=False: NOT filtering on `clean` flag")
+    examples = []
+    for r in sel:
+        ex = build_example(tok, r.get("conversations", []), args.max_seq,
+                           supervise_all_turns=not args.supervise_last_only)
+        if ex:
+            examples.append(ex)
+    log(f"built {len(examples)} training examples")
+    if not examples:
+        log("no examples; abort")
+        return 1
+    random.Random(args.seed).shuffle(examples)
+
+    def collate(batch):
+        maxlen = max(len(b["input_ids"]) for b in batch)
+        ii, lab, am = [], [], []
+        for b in batch:
+            pad = maxlen - len(b["input_ids"])
+            ii.append(b["input_ids"] + [tok.pad_token_id] * pad)
+            lab.append(b["labels"] + [-100] * pad)
+            am.append([1] * len(b["input_ids"]) + [0] * pad)
+        return (torch.tensor(ii), torch.tensor(lab), torch.tensor(am))
+
+    dl = DataLoader(examples, batch_size=args.batch_size, shuffle=True, collate_fn=collate)
+
+    # ---- held-out val set (leak-free; excluded from --data upstream) ----
+    val_dl = None
+    if args.val_data:
+        vrecs = [json.loads(l) for l in open(args.val_data) if l.strip()]
+        vex = []
+        for r in vrecs:
+            ex = build_example(tok, r.get("conversations", []), args.max_seq,
+                           supervise_all_turns=not args.supervise_last_only)
+            if ex:
+                vex.append(ex)
+        log(f"val: {len(vex)} examples from {args.val_data}")
+        if vex:
+            val_dl = DataLoader(vex, batch_size=args.batch_size, shuffle=False, collate_fn=collate)
+
+    model = AutoModelForCausalLM.from_pretrained(
+        args.model, dtype=torch.bfloat16, trust_remote_code=True,
+        attn_implementation="sdpa")
+    if os.environ.get("SFT_NO_GRAD_CKPT") == "1":
+        log("gradient checkpointing DISABLED (SFT_NO_GRAD_CKPT=1)")
+    else:
+        model.gradient_checkpointing_enable(gradient_checkpointing_kwargs={"use_reentrant": False})
+    model.config.use_cache = False
+    opt = torch.optim.AdamW(model.parameters(), lr=args.lr, weight_decay=0.0)
+
+    model, opt, dl = accelerator.prepare(model, opt, dl)
+    if val_dl is not None:
+        val_dl = accelerator.prepare(val_dl)
+
+    @torch.no_grad()
+    def evaluate():
+        # Corpus-level mean token loss over the holdout. FSDP needs every rank to
+        # run the forward collectively, so each rank evals its shard and we reduce
+        # token-weighted sums (not a mean-of-batch-means, which would mis-weight).
+        model.eval()
+        tot_loss = torch.zeros((), device=accelerator.device)
+        tot_tok = torch.zeros((), device=accelerator.device)
+        for ii, lab, am in val_dl:
+            out = model(input_ids=ii, attention_mask=am, labels=lab)
+            ntok = (lab != -100).sum()
+            tot_loss += out.loss.detach() * ntok
+            tot_tok += ntok
+        tot_loss = accelerator.reduce(tot_loss, reduction="sum")
+        tot_tok = accelerator.reduce(tot_tok, reduction="sum")
+        model.train()
+        return (tot_loss / tot_tok.clamp(min=1)).item()
+
+    steps_per_epoch = math.ceil(len(dl) / args.grad_accum)
+    total_steps = max(1, int(steps_per_epoch * args.epochs))
+    warmup = max(1, int(total_steps * args.warmup_ratio))
+
+    def lr_at(s):
+        if s < warmup:
+            return s / warmup
+        return 0.5 * (1 + math.cos(math.pi * (s - warmup) / max(1, total_steps - warmup)))
+
+    use_wandb = False
+    if is_main:
+        try:
+            import wandb
+            base_name = args.wandb_name or f"{Path(args.model).name}-{args.variant}-fsdp"
+            run_name = f"{base_name}-{time.strftime('%m%d-%H%M', time.gmtime())}"
+            wandb.init(project=args.wandb_project,
+                       name=run_name,
+                       config=vars(args) | {"n_examples": len(examples), "total_steps": total_steps})
+            use_wandb = True
+        except Exception as e:
+            log(f"wandb off ({e})")
+
+    log(f"total_steps={total_steps} steps/epoch={steps_per_epoch} warmup={warmup}")
+
+    def _save_ckpt(out_path):
+        # Full-model checkpoint (gathers FSDP shards to rank0). Called per-epoch
+        # so a long run leaves usable partial models + lets us eval intermediate
+        # checkpoints (e.g. epoch1/epoch2) for the data-scaling sweep.
+        accelerator.wait_for_everyone()
+        unwrapped = accelerator.unwrap_model(model)
+        p = Path(out_path)
+        if is_main:
+            p.mkdir(parents=True, exist_ok=True)
+            tok.save_pretrained(str(p))
+        unwrapped.save_pretrained(
+            str(p), is_main_process=is_main,
+            save_function=accelerator.save,
+            state_dict=accelerator.get_state_dict(model))
+        if is_main:
+            log(f"checkpoint saved -> {p}")
+
+    model.train()
+    gstep = 0
+    t0 = time.time()
+    if val_dl is not None:
+        vloss = evaluate()
+        log(f"val/loss (baseline, step 0) = {vloss:.4f}")
+        if use_wandb:
+            wandb.log({"val/loss": vloss}, step=0)
+    for epoch in range(math.ceil(args.epochs)):
+        for ii, lab, am in dl:
+            with accelerator.accumulate(model):
+                out = model(input_ids=ii, attention_mask=am, labels=lab)
+                accelerator.backward(out.loss)
+                if accelerator.sync_gradients:
+                    lr = args.lr * lr_at(gstep)
+                    for g in opt.param_groups:
+                        g["lr"] = lr
+                    accelerator.clip_grad_norm_(model.parameters(), 1.0)
+                opt.step()
+                opt.zero_grad()
+            if accelerator.sync_gradients:
+                gstep += 1
+                # accelerator.gather() is a COLLECTIVE — EVERY rank must call it.
+                # Guarding it behind `is_main` made only rank0 run the [1]-float
+                # all-gather while ranks 1-3 marched on to the next step's FSDP
+                # param all-gather, desyncing the process group and hanging the job
+                # at the step1->step2 boundary (NCCL collective-shape mismatch).
+                loss = accelerator.gather(out.loss.detach()).mean().item()
+                if is_main:
+                    log(f"step {gstep}/{total_steps} loss={loss:.4f} lr={lr:.2e}")
+                    if use_wandb:
+                        wandb.log({"train/loss": loss, "train/lr": lr}, step=gstep)
+            if gstep >= total_steps:
+                break
+        _save_ckpt(Path(args.out) / f"epoch{epoch + 1}")
+        if val_dl is not None:
+            vloss = evaluate()
+            log(f"val/loss (epoch {epoch + 1}) = {vloss:.4f}")
+            if use_wandb:
+                wandb.log({"val/loss": vloss, "epoch": epoch + 1}, step=gstep)
+        if gstep >= total_steps:
+            break
+
+    accelerator.wait_for_everyone()
+    log("saving full model...")
+    unwrapped = accelerator.unwrap_model(model)
+    out_dir = Path(args.out)
+    if is_main:
+        out_dir.mkdir(parents=True, exist_ok=True)
+        tok.save_pretrained(str(out_dir))
+    unwrapped.save_pretrained(
+        str(out_dir), is_main_process=is_main,
+        save_function=accelerator.save,
+        state_dict=accelerator.get_state_dict(model))
+    if is_main:
+        log(f"saved -> {out_dir}")
+        if use_wandb:
+            try:
+                wandb.finish()
+            except Exception:
+                pass
+        print("FSDP_SFT_DONE " + json.dumps({"variant": args.variant, "steps": gstep,
+              "wall_s": round(time.time() - t0, 1), "out": str(out_dir)}))
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

scripts/orchestrator/sft_configs/fsdp_modal.yaml

@@ -0,0 +1,13 @@
+compute_environment: LOCAL_MACHINE
+distributed_type: FSDP
+mixed_precision: bf16
+num_machines: 1
+num_processes: 4
+fsdp_config:
+  fsdp_sharding_strategy: FULL_SHARD
+  fsdp_auto_wrap_policy: SIZE_BASED_WRAP
+  fsdp_min_num_params: 100000000
+  fsdp_state_dict_type: FULL_STATE_DICT
+  fsdp_use_orig_params: true
+  fsdp_cpu_ram_efficient_loading: false
+  fsdp_sync_module_states: false

scripts/orchestrator/sft_tokenize.py

@@ -0,0 +1,148 @@
+#!/usr/bin/env python
+"""Conversation->tokens + assistant-only masking shared by the SFT trainers.
+
+Extracted from the old LoRA trainer so ``run_sft_fsdp.py`` (full-parameter FSDP,
+the path we actually use) doesn't depend on it. ``build_example`` tokenizes one
+``conversations`` record and returns ``input_ids`` + ``labels`` with everything
+but the supervised assistant turns masked to -100.
+"""
+from __future__ import annotations
+
+from typing import Any, Dict, List, Optional
+
+
+def normalize_messages(conversations: List[Dict[str, Any]]) -> List[Dict[str, str]]:
+    """Map raw conversation turns to {role, content} with role in
+    system/user/assistant. tool turns fold into user (matches the native
+    OrchestratorSFTDataset fallback)."""
+    msgs: List[Dict[str, str]] = []
+    for turn in conversations:
+        role = turn.get("role") or turn.get("from", "")
+        content = turn.get("content") or turn.get("value", "") or ""
+        if role in ("human", "user"):
+            role = "user"
+        elif role in ("gpt", "assistant"):
+            role = "assistant"
+        elif role == "tool":
+            content = f"[Tool '{turn.get('name', 'tool')}' returned]: {content}"
+            role = "user"
+        if role in ("system", "user", "assistant"):
+            msgs.append({"role": role, "content": str(content)})
+    return msgs
+
+
+def _lcp_len(a: List[int], b: List[int]) -> int:
+    """Length of the longest common prefix of two token-id lists."""
+    n = 0
+    for x, y in zip(a, b):
+        if x != y:
+            break
+        n += 1
+    return n
+
+
+def _chatml_assistant_spans(ids: List[int], tok) -> Optional[List[tuple]]:
+    """``(start, end)`` token spans of each assistant turn's content in a
+    ChatML-rendered sequence, located by ``<|im_start|>assistant ... <|im_end|>``
+    markers (the ``<|im_end|>`` is included so the model learns to stop).
+
+    Operates on the FINAL token stream, so it is correct even when the chat
+    template strips prior-turn ``<think>`` reasoning from history (Qwen3.x does),
+    which breaks any incremental re-rendering / prefix approach. Returns None for
+    non-ChatML templates (e.g. gemma) so the caller can fall back."""
+    im_start = tok.convert_tokens_to_ids("<|im_start|>")
+    im_end = tok.convert_tokens_to_ids("<|im_end|>")
+    asst = tok.convert_tokens_to_ids("assistant")
+    unk = tok.unk_token_id
+    if im_start is None or im_end is None or im_start == unk or im_end == unk:
+        return None
+    nl = tok.convert_tokens_to_ids("Ċ")  # the '\n' after the role header
+    spans: List[tuple] = []
+    i, n = 0, len(ids)
+    while i < n:
+        if ids[i] == im_start and i + 1 < n and ids[i + 1] == asst:
+            j = i + 2
+            if j < n and ids[j] == nl:
+                j += 1
+            k = j
+            while k < n and ids[k] != im_end:
+                k += 1
+            end = min(k + 1, n)
+            if end > j:
+                spans.append((j, end))
+            i = end
+        else:
+            i += 1
+    return spans
+
+
+def build_example(
+    tok,
+    conversations,
+    max_seq: int,
+    supervise_all_turns: bool = True,
+) -> Optional[Dict[str, List[int]]]:
+    """Tokenize one record -> input_ids + labels.
+
+    ``supervise_all_turns=True`` (default): supervise EVERY assistant turn — the
+    intermediate routing ``<tool_call>`` turns AND the final answer; only
+    system/user/tool tokens are masked. This teaches the model to actually route,
+    not just synthesize the last answer given someone else's tool calls.
+
+    ``supervise_all_turns=False`` (legacy): supervise only the final assistant
+    turn; everything before it is masked to -100.
+
+    Assistant spans are found by the ChatML turn markers on the rendered stream
+    (robust to Qwen3's prior-turn think-stripping). For non-ChatML templates
+    (gemma) we fall back to the proven last-turn longest-common-prefix boundary;
+    all-turns supervision there needs per-template markers and is not yet wired,
+    so it degrades to last-turn. Records whose last turn isn't assistant, that
+    supervise nothing, or whose final turn alone overflows max_seq are skipped."""
+    msgs = normalize_messages(conversations)
+    if len(msgs) < 2 or msgs[-1]["role"] != "assistant":
+        return None
+    try:
+        full_text = tok.apply_chat_template(msgs, tokenize=False,
+                                            add_generation_prompt=False)
+        full = tok(full_text, add_special_tokens=False)["input_ids"]
+    except Exception:
+        return None
+    if not full:
+        return None
+
+    spans = _chatml_assistant_spans(full, tok)
+    if spans:
+        selected = spans if supervise_all_turns else spans[-1:]
+        labels = [-100] * len(full)
+        for (s, e) in selected:
+            for k in range(s, e):
+                labels[k] = full[k]
+        last_start = selected[-1][0]
+    else:
+        # Non-ChatML template: proven last-turn boundary via longest-common-prefix
+        # (robust to templates whose add_generation_prompt emits extra preamble).
+        try:
+            prompt_text = tok.apply_chat_template(msgs[:-1], tokenize=False,
+                                                  add_generation_prompt=True)
+            prompt = tok(prompt_text, add_special_tokens=False)["input_ids"]
+        except Exception:
+            return None
+        boundary = _lcp_len(prompt, full)
+        if boundary >= len(full):
+            return None
+        labels = list(full)
+        for i in range(min(boundary, len(labels))):
+            labels[i] = -100
+        last_start = boundary
+
+    if all(v == -100 for v in labels):  # nothing left to supervise
+        return None
+
+    # Keep the final assistant turn if too long: left-truncate from the front.
+    if len(full) > max_seq:
+        if len(full) - last_start >= max_seq:  # final answer alone overflows -> drop
+            return None
+        cut = len(full) - max_seq
+        full = full[cut:]
+        labels = labels[cut:]
+    return {"input_ids": full, "labels": labels}

scripts/orchestrator/upload_to_braintrust.py

@@ -0,0 +1,361 @@
+#!/usr/bin/env python
+"""Upload orchestrator-SFT JSONL datasets to Braintrust so they're browsable/
+filterable in the UI (by area / difficulty / dataset / routed-model / correct).
+
+Datasets land in the SAME Braintrust project as the rollout TRACES (the
+``research`` project) so a run's data + traces sit side by side. The target is
+resolved from ``OJ_BRAINTRUST_PROJECT_ID`` (default: the research project id);
+``--project`` overrides by NAME if you want a different project.
+
+Each record -> a Braintrust dataset row:
+  input    = the problem/question
+  expected = the FINAL_ANSWER value
+  tags     = [gen_model, domain, correct|incorrect, clean|dirty, kept|dropped]
+  metadata = domain, area, difficulty, dataset, subsector, task_id, correct,
+             clean, kept, reward, gen_model, orchestrator_model, num_tool_calls,
+             n_turns, routed_models (real names)
+  (the full conversation is kept under metadata.conversation for inspection)
+
+The dataset itself carries run-level metadata (run_label, specific gen_model,
+git sha, config knobs, counts, domain distribution) so future experiments are
+distinguishable in the UI.
+
+Usage (CLI):
+  .venv/bin/python scripts/orchestrator/upload_to_braintrust.py \
+      data/sft_variants/qwen_orch_train_0707.jsonl=qwen_train_0707 \
+      data/sft_variants/qwen_orch_holdout_0707.jsonl=qwen_holdout_0707
+
+  # name is optional; defaults to {model_short}_{split}_{date}
+  .venv/bin/python scripts/orchestrator/upload_to_braintrust.py \
+      data/sft_variants/qwen_orch_holdout_0707.jsonl
+
+Programmatic (used by the pipeline auto-upload hook in make_splits.py):
+  from upload_to_braintrust import autoupload
+  autoupload(["data/sft_variants/qwen_orch_train_0707.jsonl=qwen_train_0707"],
+             run_label="...")
+"""
+import argparse
+import json
+import logging
+import os
+import re
+import subprocess
+from collections import Counter
+from datetime import datetime
+from pathlib import Path
+
+logger = logging.getLogger(__name__)
+
+# The `research` project — datasets go here, same place the rollout traces land.
+DEFAULT_PROJECT_ID = "c707124e-ce9d-4187-ad11-f49f19f777ad"
+
+_FA = re.compile(r"(?im)FINAL[_\s]?ANSWER\s*:?")
+
+
+def _ordered_turns(convs, anon_map, orchestrator):
+    """Rebuild each turn as role/model/content, in that display order.
+
+    Braintrust sorts object keys alphabetically on write, which would push the
+    (very long) `content` to the top. We prefix the keys (`1_role`, `2_model`,
+    `3_content`) so they sort into role -> model -> content and the content
+    renders last. `model` is the model behind the turn:
+      * assistant turns  -> the orchestrator model
+      * tool turns       -> the expert model that answered (de-anonymized)
+      * system/user      -> None
+    """
+    out = []
+    for c in convs:
+        role = c.get("role")
+        if role == "tool":
+            model = anon_map.get(c.get("name")) or c.get("name")
+        elif role == "assistant":
+            model = orchestrator
+        else:
+            model = None
+        out.append({"1_role": role, "2_model": model, "3_content": c.get("content")})
+    return out
+
+
+def _dataset_desc(dataset):
+    """Fallback dataset blurb for records generated before the field existed."""
+    try:
+        from openjarvis.learning.intelligence.orchestrator.sft_data.reject_sample import (
+            dataset_description,
+        )
+        return dataset_description(dataset)
+    except Exception:
+        return ""
+
+
+def _git_sha():
+    try:
+        return subprocess.check_output(
+            ["git", "rev-parse", "--short", "HEAD"],
+            cwd=Path(__file__).resolve().parent, stderr=subprocess.DEVNULL,
+        ).decode().strip() or None
+    except Exception:
+        return None
+
+
+def _model_short(rows, path):
+    """Short family tag for the default dataset name (qwen / gemma / ...)."""
+    stem = Path(path).stem.lower()
+    for fam in ("qwen", "gemma"):
+        if stem.startswith(fam) or fam in stem:
+            return fam
+    gm = next((r.get("gen_model") for r in rows if r.get("gen_model")), "") or ""
+    gm = gm.lower()
+    for fam in ("qwen", "gemma"):
+        if fam in gm:
+            return fam
+    return "orch"
+
+
+def _split_of(path):
+    stem = Path(path).stem.lower()
+    for s in ("holdout", "train", "clean", "overfit", "partial", "8k", "4k", "2k", "1k"):
+        if s in stem:
+            return s
+    return "data"
+
+
+def _default_name(rows, path):
+    return f"{_model_short(rows, path)}_{_split_of(path)}_{datetime.now():%m%d}"
+
+
+def _config_from_env():
+    """Config knobs from the run env (set by build_orchestrator_sft.py). Omit unset."""
+    cfg = {}
+    for env_key, meta_key, cast in [
+        ("OJ_CFG_TEMPERATURE", "temperature", float),
+        ("OJ_CFG_MAX_TURNS", "max_turns", int),
+        ("OJ_CFG_ANONYMIZE", "anonymize", lambda v: v.strip().lower() in ("1", "true", "yes")),
+        ("OJ_CFG_REJECTION_ONLY", "rejection_only", lambda v: v.strip().lower() in ("1", "true", "yes")),
+    ]:
+        v = os.getenv(env_key)
+        if v not in (None, ""):
+            try:
+                cfg[meta_key] = cast(v)
+            except Exception:
+                cfg[meta_key] = v
+    return cfg
+
+
+def _row(rec):
+    convs = rec.get("conversations", [])
+    user = next((c["content"] for c in convs if c["role"] == "user"), "")
+    question = re.sub(r"^\s*Problem:\s*", "", user, count=1).strip()
+    asst = [c["content"] for c in convs if c["role"] == "assistant"]
+    fa = asst[-1] if asst else ""
+    m = list(_FA.finditer(fa))
+    model_answer = fa[m[-1].end():].strip() if m else fa.strip()
+    # Gold reference the verifier graded against (stamped at generation time as
+    # `gold_answer`). `expected` in Braintrust is the GOLD so the UI shows
+    # gold-vs-model; the model's own answer goes to metadata.model_answer.
+    gold = rec.get("gold_answer") or None
+    has_gold = gold is not None
+    am = (rec.get("metrics", {}) or {}).get("anon_map", {}) or {}
+    routed = []
+    for c in convs:
+        if c["role"] == "assistant":
+            for t in re.findall(r"<tool_call>(\{.*?\})</tool_call>", c["content"], re.DOTALL):
+                try:
+                    real = am.get(json.loads(t).get("name"))
+                    if real:
+                        routed.append(real)
+                except Exception:
+                    pass
+    metrics = rec.get("metrics", {}) or {}
+    gen_model = rec.get("gen_model")
+    domain = rec.get("domain")
+    correct = rec.get("correct")
+    clean = rec.get("clean")
+    kept = rec.get("kept")
+    tags = [t for t in [
+        gen_model,
+        f"domain:{domain}" if domain else None,
+        "correct" if correct else "incorrect",
+        "clean" if clean else "dirty",
+        "kept" if kept else "dropped",
+        "gold" if has_gold else "no_gold",
+    ] if t]
+    return {
+        "input": question,
+        "expected": gold,
+        "tags": tags,
+        "metadata": {
+            "gold_answer": gold,
+            "model_answer": model_answer,
+            "has_gold": has_gold,
+            "domain": domain,
+            "area": rec.get("area"),
+            "difficulty": rec.get("difficulty") or None,
+            "dataset": rec.get("dataset"),
+            "dataset_description": rec.get("dataset_description") or _dataset_desc(rec.get("dataset")),
+            "subsector": rec.get("subsector"),
+            "task_id": rec.get("task_id"),
+            "correct": correct,
+            "clean": clean,
+            "kept": kept,
+            "reward": rec.get("reward"),
+            "gen_model": gen_model,
+            "orchestrator_model": rec.get("orchestrator_model"),
+            "num_tool_calls": metrics.get("num_tool_calls"),
+            "n_turns": metrics.get("num_turns"),
+            "routed_models": routed,
+            "conversation": _ordered_turns(convs, am, rec.get("orchestrator_model")),
+        },
+    }
+
+
+def _dist(rows, key):
+    """{value: count} for a record field, most-common first, None/'' dropped."""
+    c = Counter(r.get(key) for r in rows if (r.get(key) or "") != "")
+    return {str(k): v for k, v in sorted(c.items(), key=lambda x: -x[1])}
+
+
+def _routed_dist(rows):
+    """Distribution of real (de-anonymized) expert models routed to."""
+    c = Counter()
+    for r in rows:
+        am = (r.get("metrics", {}) or {}).get("anon_map", {}) or {}
+        for cc in r.get("conversations", []):
+            if cc.get("role") != "assistant":
+                continue
+            for t in re.findall(r"<tool_call>(\{.*?\})</tool_call>", cc.get("content", ""), re.DOTALL):
+                try:
+                    real = am.get(json.loads(t).get("name"))
+                    if real:
+                        c[real] += 1
+                except Exception:
+                    pass
+    return {k: v for k, v in sorted(c.items(), key=lambda x: -x[1])}
+
+
+def _avg(rows, *path):
+    vals = []
+    for r in rows:
+        v = r.get("metrics", {}) or {}
+        for p in path:
+            v = (v or {}).get(p) if isinstance(v, dict) else None
+        if isinstance(v, (int, float)):
+            vals.append(v)
+    return round(sum(vals) / len(vals), 2) if vals else None
+
+
+def _dataset_metadata(rows, path, run_label, gen_model):
+    n = len(rows)
+    meta = {
+        "model": _model_short(rows, path),
+        "split": _split_of(path),
+        "task": "orchestrator-SFT unified-tool routing traces",
+        "date": "2026-07-07",
+        "run_label": run_label,
+        "gen_model": gen_model,
+        "git_sha": _git_sha(),
+        "source_file": str(path),
+        "source_datasets": ["GeneralThought-430K-filtered", "OpenThoughts3-1.2M"],
+        "task_mix": "balanced (GeneralThought + OpenThoughts code/math/science)",
+        "filter": "correct + clean (rejects file-write echoes, garble, "
+                  "reasoning-degeneration, truncated tails, essays/markdown dumps)",
+        "leak_free": "train/holdout disjoint by task_id",
+        "n_total": n,
+        "n_correct": sum(1 for r in rows if r.get("correct")),
+        "n_clean": sum(1 for r in rows if r.get("clean")),
+        "n_kept": sum(1 for r in rows if r.get("kept")),
+        "n_with_gold": sum(1 for r in rows if (r.get("gold_answer") or "").strip()),
+        "avg_tool_calls": _avg(rows, "num_tool_calls"),
+        "avg_turns": _avg(rows, "num_turns"),
+        "area_distribution": _dist(rows, "area"),
+        "difficulty_distribution": _dist(rows, "difficulty"),
+        "dataset_distribution": _dist(rows, "dataset"),
+        "routed_model_distribution": _routed_dist(rows),
+    }
+    cfg = _config_from_env()
+    if cfg:
+        meta["config"] = cfg
+    return {k: v for k, v in meta.items() if v is not None}
+
+
+def upload_dataset(path, name=None, *, project_id=None, project=None,
+                   run_label=None, gen_model=None, description=""):
+    """Upload one JSONL file as a Braintrust dataset. Returns (name, n_rows, url).
+
+    Target project: ``project`` (name) if given, else ``project_id`` /
+    ``OJ_BRAINTRUST_PROJECT_ID`` / the research default.
+    """
+    import braintrust
+
+    rows = [json.loads(l) for l in open(path) if l.strip()]
+    name = name or _default_name(rows, path)
+    gen_model = gen_model or os.getenv("OJ_GEN_MODEL") or \
+        next((r.get("gen_model") for r in rows if r.get("gen_model")), None)
+    run_label = run_label or os.getenv("OJ_RUN_LABEL") or name
+
+    init_kwargs = {"name": name, "description": description or None,
+                   "metadata": _dataset_metadata(rows, path, run_label, gen_model)}
+    if project:
+        init_kwargs["project"] = project
+    else:
+        init_kwargs["project_id"] = project_id or os.getenv(
+            "OJ_BRAINTRUST_PROJECT_ID", DEFAULT_PROJECT_ID)
+
+    ds = braintrust.init_dataset(**init_kwargs)
+    for r in rows:
+        ds.insert(**_row(r))
+    ds.flush()
+    summ = ds.summarize()
+    url = getattr(summ, "dataset_url", None) or (project or init_kwargs.get("project_id"))
+    return name, len(rows), url
+
+
+def autoupload(specs, *, run_label=None, gen_model=None, description=""):
+    """No-op-safe wrapper for pipeline hooks. Honors OJ_BRAINTRUST_AUTOUPLOAD
+    (default ON) and never raises: any failure (missing key/pkg, network) is
+    logged and swallowed so the data pipeline is never broken by telemetry."""
+    if os.getenv("OJ_BRAINTRUST_AUTOUPLOAD", "1").strip().lower() in ("0", "false", "no", "off", ""):
+        logger.info("[braintrust] autoupload disabled (OJ_BRAINTRUST_AUTOUPLOAD)")
+        return
+    if not os.getenv("BRAINTRUST_API_KEY"):
+        logger.info("[braintrust] autoupload skipped — BRAINTRUST_API_KEY unset")
+        return
+    try:
+        import braintrust  # noqa: F401
+    except Exception:
+        logger.info("[braintrust] autoupload skipped — braintrust not installed")
+        return
+    for spec in specs:
+        path, _, name = spec.partition("=")
+        try:
+            nm, n, url = upload_dataset(path, name or None, run_label=run_label,
+                                        gen_model=gen_model, description=description)
+            logger.info("[braintrust] uploaded %s: %d rows -> %s", nm, n, url)
+            print(f"[braintrust] uploaded {nm}: {n} rows -> {url}")
+        except Exception as exc:
+            logger.warning("[braintrust] autoupload FAILED for %s (%s) — continuing", path, exc)
+            print(f"[braintrust] autoupload FAILED for {path} ({exc}) — pipeline unaffected")
+
+
+def main():
+    logging.basicConfig(level=logging.INFO, format="%(message)s")
+    p = argparse.ArgumentParser()
+    p.add_argument("--project", default=None,
+                   help="Target project by NAME (overrides OJ_BRAINTRUST_PROJECT_ID).")
+    p.add_argument("--project-id", default=None,
+                   help="Target project by id (default: OJ_BRAINTRUST_PROJECT_ID or research).")
+    p.add_argument("--run-label", default=None)
+    p.add_argument("--gen-model", default=None, help="Specific gen model id override.")
+    p.add_argument("--description", default="")
+    p.add_argument("specs", nargs="+", help="path.jsonl[=dataset_name]")
+    args = p.parse_args()
+    for spec in args.specs:
+        path, _, name = spec.partition("=")
+        nm, n, url = upload_dataset(
+            path, name or None, project_id=args.project_id, project=args.project,
+            run_label=args.run_label, gen_model=args.gen_model,
+            description=args.description)
+        print(f"[braintrust] {nm}: {n} rows -> {url}")
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

scripts/train/eval_orch_gemma_matx.sbatch

@@ -0,0 +1,90 @@
+#!/bin/bash
+# Serve a trained gemma-4-26B-A4B SFT checkpoint (orchestrator) on 2xL40S (mkt1),
+# then run eval_orchestrator over benchmarks. Self-contained: serve -> health -> eval -> teardown.
+# Usage: sbatch eval_orch_gemma_matx.sbatch <1k|2k> <benchmarks> <N> <tag>
+#SBATCH --job-name=eval_gemma
+#SBATCH --account=mkt
+#SBATCH --partition=mkt
+#SBATCH -w mkt1
+#SBATCH --gres=gpu:2
+#SBATCH --cpus-per-gpu=10
+#SBATCH --mem=200G
+#SBATCH -t 24:00:00
+#SBATCH --output=logs/eval_gemma_matx_%j.log
+
+set -uo pipefail
+SIZE=${1:?usage: <1k|2k> <benchmarks> <N> <tag>}
+BENCHES=${2:-gaia}
+N=${3:-2}
+TAG=${4:-smoke}
+PORT=8123
+
+# Paths are resolved relative to the repo, or overridden by env:
+#   OJ_ROOT       repo checkout           (default: two levels up from this script)
+#   OJ_WORK       working-state dir       (default: $HOME/.openjarvis)
+#   OJ_CKPT_ROOT  trained SFT checkpoints (default: $OJ_WORK/lambda_ckpts)
+OJ_ROOT=${OJ_ROOT:-$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)}
+OJ_WORK=${OJ_WORK:-$HOME/.openjarvis}
+OJ_CKPT_ROOT=${OJ_CKPT_ROOT:-$OJ_WORK/lambda_ckpts}
+
+cd "$OJ_ROOT"
+set -a
+source .env
+[ -f "$OJ_WORK/eval_key_override.sh" ] && source "$OJ_WORK/eval_key_override.sh"
+set +a
+export HF_HOME=${HF_HOME:?set HF_HOME to a cache dir with room for the weights}
+export PYTHONPATH="$PWD/src"
+# judge/expert cloud engine needs a config with an [engine.cloud] section
+export OPENJARVIS_CONFIG=${OPENJARVIS_CONFIG:?set OPENJARVIS_CONFIG to a cell config.toml}
+source .venv/bin/activate
+
+if [ "$SIZE" = "base" ]; then
+  CKPT=google/gemma-4-26B-A4B-it   # un-fine-tuned base orchestrator (from HF cache)
+else
+  CKPT=$OJ_CKPT_ROOT/sft_gemma_${SIZE}_bf16/epoch3
+fi
+OUT=results/orch-eval-gemma-${SIZE}-${TAG}
+mkdir -p "$OUT" logs
+echo "=== [$(date)] eval_gemma SIZE=$SIZE BENCHES=$BENCHES N=$N TAG=$TAG on $(hostname) ==="
+nvidia-smi --query-gpu=index,name,memory.total --format=csv,noheader
+
+# ---- serve: TP=2 bf16 + MANDATORY no-NVLink L40S flags (else NCCL hangs) ----
+echo "=== [$(date)] launching vLLM (TP=2) serve of $CKPT ==="
+NCCL_P2P_DISABLE=1 NCCL_CUMEM_HOST_ENABLE=0 VLLM_WORKER_MULTIPROC_METHOD=spawn \
+PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True \
+python -m vllm.entrypoints.openai.api_server \
+  --model "$CKPT" --served-model-name gemma-sft \
+  --port $PORT --tensor-parallel-size 2 --dtype bfloat16 \
+  --gpu-memory-utilization 0.90 --trust-remote-code \
+  --max-model-len 32768 --enforce-eager --disable-custom-all-reduce \
+  --enable-auto-tool-choice --tool-call-parser gemma4 \
+  --max-num-batched-tokens 8192 --safetensors-load-strategy eager \
+  --enable-prefix-caching > logs/serve_gemma_${SIZE}_${SLURM_JOB_ID}.log 2>&1 &
+SERVE_PID=$!
+echo "serve pid=$SERVE_PID log=logs/serve_gemma_${SIZE}_${SLURM_JOB_ID}.log"
+cleanup(){ echo "=== [$(date)] teardown kill $SERVE_PID ==="; kill $SERVE_PID 2>/dev/null; sleep 5; kill -9 $SERVE_PID 2>/dev/null; }
+trap cleanup EXIT
+
+# ---- health poll (up to 25 min; TP + 52GB load is slow on L40S) ----
+echo "=== [$(date)] waiting for /v1/models ==="
+READY=0
+for i in $(seq 1 150); do
+  if ! kill -0 $SERVE_PID 2>/dev/null; then echo "!! serve died during startup"; tail -40 logs/serve_gemma_${SIZE}_${SLURM_JOB_ID}.log; exit 3; fi
+  if curl -sf http://localhost:$PORT/v1/models 2>/dev/null | grep -q gemma-sft; then READY=1; echo "=== [$(date)] READY after ${i}0s ==="; break; fi
+  sleep 10
+done
+[ $READY -eq 1 ] || { echo "!! not ready in 25min"; tail -40 logs/serve_gemma_${SIZE}_${SLURM_JOB_ID}.log; exit 4; }
+
+# ---- eval ----
+echo "=== [$(date)] running eval_orchestrator benches=$BENCHES n=$N ==="
+python scripts/orchestrator/eval_orchestrator.py \
+  --benchmarks "$BENCHES" --n "$N" --seed 42 \
+  --orchestrator-endpoint http://localhost:$PORT/v1 \
+  --orchestrator-model gemma-sft --orchestrator-api-key EMPTY \
+  --judge-model gemini-2.5-flash --judge-engine cloud \
+  --max-workers 4 --max-turns 8 \
+  --output-dir "$OUT"
+RC=$?
+echo "=== [$(date)] eval exit=$RC summary=$OUT/summary.json ==="
+echo "EVAL_GEMMA_DONE size=$SIZE tag=$TAG rc=$RC"
+exit $RC

scripts/train/fsdp4.yaml

@@ -0,0 +1,25 @@
+compute_environment: LOCAL_MACHINE
+distributed_type: FSDP
+downcast_bf16: 'no'
+machine_rank: 0
+main_process_ip: null
+main_process_port: null
+main_training_function: main
+mixed_precision: bf16
+num_machines: 1
+num_processes: 4
+rdzv_backend: static
+same_network: true
+use_cpu: false
+fsdp_config:
+  fsdp_version: 1
+  fsdp_auto_wrap_policy: SIZE_BASED_WRAP
+  fsdp_min_num_params: 100000000
+  fsdp_sharding_strategy: FULL_SHARD
+  fsdp_state_dict_type: FULL_STATE_DICT
+  fsdp_offload_params: false
+  fsdp_backward_prefetch: BACKWARD_PRE
+  fsdp_forward_prefetch: false
+  fsdp_use_orig_params: true
+  fsdp_cpu_ram_efficient_loading: true
+  fsdp_sync_module_states: true

scripts/train/fsdp8.yaml

@@ -0,0 +1,25 @@
+compute_environment: LOCAL_MACHINE
+distributed_type: FSDP
+downcast_bf16: 'no'
+machine_rank: 0
+main_process_ip: null
+main_process_port: null
+main_training_function: main
+mixed_precision: bf16
+num_machines: 1
+num_processes: 8
+rdzv_backend: static
+same_network: true
+use_cpu: false
+fsdp_config:
+  fsdp_version: 1
+  fsdp_auto_wrap_policy: SIZE_BASED_WRAP
+  fsdp_min_num_params: 100000000
+  fsdp_sharding_strategy: FULL_SHARD
+  fsdp_state_dict_type: FULL_STATE_DICT
+  fsdp_offload_params: false
+  fsdp_backward_prefetch: BACKWARD_PRE
+  fsdp_forward_prefetch: false
+  fsdp_use_orig_params: true
+  fsdp_cpu_ram_efficient_loading: true
+  fsdp_sync_module_states: true

src/openjarvis/agents/hybrid/_base.py

@@ -394,8 +394,8 @@ class LocalCloudAgent(BaseAgent):
         tools: Optional[list] = None,
         tool_choice: Optional[dict] = None,
         output_config: Optional[dict] = None,
-        timeout: float = 600.0,
-        max_retries: int = 12,
+        timeout: float = 60.0,
+        max_retries: int = 2,
         trace_role: str = "cloud",
     ) -> Tuple[str, int, int, int]:
         """Single Anthropic call. Returns (text, p_tok, c_tok, n_web_searches).
@@ -471,7 +471,7 @@ class LocalCloudAgent(BaseAgent):
         response_format: Optional[dict] = None,
         tools: Optional[list] = None,
         tool_choice: Optional[Any] = None,
-        timeout: float = 600.0,
+        timeout: float = 60.0,
         trace_role: str = "cloud",
     ) -> Tuple[str, int, int]:
         """Single OpenAI call. Returns (text, p_tok, c_tok). Trace-captured;
@@ -537,7 +537,7 @@ class LocalCloudAgent(BaseAgent):
         system: Optional[str] = None,
         max_tokens: int = 4096,
         temperature: float = 0.0,
-        timeout: float = 600.0,
+        timeout: float = 60.0,
         trace_role: str = "cloud",
         extra_body: Optional[Dict[str, Any]] = None,
     ) -> Tuple[str, int, int]:

src/openjarvis/agents/hybrid/_prices.py

@@ -32,6 +32,14 @@ PRICES: dict[str, tuple[float, float]] = {
     "qwen/qwen-2.5-coder-32b-instruct":  (0.08, 0.18),
     "qwen/qwen3-32b":                    (0.10, 0.30),
     "meta-llama/llama-3.3-70b-instruct": (0.13, 0.39),
+    # OpenRouter slugs for the orchestrator's local-OSS class when routed via
+    # OpenRouter instead of self-hosted vLLM. ESTIMATES — no public list price
+    # exists yet for these Qwen3.5/3.6 builds; scaled by active-param size.
+    # VERIFY against openrouter.ai before trusting the cost-aware reward numbers.
+    "qwen/qwen3.5-9b":        (0.05, 0.10),
+    "qwen/qwen3.6-27b":       (0.10, 0.30),
+    "qwen/qwen3.5-122b-a10b": (0.20, 0.60),
+    "qwen/qwen3.5-397b-a17b": (0.40, 1.20),
 }
 
 # Models whose API rejects an explicit `temperature` param — callers should
@@ -40,6 +48,10 @@ NO_TEMP_PREFIXES: tuple[str, ...] = (
     "claude-opus-4-7",
     "claude-sonnet-4-7",
     "claude-haiku-4-7",
+    # 4-8 family also rejects `temperature` ("deprecated for this model").
+    "claude-opus-4-8",
+    "claude-sonnet-4-8",
+    "claude-haiku-4-8",
 )
 
 

src/openjarvis/agents/hybrid/expert_registry.py

@@ -23,6 +23,7 @@ circular import.
 
 from __future__ import annotations
 
+import os
 import random
 import re
 from dataclasses import dataclass, field
@@ -42,15 +43,15 @@ VALID_KINDS = (KIND_MODEL, KIND_WEB_SEARCH, KIND_LOCAL_SEARCH, KIND_CODE, KIND_T
 # Flat-catalog category label, surfaced in each tool's spec so the orchestrator
 # can tell tool types apart without us imposing any hierarchy (the menu stays
 # flat — this is just a tag).
-CATEGORY_GENERALIST = "generalist_model"
-CATEGORY_SPECIALIZED = "specialized_model"
 CATEGORY_BASIC = "basic_tool"
-CATEGORY_SMALL_GENERALIST = "small_generalist"
-CATEGORY_STRONG_GENERALIST = "strong_generalist"
-# Two-model-class taxonomy for the orchestrator catalog (the specialized /
-# small-vs-strong generalist tiers are superseded).
+# Two-model-class taxonomy for the orchestrator catalog — the only model tiers.
+# (The legacy default_catalog still uses CATEGORY_GENERALIST/SPECIALIZED below.)
 CATEGORY_CLOUD_FRONTIER = "cloud_frontier"
 CATEGORY_LOCAL_OSS = "local_open_source"
+# Legacy tiers kept only for default_catalog (build_unified_sft.py); not used by
+# orchestrator_catalog. Superseded by the two-class taxonomy above.
+CATEGORY_GENERALIST = "generalist_model"
+CATEGORY_SPECIALIZED = "specialized_model"
 
 # Backend dispatch types understood by ``toolorchestra._call_worker`` (plus the
 # ``openjarvis-tool`` bridge, dispatched in ``unified.make_dispatch`` via the
@@ -81,11 +82,14 @@ class ExpertTool:
     price_in: float = 0.0
     price_out: float = 0.0
     latency_s: float = 5.0
-    category: str = ""  # generalist_model | specialized_model | basic_tool
+    category: str = ""  # cloud_frontier | local_open_source | basic_tool
     # Optional explicit JSON-schema for the tool's arguments. Set for bridged
     # real OpenJarvis tools (``openjarvis-tool`` backend) whose params don't fit
     # the fixed kind-based schemas; takes precedence over the kind default.
     param_schema: Optional[dict] = None
+    # When True, ``description()`` omits the price/latency line — used by the
+    # anonymized catalog so the orchestrator can't route on cost/identity.
+    hide_cost: bool = False
 
     def __post_init__(self) -> None:
         if self.kind not in VALID_KINDS:
@@ -141,6 +145,8 @@ class ExpertTool:
 
     def description(self) -> str:
         """Full description incl. the price/latency line (paper bakes this in)."""
+        if self.hide_cost:
+            return self.summary
         if self.kind == KIND_MODEL:
             cost_line = (
                 f" Pricing: ${self.price_in:.2f}/1M input, "
@@ -181,6 +187,103 @@ def _tool_name(model: str) -> str:
     return safe or "local_model"
 
 
+_ANON_ALPHABET = "abcdefghijklmnopqrstuvwxyz0123456789"
+
+# Rough size/scale hint per model — a capability signal WITHOUT the brand name,
+# so the orchestrator can route big-vs-small on task difficulty but can't route on
+# a proprietary name/tier prior. Closed models have no public param count, so they
+# get a scale tier; open models get their param count (else parsed from the name).
+_MODEL_SIZE = {
+    # The real orchestrator_catalog: 2 cloud frontier + 4 local OSS Qwen.
+    "gpt-5.5": "frontier-scale",
+    "claude-opus-4-8": "frontier-scale",
+    "Qwen/Qwen3.5-9B": "~9B params",
+    "Qwen/Qwen3.6-27B-FP8": "~27B params",
+    "Qwen/Qwen3.5-122B-A10B-FP8": "~122B total, ~10B active (MoE)",
+    "Qwen/Qwen3.5-397B-A17B-FP8": "~397B total, ~17B active (MoE)",
+    # OpenRouter slugs (the tool.model id when these route through OpenRouter).
+    "qwen/qwen3.5-122b-a10b": "~122B total, ~10B active (MoE)",
+    "qwen/qwen3.5-397b-a17b": "~397B total, ~17B active (MoE)",
+}
+
+
+def _size_hint(model: str) -> str:
+    """Anonymized scale descriptor for an expert (no brand). Falls back to a
+    param count parsed from the model id (``...-9b`` -> ``~9B params``)."""
+    if model in _MODEL_SIZE:
+        return _MODEL_SIZE[model]
+    m = re.search(r"(\d+(?:\.\d+)?)\s*b\b", model.lower())
+    return f"~{m.group(1)}B params" if m else "unspecified scale"
+
+
+def _class_hint(tool) -> str:
+    """Which of the two model classes this expert belongs to (anonymized)."""
+    cat = getattr(tool, "category", "")
+    if cat == CATEGORY_CLOUD_FRONTIER:
+        return "cloud frontier"
+    if cat == CATEGORY_LOCAL_OSS:
+        return "local open-source"
+    return ""
+
+
+def _cost_tier_hint(tool) -> str:
+    """Coarse cost tier (not exact pricing — that's the bias we anonymize away).
+    Shown so the policy learns the strong-but-expensive vs cheap tradeoff instead
+    of always delegating to the biggest model. Driven by class: cloud frontier is
+    expensive, self-hosted open-source is cheap."""
+    cat = getattr(tool, "category", "")
+    if cat == CATEGORY_CLOUD_FRONTIER:
+        return "expensive"
+    if cat == CATEGORY_LOCAL_OSS:
+        return "cheap"
+    if getattr(tool, "backend_type", "") == "vllm" or (tool.price_out or 0) <= 0:
+        return "cheap"
+    po = tool.price_out
+    return "cheap" if po < 5 else "moderate" if po < 20 else "expensive"
+
+
+def anonymize_tools(tools, rng):
+    """Strip model identity for unbiased routing data.
+
+    Each MODEL expert is replaced with an opaque random label
+    (``expert_<4 rand>``), a uniform description, a uniform category and no
+    price/latency line — so the orchestrator cannot route on a model's name,
+    position, cost, or tier (all of which we found dominate the choice). The full
+    list is shuffled to kill position bias. Basic tools (calculator, web_search,
+    …) keep their real names — the policy must still know what they do.
+
+    Returns ``(anon_tools, anon_to_real)`` where ``anon_to_real`` maps each opaque
+    label back to the real tool name (for offline analysis only; never shown to
+    the model). ``.model`` is preserved on each tool so dispatch still reaches the
+    right backend. Pass a fresh ``rng`` per rollout so labels don't stabilise.
+    """
+    from dataclasses import replace
+
+    anon_to_real: Dict[str, str] = {}
+    experts: List[ExpertTool] = []
+    basics: List[ExpertTool] = []
+    for t in tools:
+        if t.kind == KIND_MODEL:
+            tag = "model_" + "".join(rng.choice(_ANON_ALPHABET) for _ in range(4))
+            while tag in anon_to_real:
+                tag = "model_" + "".join(rng.choice(_ANON_ALPHABET) for _ in range(4))
+            anon_to_real[tag] = t.name
+            bits = [b for b in (_class_hint(t), _size_hint(t.model), _cost_tier_hint(t)) if b]
+            experts.append(replace(
+                t, name=tag,
+                summary="Another model — " + ", ".join(bits) + ". Send it a sub-question.",
+                category="model", hide_cost=True,
+            ))
+        else:
+            basics.append(t)
+    # Shuffle WITHIN the experts to kill per-expert position bias, but keep all
+    # experts as a block on top and the basic tools underneath — a clean split
+    # (experts first) rather than experts and utilities interleaved.
+    rng.shuffle(experts)
+    out = experts + basics
+    return out, anon_to_real
+
+
 # Default catalog: the paper's tool categories, mapped onto the models/tools
 # OpenJarvis can actually call. One named tool per model (faithful §3.1).
 def default_catalog(
@@ -414,62 +517,149 @@ def _openjarvis_basic_tools() -> List[ExpertTool]:
 
 
 # Local-Cloud Hybrid orchestrator catalog. The menu is two model classes — cloud
-# frontier models and locally-served open-source models — plus the basic tools
-# (web search, code interpreter, and the bridged real OpenJarvis tools). The
-# orchestrator model itself is NOT in the catalog.
+# frontier models and open-source models — plus the basic tools (web search, code
+# interpreter, and the bridged real OpenJarvis tools). The orchestrator model
+# itself is NOT in the catalog.
 #
-# Local model ids -> vLLM base_url come from ``local_endpoints``; unmapped ids
-# get ``base_url=None`` (tool still listed, dispatch wires the endpoint later).
+# Routing default is OpenRouter for every model tool, so the catalog works with
+# no self-hosted servers. A model is routed to local vLLM instead when its id is
+# in ``local_endpoints`` or ``model_backends`` overrides it. The
+# orchestrator-visible tool *name* is always derived from the canonical id, so
+# routing can change (vLLM <-> OpenRouter <-> native API) without shifting the
+# menu the policy was trained on.
 _LOCAL_OSS_MODELS = (
-    "Qwen/Qwen3.5-9B",
-    "Qwen/Qwen3.6-27B-FP8",
-    "Qwen/Qwen3.5-122B-A10B-FP8",
-    "Qwen/Qwen3.5-397B-A17B-FP8",
+    # (canonical_id, openrouter_slug)
+    ("Qwen/Qwen3.5-9B", "qwen/qwen3.5-9b"),
+    ("Qwen/Qwen3.6-27B-FP8", "qwen/qwen3.6-27b"),
+    ("Qwen/Qwen3.5-122B-A10B-FP8", "qwen/qwen3.5-122b-a10b"),
+    ("Qwen/Qwen3.5-397B-A17B-FP8", "qwen/qwen3.5-397b-a17b"),
 )
 
 _CLOUD_FRONTIER_MODELS = (
-    # (model, backend, summary, latency_s)
-    ("gpt-5.5", "openai",
-     "Cloud frontier generalist (GPT-5.5). Strongest reasoning across domains.", 30.0),
-    ("claude-opus-4-8", "anthropic",
-     "Cloud frontier generalist (Claude Opus 4.8). Strong long-horizon reasoning.", 26.0),
+    # (canonical, native_backend, openrouter_slug, summary, latency_s)
+    # Neutral, uniform summaries (no capability ranking) so the orchestrator
+    # doesn't just pick whichever model is labelled "strongest" — routing should
+    # be learned from the reward, not hand-labelled here.
+    ("gpt-5.5", "openai", "openai/gpt-5.5",
+     "Expert model (GPT-5.5).", 30.0),
+    ("claude-opus-4-8", "anthropic", "anthropic/claude-opus-4.8",
+     "Expert model (Claude Opus 4.8).", 26.0),
 )
 
 
+def _model_tool(
+    canonical: str,
+    *,
+    native_backend: str,
+    or_slug: str,
+    summary: str,
+    lat: float,
+    category: str,
+    local_endpoints: Dict[str, str],
+    model_backends: Dict[str, str],
+    openrouter_slugs: Dict[str, str],
+) -> ExpertTool:
+    """Build one model tool, resolving its backend.
+
+    Backend precedence: explicit ``model_backends[canonical]`` > vLLM if the model
+    has a ``local_endpoints`` entry > the model's NATIVE provider (openai /
+    anthropic / gemini) when it has one > OpenRouter. Frontier models thus hit
+    their first-party API by default (OpenRouter's gpt-5.5 was returning 400s);
+    OSS Qwen experts (native_backend="vllm") fall through to OpenRouter.
+    """
+    backend = (
+        model_backends.get(canonical)
+        or ("vllm" if canonical in local_endpoints else None)
+        or (native_backend if native_backend in ("openai", "anthropic", "gemini")
+            else "openrouter")
+    )
+    name = _tool_name(canonical)
+    if backend == "vllm":
+        # Self-hosted: free per the cost model.
+        return ExpertTool(
+            name=name, kind=KIND_MODEL, backend_type="vllm", summary=summary,
+            model=canonical, base_url=local_endpoints.get(canonical),
+            price_in=0.0, price_out=0.0, latency_s=lat, category=category,
+        )
+    if backend == "openrouter":
+        slug = openrouter_slugs.get(canonical, or_slug)
+        pi, po = _price(slug)
+        if (pi, po) == (0.0, 0.0):  # fall back to the canonical id's price
+            pi, po = _price(canonical)
+        return ExpertTool(
+            name=name, kind=KIND_MODEL, backend_type="openrouter", summary=summary,
+            model=slug, base_url=None, price_in=pi, price_out=po,
+            latency_s=lat, category=category,
+        )
+    # native provider API (openai / anthropic / gemini)
+    pi, po = _price(canonical)
+    return ExpertTool(
+        name=name, kind=KIND_MODEL, backend_type=backend, summary=summary,
+        model=canonical, price_in=pi, price_out=po, latency_s=lat, category=category,
+    )
+
+
 def orchestrator_catalog(
     *,
     local_endpoints: Optional[Dict[str, str]] = None,
+    model_backends: Optional[Dict[str, str]] = None,
+    openrouter_slugs: Optional[Dict[str, str]] = None,
     include_tools: bool = True,
 ) -> List[ExpertTool]:
     """Return the orchestrator's tool catalog: two model classes + basic tools.
 
-    ``local_endpoints`` maps a local model id (e.g. ``"Qwen/Qwen3.5-9B"``) to its
-    vLLM ``base_url``; ids not present get ``base_url=None``. ``include_tools``
-    (default True) appends the basic tools — web search, code interpreter, and the
-    bridged real OpenJarvis tools (calculator, shell_exec, file_read, file_write,
-    http_request).
+    Routing for the model tools defaults to **OpenRouter** (so the catalog works
+    with no self-hosted servers). Overrides, in precedence order:
+
+    * ``model_backends`` maps a canonical model id -> ``"vllm" | "openrouter" |
+      "openai" | "anthropic" | "gemini"`` to force that model's backend.
+    * ``local_endpoints`` maps a canonical id (e.g. ``"Qwen/Qwen3.5-9B"``) to a
+      vLLM ``base_url``; a model present here routes to vLLM (free) unless
+      ``model_backends`` says otherwise.
+    * ``openrouter_slugs`` overrides the per-model OpenRouter slug used when a
+      model routes through OpenRouter.
+
+    ``include_tools`` (default True) appends the basic tools — web search, code
+    interpreter, and the bridged real OpenJarvis tools (calculator, shell_exec,
+    file_read, file_write, http_request).
     """
     local_endpoints = local_endpoints or {}
+    model_backends = model_backends or {}
+    openrouter_slugs = openrouter_slugs or {}
     cat: List[ExpertTool] = []
 
+    # Env-gated expert exclusion: OJ_EXCLUDE_EXPERTS is a comma-separated list of
+    # case-insensitive substrings matched against a model's canonical id. Any
+    # match is skipped from the catalog. Used to temporarily drop unreliable
+    # experts (e.g. OpenRouter giants during a provider outage) without editing
+    # the registry — unset the var to restore them.
+    _excl = {s.strip().lower() for s in os.environ.get("OJ_EXCLUDE_EXPERTS", "").split(",") if s.strip()}
+    def _excluded(canonical: str) -> bool:
+        c = canonical.lower()
+        return any(x in c for x in _excl)
+
     # ---- cloud frontier models ----
-    for model, backend, summary, lat in _CLOUD_FRONTIER_MODELS:
-        pi, po = _price(model)
-        cat.append(ExpertTool(
-            name=_tool_name(model), kind=KIND_MODEL, backend_type=backend,
-            summary=summary, model=model, price_in=pi, price_out=po,
-            latency_s=lat, category=CATEGORY_CLOUD_FRONTIER,
+    for canonical, native_backend, or_slug, summary, lat in _CLOUD_FRONTIER_MODELS:
+        if _excluded(canonical):
+            continue
+        cat.append(_model_tool(
+            canonical, native_backend=native_backend, or_slug=or_slug,
+            summary=summary, lat=lat, category=CATEGORY_CLOUD_FRONTIER,
+            local_endpoints=local_endpoints, model_backends=model_backends,
+            openrouter_slugs=openrouter_slugs,
         ))
 
-    # ---- local open-source models (served via vLLM, price 0/0) ----
-    for model in _LOCAL_OSS_MODELS:
-        cat.append(ExpertTool(
-            name=_tool_name(model), kind=KIND_MODEL, backend_type="vllm",
-            summary=(f"Locally-served open-source model ({model}). Cheap and "
-                     "private; no per-token cost."),
-            model=model, base_url=local_endpoints.get(model),
-            price_in=0.0, price_out=0.0, latency_s=4.0,
-            category=CATEGORY_LOCAL_OSS,
+    # ---- open-source models (OpenRouter by default; vLLM when an endpoint or
+    #      a model_backends override is supplied) ----
+    for canonical, or_slug in _LOCAL_OSS_MODELS:
+        if _excluded(canonical):
+            continue
+        cat.append(_model_tool(
+            canonical, native_backend="vllm", or_slug=or_slug,
+            summary=f"Expert model ({canonical}).",
+            lat=4.0, category=CATEGORY_LOCAL_OSS,
+            local_endpoints=local_endpoints, model_backends=model_backends,
+            openrouter_slugs=openrouter_slugs,
         ))
 
     if include_tools:
@@ -576,9 +766,7 @@ __all__ = [
     "CATEGORY_CLOUD_FRONTIER",
     "CATEGORY_GENERALIST",
     "CATEGORY_LOCAL_OSS",
-    "CATEGORY_SMALL_GENERALIST",
     "CATEGORY_SPECIALIZED",
-    "CATEGORY_STRONG_GENERALIST",
     "ExpertTool",
     "KIND_CODE",
     "KIND_LOCAL_SEARCH",

src/openjarvis/agents/hybrid/toolorchestra/parsing.py

@@ -13,6 +13,27 @@ _TOOL_CALL_TAG_RE = re.compile(
     r"<tool_call>\s*(\{.*?\})\s*</tool_call>", re.DOTALL
 )
 
+# Fallback for a known failure mode: the SFT'd orchestrator often emits its
+# delegation as ``\boxed{expert_ab12: <sub-question>}`` (math-data habit bleeding
+# into routing) instead of a real ``<tool_call>`` JSON block. The name before the
+# colon is a valid (anonymized) tool label; the rest is the sub-question/args.
+#
+# Real outputs use several shapes, all handled here:
+#   \boxed{web_search: <query>}              -> name, args
+#   \boxed{web_search query: <query>}        -> the word "query" is dropped
+#   \boxed{file_read, path: <path>}          -> the ", <key>" hint sets the arg key
+# We require a ``name <optional , key | query> : <args>`` structure, so a genuine
+# answer like ``\boxed{10}``, ``\boxed{b,e}`` or ``\boxed{The answer is: 42}``
+# (first token isn't followed by a bare ``:`` / ``, key:`` / ``query:``) is left
+# alone. ``group("key")`` is the explicit arg key when the model wrote ``, key:``.
+_BOXED_DELEGATION_RE = re.compile(
+    r"\\boxed\{\s*([A-Za-z_][\w-]*)\s*"
+    r"(?:,\s*(?P<key>\w+)\s*)?"   # optional ", key" hint (e.g. file_read, path:)
+    r"(?:query\s*)?"             # optional literal "query" word before the colon
+    r":\s*(.+?)\s*\}\s*$",
+    re.DOTALL,
+)
+
 
 def _parse_rl_tool_call(content: str, sdk_tool_calls: Any) -> Optional[Dict[str, Any]]:
     """Return ``{"name": str, "arguments": dict}`` or None.
@@ -37,19 +58,23 @@ def _parse_rl_tool_call(content: str, sdk_tool_calls: Any) -> Optional[Dict[str,
     if not isinstance(content, str):
         return None
     m = _TOOL_CALL_TAG_RE.search(content)
-    if not m:
-        return None
-    try:
-        obj = json.loads(m.group(1))
-    except json.JSONDecodeError:
-        return None
-    if not isinstance(obj, dict):
-        return None
-    name = obj.get("name")
-    args = obj.get("arguments", {})
-    if not isinstance(name, str) or not isinstance(args, dict):
-        return None
-    return {"name": name, "arguments": args}
+    if m:
+        try:
+            obj = json.loads(m.group(1))
+        except json.JSONDecodeError:
+            obj = None
+        if isinstance(obj, dict):
+            name = obj.get("name")
+            args = obj.get("arguments", {})
+            if isinstance(name, str) and isinstance(args, dict):
+                return {"name": name, "arguments": args}
+    # \boxed{name[, key][ query]: <args>} fallback (see _BOXED_DELEGATION_RE).
+    bm = _BOXED_DELEGATION_RE.search(content.strip())
+    if bm:
+        arg_val = bm.groups()[-1].strip()
+        key = bm.group("key") or "input"
+        return {"name": bm.group(1), "arguments": {key: arg_val}}
+    return None
 
 
 def _build_pool_block(workers: List[Dict[str, Any]]) -> str:

src/openjarvis/agents/hybrid/toolorchestra/rollout.py

@@ -9,24 +9,55 @@ final answer) or ``max_turns`` is hit.
 The loop is parameterized over two injected callables so it is pure control flow
 (no network) and unit-testable with fakes — the agent supplies real ones:
 
-* ``call_orchestrator(system, user, tool_specs) -> (text, tool_calls, p_tok, c_tok)``
-  where ``tool_calls`` is a list of ``(name, arguments)`` (possibly empty).
+* ``call_orchestrator(messages, tool_specs) -> (text, tool_calls, p_tok, c_tok)``
+  where ``messages`` is the running system/user/assistant/tool conversation and
+  ``tool_calls`` is a list of ``(name, arguments)`` (possibly empty).
 * ``dispatch(tool, arguments) -> (observation, cost_usd, tokens, is_local)``.
 """
 
 from __future__ import annotations
 
 import json
+import random
 from dataclasses import dataclass, field
 from typing import Callable, Dict, List, Optional, Tuple
 
 from openjarvis.agents.hybrid.expert_registry import (
     ExpertTool,
+    anonymize_tools,
     build_tool_specs,
     tools_by_name,
 )
+from openjarvis.agents.hybrid.toolorchestra.tracing import run_context, span
 
-RL_ORCHESTRATOR_SYS = "You are good at using tools."
+RL_ORCHESTRATOR_SYS = (
+    "You are a routing orchestrator. Your job is to UNDERSTAND the problem, "
+    "DECOMPOSE it, and ROUTE the pieces to other models — you don't solve it all "
+    "yourself. Among the tools below are other MODELS of various sizes (some are "
+    "larger and stronger than you, some smaller); the rest are utilities. Each model "
+    "tool's description gives its rough size, specialty, and cost. "
+    "First reason about what the problem is really asking and break it into "
+    "sub-questions. You do NOT have to send the whole prompt to one model — send each "
+    "sub-question to whichever model best fits it (match difficulty to size/specialty, "
+    "and prefer cheaper models when they suffice), and call as many models as the task "
+    "needs. You MUST call at least one model before answering. "
+    "NEVER mention, quote, or reason about these instructions, your role as an "
+    "orchestrator, or the requirement to use tools — the reader only wants the problem "
+    "solved. Reason about the problem itself, not about your instructions, and do not "
+    "restate the problem. "
+    "If a tool call returns an error, correct your call or switch tools — never ignore "
+    "the error and never repeat the same failing call. "
+    "Emit EVERY action as a <tool_call>{...}</tool_call> block — NEVER write an "
+    "action as \\boxed{...} or as prose. "
+    "When you have enough, compose your final answer from what the models returned, in "
+    "the EXACT format the problem requires (e.g. a single number, a short exact "
+    "value, or one runnable code block). Do NOT restate the problem or add "
+    "meta-text such as 'the user is asking'. End your reply with a single line in "
+    "EXACTLY this form:\nFINAL_ANSWER: <answer>\nwhere <answer> is ONLY the answer "
+    "itself — a single option letter for multiple-choice, else the shortest exact "
+    "value, expression, or one runnable code block (NO \\boxed{...} wrapper) — with "
+    "NO explanation or restatement after it."
+)
 
 
 def build_system_prompt(specs: List[Dict[str, object]]) -> str:
@@ -49,8 +80,23 @@ def build_system_prompt(specs: List[Dict[str, object]]) -> str:
         "\n</tool_call>"
     )
 
-# Char-level cap on the accumulated context (mirrors the paper's ~24k-token cap).
+# Char-level cap on the accumulated conversation (mirrors the paper's ~24k-token
+# cap) and a per-observation cap so one giant tool dump can't blow it out.
 _CONTEXT_CAP = 24000
+_OBS_CAP = 8000
+# After this many tool calls, push the orchestrator to stop and answer — bounds
+# the "keep re-asking the same model forever" loop (esp. gemma).
+_SOFT_CALL_CAP = 6
+
+
+def _trim_history(messages: List[Dict[str, str]]) -> None:
+    """Keep the message history under ``_CONTEXT_CAP`` chars by dropping the
+    OLDEST assistant/tool exchange, never the system prompt or the problem."""
+    def total() -> int:
+        return sum(len(m.get("content") or "") for m in messages)
+    # messages[0]=system, messages[1]=user problem — always keep those two.
+    while total() > _CONTEXT_CAP and len(messages) > 4:
+        del messages[2:4]
 
 
 @dataclass
@@ -71,6 +117,16 @@ class UnifiedRollout:
     tokens: int = 0
     num_tool_calls: int = 0
     parse_failures: int = 0
+    # When experts were anonymized for this rollout, maps the opaque label the
+    # policy saw (e.g. ``expert_a3f9``) back to the real tool name (``gpt_5_5``).
+    anon_map: Optional[Dict[str, str]] = None
+    # The EXACT tool specs the policy saw this rollout (anonymized when
+    # ``anonymize=True``). Serialization MUST build the saved system prompt from
+    # these — not from the real registry — or the prompt's ``<tools>`` block ends
+    # up with real model names+pricing while the assistant ``<tool_call>`` tags
+    # use the anon labels, which both breaks SFT (calls a tool absent from its
+    # list) and re-injects the name/cost bias anonymization removed.
+    tool_specs: Optional[List[Dict[str, object]]] = None
 
     def tool_calls(self) -> List[Tuple[str, Dict[str, object]]]:
         return [(t.tool_name, t.arguments) for t in self.turns if t.tool_name]
@@ -85,7 +141,7 @@ def _tool_prompt(tool: ExpertTool, arguments: Dict[str, object], question: str)
     return question
 
 
-def run_unified_rollout(
+def _run_unified_rollout_inner(
     question: str,
     tools: List[ExpertTool],
     *,
@@ -93,43 +149,90 @@ def run_unified_rollout(
     dispatch: Callable[[ExpertTool, Dict[str, object]], Tuple[str, float, int, bool]],
     max_turns: int = 50,
     system: str = RL_ORCHESTRATOR_SYS,
+    anonymize: bool = False,
 ) -> UnifiedRollout:
-    """Drive the faithful unified-tool rollout for one task."""
+    """Drive the faithful unified-tool rollout for one task.
+
+    ``anonymize``: replace each model expert with an opaque random label, a
+    uniform description and no cost line, shuffled — so the policy can't route on
+    a model's name/position/cost (which we found dominate the choice). The
+    anon->real mapping is returned on the rollout for offline analysis.
+    """
+    anon_map: Optional[Dict[str, str]] = None
+    if anonymize:
+        tools, anon_map = anonymize_tools(tools, random.Random())
     specs = build_tool_specs(tools)
     by_name = tools_by_name(tools)
 
-    context = ""
+    # Proper multi-turn conversation (NOT a flattened user blob): the orchestrator
+    # sees its own calls as `assistant` turns and each observation as a `tool`
+    # turn, so it can tell a tool result from user input and doesn't re-derive the
+    # whole problem every turn. This mirrors the serialized SFT format exactly.
+    messages: List[Dict[str, str]] = [
+        {"role": "system", "content": system},
+        {"role": "user", "content": f"Problem: {question}"},
+    ]
     turns: List[UnifiedTurn] = []
     cost = 0.0
     tokens = 0
     n_tool_calls = 0
     parse_failures = 0
+    nudges = 0
+    empty_input_nudges = 0
     final_answer = ""
 
     for _ in range(max_turns):
-        user = (
-            f"Problem: {question}\n\n{context or '(no context yet)'}\n\n"
-            "Choose an appropriate tool, or answer directly if you have enough."
-        )
-        text, tool_calls, p_tok, c_tok = call_orchestrator(system, user, specs)
+        text, tool_calls, p_tok, c_tok = call_orchestrator(messages, specs)
         tokens += int(p_tok) + int(c_tok)
 
         if not tool_calls:
+            # Enforce the "MUST delegate to a model" rule: if the orchestrator
+            # tries to answer before ANY tool call, nudge it to route instead of
+            # accepting a solve-it-yourself answer (which reject-sampling drops).
+            if n_tool_calls == 0 and nudges < 2:
+                nudges += 1
+                messages.append({"role": "assistant", "content": text or ""})
+                messages.append({"role": "user", "content": (
+                    "Make progress by delegating a concrete sub-question to one of the "
+                    "models now via a <tool_call>.")})
+                continue
             # No tool call -> the orchestrator is answering. Terminate.
             final_answer = (text or "").strip()
             turns.append(UnifiedTurn(reasoning=text or "", tool_name=None))
+            messages.append({"role": "assistant", "content": text or ""})
             break
 
         name, arguments = tool_calls[0]
         if name not in by_name:
             parse_failures += 1
-            context = (context + f"\n[invalid tool {name!r} — choose from the list]")[-_CONTEXT_CAP:]
+            messages.append({"role": "assistant", "content": text or ""})
+            messages.append({"role": "user",
+                             "content": f"[invalid tool {name!r} — choose one from the provided tool list]"})
             if parse_failures >= 2:
                 final_answer = (text or "").strip()
                 break
             continue
 
         tool = by_name[name]
+        # Empty-input guard: the small orchestrator often emits a <tool_call> with
+        # a blank/missing input on harder multi-turn tasks. Dispatching it returns
+        # a "no input provided" error observation that poisons the whole (often
+        # otherwise-correct) trajectory — the dominant clean-yield killer on hard
+        # tasks. Instead, nudge the model to resend WITH input and drop the
+        # malformed turn (don't record it), so it never enters the training data.
+        if tool.kind == "model" or tool.backend_type != "openjarvis-tool":
+            _has_input = any(
+                isinstance(arguments.get(k), str) and arguments.get(k).strip()
+                for k in ("input", "query", "code")
+            )
+            if not _has_input and empty_input_nudges < 3:
+                empty_input_nudges += 1
+                messages.append({"role": "assistant", "content": text or ""})
+                messages.append({"role": "user", "content": (
+                    f"Your call to {name} had an empty 'input'. Resend the "
+                    "<tool_call> with a non-empty 'input' field containing the "
+                    "concrete sub-question to delegate.")})
+                continue
         obs, dcost, dtok, _is_local = dispatch(tool, arguments)
         cost += float(dcost)
         tokens += int(dtok)
@@ -138,7 +241,22 @@ def run_unified_rollout(
             reasoning=text or "", tool_name=name, arguments=dict(arguments),
             observation=obs,
         ))
-        context = (context + f"\n[{name}] {obs}")[-_CONTEXT_CAP:]
+        # The model's own action as an assistant turn (the <tool_call> tag in
+        # content, matching the SFT serialization), then the observation as a
+        # distinct `tool` turn the model reads as a tool response.
+        call_content = text or ""
+        if "<tool_call>" not in call_content:
+            call_content = (call_content + "\n" + tool_call_tag(name, arguments)).strip()
+        messages.append({"role": "assistant", "content": call_content})
+        obs_text = obs or ""
+        if len(obs_text) > _OBS_CAP:
+            obs_text = obs_text[:_OBS_CAP] + "\n…[truncated]"
+        messages.append({"role": "tool", "name": name, "content": obs_text})
+        if n_tool_calls >= _SOFT_CALL_CAP:
+            messages.append({"role": "user", "content": (
+                "You now have enough information from the models. Do NOT call any "
+                "more tools — reply with your FINAL_ANSWER line only.")})
+        _trim_history(messages)
     else:
         # Hit max_turns with no explicit answer: use the last observation/text.
         final_answer = (turns[-1].observation or turns[-1].reasoning).strip() if turns else ""
@@ -146,9 +264,51 @@ def run_unified_rollout(
     return UnifiedRollout(
         turns=turns, final_answer=final_answer, cost_usd=cost, tokens=tokens,
         num_tool_calls=n_tool_calls, parse_failures=parse_failures,
+        anon_map=anon_map, tool_specs=specs,
     )
 
 
+def run_unified_rollout(
+    question: str,
+    tools: List[ExpertTool],
+    *,
+    call_orchestrator: Callable[..., Tuple[str, List[Tuple[str, Dict[str, object]]], int, int]],
+    dispatch: Callable[[ExpertTool, Dict[str, object]], Tuple[str, float, int, bool]],
+    max_turns: int = 50,
+    system: str = RL_ORCHESTRATOR_SYS,
+    anonymize: bool = False,
+) -> UnifiedRollout:
+    """One ToolOrchestra rollout = one Braintrust trace. Opens the root span,
+    runs the loop (orchestrator turns + expert/tool dispatches nest inside as
+    child spans), then logs the final answer + cost/tokens. No-op when tracing
+    is disabled (see :mod:`.tracing`)."""
+    _run_meta, _run_tags = run_context()
+    _fields = dict(
+        input={"question": question},
+        metadata={"max_turns": max_turns, "anonymize": anonymize,
+                  "n_tools": len(tools), **_run_meta},
+    )
+    if _run_tags:
+        _fields["tags"] = _run_tags
+    with span("toolorchestra.rollout", span_type="task", **_fields) as _s:
+        roll = _run_unified_rollout_inner(
+            question, tools, call_orchestrator=call_orchestrator, dispatch=dispatch,
+            max_turns=max_turns, system=system, anonymize=anonymize,
+        )
+        _s.log(
+            output=roll.final_answer,
+            metrics={"cost_usd": roll.cost_usd, "tokens": roll.tokens,
+                     "num_tool_calls": roll.num_tool_calls,
+                     "parse_failures": roll.parse_failures},
+            metadata={"n_experts_available": len(roll.anon_map or {}),
+                      "answered": bool(roll.final_answer),
+                      # label -> real model, so every anonymized route span in this
+                      # trace can be decoded back to the model that actually ran.
+                      "anon_map": roll.anon_map or {}},
+        )
+        return roll
+
+
 def tool_call_tag(name: str, arguments: Dict[str, object]) -> str:
     """Render a tool call as the ``<tool_call>{...}</tool_call>`` text the model emits."""
     return f"<tool_call>{json.dumps({'name': name, 'arguments': arguments})}</tool_call>"

src/openjarvis/agents/hybrid/toolorchestra/tracing.py

@@ -0,0 +1,161 @@
+"""Braintrust telemetry for the ToolOrchestra rollout.
+
+Each ``run_unified_rollout`` becomes ONE Braintrust trace (root span); the
+orchestrator turns and every expert/tool dispatch nest inside it, and the
+underlying OpenAI/Anthropic calls (wrapped clients) nest one level deeper — so
+you see the full routing tree with inputs/outputs/tokens/cost per node.
+
+On by default. Degrades to a total no-op (never raises, never changes behavior)
+when: ``OJ_BRAINTRUST=0``, the ``braintrust`` package isn't installed, or
+``BRAINTRUST_API_KEY`` is unset. Concurrency: rollouts run one-per-thread
+(ThreadPoolExecutor); threads don't inherit contextvars, so each rollout opens
+its own independent root trace and same-thread child calls nest correctly.
+"""
+
+from __future__ import annotations
+
+import contextlib
+import logging
+import os
+from typing import Any, Optional
+
+logger = logging.getLogger(__name__)
+
+_STATE: dict = {"resolved": False, "enabled": False, "bt": None}
+
+
+def _truthy(v: str) -> bool:
+    return v.strip().lower() not in ("0", "false", "no", "off", "")
+
+
+def _resolve() -> bool:
+    """Lazily decide whether tracing is active and init the logger once."""
+    if _STATE["resolved"]:
+        return _STATE["enabled"]
+    _STATE["resolved"] = True
+    if not _truthy(os.getenv("OJ_BRAINTRUST", "1")):  # on by default
+        return False
+    if not os.getenv("BRAINTRUST_API_KEY"):
+        logger.info("braintrust on-by-default but BRAINTRUST_API_KEY unset — tracing disabled")
+        return False
+    try:
+        import braintrust as _bt
+
+        proj_id = os.getenv("OJ_BRAINTRUST_PROJECT_ID")
+        if proj_id:
+            _bt.init_logger(project_id=proj_id)
+        else:
+            _bt.init_logger(project=os.getenv("OJ_BRAINTRUST_PROJECT", "toolorchestra"))
+        _STATE["bt"] = _bt
+        _STATE["enabled"] = True
+        logger.info("braintrust tracing ENABLED (%s)",
+                    f"project_id={proj_id}" if proj_id
+                    else f"project={os.getenv('OJ_BRAINTRUST_PROJECT', 'toolorchestra')}")
+    except Exception as exc:  # missing pkg / bad key / init failure — never crash
+        logger.warning("braintrust init failed (%s) — tracing disabled", exc)
+    return _STATE["enabled"]
+
+
+def enabled() -> bool:
+    return _resolve()
+
+
+def run_context() -> tuple[dict, list]:
+    """Run-level metadata + tags for the ROOT rollout span, sourced from env
+    (set by the generation driver). No-op-safe: returns ({}, []) if nothing is
+    set, and never raises. Env keys:
+
+    - ``OJ_RUN_LABEL``   — human run label (also stamped on the uploaded dataset).
+    - ``OJ_GEN_MODEL``   — SPECIFIC gen model id (e.g. ``Qwen/Qwen3.5-9B``).
+    - ``OJ_RUN_STAGE``   — ``prod``|``smoke`` (inferred from the label if unset).
+    - ``OJ_CFG_*``       — config knobs (temperature/max_turns/anonymize/...).
+    """
+    import datetime
+
+    meta: dict = {}
+    tags: list = []
+    try:
+        label = os.getenv("OJ_RUN_LABEL")
+        gen_model = os.getenv("OJ_GEN_MODEL")
+        stage = os.getenv("OJ_RUN_STAGE")
+        if not stage and label:
+            stage = "smoke" if "smoke" in label.lower() else "prod"
+        if label:
+            meta["run_label"] = label
+        if gen_model:
+            meta["gen_model"] = gen_model
+        if stage:
+            meta["stage"] = stage
+        cfg = {}
+        for env_key, key in (("OJ_CFG_TEMPERATURE", "temperature"),
+                             ("OJ_CFG_MAX_TURNS", "max_turns"),
+                             ("OJ_CFG_ANONYMIZE", "anonymize"),
+                             ("OJ_CFG_REJECTION_ONLY", "rejection_only")):
+            v = os.getenv(env_key)
+            if v not in (None, ""):
+                cfg[key] = v
+        if cfg:
+            meta["config"] = cfg
+        date = datetime.date.today().isoformat()
+        tags = [t for t in (gen_model, stage, date) if t]
+    except Exception:  # never let telemetry enrichment break a rollout
+        return {}, []
+    return meta, tags
+
+
+def wrap_client(client: Any) -> Any:
+    """Wrap an OpenAI/Anthropic client so its calls auto-log under the current
+    span. Pass-through (unchanged client) when tracing is off or on any error —
+    so this is always safe to call at client construction."""
+    if not _resolve():
+        return client
+    try:
+        bt = _STATE["bt"]
+        mod = type(client).__module__.lower()
+        if "anthropic" in mod:
+            return bt.wrap_anthropic(client)
+        return bt.wrap_openai(client)
+    except Exception as exc:
+        logger.warning("braintrust wrap_client failed (%s) — using raw client", exc)
+        return client
+
+
+class _NullSpan:
+    """No-op span used when tracing is disabled (keeps call sites branch-free)."""
+
+    def log(self, **_kw: Any) -> None:
+        pass
+
+    def __enter__(self) -> "_NullSpan":
+        return self
+
+    def __exit__(self, *_a: Any) -> bool:
+        return False
+
+
+@contextlib.contextmanager
+def span(name: str, *, span_type: str = "task", **fields: Any):
+    """Open a Braintrust span (or a no-op). Use as::
+
+        with span("toolorchestra.rollout", input=...) as s:
+            ...
+            s.log(output=..., metrics=..., metadata=...)
+
+    ``fields`` (input/metadata/...) are forwarded to ``start_span``.
+    """
+    if not _resolve():
+        yield _NullSpan()
+        return
+    # Guard only span CREATION (so a Braintrust hiccup degrades to a no-op). Do
+    # NOT wrap the yielded body in try/except: an exception from the rollout would
+    # be thrown into this generator, caught here, and masked by a second yield
+    # ("generator didn't stop after throw()"). Let the body's own exceptions
+    # propagate through the `with` untouched — Braintrust records + re-raises.
+    try:
+        cm = _STATE["bt"].start_span(name=name, type=span_type, **fields)
+    except Exception as exc:  # span creation failed — run trace-less, never break
+        logger.warning("braintrust start_span(%s) failed (%s) — continuing untraced", name, exc)
+        yield _NullSpan()
+        return
+    with cm as s:
+        yield s

src/openjarvis/agents/hybrid/toolorchestra/unified.py

@@ -11,16 +11,25 @@ network or keys.
 from __future__ import annotations
 
 import json
+import threading
 from typing import Any, Callable, Dict, List, Optional, Tuple
 
+# Guards the lazy, one-time build of the shared ToolExecutor in
+# ``_dispatch_openjarvis_tool``: with concurrent rollouts (parallel rejection
+# sampling) several threads can reach the build at once and would each instantiate
+# the full tool registry. The lock makes it build-once.
+_EXECUTOR_BUILD_LOCK = threading.Lock()
+
 from openjarvis.agents.hybrid._prices import cost as _model_cost
 from openjarvis.agents.hybrid.expert_registry import ExpertTool, to_worker_dict
 from openjarvis.agents.hybrid.toolorchestra.parsing import _parse_rl_tool_call
 from openjarvis.agents.hybrid.toolorchestra.rollout import (
     UnifiedRollout,
+    build_system_prompt,
     run_unified_rollout,
 )
 from openjarvis.agents.hybrid.toolorchestra.workers import _call_worker
+from openjarvis.agents.hybrid.toolorchestra.tracing import span
 
 
 def make_call_orchestrator(
@@ -31,24 +40,68 @@ def make_call_orchestrator(
     temperature: float = 1.0,
     max_tokens: int = 4096,
     timeout: float = 600.0,
+    native_tools: bool = True,
+    frequency_penalty: float = 0.2,
+    presence_penalty: float = 0.0,
+    repetition_penalty: float = 1.15,
 ) -> Callable[..., Tuple[str, List[Tuple[str, Dict[str, Any]]], int, int]]:
     """Teacher-orchestrator caller. ``base_url=None`` → OpenAI cloud; set it to a
     vLLM endpoint (with ``api_key="EMPTY"``) to drive a local served teacher.
+
+    ``native_tools`` picks the tool-calling convention, and the two modes MUST NOT
+    be mixed on the same model:
+
+    * ``True`` (default — DATA GENERATION with a *base* model): pass the OpenAI
+      ``tools=`` param so the server's native tool template + parser (gemma4 /
+      qwen3_xml / hermes) drive the base model, which only emits reliable tool
+      calls that way. The rollout captures ``(name, arguments)`` and the serializer
+      re-renders them into the canonical JSON ``<tool_call>`` form regardless.
+    * ``False`` (SERVING a *fine-tuned* model): bake the ``<tools>`` block + JSON
+      ``<tool_call>`` format into the system prompt (exactly what the serializer
+      trained on) and DROP ``tools=``. If ``tools=`` is set here, the served chat
+      template injects its own XML ``<function=>`` instructions, which conflict
+      with the JSON format the model learned -> it falls back to ``\\boxed{}`` and
+      never routes. ``_parse_rl_tool_call`` scrapes the JSON tag from raw text.
     """
 
-    def call_orchestrator(system: str, user: str, specs: List[Dict[str, Any]]):
-        from openai import OpenAI
+    # Create the client ONCE and reuse it for every turn. Creating a fresh OpenAI
+    # client per call (as before) leaks an httpx connection pool each time -> under
+    # heavy parallel generation, sockets pile up in CLOSE-WAIT, the run degrades and
+    # has to be restarted. The orchestrator is the highest-frequency call, so reusing
+    # one client here removes the dominant leak. httpx clients are thread-safe.
+    from openai import OpenAI
 
-        client = OpenAI(base_url=base_url, api_key=api_key or "EMPTY", timeout=timeout)
+    from openjarvis.agents.hybrid.toolorchestra.tracing import wrap_client
+
+    client = wrap_client(OpenAI(base_url=base_url, api_key=api_key or "EMPTY", timeout=timeout))
+
+    def call_orchestrator(messages: List[Dict[str, Any]], specs: List[Dict[str, Any]]):
+        # ``messages`` is the full running conversation (system/user/assistant/tool)
+        # built by run_unified_rollout — pass it through so the model sees its own
+        # prior calls + tool responses, not a flattened user blob.
+        if native_tools:
+            # Base-model generation: native tool template drives the tool calls.
+            send = messages
+            kwargs = {"tools": specs} if specs else {}
+        else:
+            # Fine-tuned serving: JSON format is baked into the system prompt and
+            # tools= is dropped (see make_call_orchestrator docstring).
+            send = list(messages)
+            if specs and send and send[0].get("role") == "system":
+                send[0] = {"role": "system", "content": build_system_prompt(specs)}
+            kwargs = {}
+        # repetition_penalty is a vLLM extra (not OpenAI-native); only send it to a
+        # local vLLM endpoint (base_url set), never to the cloud frontier APIs.
+        if base_url and repetition_penalty and repetition_penalty != 1.0:
+            kwargs.setdefault("extra_body", {})["repetition_penalty"] = repetition_penalty
         resp = client.chat.completions.create(
             model=model,
-            messages=[
-                {"role": "system", "content": system},
-                {"role": "user", "content": user},
-            ],
+            messages=send,
             temperature=temperature,
             max_tokens=max_tokens,
-            tools=specs,
+            frequency_penalty=frequency_penalty,
+            presence_penalty=presence_penalty,
+            **kwargs,
         )
         msg = resp.choices[0].message
         text = msg.content or ""
@@ -78,26 +131,34 @@ def _dispatch_openjarvis_tool(
     try:
         executor = executor_holder.get("executor")
         if executor is None:
-            from openjarvis.core.registry import ToolRegistry
-            from openjarvis.tools._stubs import ToolExecutor
+            with _EXECUTOR_BUILD_LOCK:
+                # Re-check inside the lock: another thread may have built it while
+                # we waited.
+                executor = executor_holder.get("executor")
+                if executor is None:
+                    from openjarvis.core.registry import ToolRegistry
+                    from openjarvis.tools._stubs import ToolExecutor
 
-            instances = []
-            for name in ToolRegistry.keys():
-                entry = ToolRegistry.get(name)
-                try:
-                    instances.append(entry() if isinstance(entry, type) else entry)
-                except Exception:
-                    continue
-            # Auto-approve confirmation-gated tools (shell_exec, git_commit, ...):
-            # rollouts/eval run headless in a sandbox, so there is no human to
-            # confirm. Without this, those tools return a "requires confirmation"
-            # error instead of executing — which would silently break TerminalBench.
-            executor = ToolExecutor(
-                instances,
-                interactive=True,
-                confirm_callback=lambda _prompt: True,
-            )
-            executor_holder["executor"] = executor
+                    instances = []
+                    for name in ToolRegistry.keys():
+                        entry = ToolRegistry.get(name)
+                        try:
+                            instances.append(
+                                entry() if isinstance(entry, type) else entry
+                            )
+                        except Exception:
+                            continue
+                    # Auto-approve confirmation-gated tools (shell_exec,
+                    # git_commit, ...): rollouts/eval run headless in a sandbox, so
+                    # there is no human to confirm. Without this, those tools return
+                    # a "requires confirmation" error instead of executing — which
+                    # would silently break TerminalBench.
+                    executor = ToolExecutor(
+                        instances,
+                        interactive=True,
+                        confirm_callback=lambda _prompt: True,
+                    )
+                    executor_holder["executor"] = executor
 
         from openjarvis.core.types import ToolCall
 
@@ -126,7 +187,7 @@ def make_dispatch(
     cfg = cfg or {}
     executor_holder: Dict[str, Any] = {}
 
-    def dispatch(tool: ExpertTool, arguments: Dict[str, Any]):
+    def _dispatch_inner(tool: ExpertTool, arguments: Dict[str, Any]):
         # Bridged real OpenJarvis tools run through the ToolExecutor, not
         # the model/worker path.
         if tool.backend_type == "openjarvis-tool":
@@ -139,10 +200,32 @@ def make_dispatch(
             if isinstance(val, str) and val.strip():
                 prompt = val
                 break
+        if not prompt.strip():
+            # The orchestrator emitted a tool call with no/empty input. Don't hit
+            # the API (it 400s on empty content) — return a usable error so the
+            # rollout keeps going instead of dropping.
+            return (f"[{tool.name}: no input provided — supply a non-empty "
+                    "'input' to delegate]", 0.0, 0, True)
         text, p, c, is_local, extra_cost, _n = _call_worker(worker, prompt, cfg)
         usd = (0.0 if is_local else _model_cost(str(tool.model), p, c)) + float(extra_cost)
         return text, usd, int(p) + int(c), bool(is_local)
 
+    def dispatch(tool: ExpertTool, arguments: Dict[str, Any]):
+        # One span per route so the trace tree shows which expert/tool was called,
+        # with the delegated input, the returned observation, and cost/tokens. The
+        # wrapped OpenAI/Anthropic call (if any) nests one level deeper.
+        # Span is titled with the REAL model that actually ran (``tool.model``) so
+        # the trace reads clearly even under anonymization; ``anon_label`` in the
+        # metadata records the opaque label the orchestrator actually saw/chose.
+        real_model = str(tool.model)
+        with span(f"route:{real_model}", span_type="tool", input=arguments,
+                  metadata={"real_model": real_model, "anon_label": tool.name,
+                            "backend": tool.backend_type}) as s:
+            obs, usd, toks, is_local = _dispatch_inner(tool, arguments)
+            s.log(output=obs, metrics={"cost_usd": usd, "tokens": toks},
+                  metadata={"is_local": is_local})
+            return obs, usd, toks, is_local
+
     return dispatch
 
 

src/openjarvis/agents/hybrid/toolorchestra/workers.py

@@ -2,6 +2,7 @@
 
 from __future__ import annotations
 
+import os
 from pathlib import Path
 from typing import Any, Dict, List, Optional, Tuple
 
@@ -285,7 +286,7 @@ def _call_worker(
 ) -> Tuple[str, int, int, bool, float, int]:
     """Returns (text, p_tok, c_tok, is_local, extra_cost, n_web_searches)."""
     wtype = worker.get("type", "openai")
-    max_tok = int(cfg.get("worker_max_tokens", 4096))
+    max_tok = int(cfg.get("worker_max_tokens") or os.environ.get("OJ_WORKER_MAX_TOKENS", "4096"))
     temp = float(cfg.get("worker_temperature", 0.2))
 
     if wtype == "vllm":

src/openjarvis/agents/orchestrator.py

@@ -13,6 +13,7 @@ Supports two modes:
 from __future__ import annotations
 
 import concurrent.futures
+import json
 import re
 from typing import Any, List, Optional
 
@@ -201,6 +202,70 @@ class OrchestratorAgent(ToolUsingAgent):
 
         return result
 
+    @staticmethod
+    def _boxed_tool_call(content, openai_tools, call_id):
+        r"""Salvage a tool call the model wrote as ``\boxed{tool: query}``.
+
+        Some SFT'd checkpoints (Qwen reasoning-mode habit) emit their action
+        as a boxed string instead of a real ``<tool_call>`` — e.g.
+        ``\boxed{web_search: latest news}`` or ``\boxed{web_search query: ...}``.
+        The engine's tool-call parser sees no structured call, so without this
+        the agent would grade the half-finished thought as its final answer and
+        never actually act. Here we recover ``(tool, args)`` and build a real
+        ``ToolCall`` keyed on the tool's primary parameter. Returns ``None`` when
+        the boxed content isn't a recognisable tool action (leaving genuine
+        boxed final answers untouched).
+        """
+        if not openai_tools or "\\boxed" not in content:
+            return None
+
+        # tool name -> primary parameter (first required, else first property)
+        param_of: dict[str, str] = {}
+        for t in openai_tools:
+            fn = t.get("function", t)
+            name = fn.get("name")
+            if not name:
+                continue
+            params = fn.get("parameters") or {}
+            props = list((params.get("properties") or {}).keys())
+            req = params.get("required") or []
+            param_of[name] = req[0] if req else (props[0] if props else "query")
+
+        # The action is the LAST boxed expression in the response.
+        boxed = re.findall(r"\\boxed\{([^{}]*)\}", content)
+        if not boxed:
+            return None
+        inner = boxed[-1].strip()
+
+        # Match a leading tool name (longest first to avoid prefix collisions),
+        # then strip the separator between name and args. Handles
+        # ``tool: q`` / ``tool query: q`` / ``tool q`` / ``tool(q)``.
+        for name in sorted(param_of, key=len, reverse=True):
+            m = re.match(
+                r"^\s*" + re.escape(name) + r"\b(.*)$",
+                inner,
+                re.DOTALL | re.IGNORECASE,
+            )
+            if not m:
+                continue
+            rest = m.group(1).strip()
+            if rest.startswith("(") and rest.endswith(")"):
+                rest = rest[1:-1]  # tool(args) wrapper -> drop matching parens
+            else:
+                # optional "query" word, then a ":" / "-" separator
+                rest = re.sub(
+                    r"^\s*(?:query\s*)?[:\-]\s*", "", rest, flags=re.IGNORECASE
+                )
+            arg = rest.strip().strip("\"'").strip()
+            if not arg:
+                return None
+            return ToolCall(
+                id=call_id,
+                name=name,
+                arguments=json.dumps({param_of[name]: arg}),
+            )
+        return None
+
     # ------------------------------------------------------------------
     # Function-calling mode (original behaviour)
     # ------------------------------------------------------------------
@@ -245,8 +310,31 @@ class OrchestratorAgent(ToolUsingAgent):
             content = result.get("content", "")
             raw_tool_calls = result.get("tool_calls", [])
 
-            # No tool calls -> check continuation, then final answer
+            # No tool calls -> try to salvage a \boxed{tool: query} action the
+            # model wrote instead of a real <tool_call>; else final answer.
             if not raw_tool_calls:
+                boxed_call = self._boxed_tool_call(
+                    content, openai_tools, f"boxed_{turns}"
+                )
+                if boxed_call is not None:
+                    messages.append(
+                        Message(
+                            role=Role.ASSISTANT,
+                            content=content,
+                            tool_calls=[boxed_call],
+                        )
+                    )
+                    tool_result = self._executor.execute(boxed_call)
+                    all_tool_results.append(tool_result)
+                    messages.append(
+                        Message(
+                            role=Role.TOOL,
+                            content=tool_result.content,
+                            tool_call_id=boxed_call.id,
+                            name=boxed_call.name,
+                        )
+                    )
+                    continue
                 content = self._check_continuation(result, messages)
                 content = self._strip_think_tags(content)
                 self._emit_turn_end(turns=turns, content_length=len(content))

src/openjarvis/evals/core/scorer.py

@@ -2,12 +2,44 @@
 
 from __future__ import annotations
 
+import logging
+import random
+import time
 from abc import ABC, abstractmethod
 from typing import Any, Dict, Optional, Tuple
 
 from openjarvis.evals.core.backend import InferenceBackend
 from openjarvis.evals.core.types import EvalRecord
 
+LOGGER = logging.getLogger(__name__)
+
+# Transient failures worth retrying the judge call on. A single rate-limit
+# (429) used to fall straight through to `llm_fallback_error` and score the
+# sample WRONG — e.g. 93/100 GAIA judge calls 429'd on one run and zeroed the
+# whole bench. Retry with exponential backoff so a busy judge endpoint doesn't
+# get mis-scored as a wrong answer.
+_JUDGE_MAX_RETRIES = 6
+_JUDGE_BASE_DELAY_S = 2.0
+_JUDGE_MAX_DELAY_S = 60.0
+_RETRYABLE_MARKERS = (
+    "429",
+    "rate_limit",
+    "rate limit",
+    "overloaded",
+    "timeout",
+    "timed out",
+    "503",
+    "502",
+    "500",
+    "connection",
+    "temporarily unavailable",
+)
+
+
+def _is_retryable_judge_error(exc: Exception) -> bool:
+    msg = str(exc).lower()
+    return any(marker in msg for marker in _RETRYABLE_MARKERS)
+
 
 class Scorer(ABC):
     """Base class for all scorers."""
@@ -42,14 +74,36 @@ class LLMJudgeScorer(Scorer):
         temperature: float = 0.0,
         max_tokens: int = 2048,
     ) -> str:
-        """Send a prompt to the judge LLM and return the response text."""
-        return self._judge_backend.generate(
-            prompt,
-            model=self._judge_model,
-            system=system,
-            temperature=temperature,
-            max_tokens=max_tokens,
-        )
+        """Send a prompt to the judge LLM and return the response text.
+
+        Retries transient failures (429 / rate-limit / 5xx / timeout) with
+        exponential backoff + jitter so a busy judge endpoint doesn't get
+        mis-scored as a wrong answer. Non-retryable errors, or exhaustion of
+        the retry budget, re-raise to the caller (which records the failure).
+        """
+        last_exc: Optional[Exception] = None
+        for attempt in range(_JUDGE_MAX_RETRIES):
+            try:
+                return self._judge_backend.generate(
+                    prompt,
+                    model=self._judge_model,
+                    system=system,
+                    temperature=temperature,
+                    max_tokens=max_tokens,
+                )
+            except Exception as exc:  # noqa: BLE001 - re-raised below
+                last_exc = exc
+                if attempt == _JUDGE_MAX_RETRIES - 1 or not _is_retryable_judge_error(exc):
+                    raise
+                delay = min(_JUDGE_BASE_DELAY_S * (2 ** attempt), _JUDGE_MAX_DELAY_S)
+                delay += random.uniform(0.0, delay * 0.25)  # jitter
+                LOGGER.warning(
+                    "judge call failed (attempt %d/%d): %s — retrying in %.1fs",
+                    attempt + 1, _JUDGE_MAX_RETRIES, exc, delay,
+                )
+                time.sleep(delay)
+        # Unreachable (loop either returns or raises), but keeps type-checkers happy.
+        raise last_exc  # type: ignore[misc]
 
 
 __all__ = ["LLMJudgeScorer", "Scorer"]

src/openjarvis/evals/scorers/gaia_exact.py

@@ -140,7 +140,14 @@ class GAIAScorer(LLMJudgeScorer):
             if structured_match:
                 is_correct = structured_match.group(1).lower() == "yes"
             else:
-                is_correct = "CORRECT" in raw.upper() and "INCORRECT" not in raw.upper()
+                up = raw.upper()
+                # "not correct" / "is not correct" contain "CORRECT" and lack
+                # "INCORRECT" — guard the negations explicitly or they read True.
+                is_correct = (
+                    "CORRECT" in up
+                    and "INCORRECT" not in up
+                    and "NOT CORRECT" not in up
+                )
 
             meta: Dict[str, Any] = {
                 "match_type": "llm_fallback",

src/openjarvis/evals/scorers/mmlu_pro_mcq.py

@@ -27,6 +27,25 @@ class MMLUProScorer(LLMJudgeScorer):
             return "".join(chr(ord("A") + i) for i in range(n))
         return "ABCDEFGHIJ"
 
+    def _extract_answer_direct(
+        self,
+        model_answer: str,
+        valid_letters: str,
+    ) -> Optional[str]:
+        """High-confidence, unambiguous letter straight from the output.
+
+        Only an explicit ``\\boxed{X}`` counts here. We deliberately do NOT regex
+        prose like "the answer is X": the letters "I" and "A" are real English
+        words, so "FINAL_ANSWER: I need ..." would grab "I" — the exact bug that
+        mislabels answers. Semantic extraction is left to the judge below.
+        """
+        if not model_answer:
+            return None
+        for cand in reversed(re.findall(r"\\boxed\{\s*([A-Za-z])\s*\}", model_answer)):
+            if cand.upper() in valid_letters:
+                return cand.upper()
+        return None
+
     def _extract_answer_with_llm(
         self,
         problem: str,
@@ -56,10 +75,13 @@ class MMLUProScorer(LLMJudgeScorer):
             )
 
             extracted = raw_response.strip().upper()
+            if "NONE" in extracted:
+                return None
 
-            # Handle "The answer is: A" etc.
+            # Accept only an ISOLATED letter — never the first cap of a word like
+            # "It"/"None". Prefer an explicit "the answer is X" phrasing.
             answer_match = re.search(
-                r"(?:THE ANSWER IS:?\s*)?([A-Z])",
+                r"(?:THE ANSWER IS:?\s*)?\b([A-Z])\b(?![A-Za-z'])",
                 extracted,
                 re.IGNORECASE,
             )
@@ -86,11 +108,17 @@ class MMLUProScorer(LLMJudgeScorer):
 
         valid_letters = self._valid_letters_from_options(record.metadata)
 
-        candidate = self._extract_answer_with_llm(
-            record.problem,
-            model_answer,
-            valid_letters,
-        )
+        # Parse straight from the output first (fast, deterministic, no "I" bug);
+        # only fall back to the judge when no isolated letter is present.
+        method = "direct"
+        candidate = self._extract_answer_direct(model_answer, valid_letters)
+        if not candidate:
+            method = "llm"
+            candidate = self._extract_answer_with_llm(
+                record.problem,
+                model_answer,
+                valid_letters,
+            )
         if not candidate:
             return None, {"reason": "no_choice_letter_extracted"}
 
@@ -99,6 +127,7 @@ class MMLUProScorer(LLMJudgeScorer):
             "reference_letter": ref,
             "candidate_letter": candidate,
             "valid_letters": valid_letters,
+            "extract_method": method,
         }
         return is_correct, meta
 

src/openjarvis/evals/scorers/supergpqa_mcq.py

@@ -27,6 +27,25 @@ class SuperGPQAScorer(LLMJudgeScorer):
             return "".join(chr(ord("A") + i) for i in range(n))
         return "ABCD"
 
+    def _extract_answer_direct(
+        self,
+        model_answer: str,
+        valid_letters: str,
+    ) -> Optional[str]:
+        """High-confidence, unambiguous letter straight from the output.
+
+        Only an explicit ``\\boxed{X}`` counts here. We deliberately do NOT regex
+        prose like "the answer is X": the letters "I" and "A" are real English
+        words, so "FINAL_ANSWER: I need ..." would grab "I" — the exact bug that
+        mislabels answers. Semantic extraction is left to the judge below.
+        """
+        if not model_answer:
+            return None
+        for cand in reversed(re.findall(r"\\boxed\{\s*([A-Za-z])\s*\}", model_answer)):
+            if cand.upper() in valid_letters:
+                return cand.upper()
+        return None
+
     def _extract_answer_with_llm(
         self,
         problem: str,
@@ -56,10 +75,13 @@ class SuperGPQAScorer(LLMJudgeScorer):
             )
 
             extracted = raw_response.strip().upper()
+            if "NONE" in extracted:
+                return None
 
-            # Handle "The answer is: A" etc.
+            # Accept only an ISOLATED letter — never the first cap of a word like
+            # "It"/"None". Prefer an explicit "the answer is X" phrasing.
             answer_match = re.search(
-                r"(?:THE ANSWER IS:?\s*)?([A-Z])",
+                r"(?:THE ANSWER IS:?\s*)?\b([A-Z])\b(?![A-Za-z'])",
                 extracted,
                 re.IGNORECASE,
             )
@@ -86,11 +108,17 @@ class SuperGPQAScorer(LLMJudgeScorer):
 
         valid_letters = self._valid_letters_from_options(record.metadata)
 
-        candidate = self._extract_answer_with_llm(
-            record.problem,
-            model_answer,
-            valid_letters,
-        )
+        # Parse straight from the output first (fast, deterministic, no "I" bug);
+        # only fall back to the judge when no isolated letter is present.
+        method = "direct"
+        candidate = self._extract_answer_direct(model_answer, valid_letters)
+        if not candidate:
+            method = "llm"
+            candidate = self._extract_answer_with_llm(
+                record.problem,
+                model_answer,
+                valid_letters,
+            )
         if not candidate:
             return None, {"reason": "no_choice_letter_extracted"}
 
@@ -99,6 +127,7 @@ class SuperGPQAScorer(LLMJudgeScorer):
             "reference_letter": ref,
             "candidate_letter": candidate,
             "valid_letters": valid_letters,
+            "extract_method": method,
         }
         return is_correct, meta
 

src/openjarvis/learning/intelligence/orchestrator/eval_backend.py

@@ -18,9 +18,28 @@ inside ``call_orchestrator`` at generate time (lazy import in ``unified.py``).
 
 from __future__ import annotations
 
+import re
 import time
 from typing import Any, Dict, Optional
 
+# The served fine-tuned model sometimes over-emits the answer marker, e.g.
+# "FINAL_ANSWER: FINAL_ANSWER: 42" — a doubled prefix that breaks answer
+# extraction and auto-scores the sample 0. Collapse any run of FINAL_ANSWER
+# markers to a single one, keeping the text after the LAST marker as the answer.
+_FA_MARK = re.compile(r"(?im)FINAL[_\s]?ANSWER\s*:?")
+
+
+def _clean_final_answer(text: str) -> str:
+    t = (text or "").strip()
+    marks = list(_FA_MARK.finditer(t))
+    if not marks:
+        return t
+    answer = t[marks[-1].end():].strip()
+    return f"FINAL_ANSWER: {answer}" if answer else t
+
+
+_OBS_CAP = 4000  # cap persisted observations so the eval JSONL stays readable
+
 from openjarvis.agents.hybrid.expert_registry import orchestrator_catalog
 from openjarvis.agents.hybrid.toolorchestra import rollout as _rollout_mod
 from openjarvis.agents.hybrid.toolorchestra.unified import (
@@ -138,6 +157,9 @@ class OrchestratorBackend(InferenceBackend):
                 api_key=self.api_key,
                 temperature=temp,
                 max_tokens=max_tokens,
+                # Serving a fine-tuned model: JSON <tool_call> format in the system
+                # prompt, no tools= param (matches how it was trained/serialized).
+                native_tools=False,
             )
             dispatch = make_dispatch(self._dispatch_cfg)
             rollout = _rollout_mod.run_unified_rollout(
@@ -146,6 +168,10 @@ class OrchestratorBackend(InferenceBackend):
                 call_orchestrator=call_orch,
                 dispatch=dispatch,
                 max_turns=self.max_turns,
+                # Match TRAINING: the model was fine-tuned on anonymized expert
+                # labels (expert_xxxx), so it only picks valid labels / routes
+                # correctly when it sees the same anonymized names at inference.
+                anonymize=True,
             )
         except Exception as exc:  # noqa: BLE001 - surface as a failed sample
             return {
@@ -162,8 +188,9 @@ class OrchestratorBackend(InferenceBackend):
 
         latency = time.time() - started
         total_tokens = int(getattr(rollout, "tokens", 0) or 0)
+        anon_map = getattr(rollout, "anon_map", None) or {}
         return {
-            "content": rollout.final_answer or "",
+            "content": _clean_final_answer(rollout.final_answer or ""),
             # The rollout reports a single combined token count; we surface it
             # as completion_tokens so it still flows into total-token telemetry.
             "usage": {
@@ -181,6 +208,23 @@ class OrchestratorBackend(InferenceBackend):
                 "tool_calls": [
                     {"name": n, "arguments": a} for n, a in rollout.tool_calls()
                 ],
+                # Full step-by-step trace so eval samples are inspectable
+                # (format_eval_sample.py renders this). Observations capped.
+                "anon_map": anon_map,
+                "turns": [
+                    {
+                        "reasoning": t.reasoning or "",
+                        "tool_name": t.tool_name,
+                        "real_model": anon_map.get(t.tool_name) if t.tool_name else None,
+                        "arguments": t.arguments,
+                        "observation": (
+                            (t.observation or "")[:_OBS_CAP]
+                            + ("…[truncated]" if t.observation and len(t.observation) > _OBS_CAP else "")
+                        ),
+                    }
+                    for t in (getattr(rollout, "turns", []) or [])
+                ],
+                "final_answer": _clean_final_answer(rollout.final_answer or ""),
             },
         }
 

src/openjarvis/learning/intelligence/orchestrator/grpo_trainer.py

@@ -381,10 +381,10 @@ class OrchestratorGRPOTrainer:
         return records
 
     def _make_policy_caller(self, turn_buffer):
-        """Build a policy-driven ``call_orchestrator(system, user, specs)``.
+        """Build a policy-driven ``call_orchestrator(messages, specs)``.
 
-        Tokenizes ``system+user`` with the model's chat template, generates ONE
-        assistant turn with the current policy, parses any
+        Tokenizes the running ``messages`` conversation with the model's chat
+        template, generates ONE assistant turn with the current policy, parses any
         ``<tool_call>{...}</tool_call>``, appends the per-turn
         ``(input_ids, generated_ids)`` to ``turn_buffer`` (for the later
         log-prob pass), and returns
@@ -397,11 +397,7 @@ class OrchestratorGRPOTrainer:
 
         tok = self.policy.tokenizer
 
-        def call_orchestrator(system: str, user: str, specs):
-            messages = [
-                {"role": "system", "content": system},
-                {"role": "user", "content": user},
-            ]
+        def call_orchestrator(messages, specs):
             input_ids = tok.apply_chat_template(
                 messages, add_generation_prompt=True, tokenize=True
             )

src/openjarvis/learning/intelligence/orchestrator/sft_data/datasets.py

@@ -38,7 +38,10 @@ class Task:
     task_id: str
     question: str
     answer: str
-    domain: str
+    domain: str            # coarse area: math / code / science / medical / chat / misc
+    difficulty: str = ""   # OpenThoughts difficulty tier (GeneralThought has none)
+    dataset: str = ""      # source dataset: GeneralThought / OpenThoughts3
+    subsector: str = ""    # fine source: NuminaMath / NHSQA / TACO / glaive / ...
 
     @property
     def instruction(self) -> str:
@@ -90,6 +93,9 @@ def _normalize_generalthought(row: Dict[str, Any], *, index: int = 0) -> Optiona
         question=question,
         answer=answer,
         domain=_generalthought_domain(row),
+        difficulty=str(row.get("difficulty") or "").strip(),  # usually absent for GT
+        dataset="GeneralThought",
+        subsector=str(row.get("question_source") or "").strip(),
     )
 
 
@@ -98,12 +104,15 @@ def load_generalthought(
     n: int = 2000,
     seed: int = 42,
     source: Optional[Iterable[Dict[str, Any]]] = None,
+    buffer: Optional[int] = None,
 ) -> Iterator[Task]:
     """Yield up to ``n`` GeneralThought tasks, randomly mixed across categories.
 
     ``source`` overrides the HF stream with an iterable of raw row dicts (tests).
     We over-stream a buffer and shuffle it so the yielded tasks are a random mix
-    of the source's categories rather than a single leading shard.
+    of the source's categories rather than a single leading shard. ``buffer``
+    overrides the shuffle-buffer size; pass a small value (e.g. for smoke runs)
+    to avoid streaming the default 6000-row floor.
     """
     if source is None:
         from datasets import load_dataset  # lazy: optional dep / network
@@ -113,7 +122,7 @@ def load_generalthought(
     rng = random.Random(seed)
     # Buffer a generous multiple of n so the shuffle actually mixes categories,
     # but stay bounded for the multi-hundred-K streams.
-    buf_cap = max(n * 6, 6000)
+    buf_cap = buffer if buffer is not None else max(n * 6, 6000)
     buf: List[Task] = []
     for i, row in enumerate(source):
         task = _normalize_generalthought(dict(row), index=i)
@@ -186,7 +195,29 @@ def _normalize_openthoughts(row: Dict[str, Any], *, index: int = 0) -> Optional[
         return None
     domain = str(row.get("domain") or "unknown").strip().lower() or "unknown"
     task_id = str(row.get("id") or row.get("source") or f"openthoughts-{index}") + f"-{index}"
-    return Task(task_id=task_id, question=question, answer=answer, domain=domain)
+    return Task(
+        task_id=task_id,
+        question=question,
+        answer=answer,
+        domain=domain,
+        difficulty=str(row.get("difficulty") or "").strip(),
+        dataset="OpenThoughts3",
+        subsector=str(row.get("source") or "").strip(),
+    )
+
+
+# The 1.2M set ships as 120 uniform parquet shards with no domain in the file
+# names, but the rows are front-ordered by domain. Probing one row per shard puts
+# code in shards ~0-30, math ~32-104, science ~105-119. Streaming the single
+# combined split therefore has to read the entire code (+ math) prefix before it
+# reaches math/science — minutes of wasted I/O. Instead we stream each domain from
+# shards *inside* its own region. A few shards (~10K rows each) cover any quota.
+_OT_SHARD_FMT = "data/train-{:05d}-of-00120.parquet"
+_OT_DOMAIN_SHARDS: Dict[str, List[int]] = {
+    "code": [0, 1, 2, 3],
+    "math": [45, 46, 47, 48],
+    "science": [112, 113, 114, 115, 116, 117, 118, 119],
+}
 
 
 def load_openthoughts(
@@ -199,26 +230,43 @@ def load_openthoughts(
 ) -> Iterator[Task]:
     """Yield OpenThoughts3 tasks balanced across code / math / science.
 
-    The HF stream is sharded by domain (all-code first), so we route rows into
-    per-domain buffers and stop once every quota is met. ``source`` overrides the
-    stream with raw row dicts (tests).
+    ``source`` overrides the stream with raw row dicts (tests): rows are routed
+    into per-domain buffers, stopping once every quota is met. With no override we
+    stream each domain from its own shard region (see ``_OT_DOMAIN_SHARDS``) so a
+    balanced pull never has to read the entire front-ordered code/math prefix.
     """
-    if source is None:
-        from datasets import load_dataset  # lazy: optional dep / network
-
-        source = load_dataset(OPENTHOUGHTS_ID, split="train", streaming=True)
-
     quotas = {"code": n_code, "math": n_math, "science": n_science}
     bufs: Dict[str, List[Task]] = {k: [] for k in quotas}
-    for i, row in enumerate(source):
-        task = _normalize_openthoughts(dict(row), index=i)
-        if task is None:
-            continue
-        dom = task.domain
-        if dom in bufs and len(bufs[dom]) < quotas[dom]:
-            bufs[dom].append(task)
-        if all(len(bufs[k]) >= quotas[k] for k in quotas):
-            break
+
+    if source is not None:
+        for i, row in enumerate(source):
+            task = _normalize_openthoughts(dict(row), index=i)
+            if task is None:
+                continue
+            dom = task.domain
+            if dom in bufs and len(bufs[dom]) < quotas[dom]:
+                bufs[dom].append(task)
+            if all(len(bufs[k]) >= quotas[k] for k in quotas):
+                break
+    else:
+        from datasets import load_dataset  # lazy: optional dep / network
+
+        idx = 0
+        for dom, quota in quotas.items():
+            if quota <= 0:
+                continue
+            files = [_OT_SHARD_FMT.format(s) for s in _OT_DOMAIN_SHARDS[dom]]
+            ds = load_dataset(
+                OPENTHOUGHTS_ID, data_files=files, split="train", streaming=True
+            )
+            for row in ds:
+                task = _normalize_openthoughts(dict(row), index=idx)
+                idx += 1
+                if task is None or task.domain != dom:
+                    continue
+                bufs[dom].append(task)
+                if len(bufs[dom]) >= quota:
+                    break
 
     rng = random.Random(seed)
     out: List[Task] = []
@@ -232,8 +280,33 @@ def load_openthoughts(
 # --- combined SFT / GRPO sets -------------------------------------------------
 
 
-def load_sft_tasks(*, seed: int = 42) -> List[Task]:
-    """The 8K cold-start SFT set: 2K GeneralThought + 2K code + 2K math + 2K science."""
+def load_sft_tasks(
+    *, seed: int = 42, cap: Optional[int] = None, balanced: bool = True
+) -> List[Task]:
+    """The 8K cold-start SFT set: 2K GeneralThought + 2K code + 2K math + 2K science.
+
+    ``cap`` (smoke runs): when set, draw ~``cap`` tasks total.
+      - default (``balanced=False``): GeneralThought only with a small stream
+        buffer — fastest, but the domain mix is whatever GeneralThought yields
+        (math/code/medical/chat), so a given sample can skew (e.g. all-medical).
+      - ``balanced=True``: ~cap/4 from each of GeneralThought + OpenThoughts
+        code/math/science, so the smoke is representative of the real run. Now
+        cheap because OpenThoughts streams from per-domain shards (no front-prefix).
+    """
+    if cap is not None and cap > 0:
+        if balanced:
+            per = max(cap // 4, 1)
+            tasks = list(load_generalthought(n=per, seed=seed, buffer=max(per * 6, 64)))
+            tasks.extend(
+                load_openthoughts(n_code=per, n_math=per, n_science=per, seed=seed)
+            )
+            rng = random.Random(seed)
+            rng.shuffle(tasks)
+            return tasks[:cap]
+        buf = max(cap * 4, 64)
+        tasks = list(load_generalthought(n=cap, seed=seed, buffer=buf))
+        random.Random(seed).shuffle(tasks)
+        return tasks[:cap]
     tasks: List[Task] = []
     tasks.extend(load_generalthought(n=2000, seed=seed))
     tasks.extend(load_openthoughts(n_code=2000, n_math=2000, n_science=2000, seed=seed))

src/openjarvis/learning/intelligence/orchestrator/sft_data/reject_sample.py

@@ -19,6 +19,7 @@ from __future__ import annotations
 
 import json
 import logging
+import re
 from collections import Counter
 from pathlib import Path
 from typing import Callable, Iterable, List, Optional
@@ -28,6 +29,8 @@ from typing import Any
 from openjarvis.agents.hybrid.expert_registry import ExpertTool
 from openjarvis.agents.hybrid.toolorchestra.rollout import UnifiedRollout
 from openjarvis.learning.intelligence.orchestrator.sft_data.unified_serialize import (
+    _CONTROL_TOKEN_RE,
+    _strip_control_tokens,
     trajectory_to_record,
 )
 
@@ -42,6 +45,177 @@ TaskLike = Any
 RolloutFn = Callable[[TaskLike], Optional[UnifiedRollout]]
 VerifyFn = Callable[[TaskLike, UnifiedRollout], bool]
 
+# Human-readable blurb for each source dataset, stamped onto every record as
+# ``dataset_description`` so a row is self-describing (what corpus the question
+# came from) without a lookup. Keyed on the record's ``dataset`` value.
+DATASET_DESCRIPTIONS = {
+    "GeneralThought": (
+        "natolambert/GeneralThought-430K-filtered — filtered reasoning Q&A from "
+        "gr.inc; each item has a question and a short reference (gold) answer "
+        "distilled from DeepSeek-R1 traces."
+    ),
+    "OpenThoughts3": (
+        "open-thoughts/OpenThoughts3-1.2M — code/math/science reasoning tasks; "
+        "the gold answer is extracted from the boxed final solution."
+    ),
+}
+
+
+def dataset_description(dataset: str) -> str:
+    """Blurb for a source dataset name (empty string if unknown)."""
+    return DATASET_DESCRIPTIONS.get((dataset or "").strip(), "")
+
+
+# Markers of a broken expert/observation that must never become a training
+# target (empty turns, tracebacks, truncated tool errors, dead-provider strings).
+_ERR_MARKERS = (
+    "Traceback (most recent call last)",
+    "insufficient_quota",
+    "rate limit",
+    "RateLimitError",
+    "[invalid tool",
+    "Error code: 4",
+    "Error code: 5",
+    "InternalServerError",
+    "ConnectionError",
+    "timed out",
+    # A tool call that reached dispatch with no/empty input, or a bridged-tool
+    # crash — the call was malformed. The trajectory routed garbage; drop it.
+    "no input provided",
+    "[openjarvis-tool error",
+)
+
+# A final answer cut off mid-expression ends on a dangling connective
+# (``a=1, b=``, ``result = ``, ``compute (``) — the decode hit the token cap
+# before finishing. A real distilled answer never ends this way.
+_TRUNCATED_TAIL_RE = re.compile(r"[=,(\[{+\-*/]\s*$")
+
+
+def _target_is_clean(roll: UnifiedRollout) -> bool:
+    """Structural gate on whether a trajectory is fit to be an SFT *target*.
+
+    The judge decides semantic correctness; this catches the ~10-12% of records
+    whose targets are empty / a Traceback / a truncated tool error / unrouted —
+    garbage the model must never be trained to imitate. Requirements:
+      * non-empty final answer, no error marker, balanced ``<think>`` (not
+        truncated mid-thought);
+      * at least one model-expert tool call (it actually *routed*);
+      * no error-marker / empty observation in the kept trajectory.
+    """
+    fa = (roll.final_answer or "").strip()
+    if not fa:
+        return False
+    if any(m in fa for m in _ERR_MARKERS):
+        return False
+    if fa.count("<think>") != fa.count("</think>"):  # unbalanced think tags (truncated OR stray </think>)
+        return False
+    if _TRUNCATED_TAIL_RE.search(fa):  # truncated mid-expression (e.g. "a=1, b=")
+        return False
+    # Malformed final: a proper final answer is plain text, NOT a stray/broken
+    # <tool_call> tag (the model's answer leaking inside a tool call that failed
+    # to parse). (\boxed{} is NOT rejected here — the serializer de-boxes it, so
+    # an otherwise-good boxed answer is salvaged rather than thrown away.)
+    if "<tool_call>" in fa or "</tool_call>" in fa:
+        return False
+    # Control-token backstop (defense in depth). The serializer strips leaked
+    # control/special tokens (<|im_end|>, <|tool_call>, <start_of_turn>, …) to
+    # salvage good answers, so we mirror that strip and gate on the RESULT:
+    #   * strips to empty  -> the answer WAS nothing but control tokens -> drop.
+    #   * a token survives the strip -> something malformed the stripper can't
+    #     safely delete mid-answer -> drop rather than train on it.
+    # A clean answer (or one the stripper fully salvages) sails through.
+    fa_stripped = _strip_control_tokens(fa)
+    if not fa_stripped:
+        return False
+    if _CONTROL_TOKEN_RE.search(fa_stripped):
+        return False
+    # Degenerate repetition: the same substantive line emitted many times (the
+    # small-model decode loop). Reject rather than train on it.
+    _lines = [ln.strip() for ln in fa.split("\n") if len(ln.strip()) > 30]
+    if _lines and Counter(_lines).most_common(1)[0][1] >= 4:
+        return False
+    # Word-salad run-on: a giant unbroken line (no newline) is the other decode
+    # collapse — the model spraying novel tokens instead of repeating. Reject.
+    if any(len(ln) > 2000 for ln in fa.split("\n")):
+        return False
+    # Markdown-table dump: the model wrote a formatted table/essay instead of the
+    # short exact value the format demands. A ``|---|`` separator row is the tell.
+    if re.search(r"\|\s*:?-{2,}", fa):
+        return False
+    # Essay-style final: the format wants a distilled answer, not a multi-section
+    # writeup. Check the answer AFTER the FINAL_ANSWER marker (reasoning before it
+    # is fine). NOTE: length/bold limits relaxed for the BALANCED/harder mix —
+    # code & multi-step math answers are legitimately longer and use **bold** for
+    # the key result, so the old 700-char + any-bold rejects were dropping ~half
+    # the correct hard-task answers. Keep the STRUCTURAL essay signals (many
+    # numbered sections, markdown headers, tables) which still catch real essays.
+    _marks = list(re.finditer(r"(?im)FINAL[_\s]?ANSWER\s*:?", fa))
+    _ans = fa[_marks[-1].end():].strip() if _marks else fa
+    if len(_ans) > 2000:                                 # was 700 — too tight for code/math
+        return False
+    # Tool-status echo as the final answer: on code tasks the orchestrator
+    # sometimes ends with a file_write/shell status line ("Successfully wrote to
+    # /tmp/x.py") instead of the actual answer — a non-answer that slips the
+    # length/format checks. Reject. (Audit: ~code/math trajectories where the
+    # trace collapsed but the row was still scored correct.)
+    if re.search(r"(?i)\b(successfully (wrote|created|saved|executed|ran)|written to /|"
+                 r"file (written|saved|created)|no further actions?)\b", _ans):
+        return False
+    if len(re.findall(r"(?m)^\s*\d+\.\s", _ans)) >= 6:   # many numbered sections = essay (was 4)
+        return False
+    if re.search(r"(?m)^\s*#{1,6}\s", _ans):             # markdown headers = essay
+        return False
+    # Garbled / shouty final: a multi-word answer that's mostly UPPERCASE, or has
+    # an absurdly long merged all-letter token, is decode garble — the audit found
+    # e.g. "RABES PEPTETANUS BOOSTERS" (for "rabies PEP, tetanus boosters") passing.
+    if len(_ans) > 20:
+        _alpha = [c for c in _ans if c.isalpha()]
+        if (_alpha and sum(c.isupper() for c in _alpha) / len(_alpha) > 0.7
+                and len(_ans.split()) >= 3):
+            return False
+        if any(len(w) > 25 and w.isalpha() for w in _ans.split()):
+            return False
+    # Reject runaway final answers — a distilled answer, not a multi-KB essay or
+    # a verbatim dump of an expert observation. (Audit: 279 finals >4k chars, the
+    # worst a 409k-char wordlist; ~363 were prefix-identical to a tool obs.)
+    if len(fa) > 8000:
+        return False
+    fa_head = re.sub(r"\s+", " ", fa[:200]).strip()
+    for t in roll.turns:
+        if t.observation:
+            obs_head = re.sub(r"\s+", " ", t.observation[:200]).strip()
+            if fa_head and fa_head == obs_head and len(fa) > 200:  # long final = copied tool dump (short relays are legit)
+                return False
+    # "routed" = delegated to a model EXPERT (not just a utility like web_search).
+    # When anonymized, expert calls are the anon labels in anon_map; otherwise
+    # fall back to "made any tool call".
+    expert_names = set(roll.anon_map or {})
+    called = {name for name, _ in roll.tool_calls()}
+    if expert_names:
+        if not (called & expert_names):
+            return False
+    elif roll.num_tool_calls < 1:
+        return False
+    for t in roll.turns:
+        if t.tool_name is not None:
+            obs = (t.observation or "").strip()
+            if not obs or any(m in obs for m in _ERR_MARKERS):
+                return False
+    # Reasoning-side decode collapse: an intermediate turn whose reasoning is a
+    # giant unbroken line, or is dominated by exotic unicode / emoji spam, is
+    # garbage even when the final answer looks fine (the answer-only guards above
+    # miss it). The audit found a trajectory with a thousands-char symbol blob in
+    # its reasoning that scored correct and slipped through.
+    for t in roll.turns:
+        r = t.reasoning or ""
+        if any(len(ln) > 2000 for ln in r.split("\n")):
+            return False
+        if len(r) > 200:
+            weird = sum(1 for ch in r if ord(ch) > 0x2100 and not ch.isalnum())
+            if weird / len(r) > 0.10:  # >10% exotic-unicode/emoji = decode spam
+                return False
+    return True
+
 
 def gold_coverage_verify(task: TaskLike, rollout: UnifiedRollout) -> bool:
     """Dependency-free proxy verifier: trajectory must (a) produce a non-empty
@@ -59,6 +233,43 @@ def gold_coverage_verify(task: TaskLike, rollout: UnifiedRollout) -> bool:
     return gold.issubset(called)
 
 
+def _process_task(
+    task: TaskLike,
+    *,
+    rollout_fn: RolloutFn,
+    verify_fn: VerifyFn,
+    samples_per_task: int,
+    max_keep_per_task: int,
+    stop_at_keep: bool = False,
+    keep_all: bool = False,
+) -> List[tuple[UnifiedRollout, bool]]:
+    """One task's rejection-sampling unit of work: roll out ``samples_per_task``
+    times and verify each. Returns ``(rollout, is_correct)`` for **every** sample
+    (the caller decides what to write). Runs in a worker thread, so the only
+    shared state it touches is the injected fns (each rollout builds its own
+    OpenAI client; the tool executor is built under a lock).
+
+    ``stop_at_keep``: short-circuit as soon as ``max_keep_per_task`` passing
+    trajectories are found, instead of always exhausting ``samples_per_task``.
+    Big throughput win (no wasted rollouts on already-solved tasks), but it
+    trades away the cheapest-of-N cost optimisation (we keep the first passers,
+    not the cheapest). Ignored when ``keep_all`` is set (we want every sample).
+    Default off preserves the original semantics."""
+    results: List[tuple[UnifiedRollout, bool]] = []
+    n_correct = 0
+    for _ in range(samples_per_task):
+        roll = rollout_fn(task)
+        if roll is None:
+            continue
+        ok = bool(verify_fn(task, roll))
+        results.append((roll, ok))
+        if ok:
+            n_correct += 1
+            if stop_at_keep and not keep_all and n_correct >= max_keep_per_task:
+                break
+    return results
+
+
 def generate_sft_dataset(
     out_path: str,
     *,
@@ -69,55 +280,153 @@ def generate_sft_dataset(
     samples_per_task: int = 4,
     max_keep_per_task: int = 1,
     reward_fn: Optional[Callable[[UnifiedRollout], float]] = None,
+    concurrency: int = 1,
+    stop_at_keep: bool = False,
+    keep_all: bool = False,
+    record_extra: Optional[dict] = None,
 ) -> dict:
     """Run rejection sampling over ``tasks`` and write the SFT JSONL.
 
     ``max_keep_per_task`` caps records kept per task; when >1 the cheapest
-    passing trajectories are kept first. Returns stats + writes a ``.stats.json``.
+    passing trajectories are kept first. ``concurrency`` rolls out that many tasks
+    in parallel (each task issues its samples sequentially, so ~``concurrency``
+    requests hit the served model at once) — set 1 for the original sequential
+    behaviour. Records are written as each task finishes, so peak memory is bounded
+    by the in-flight tasks, not the whole dataset. Returns stats + writes a
+    ``.stats.json``.
+
+    ``keep_all``: write **every** rolled-out trajectory (correct and incorrect)
+    rather than dropping the failures. Each record gets ``correct`` (verifier
+    verdict) and ``kept`` (True on the cheapest-correct sample — the one the
+    rejection sampler would have selected). This preserves the full sample set so
+    you can compute accuracy / inspect failures; the SFT trainer should filter on
+    ``correct`` (or ``kept``). Default off = original drop-the-failures behaviour.
     """
     out = Path(out_path)
     out.parent.mkdir(parents=True, exist_ok=True)
 
-    seen = 0
+    tasks = list(tasks)
+    seen = len(tasks)
     written = 0
-    dropped = 0
+    correct_written = 0
+    incorrect_written = 0
+    dropped = 0          # tasks that produced no kept record
+    tasks_solved = 0     # tasks with >=1 correct sample
     domain_counts: Counter[str] = Counter()
 
-    with out.open("w") as fh:
-        for task in tasks:
-            seen += 1
-            passing: List[UnifiedRollout] = []
-            for _ in range(samples_per_task):
-                roll = rollout_fn(task)
-                if roll is None:
-                    continue
-                if verify_fn(task, roll):
-                    passing.append(roll)
-            if not passing:
+    def _work(task: TaskLike) -> tuple[TaskLike, List[tuple[UnifiedRollout, bool]]]:
+        return task, _process_task(
+            task,
+            rollout_fn=rollout_fn,
+            verify_fn=verify_fn,
+            samples_per_task=samples_per_task,
+            max_keep_per_task=max_keep_per_task,
+            stop_at_keep=stop_at_keep,
+            keep_all=keep_all,
+        )
+
+    def _emit(fh, task: TaskLike, roll: UnifiedRollout, ok: bool, kept: bool) -> None:
+        nonlocal written, correct_written, incorrect_written
+        reward = reward_fn(roll) if reward_fn else 0.0
+        record = trajectory_to_record(
+            task.task_id, task.instruction, tools, roll,
+            reward=reward, domain=task.domain,
+        )
+        record["correct"] = ok
+        record["kept"] = kept
+        record["clean"] = _target_is_clean(roll)
+        # Task provenance so every record is self-describing: area (domain),
+        # difficulty tier, source dataset, and fine subsector. Lets us slice
+        # train/holdout/analysis by any of these later.
+        record["area"] = task.domain
+        record["difficulty"] = getattr(task, "difficulty", "") or ""
+        record["dataset"] = getattr(task, "dataset", "") or ""
+        record["dataset_description"] = dataset_description(record["dataset"])
+        record["subsector"] = getattr(task, "subsector", "") or ""
+        # Gold reference answer the verifier graded against. Persisted so every
+        # record supports gold-vs-model inspection downstream (Braintrust, error
+        # analysis) instead of only a bare `correct` boolean.
+        record["gold_answer"] = getattr(task, "answer", "") or ""
+        # Stamp provenance (which model/orchestrator generated this trajectory)
+        # so records are self-identifying when pooled across model families.
+        if record_extra:
+            record.update(record_extra)
+        fh.write(json.dumps(record) + "\n")
+        fh.flush()
+        written += 1
+        correct_written += int(ok)
+        incorrect_written += int(not ok)
+        domain_counts[task.domain] += 1
+
+    def _write(fh, task: TaskLike, results: List[tuple[UnifiedRollout, bool]]) -> None:
+        nonlocal dropped, tasks_solved
+        # A trajectory is only eligible to be *kept* as a training target if the
+        # judge passed it AND it's structurally clean (non-error, routed, not
+        # truncated). Unclean-but-correct rollouts are still written in keep_all
+        # mode (for analysis) but never marked kept.
+        correct = [r for r in results if r[1] and _target_is_clean(r[0])]
+        cheapest = min((r for r, _ in correct), key=lambda r: r.cost_usd, default=None)
+        if correct:
+            tasks_solved += 1
+        if keep_all:
+            # Write every sample; mark the cheapest-correct one as kept.
+            if not results:
                 dropped += 1
-                continue
-            # Keep cheapest-first.
-            passing.sort(key=lambda r: r.cost_usd)
-            for roll in passing[:max_keep_per_task]:
-                reward = reward_fn(roll) if reward_fn else 0.0
-                record = trajectory_to_record(
-                    task.task_id, task.instruction, tools, roll,
-                    reward=reward, domain=task.domain,
-                )
-                fh.write(json.dumps(record) + "\n")
-                written += 1
-                domain_counts[task.domain] += 1
+                return
+            for roll, ok in results:
+                _emit(fh, task, roll, ok, kept=(roll is cheapest))
+        else:
+            # Original behaviour: keep only cheapest-correct, capped.
+            if not correct:
+                dropped += 1
+                return
+            for roll in sorted((r for r, _ in correct),
+                               key=lambda r: r.cost_usd)[:max_keep_per_task]:
+                _emit(fh, task, roll, True, kept=(roll is cheapest))
+
+    with out.open("w") as fh:
+        if concurrency <= 1:
+            for task in tasks:
+                _, results = _work(task)
+                _write(fh, task, results)
+        else:
+            from concurrent.futures import ThreadPoolExecutor, as_completed
+
+            done = 0
+            with ThreadPoolExecutor(max_workers=concurrency) as pool:
+                futures = {pool.submit(_work, t): t for t in tasks}
+                for fut in as_completed(futures):
+                    task = futures[fut]
+                    try:
+                        _, results = fut.result()
+                    except Exception as exc:  # a task's worker died; count + skip
+                        logger.warning("task %s failed: %s",
+                                       getattr(task, "task_id", "?"), exc)
+                        results = []
+                    _write(fh, task, results)
+                    done += 1
+                    if done % 50 == 0 or done == seen:
+                        logger.info("rejection-sampling: %d/%d tasks done "
+                                    "(%d written, %d solved, %d dropped)",
+                                    done, seen, written, tasks_solved, dropped)
 
     stats = {
         "out_path": str(out),
         "tasks_seen": seen,
         "records_written": written,
+        "records_correct": correct_written,
+        "records_incorrect": incorrect_written,
+        "tasks_solved": tasks_solved,
         "tasks_dropped": dropped,
+        "task_accuracy": round(tasks_solved / seen, 4) if seen else 0.0,
         "samples_per_task": samples_per_task,
+        "keep_all": keep_all,
+        "concurrency": concurrency,
         "domain_distribution": dict(domain_counts),
     }
     out.with_suffix(out.suffix + ".stats.json").write_text(json.dumps(stats, indent=2))
-    logger.info("Wrote %d SFT records to %s", written, out)
+    logger.info("Wrote %d SFT records to %s (%d correct, %d incorrect)",
+                written, out, correct_written, incorrect_written)
     return stats
 
 

src/openjarvis/learning/intelligence/orchestrator/sft_data/unified_serialize.py

@@ -13,6 +13,7 @@ prompt), ``assistant`` (reasoning + a ``<tool_call>`` tag, or the final answer),
 from __future__ import annotations
 
 import json
+import re
 from typing import Any, Dict, List
 
 from openjarvis.agents.hybrid.expert_registry import ExpertTool, build_tool_specs
@@ -23,9 +24,170 @@ from openjarvis.agents.hybrid.toolorchestra.rollout import (
 )
 
 
-def _system_prompt(tools: List[ExpertTool]) -> str:
+def _system_prompt(tools: List[ExpertTool], rollout: UnifiedRollout) -> str:
     # Faithful ToolOrchestra system prompt (paper's prepare_sft_data.py format).
-    return build_system_prompt(build_tool_specs(tools))
+    # Prefer the EXACT specs the policy saw this rollout (anonymized labels when
+    # anonymize=True) so the saved <tools> block matches the assistant's
+    # <tool_call> tags. Falling back to the real registry here is the bug that
+    # leaked real model names+pricing into anonymized records.
+    specs = rollout.tool_specs if rollout.tool_specs else build_tool_specs(tools)
+    return build_system_prompt(specs)
+
+
+def _normalize_think(text: str) -> str:
+    """Ensure a reasoning block has matched tags. Qwen rollouts routinely yield a
+    dangling ``</think>`` with NO opening ``<think>`` (the chat template consumes
+    the opener in the prompt, so only the closer survives in the completion). Add
+    the opener back so the trained turn is well-formed."""
+    t = (text or "").lstrip()
+    if "</think>" in t and "<think>" not in t:
+        t = "<think>\n" + t
+    return t
+
+
+def _debox(text: str) -> str:
+    """Unwrap every ``\\boxed{X}`` -> ``X`` (balanced braces). The format kills
+    \\boxed everywhere, but small models keep emitting it in otherwise-correct
+    answers — de-box to salvage them rather than reject the whole trajectory."""
+    out, i = [], 0
+    marker = r"\boxed{"
+    while i < len(text):
+        j = text.find(marker, i)
+        if j == -1:
+            out.append(text[i:])
+            break
+        out.append(text[i:j])
+        k = j + len(marker)
+        depth, inner = 1, []
+        while k < len(text) and depth:
+            c = text[k]
+            if c == "{":
+                depth += 1
+            elif c == "}":
+                depth -= 1
+                if depth == 0:
+                    break
+            inner.append(c)
+            k += 1
+        out.append("".join(inner))
+        i = k + 1
+    return "".join(out)
+
+
+# Reasoning that PAROTS the training constraint back ("I must call a model per
+# my instructions", "this is testing whether I need to route") — the model
+# reasoning about its own harness. Never a good target: drop the whole line.
+_LEAK_RE = re.compile(
+    r"(?im)^.*\b(?:"
+    r"the user (?:is|was|wants|has|had|needs|'s|would|will|asked|'ve asked|has asked)\b|"
+    r"asked me to\b|"                                    # echoes of an injected nudge turn
+    r"make progress by delegat|"
+    r"delegat\w+[^\n]*sub-?question|"
+    r"(?:to |ask |send .{0,20}to )?one of the models\b|"
+    r"instructions?|"                                    # any self-reference to the rules
+    r"requirement|"
+    r"at least one (?:model|tool|expert)|"               # "(call|invoke) at least one model"
+    r"(?:call|invoke|route to|delegate to|use)\s+(?:a|one|at least one|the)\s+model\b|"
+    r"route\s+(?:this|it|that)\b[^\n]{0,40}\bthrough\s+(?:a|one|the)\s+model\b|"
+    r"without mentioning (?:any )?(?:tools?|reasoning|meta|steps)|"
+    r"in the required format\b|"
+    r"based on the (?:model|response|analysis|expert)|"        # response-acknowledgment
+    r"the model (?:provided|confirmed|gave|returned|correctly|analy\w+|respon\w+|said|indicated|identified)|"
+    r"i (?:got|received|have) (?:a |the )?(?:comprehensive |clear |good |detailed )?answer\b|"
+    r"now i can (?:confidently )?(?:give|provide|state|answer)|"
+    r"perfect[!,]|great[!,]|excellent[!,]|"                    # narration exclamations
+    r"as an orchestrator|"
+    r"whose job is to (?:route|delegate|orchestrate)|"
+    r"testing whether i (?:need|have|should|must)\b|"
+    r"i(?:'m| am)? (?:required|instructed|supposed|expected) to\b"
+    r")[^\n]*(?:\n|$)"
+)
+# A leading task-restatement opener ("The user is asking…") — the format
+# explicitly bans this meta-text; drop it when it opens the reasoning.
+_META_OPEN_RE = re.compile(
+    r"(?is)^\s*(?:the user (?:is|was|wants|has|needs|would|'s)\b[^.\n]*[.\n]+\s*)+"
+)
+
+
+def _scrub_meta(text: str) -> str:
+    """Remove harness-leakage lines and a leading 'The user is asking…' meta
+    opener from a reasoning block. Both are disallowed by the system prompt, so
+    they must not survive into the supervised target. Preserves a leading
+    ``<think>`` marker so the block stays well-formed."""
+    if not text:
+        return text
+    think = ""
+    body = text
+    m = re.match(r"(?is)^\s*(<think>)\s*", body)
+    if m:
+        think = "<think>\n"
+        body = body[m.end():]
+    body = _LEAK_RE.sub("", body)
+    body = _META_OPEN_RE.sub("", body).lstrip()
+    return (think + body).strip() if think else body.strip()
+
+
+# Control / special tokens that must never survive into a training target's
+# final answer. Three shapes, in order of alternation:
+#   1. closed pipe token  — ``<|im_end|>``, ``<|im_start|>``, ``<|eot_id|>``,
+#      ``<|end_of_turn|>``, ``<|"|>`` (the stray-quote leak).
+#   2. UNCLOSED pipe token — ``<|tool_call>`` (opened with ``<|`` but closed on a
+#      bare ``>`` because the decode was cut before the second pipe).
+#   3. named angle special tokens — gemma ``<start_of_turn>`` / ``<end_of_turn>``
+#      and the sentinel set ``<eos> <bos> <pad> <unk> <s> </s>``.
+# Deliberately narrow: only the ``<|...|>`` pipe form and this known name list
+# match, so legitimate ``<``/``>`` in math/code (``x < 3 and y > 2``) is left
+# untouched.
+_CONTROL_TOKEN_RE = re.compile(
+    r"<\|[^>]*?\|>"                                                  # closed pipe token
+    r"|<\|[^|>]*>"                                                   # unclosed pipe token
+    r"|</?(?:start_of_turn|end_of_turn|eos|bos|pad|unk|s)>"         # named angle tokens
+)
+
+
+def _strip_control_tokens(text: str) -> str:
+    """Delete residual control/special tokens from an answer so a good answer
+    with a stray token is salvaged rather than dropped. Loops (bounded) because
+    removing one match can expose a nested/overlapping leftover (e.g.
+    ``<|<|im_end|>|>``). Re-strips whitespace at the end."""
+    if not text:
+        return text
+    out = text
+    for _ in range(4):
+        stripped = _CONTROL_TOKEN_RE.sub("", out)
+        if stripped == out:
+            break
+        out = stripped
+    return out.strip()
+
+
+def _final_answer_block(text: str) -> str:
+    """Render the final-answer assistant message with a single clean
+    ``FINAL_ANSWER: <value>`` line. De-boxes the answer, strips leaked control
+    tokens, and normalizes whatever spelling the model used (``FINALANSWER``,
+    ``FINAL ANSWER``, no colon, …) to the exact tag. Keeps the think block at
+    most once."""
+    text = _debox(_normalize_think((text or "").strip()))
+    # Normalize any final-answer marker spelling to the canonical form; take the
+    # LAST one as the real answer (idempotent — never emits two tags).
+    marks = list(re.finditer(r"(?im)FINAL[_\s]?ANSWER\s*:?", text))
+    if marks:
+        last = marks[-1]
+        answer = _strip_control_tokens(text[last.end():].strip())
+        # Final turn = the bare short answer only. Drop EVERYTHING before
+        # FINAL_ANSWER — both the visible "Perfect! Based on the model's response…"
+        # narration AND the final-turn <think>, which is just confirmatory
+        # "both models agree… I've verified…" fluff. The routing reasoning already
+        # lives in the earlier tool-call turns, so nothing of value is lost, and
+        # this kills all final-turn narration deterministically (any wording).
+        return f"FINAL_ANSWER: {answer}"
+    if "</think>" in text:
+        # No explicit marker: the answer is whatever follows the final </think>.
+        # Drop the think block (same rationale as above) — keep only the answer.
+        _, _, answer = text.rpartition("</think>")
+        answer = _strip_control_tokens(answer.strip())
+        return f"FINAL_ANSWER: {answer}" if answer else f"FINAL_ANSWER: {_strip_control_tokens(_scrub_meta(text))}"
+    return f"FINAL_ANSWER: {_strip_control_tokens(text)}"
 
 
 def trajectory_to_record(
@@ -39,21 +201,21 @@ def trajectory_to_record(
 ) -> Dict[str, Any]:
     """Convert a passing :class:`UnifiedRollout` into one SFT JSONL record."""
     conversations: List[Dict[str, str]] = [
-        {"role": "system", "content": _system_prompt(tools)},
+        {"role": "system", "content": _system_prompt(tools, rollout)},
         {"role": "user", "content": f"Problem: {question}"},
     ]
 
     for turn in rollout.turns:
         if turn.tool_name is None:
-            # Final-answer turn.
+            # Final-answer turn. turn.reasoning is the model's actual output for
+            # this turn (which already contains the answer); render it once.
             conversations.append({
                 "role": "assistant",
-                "content": (turn.reasoning or "").rstrip()
-                + f"\nFINAL_ANSWER: {rollout.final_answer}",
+                "content": _final_answer_block(turn.reasoning or rollout.final_answer),
             })
             continue
         tag = tool_call_tag(turn.tool_name, turn.arguments)
-        reasoning = (turn.reasoning or "").rstrip()
+        reasoning = _scrub_meta(_normalize_think((turn.reasoning or "").rstrip()))
         conversations.append({
             "role": "assistant",
             "content": (reasoning + "\n" + tag).strip(),
@@ -68,7 +230,7 @@ def trajectory_to_record(
     if not rollout.turns or rollout.turns[-1].tool_name is not None:
         conversations.append({
             "role": "assistant",
-            "content": f"FINAL_ANSWER: {rollout.final_answer}",
+            "content": _final_answer_block(rollout.final_answer),
         })
 
     return {
@@ -81,6 +243,9 @@ def trajectory_to_record(
             "tokens": rollout.tokens,
             "num_tool_calls": rollout.num_tool_calls,
             "num_turns": len(rollout.turns),
+            # Present only when experts were anonymized: opaque label -> real
+            # tool name, so analysis can recover which model was actually picked.
+            **({"anon_map": rollout.anon_map} if rollout.anon_map else {}),
         },
     }
 

src/openjarvis/learning/intelligence/orchestrator/sft_data/verify.py

@@ -29,6 +29,12 @@ from .datasets import Task
 
 GEMINI_BASE_URL = "https://generativelanguage.googleapis.com/v1beta/openai/"
 GEMINI_MODEL = "gemini-2.5-flash"
+# LLM judge model. Switched from Gemini (free-tier rate limit stalled parallel gen)
+# to Anthropic Haiku: fast (~0.6s), direct PASS/FAIL, high rate limits.
+JUDGE_MODEL = "claude-haiku-4-5-20251001"
+# Reused across calls — creating a fresh Anthropic client per judge call leaks an
+# httpx connection pool (CLOSE-WAIT pileup under parallel generation). Thread-safe.
+_JUDGE_CLIENT = None
 
 
 # --- normalization (copied from hotpotqa.py — keep self-contained) -----------
@@ -136,29 +142,11 @@ def _math_equal(prediction: str, gold: str) -> bool:
         if abs(pf - gf) <= 1e-6 * max(1.0, abs(gf)):
             return True
 
-    # Symbolic / exact equality via sympy when available.
-    try:
-        import sympy
-        from sympy.parsing.latex import parse_latex  # noqa: F401  (optional)
-
-        def _parse(x: str):
-            try:
-                return sympy.sympify(x.replace("^", "**"))
-            except Exception:
-                try:
-                    return sympy.parsing.latex.parse_latex(x)
-                except Exception:
-                    return None
-
-        pe, ge = _parse(pred), _parse(g)
-        if pe is not None and ge is not None:
-            try:
-                if sympy.simplify(pe - ge) == 0:
-                    return True
-            except Exception:
-                pass
-    except Exception:
-        pass
+    # NOTE: a sympy/parse_latex symbolic-equality block used to live here, but
+    # sympy.simplify / antlr parse_latex can hang on pathological \boxed{} answers
+    # while holding the GIL -> deadlocks the whole threaded rejection sampler
+    # (all worker threads freeze in futex_wait, server goes idle, 0 records).
+    # Symbolic equivalence is now deferred to the Gemini judge in verify_answer().
 
     # Last resort: whole-word substring of the gold in the prediction.
     return _string_or_f1(prediction, g, f1_threshold=0.9)
@@ -169,16 +157,24 @@ def _math_equal(prediction: str, gold: str) -> bool:
 
 def _gemini_judge(task: Task, prediction: str) -> Optional[bool]:
     """Ask Gemini PASS/FAIL. Returns None if unavailable (caller falls back)."""
-    api_key = os.environ.get("GEMINI_API_KEY")
+    api_key = os.environ.get("ANTHROPIC_API_KEY")
     if not api_key:
         return None
     try:
-        from openai import OpenAI
+        import anthropic
     except Exception:
         return None
 
     try:
-        client = OpenAI(api_key=api_key, base_url=GEMINI_BASE_URL)
+        # Judge = Anthropic Haiku (fast ~0.6s, direct PASS/FAIL, high rate limits).
+        # Switched off the Gemini judge: its free-tier rate limit + SDK retry/backoff
+        # blocked worker threads for minutes under parallel generation and stalled
+        # the run. Fail-fast (no retries, short timeout): on error return None and the
+        # caller falls back to string/f1.
+        global _JUDGE_CLIENT
+        if _JUDGE_CLIENT is None:
+            _JUDGE_CLIENT = anthropic.Anthropic(api_key=api_key, max_retries=0, timeout=15)
+        client = _JUDGE_CLIENT
         prompt = (
             "You are grading a candidate answer against a gold reference.\n"
             "Reply with exactly one word: PASS if the candidate is correct and "
@@ -188,13 +184,12 @@ def _gemini_judge(task: Task, prediction: str) -> Optional[bool]:
             f"Candidate answer:\n{prediction}\n\n"
             "Verdict (PASS or FAIL):"
         )
-        resp = client.chat.completions.create(
-            model=GEMINI_MODEL,
-            messages=[{"role": "user", "content": prompt}],
-            temperature=0.0,
+        resp = client.messages.create(
+            model=JUDGE_MODEL,
             max_tokens=8,
+            messages=[{"role": "user", "content": prompt}],
         )
-        verdict = (resp.choices[0].message.content or "").strip().upper()
+        verdict = (resp.content[0].text or "").strip().upper()
         if "PASS" in verdict:
             return True
         if "FAIL" in verdict:
@@ -216,6 +211,10 @@ def verify_answer(task: Task, prediction: str) -> bool:
     domain = (task.domain or "").lower()
 
     if domain == "math":
+        # string + numeric + f1 only. sympy removed (GIL-deadlocked the threaded
+        # sampler); Gemini judge NOT used here either (32 tasks x N samples of math
+        # judge calls throttle Gemini and stall the whole run). Accept slightly lower
+        # math yield for a fast, hang-free verifier.
         return _math_equal(pred, task.answer)
 
     if domain == "code":

src/openjarvis/tools/calculator.py

@@ -42,8 +42,23 @@ _MATH_FUNCS = {
     "sin": math.sin,
     "cos": math.cos,
     "tan": math.tan,
+    "asin": math.asin,
+    "acos": math.acos,
+    "atan": math.atan,
+    "atan2": math.atan2,
+    "sinh": math.sinh,
+    "cosh": math.cosh,
+    "tanh": math.tanh,
+    "radians": math.radians,
+    "degrees": math.degrees,
+    "exp": math.exp,
+    "abs": abs,
+    "fabs": math.fabs,
+    "hypot": math.hypot,
+    "factorial": math.factorial,
     "pi": math.pi,
     "e": math.e,
+    "tau": math.tau,
     "ceil": math.ceil,
     "floor": math.floor,
 }

src/openjarvis/tools/web_search.py

@@ -116,11 +116,31 @@ class WebSearchTool(BaseTool):
         return text
 
     def _duckduckgo_search(self, query: str, max_results: int) -> str:
-        """Search using DuckDuckGo as fallback."""
+        """Search using DuckDuckGo as fallback.
+
+        ddgs queries several engines (yandex/yahoo/brave/...) that frequently
+        hang; with no timeout this blocks the whole rollout. Cap each engine
+        request (DDGS timeout) AND wrap the call in a hard wall-clock deadline so
+        a flaky search fast-fails instead of stalling data generation.
+        """
+        from concurrent.futures import ThreadPoolExecutor, TimeoutError as _FTO
+
         from ddgs import DDGS
 
-        ddgs = DDGS()
-        raw_results = list(ddgs.text(query, max_results=max_results))
+        def _run():
+            ddgs = DDGS(timeout=5)
+            return list(ddgs.text(query, max_results=max_results))
+
+        # Don't use `with` — its shutdown(wait=True) blocks on the hung thread,
+        # defeating the deadline. submit, wait up to 8s, then abandon the thread
+        # (it finishes in the background harmlessly) and fast-fail.
+        _ex = ThreadPoolExecutor(max_workers=1)
+        try:
+            raw_results = _ex.submit(_run).result(timeout=8)
+        except Exception:
+            _ex.shutdown(wait=False, cancel_futures=True)
+            return "[web_search: no results (search backend timed out)]"
+        _ex.shutdown(wait=False)
         results = []
         for r in raw_results:
             title = r.get("title", "Untitled")

tests/agents/test_anonymize_tools.py

@@ -0,0 +1,129 @@
+"""Unit coverage for ``anonymize_tools`` — the identity-stripping step that makes
+routing data unbiased. See ``expert_registry.anonymize_tools``.
+
+The anonymizer takes the orchestrator catalog and replaces every MODEL expert
+with an opaque ``model_xxxx`` label, a uniform brand-free description, no
+price/latency line, and shuffles the expert block — so the policy can't route on
+a model's name, position, cost, or tier. Basic tools keep their real names.
+It returns ``(anon_tools, anon_to_real)``.
+"""
+
+from __future__ import annotations
+
+import json
+import random
+
+from openjarvis.agents.hybrid.expert_registry import (
+    KIND_MODEL,
+    anonymize_tools,
+    build_tool_specs,
+    orchestrator_catalog,
+    tools_by_name,
+)
+
+# Real brand tokens that must never leak into the anonymized, model-facing specs.
+_BRANDS = ("gpt", "claude", "gemini", "qwen")
+
+
+def _model_names(cat):
+    return [t.name for t in cat if t.kind == KIND_MODEL]
+
+
+def _basic_names(cat):
+    return [t.name for t in cat if t.kind != KIND_MODEL]
+
+
+def test_no_brand_names_in_anonymized_specs():
+    cat = orchestrator_catalog()
+    anon, _ = anonymize_tools(cat, random.Random(0))
+    specs = build_tool_specs(anon)
+    # The whole model-facing payload (names + descriptions + categories) must be
+    # brand-free. `.model` is preserved on the tool for dispatch but is NOT part
+    # of the spec the orchestrator conditions on, so it's fine that it still holds
+    # the real id.
+    blob = json.dumps(specs).lower()
+    for brand in _BRANDS:
+        assert brand not in blob, f"brand {brand!r} leaked into anonymized specs"
+    # And specifically the anonymized expert descriptions carry no brand.
+    for t in anon:
+        if t.name.startswith("model_"):
+            assert not any(b in t.description().lower() for b in _BRANDS)
+
+
+def test_hide_cost_removes_price_line_from_descriptions():
+    cat = orchestrator_catalog()
+    # Sanity: the raw model tools DO surface a price line before anonymizing.
+    raw_model = next(t for t in cat if t.kind == KIND_MODEL)
+    assert "Pricing:" in raw_model.description()
+
+    anon, anon_to_real = anonymize_tools(cat, random.Random(1))
+    for t in anon:
+        if t.name in anon_to_real:  # an anonymized model expert
+            assert t.hide_cost is True
+            desc = t.description()
+            assert "Pricing:" not in desc
+            assert "$" not in desc
+            assert "/1M" not in desc
+            assert "latency" not in desc.lower()
+
+
+def test_each_model_maps_to_opaque_label_and_round_trips():
+    cat = orchestrator_catalog()
+    orig_models = _model_names(cat)
+    anon, anon_to_real = anonymize_tools(cat, random.Random(2))
+
+    # One opaque label per real model, all in the model_xxxx namespace.
+    assert len(anon_to_real) == len(orig_models)
+    assert all(lbl.startswith("model_") for lbl in anon_to_real)
+
+    # No collisions: labels unique, and each real model recovered exactly once.
+    assert len(set(anon_to_real)) == len(anon_to_real)
+    assert len(set(anon_to_real.values())) == len(anon_to_real)
+    assert set(anon_to_real.values()) == set(orig_models)
+
+    # Round-trip: every anonymized expert's label resolves back to a real name,
+    # and `.model` is preserved so dispatch still reaches the right backend.
+    by_orig = tools_by_name(cat)
+    for t in anon:
+        if t.name.startswith("model_"):
+            real = anon_to_real[t.name]
+            assert real in by_orig
+            assert t.model == by_orig[real].model  # backend id untouched
+
+    # Basic tools keep their real names and are untouched by the label map.
+    basics = _basic_names(cat)
+    anon_names = {t.name for t in anon}
+    assert set(basics) <= anon_names
+    assert not (set(basics) & set(anon_to_real))
+
+
+def test_experts_block_on_top_then_basics():
+    cat = orchestrator_catalog()
+    anon, anon_to_real = anonymize_tools(cat, random.Random(3))
+    n_models = len(anon_to_real)
+    # Experts form a contiguous block at the front, basics underneath.
+    assert all(t.name.startswith("model_") for t in anon[:n_models])
+    assert all(not t.name.startswith("model_") for t in anon[n_models:])
+
+
+def test_labels_and_order_are_shuffled_not_identity():
+    cat = orchestrator_catalog()
+    orig_models = _model_names(cat)
+
+    orderings = set()
+    labels_for_first = set()
+    for seed in range(25):
+        anon, anon_to_real = anonymize_tools(cat, random.Random(seed))
+        # Real-model order as it appears in the anonymized expert block.
+        order = tuple(anon_to_real[t.name] for t in anon if t.name.startswith("model_"))
+        orderings.add(order)
+        # Label assigned to the first catalog model varies across rngs.
+        rev = {real: lbl for lbl, real in anon_to_real.items()}
+        labels_for_first.add(rev[orig_models[0]])
+
+    # Not a fixed identity: across seeds the expert order actually varies...
+    assert len(orderings) > 1
+    # ...and at least one ordering differs from the input model order.
+    assert any(order != tuple(orig_models) for order in orderings)
+    # Labels are random per-rng, not a stable function of position.
+    assert len(labels_for_first) > 1

tests/agents/test_expert_registry.py

@@ -136,11 +136,14 @@ def test_orchestrator_catalog_two_model_classes_plus_basics():
     by = tools_by_name(cat)
     assert by["gpt_5_5"].category == CATEGORY_CLOUD_FRONTIER
     assert by["claude_opus_4_8"].category == CATEGORY_CLOUD_FRONTIER
+    # Default routing for every model tool is OpenRouter (no servers required).
     for n in ("qwen3_5_9b", "qwen3_6_27b_fp8", "qwen3_5_122b_a10b_fp8",
               "qwen3_5_397b_a17b_fp8"):
         assert by[n].category == CATEGORY_LOCAL_OSS
-        assert by[n].backend_type == "vllm"
-        assert by[n].price_in == 0.0 and by[n].price_out == 0.0
+        assert by[n].backend_type == "openrouter"
+        assert by[n].base_url is None
+        # OpenRouter routing carries the slug + a (estimated) per-token price.
+        assert "/" in by[n].model and by[n].price_in > 0.0
 
 
 def test_orchestrator_catalog_categories_present():
@@ -167,21 +170,54 @@ def test_orchestrator_specs_include_category_field():
 
 
 def test_orchestrator_local_models_get_base_url_when_provided():
+    # An endpoint switches that model from the OpenRouter default to local vLLM.
     cat = orchestrator_catalog(
         local_endpoints={
             "Qwen/Qwen3.5-9B": "http://x/v1",
             "Qwen/Qwen3.6-27B-FP8": "http://y/v1",
         })
     by = tools_by_name(cat)
+    assert by["qwen3_5_9b"].backend_type == "vllm"
     assert by["qwen3_5_9b"].base_url == "http://x/v1"
+    assert by["qwen3_5_9b"].price_in == 0.0 and by["qwen3_5_9b"].price_out == 0.0
     assert by["qwen3_6_27b_fp8"].base_url == "http://y/v1"
-    # Unmapped local model -> base_url None.
+    # Unmapped local model -> stays on the OpenRouter default (base_url None).
+    assert by["qwen3_5_122b_a10b_fp8"].backend_type == "openrouter"
     assert by["qwen3_5_122b_a10b_fp8"].base_url is None
     # Cloud frontier carries real pricing.
     assert by["claude_opus_4_8"].price_in > 0.0
     assert by["gpt_5_5"].price_in > 0.0
 
 
+def test_orchestrator_model_backends_override():
+    # Force a cloud model onto OpenRouter and a local model onto vLLM explicitly.
+    cat = orchestrator_catalog(
+        model_backends={
+            "claude-opus-4-8": "openrouter",
+            "Qwen/Qwen3.5-397B-A17B-FP8": "vllm",
+        },
+        local_endpoints={"Qwen/Qwen3.5-397B-A17B-FP8": "http://z/v1"},
+    )
+    by = tools_by_name(cat)
+    assert by["claude_opus_4_8"].backend_type == "openrouter"
+    assert by["claude_opus_4_8"].model == "anthropic/claude-opus-4.8"
+    # No override for gpt-5.5 -> it defaults to its NATIVE first-party API
+    # (openai), not OpenRouter. Frontier models hit their native provider by
+    # default; OpenRouter is only the fallback for OSS/local models or when
+    # explicitly requested via model_backends.
+    assert by["gpt_5_5"].backend_type == "openai"  # native default
+    assert by["gpt_5_5"].model == "gpt-5.5"
+    assert by["qwen3_5_397b_a17b_fp8"].backend_type == "vllm"
+    assert by["qwen3_5_397b_a17b_fp8"].base_url == "http://z/v1"
+
+
+def test_orchestrator_openrouter_slug_override():
+    cat = orchestrator_catalog(
+        openrouter_slugs={"Qwen/Qwen3.5-9B": "qwen/qwen3.5-9b-custom"})
+    by = tools_by_name(cat)
+    assert by["qwen3_5_9b"].model == "qwen/qwen3.5-9b-custom"
+
+
 def test_openjarvis_tool_bridges_real_tool_with_custom_schema():
     params = {
         "type": "object",

tests/test_orchestrator_learning/sft_data/test_rejection_pipeline.py

@@ -60,7 +60,9 @@ def test_run_unified_rollout_terminates_on_no_tool_call():
     ]
     calls = iter(scripted)
 
-    def call_orch(system, user, specs):
+    def call_orch(messages, specs):
+        # Signature is now (messages, specs): the rollout drives a running
+        # system/user/assistant/tool conversation, not a flattened (system, user).
         return next(calls)
 
     def dispatch(tool, args):
@@ -81,7 +83,11 @@ def test_serialize_record_shape_and_tool_call_tags():
         turns=[
             UnifiedTurn(reasoning="think", tool_name="qwen3_32b",
                         arguments={"input": "q"}, observation="obs"),
-            UnifiedTurn(reasoning="done", tool_name=None),
+            # The final turn's reasoning IS the model's real final output (it
+            # already contains the answer). The serializer now renders this turn
+            # via _final_answer_block(turn.reasoning), not rollout.final_answer.
+            UnifiedTurn(reasoning="The result is 42.\nFINAL_ANSWER: 42",
+                        tool_name=None),
         ],
         final_answer="42", cost_usd=0.02, tokens=30, num_tool_calls=1,
     )
@@ -131,7 +137,9 @@ def test_generate_sft_dataset_end_to_end(tmp_path):
                     UnifiedTurn("", "refund", {"user": "8612"}, "ok"),
                     UnifiedTurn("done", None),
                 ],
-                final_answer="refunded $20.90", cost_usd=0.03,
+                # num_tool_calls must reflect the two routed calls: the structural
+                # _target_is_clean gate drops a trajectory with num_tool_calls < 1.
+                final_answer="refunded $20.90", cost_usd=0.03, num_tool_calls=2,
             )
         return UnifiedRollout(turns=[UnifiedTurn("x", "cancel", {}, "ok")],
                               final_answer="nope", cost_usd=0.05)

tests/test_orchestrator_learning/sft_data/test_verify.py

@@ -35,10 +35,14 @@ def test_math_numeric_float_match():
     assert verify_answer(_math("42"), "42.0") is True
 
 
-def test_math_symbolic_equivalence():
-    # x^2 + 2x + 1 == (x+1)^2 via sympy (skips gracefully if sympy absent).
+def test_math_symbolic_equivalence_no_longer_verified_locally():
+    # The sympy symbolic-equality block was removed from _math_equal: sympy.simplify
+    # / parse_latex could hang on pathological \boxed{} answers while holding the GIL
+    # and deadlock the threaded rejection sampler. The math domain now uses
+    # string+numeric+f1 only (no judge call either), so a purely SYMBOLIC
+    # equivalence that isn't string/numeric-equal is intentionally NOT recognized.
     t = _math("(x+1)^2")
-    assert verify_answer(t, "x^2 + 2*x + 1") is True
+    assert verify_answer(t, "x^2 + 2*x + 1") is False
 
 
 def test_empty_prediction_false():

tests/test_orchestrator_learning/test_build_orchestrator_sft.py

@@ -18,7 +18,8 @@ from openjarvis.learning.intelligence.orchestrator.sft_data.reject_sample import
 )
 
 
-def _fake_tasks() -> list[Task]:
+def _fake_tasks(**_kwargs) -> list[Task]:
+    # Accepts **kwargs so it can stand in for ``load_sft_tasks(cap=..., balanced=...)``.
     return [
         Task(task_id="t-math-1", question="What is 6 * 7?", answer="42", domain="math"),
         Task(task_id="t-code-1", question="Reverse 'ab'.", answer="ba", domain="code"),
@@ -31,7 +32,7 @@ def _canned_rollout(task: Task) -> UnifiedRollout:
         turns=[
             UnifiedTurn(reasoning="let me compute", tool_name="code_interpreter",
                         arguments={"code": "print(6*7)"}, observation="42"),
-            UnifiedTurn(reasoning="that's the result", tool_name=None),
+            UnifiedTurn(reasoning=f"The tool returned {task.answer}. FINAL_ANSWER: {task.answer}", tool_name=None),
         ],
         final_answer=task.answer, cost_usd=0.01, tokens=20, num_tool_calls=1,
     )
@@ -110,14 +111,19 @@ def test_driver_runs_end_to_end_with_fakes(tmp_path, monkeypatch):
     # make_call_orchestrator builds an OpenAI client lazily, so it never connects
     # here (run_unified_rollout is stubbed out).
 
-    out = tmp_path / "v1.jsonl"
+    # The driver now treats --out as a LABEL and always writes to
+    # data/runs/<stamp>_<label>/data.jsonl (relative to cwd). chdir into tmp_path
+    # so the run folder is created under the test's temp dir, not the real repo.
+    monkeypatch.chdir(tmp_path)
     rc = drv.main([
-        "--out", str(out),
+        "--out", "v1",
         "--samples-per-task", "1",
         "--max-tasks", "2",
     ])
     assert rc == 0
-    lines = out.read_text().strip().splitlines()
+    produced = list((tmp_path / "data" / "runs").glob("*_v1/data.jsonl"))
+    assert len(produced) == 1
+    lines = produced[0].read_text().strip().splitlines()
     assert len(lines) == 2
     rec = json.loads(lines[0])
     assert rec["conversations"][0]["role"] == "system"

tests/test_orchestrator_learning/test_eval_backend.py

@@ -11,6 +11,7 @@ from __future__ import annotations
 
 import pytest
 
+from openjarvis.agents.hybrid.toolorchestra.rollout import UnifiedTurn
 from openjarvis.learning.intelligence.orchestrator import eval_backend as eb
 from openjarvis.learning.intelligence.orchestrator.eval_backend import (
     OrchestratorBackend,
@@ -26,7 +27,14 @@ class _CannedRollout:
         self.tokens = 42
         self.num_tool_calls = 1
         self.parse_failures = 0
-        self.turns = [object()]
+        self.anon_map = {}
+        # generate_full now serializes a per-turn trace (reasoning / tool_name /
+        # arguments / observation), so turns must be real UnifiedTurn-shaped
+        # objects, not bare object() placeholders.
+        self.turns = [
+            UnifiedTurn(reasoning="let me search", tool_name="web_search",
+                        arguments={"query": "x"}, observation="result"),
+        ]
 
     def tool_calls(self):
         return [("web_search", {"query": "x"})]

tests/test_orchestrator_learning/test_final_answer_sanitize.py

@@ -0,0 +1,112 @@
+"""Control-token sanitation in the SFT serializer + clean gate.
+
+Regression guard for the leak audit: leaked control/special tokens
+(``<|im_end|>``, ``<|tool_call>``, gemma ``<start_of_turn>``/``<end_of_turn>``,
+``<|"|>`` …) used to survive into the supervised final answer because the clean
+gate only rejected the bare ``<tool_call>`` form. Defense in depth now:
+
+1. ``_final_answer_block`` STRIPS residual control tokens so a good answer with a
+   stray token is salvaged (not dropped).
+2. ``_target_is_clean`` REJECTS a rollout whose final answer is nothing but
+   control tokens, or where a token survives the strip.
+
+Both must leave legitimate ``<``/``>`` in math/code untouched.
+"""
+
+from __future__ import annotations
+
+import pytest
+
+from openjarvis.agents.hybrid.toolorchestra.rollout import UnifiedRollout, UnifiedTurn
+from openjarvis.learning.intelligence.orchestrator.sft_data.reject_sample import (
+    _target_is_clean,
+)
+from openjarvis.learning.intelligence.orchestrator.sft_data.unified_serialize import (
+    _CONTROL_TOKEN_RE,
+    _final_answer_block,
+    _strip_control_tokens,
+)
+
+
+# --- serializer strips control tokens, leaving a clean FINAL_ANSWER line ------
+
+@pytest.mark.parametrize(
+    "raw, expected_answer",
+    [
+        ("FINAL_ANSWER: 42<end_of_turn>", "42"),
+        ("FINAL_ANSWER: 42<|end_of_turn|>", "42"),
+        ("FINAL_ANSWER: 42<|im_end|>", "42"),
+        ("FINAL_ANSWER: 42<|eot_id|>", "42"),
+        ("FINAL_ANSWER: The result<|tool_call>", "The result"),
+        ('FINAL_ANSWER: foo <|"|> bar', "foo  bar"),
+        ("<start_of_turn>model", "model"),
+    ],
+)
+def test_final_answer_block_strips_tokens(raw: str, expected_answer: str) -> None:
+    out = _final_answer_block(raw)
+    assert out == f"FINAL_ANSWER: {expected_answer}"
+    # No control token survives the serializer.
+    assert not _CONTROL_TOKEN_RE.search(out), out
+
+
+def test_bare_im_end_only_becomes_empty_answer() -> None:
+    # An answer that is nothing but a control token strips to empty; the clean
+    # gate (below) is what rejects such a rollout.
+    assert _final_answer_block("<|im_end|>") == "FINAL_ANSWER: "
+
+
+def test_math_and_code_angle_brackets_survive() -> None:
+    # The narrow regex must NOT eat legitimate comparisons / generics.
+    for good in [
+        "FINAL_ANSWER: x < 3 and y > 2",
+        "FINAL_ANSWER: List<int> and a<b>c",
+        "FINAL_ANSWER: if a < b: return a > 0",
+    ]:
+        out = _final_answer_block(good)
+        assert out == good, out
+        assert not _CONTROL_TOKEN_RE.search(out)
+
+
+def test_strip_helper_handles_nested_and_whitespace() -> None:
+    assert _strip_control_tokens("  hi <|im_end|>  ") == "hi"
+    assert _strip_control_tokens("a<|im_start|>b<|im_end|>c") == "abc"
+
+
+# --- clean gate: salvage vs reject -------------------------------------------
+
+def _roll(final_answer: str) -> UnifiedRollout:
+    """A minimal well-formed rollout (one expert call, clean obs) whose only
+    variable is the final answer."""
+    return UnifiedRollout(
+        turns=[
+            UnifiedTurn(reasoning="route it", tool_name="expert_a",
+                        arguments={"input": "q"}, observation="valid observation"),
+            UnifiedTurn(reasoning=final_answer, tool_name=None),
+        ],
+        final_answer=final_answer, cost_usd=0.01, tokens=10, num_tool_calls=1,
+        anon_map={"expert_a": "gpt"},
+    )
+
+
+@pytest.mark.parametrize(
+    "final_answer, clean",
+    [
+        # Salvageable: a good answer with a trailing stray token stays clean
+        # (the serializer strips the token from the emitted target).
+        ("42<end_of_turn>", True),
+        ("42<|end_of_turn|>", True),
+        ("The result<|tool_call>", True),
+        ('foo <|"|> bar', True),
+        ("<start_of_turn>model", True),
+        ("x < 3 and y > 2", True),
+        ("a normal plain answer", True),
+        # Unsalvageable: the answer is nothing but a control token -> rejected.
+        ("<|im_end|>", False),
+        ("<|eot_id|>", False),
+        ("<end_of_turn>", False),
+        # Bare tool-call tag: already rejected by the existing gate.
+        ("<tool_call>{}</tool_call>", False),
+    ],
+)
+def test_clean_gate_salvages_or_rejects(final_answer: str, clean: bool) -> None:
+    assert _target_is_clean(_roll(final_answer)) is clean