mirror of
https://github.com/open-jarvis/OpenJarvis.git
synced 2026-07-27 21:05:34 +00:00
orchestrator: track the SFT training pipeline and fix ToolOrchestra tracing
The orchestrator SFT pipeline was living entirely in the working tree and never got committed. One script in that lineage (clean_sft_data.py) has already been lost with no way to recover it, so track the rest before the same thing happens again: run_sft_fsdp.py full-parameter FSDP trainer (launched via accelerate) sft_tokenize.py conversation -> tokens + assistant-only masking render_sft_data.py record -> chat-template rendering make_splits.py train / holdout / overfit splits chat_orchestrator.py REPL for eyeballing a trained checkpoint's routing format_eval_sample.py, upload_to_braintrust.py scripts/train/ fsdp4.yaml, fsdp8.yaml, eval sbatch toolorchestra/tracing.py was also untracked despite being imported by unified.py and rollout.py, which left the branch broken for a fresh checkout. Add it along with the rest of the ToolOrchestra package split. Also here: mmlu_pro / supergpqa scorers, expert pricing for the OpenRouter Qwen builds (estimates, flagged in-file), an expanded calculator/web_search tool surface, and the reject-sampling clean gate that run_sft_fsdp reads via --require-clean. GRPO is left at its upstream state -- the RL stage hasn't been run yet, so it stays out of this change. Paths in the sbatch are env-driven rather than hardcoded to one cluster home.
This commit is contained in:
Andrew Park
@@ -133,3 +133,10 @@ oj-debug.*.json
|
||||
|
||||
# Generated orchestrator SFT/eval data artifacts
|
||||
data/
|
||||
|
||||
# Training run artifacts (model checkpoints + W&B run dirs — regenerated, large)
|
||||
checkpoints/
|
||||
wandb/
|
||||
|
||||
# Local LiteLLM spend-tracking proxy (machine-specific, not part of the package)
|
||||
litellm/
|
||||
|
||||
@@ -28,6 +28,7 @@ classifiers = [
|
||||
"Topic :: Scientific/Engineering :: Artificial Intelligence",
|
||||
]
|
||||
dependencies = [
|
||||
"braintrust>=0.0.150",
|
||||
"click>=8",
|
||||
"datasets>=4.5.0",
|
||||
"ddgs>=9.11.4",
|
||||
|
||||
@@ -42,6 +42,10 @@ from __future__ import annotations
|
||||
import argparse
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
from typing import Optional
|
||||
|
||||
from openjarvis.agents.hybrid.expert_registry import orchestrator_catalog
|
||||
@@ -63,12 +67,25 @@ def main(argv: Optional[list[str]] = None) -> int:
|
||||
p = argparse.ArgumentParser(
|
||||
description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter
|
||||
)
|
||||
p.add_argument("--out", default="data/orchestrator_sft_v1.jsonl")
|
||||
# Default output is timestamped (e.g. data/orchestrator_sft_06-22-1230am.jsonl)
|
||||
# so every run lands in its own file and concurrent runs never collide. Pass
|
||||
# --out explicitly to override (sharded runs do this, one file per shard).
|
||||
p.add_argument("--out", default=None,
|
||||
help="Output JSONL. Default: data/orchestrator_sft_<MM-DD-HHMMam>.jsonl")
|
||||
# The orchestrator == the local Qwen3-8B self-sampling over its own rollouts.
|
||||
p.add_argument("--orchestrator-endpoint", default="http://localhost:8001/v1",
|
||||
help="OpenAI-compatible vLLM base URL serving the orchestrator.")
|
||||
p.add_argument("--orchestrator-model", default="qwen3-8b")
|
||||
p.add_argument("--orchestrator-api-key", default="EMPTY")
|
||||
# Provenance stamps: when set, every written record carries these fields so
|
||||
# the JSONL is self-identifying once pooled across model families (gemma vs
|
||||
# qwen). Default None -> no stamping (existing qwen runs unaffected).
|
||||
p.add_argument("--gen-model", default=None,
|
||||
help="Full HF id of the generating orchestrator, stamped as "
|
||||
"record['gen_model'] (e.g. google/gemma-4-26B-A4B-it).")
|
||||
p.add_argument("--orchestrator-label", default=None,
|
||||
help="Short label stamped as record['orchestrator_model'] "
|
||||
"(e.g. gemma-4-26b).")
|
||||
# Local OSS model endpoints; repeatable "model_id=base_url" (e.g.
|
||||
# "Qwen/Qwen3.5-9B=http://localhost:8001/v1"). Unmapped local models are
|
||||
# still listed but served unconfigured (base_url=None).
|
||||
@@ -77,13 +94,112 @@ def main(argv: Optional[list[str]] = None) -> int:
|
||||
help="Local model id -> vLLM base URL (repeatable).")
|
||||
p.add_argument("--max-tasks", type=int, default=None,
|
||||
help="Cap on tasks (default: all of load_sft_tasks()).")
|
||||
p.add_argument("--skip-task-ids-from", action="append", default=[],
|
||||
metavar="GLOB",
|
||||
help="Glob of prior data.jsonl files; skip task_ids already "
|
||||
"generated there so this run only does unseen prompts "
|
||||
"(resume). Repeatable; applied before sharding.")
|
||||
p.add_argument("--samples-per-task", type=int, default=8)
|
||||
p.add_argument("--max-keep-per-task", type=int, default=1)
|
||||
p.add_argument("--max-turns", type=int, default=8)
|
||||
p.add_argument("--temperature", type=float, default=0.7)
|
||||
# Orchestrator completion cap per turn. The library default (4096) intermittently
|
||||
# truncates the final answer mid-sentence on longer reasoning turns; bump it so
|
||||
# the trace ends cleanly. Raise further (e.g. 16384) if traces still cut off.
|
||||
p.add_argument("--max-tokens", type=int, default=8192,
|
||||
help="Per-turn completion cap for the orchestrator (default 8192).")
|
||||
# Number of tasks rolled out concurrently against vLLM. Each in-flight task
|
||||
# issues its samples sequentially, so ~concurrency requests hit the server at
|
||||
# once. ~30-50 is comfortable on 1xL40S (prefix caching + continuous batching).
|
||||
p.add_argument("--concurrency", type=int, default=32,
|
||||
help="Tasks rolled out in parallel (default 32).")
|
||||
# Balanced-smoke pull: cap//4 from each of GeneralThought + OpenThoughts
|
||||
# code/math/science instead of the GeneralThought-only fast cap path. Use for a
|
||||
# representative smoke; the real run omits --max-tasks for the full 8K balanced set.
|
||||
p.add_argument("--balanced", action=argparse.BooleanOptionalAction, default=True,
|
||||
help="Draw an EVEN cross-domain sample (GeneralThought + "
|
||||
"OpenThoughts code/math/science). Default ON — pass "
|
||||
"--no-balanced for the old GeneralThought-only skew.")
|
||||
# Data-parallel sharding: split the (deterministic, seed-42) task list across
|
||||
# N independent driver processes, each pointed at its own orchestrator vLLM
|
||||
# replica. Shard i takes tasks[i::N] (a strided slice, so every shard stays
|
||||
# domain-balanced). Each writes its own --out file; concatenate the shard
|
||||
# JSONLs afterward. Lets the GPU-bound orchestrator scale across idle GPUs.
|
||||
p.add_argument("--shard-index", type=int, default=0,
|
||||
help="This shard's index in [0, shard-count).")
|
||||
p.add_argument("--shard-count", type=int, default=1,
|
||||
help="Total number of shards (default 1 = no sharding).")
|
||||
# Throughput knob: stop sampling a task as soon as max-keep-per-task passing
|
||||
# trajectories are found, instead of always running all --samples-per-task.
|
||||
# ~3-4x faster when most tasks solve early, but keeps the *first* passers
|
||||
# rather than the *cheapest* of N (drops the cost-optimisation signal).
|
||||
p.add_argument("--stop-at-keep", action="store_true",
|
||||
help="Short-circuit a task once max-keep passing samples found.")
|
||||
# Anonymize model experts (opaque random labels, uniform description, no cost
|
||||
# line, shuffled order) so the policy can't route on a model's name/position/
|
||||
# cost. The anon->real map is saved per record (metrics.anon_map) for analysis.
|
||||
p.add_argument("--anonymize-experts", action="store_true",
|
||||
help="Hide expert identity (random labels + shuffle) when routing.")
|
||||
# By default we keep EVERY rolled-out trajectory (correct + incorrect), each
|
||||
# tagged with ``correct`` (verifier verdict) and ``kept`` (the cheapest-correct
|
||||
# sample the rejection sampler would pick). Lets you compute accuracy / inspect
|
||||
# failures from the JSONL directly. --rejection-only restores the old
|
||||
# drop-the-failures behaviour (only cheapest-correct written).
|
||||
p.add_argument("--rejection-only", action="store_true",
|
||||
help="Drop incorrect rollouts; write only the cheapest-correct "
|
||||
"sample per task (the original behaviour).")
|
||||
# Rollouts call shell_exec / file_write with model-chosen relative paths
|
||||
# (e.g. ``solution.py``), which otherwise land in the repo root. Run them
|
||||
# from a throwaway scratch dir so generated files never dirty the tree.
|
||||
# gitignored via the existing ``scratch/`` rule.
|
||||
p.add_argument("--scratch-dir", default="scratch/sft-rollouts",
|
||||
help="CWD for rollouts; stray tool-written files go here.")
|
||||
args = p.parse_args(argv)
|
||||
|
||||
logging.basicConfig(level=logging.INFO, format="%(message)s")
|
||||
# Quiet the per-request HTTP / dataset-stream spam so the run log stays
|
||||
# readable (rollout calls and dataset shards otherwise flood it).
|
||||
for _noisy in ("httpx", "httpcore", "urllib3", "datasets", "fsspec",
|
||||
"huggingface_hub", "openai"):
|
||||
logging.getLogger(_noisy).setLevel(logging.WARNING)
|
||||
|
||||
# Every run lands in its OWN datetimed folder, all collected flat under
|
||||
# data/runs/ — so runs sort by time, never collide, and never litter data/.
|
||||
# --out is treated as an optional LABEL for the folder, not a path; the data
|
||||
# file inside is always data.jsonl (+ .stats.json/.pretty.json/.txt/.html).
|
||||
# Folder name: <MM-DD-HHMMam/pm>_<label> (local PT) e.g. 06-24-0349pm_gemma_20_anon
|
||||
stamp = time.strftime("%m-%d-%I%M%p").lower()
|
||||
label = Path(args.out).stem if args.out else "orchestrator_sft"
|
||||
run_dir = (Path("data/runs") / f"{stamp}_{label}").resolve()
|
||||
if run_dir.exists(): # parallel runs, same label+minute -> disambiguate
|
||||
run_dir = run_dir.with_name(f"{run_dir.name}-{os.getpid()}")
|
||||
out_p = run_dir / "data.jsonl"
|
||||
lock_p = out_p.with_suffix(out_p.suffix + ".lock")
|
||||
out_p.parent.mkdir(parents=True, exist_ok=True)
|
||||
lock_p.write_text(f"{stamp} pid={os.getpid()}")
|
||||
args.out = str(out_p)
|
||||
import atexit
|
||||
atexit.register(lambda: lock_p.exists() and lock_p.unlink())
|
||||
logging.info("Run dir: %s", run_dir)
|
||||
|
||||
# Enrich Braintrust rollout traces with run-level provenance (no-op-safe;
|
||||
# tracing.run_context() reads these). setdefault so an explicit env wins.
|
||||
os.environ.setdefault("OJ_RUN_LABEL", label)
|
||||
os.environ.setdefault("OJ_GEN_MODEL", args.gen_model or args.orchestrator_model)
|
||||
os.environ.setdefault("OJ_RUN_STAGE",
|
||||
"smoke" if args.max_tasks else "prod")
|
||||
os.environ["OJ_CFG_TEMPERATURE"] = str(args.temperature)
|
||||
os.environ["OJ_CFG_MAX_TURNS"] = str(args.max_turns)
|
||||
os.environ["OJ_CFG_ANONYMIZE"] = str(bool(args.anonymize_experts))
|
||||
os.environ["OJ_CFG_REJECTION_ONLY"] = str(bool(args.rejection_only))
|
||||
|
||||
# Sandbox the rollouts: chdir into a scratch dir so any file a rollout writes
|
||||
# (shell_exec redirects, file_write with a relative path) lands there instead
|
||||
# of the repo root. (--out is already absolute via run_dir.resolve() above.)
|
||||
scratch = Path(args.scratch_dir).resolve()
|
||||
scratch.mkdir(parents=True, exist_ok=True)
|
||||
os.chdir(scratch)
|
||||
logging.info("Rollout scratch dir (cwd): %s", scratch)
|
||||
|
||||
local_endpoints = {}
|
||||
for item in args.local_endpoint:
|
||||
@@ -91,6 +207,11 @@ def main(argv: Optional[list[str]] = None) -> int:
|
||||
if model_id and url:
|
||||
local_endpoints[model_id] = url
|
||||
tools = orchestrator_catalog(local_endpoints=local_endpoints or None)
|
||||
# Drop the no-op ``think`` scratchpad tool: it adds turns/cost and is a
|
||||
# harness-leakage vector (the model narrates the routing rule inside a think
|
||||
# call, which the reasoning-scrub can't reach). The model still reasons in
|
||||
# its own <think> blocks — it just can't emit reasoning AS a tool call.
|
||||
tools = [t for t in tools if t.name != "think"]
|
||||
logging.info("Tool catalog (%d): %s", len(tools), [t.name for t in tools])
|
||||
|
||||
call_orch = make_call_orchestrator(
|
||||
@@ -98,24 +219,71 @@ def main(argv: Optional[list[str]] = None) -> int:
|
||||
base_url=args.orchestrator_endpoint,
|
||||
api_key=args.orchestrator_api_key,
|
||||
temperature=args.temperature,
|
||||
max_tokens=args.max_tokens,
|
||||
)
|
||||
dispatch = make_dispatch({})
|
||||
|
||||
# Build the provenance stamp once (None if neither flag given).
|
||||
record_extra = {}
|
||||
if args.gen_model:
|
||||
record_extra["gen_model"] = args.gen_model
|
||||
if args.orchestrator_label:
|
||||
record_extra["orchestrator_model"] = args.orchestrator_label
|
||||
record_extra = record_extra or None
|
||||
|
||||
def rollout_fn(task):
|
||||
try:
|
||||
return run_unified_rollout(
|
||||
task.instruction, tools,
|
||||
call_orchestrator=call_orch, dispatch=dispatch,
|
||||
max_turns=args.max_turns,
|
||||
anonymize=args.anonymize_experts,
|
||||
)
|
||||
except Exception as exc: # network/key failures shouldn't kill the run
|
||||
logging.warning("rollout failed for %s: %s", task.task_id, exc)
|
||||
return None
|
||||
|
||||
tasks = load_sft_tasks()
|
||||
if args.max_tasks is not None:
|
||||
tasks = tasks[: args.max_tasks]
|
||||
logging.info("Loaded %d SFT tasks", len(tasks))
|
||||
# cap= caps the task count for smoke runs; --balanced makes that small sample
|
||||
# cross-domain (GeneralThought + OpenThoughts code/math/science) instead of the
|
||||
# GeneralThought-only fast path. The real run omits --max-tasks (full 8K balanced).
|
||||
tasks = load_sft_tasks(cap=args.max_tasks, balanced=args.balanced)
|
||||
# Resume on unseen prompts: drop task_ids already generated in prior runs
|
||||
# (per-track seen set via --skip-task-ids-from). Filter BEFORE sharding so the
|
||||
# remaining unseen tasks stride evenly and never re-cover finished prompts.
|
||||
if args.skip_task_ids_from:
|
||||
import glob as _glob
|
||||
skip_ids = set()
|
||||
for pattern in args.skip_task_ids_from:
|
||||
for fp in _glob.glob(pattern):
|
||||
try:
|
||||
with open(fp) as fh:
|
||||
for line in fh:
|
||||
line = line.strip()
|
||||
if not line:
|
||||
continue
|
||||
try:
|
||||
tid = json.loads(line).get("task_id")
|
||||
except Exception:
|
||||
continue
|
||||
if tid:
|
||||
skip_ids.add(tid)
|
||||
except OSError:
|
||||
continue
|
||||
if skip_ids:
|
||||
n_before = len(tasks)
|
||||
tasks = [t for t in tasks if t.task_id not in skip_ids]
|
||||
logging.info("Resume: %d done task_ids -> skipping; %d of %d tasks remain",
|
||||
len(skip_ids), len(tasks), n_before)
|
||||
if args.shard_count > 1:
|
||||
n_all = len(tasks)
|
||||
tasks = tasks[args.shard_index::args.shard_count]
|
||||
logging.info("Shard %d/%d -> %d of %d tasks",
|
||||
args.shard_index, args.shard_count, len(tasks), n_all)
|
||||
from collections import Counter as _Counter
|
||||
_dom = _Counter(getattr(t, "domain", "unknown") for t in tasks)
|
||||
logging.info("Loaded %d SFT tasks | balanced=%s | domain split: %s",
|
||||
len(tasks), args.balanced,
|
||||
", ".join(f"{d}={n}" for d, n in sorted(_dom.items())))
|
||||
|
||||
stats = generate_sft_dataset(
|
||||
args.out,
|
||||
@@ -126,7 +294,23 @@ def main(argv: Optional[list[str]] = None) -> int:
|
||||
samples_per_task=args.samples_per_task,
|
||||
max_keep_per_task=args.max_keep_per_task,
|
||||
reward_fn=lambda r: -r.cost_usd, # cheapest-correct gets highest reward
|
||||
concurrency=args.concurrency,
|
||||
stop_at_keep=args.stop_at_keep,
|
||||
keep_all=not args.rejection_only,
|
||||
record_extra=record_extra,
|
||||
)
|
||||
# Emit human-viewable companions (<out>.pretty.json + <out>.txt) so the
|
||||
# traces / full message turns can be inspected without un-escaping the JSONL.
|
||||
try:
|
||||
from render_sft_data import render # same scripts/orchestrator dir
|
||||
except ImportError:
|
||||
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
||||
from render_sft_data import render
|
||||
if stats.get("records_written", 0) > 0:
|
||||
rinfo = render(args.out)
|
||||
logging.info("Rendered viewable traces: %s + %s",
|
||||
rinfo["pretty"], rinfo["txt"])
|
||||
|
||||
print(json.dumps(stats, indent=2))
|
||||
return 0
|
||||
|
||||
|
||||
@@ -1,103 +0,0 @@
|
||||
#!/usr/bin/env python
|
||||
"""Build ToolOrchestra SFT data by rejection sampling over ToolScale.
|
||||
|
||||
For each ToolScale task, a teacher orchestrator is rolled out N times over the
|
||||
faithful unified tool catalog (one tool per model); passing trajectories are
|
||||
serialized into the conversations JSONL the SFT trainer consumes.
|
||||
|
||||
Needs API access for the teacher + expert models (set the usual env keys).
|
||||
|
||||
Example:
|
||||
uv run python scripts/orchestrator/build_unified_sft.py \
|
||||
--out data/orchestrator_unified_sft.jsonl \
|
||||
--teacher-model gpt-5 --max-tasks 200 --samples-per-task 4 \
|
||||
--local-model qwen3:8b --local-endpoint http://localhost:8001/v1
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
from typing import Optional
|
||||
|
||||
from openjarvis.agents.hybrid.expert_registry import default_catalog
|
||||
from openjarvis.agents.hybrid.toolorchestra.unified import (
|
||||
make_call_orchestrator,
|
||||
make_dispatch,
|
||||
)
|
||||
from openjarvis.agents.hybrid.toolorchestra.rollout import run_unified_rollout
|
||||
from openjarvis.learning.intelligence.orchestrator.sft_data.reject_sample import (
|
||||
generate_sft_dataset,
|
||||
gold_coverage_verify,
|
||||
)
|
||||
from openjarvis.learning.intelligence.orchestrator.sft_data.toolscale import (
|
||||
load_toolscale,
|
||||
)
|
||||
|
||||
|
||||
def main(argv: Optional[list[str]] = None) -> int:
|
||||
p = argparse.ArgumentParser(description=__doc__)
|
||||
p.add_argument("--out", default="data/orchestrator_unified_sft.jsonl")
|
||||
p.add_argument("--teacher-model", default="claude-sonnet-4-6")
|
||||
p.add_argument("--teacher-base-url", default="https://api.anthropic.com/v1/",
|
||||
help="OpenAI-compatible base URL; pass '' for OpenAI cloud.")
|
||||
p.add_argument("--max-tasks", type=int, default=200)
|
||||
p.add_argument("--samples-per-task", type=int, default=4)
|
||||
p.add_argument("--max-keep-per-task", type=int, default=1)
|
||||
p.add_argument("--max-turns", type=int, default=50)
|
||||
p.add_argument("--temperature", type=float, default=1.0)
|
||||
p.add_argument("--local-model", default=None)
|
||||
p.add_argument("--local-endpoint", default=None)
|
||||
args = p.parse_args(argv)
|
||||
|
||||
logging.basicConfig(level=logging.INFO, format="%(message)s")
|
||||
|
||||
tools = default_catalog(
|
||||
local_model=args.local_model, local_endpoint=args.local_endpoint,
|
||||
)
|
||||
logging.info("Tool catalog (%d): %s", len(tools), [t.name for t in tools])
|
||||
|
||||
base_url = args.teacher_base_url or None
|
||||
api_key = (
|
||||
os.environ.get("ANTHROPIC_API_KEY")
|
||||
if base_url and "anthropic" in base_url
|
||||
else os.environ.get("OPENAI_API_KEY")
|
||||
)
|
||||
call_orch = make_call_orchestrator(
|
||||
args.teacher_model,
|
||||
base_url=base_url,
|
||||
api_key=api_key,
|
||||
temperature=args.temperature,
|
||||
)
|
||||
dispatch = make_dispatch({})
|
||||
|
||||
def rollout_fn(task):
|
||||
try:
|
||||
return run_unified_rollout(
|
||||
task.instruction, tools,
|
||||
call_orchestrator=call_orch, dispatch=dispatch,
|
||||
max_turns=args.max_turns,
|
||||
)
|
||||
except Exception as exc: # network/key failures shouldn't kill the run
|
||||
logging.warning("rollout failed for %s: %s", task.task_id, exc)
|
||||
return None
|
||||
|
||||
tasks = load_toolscale(max_tasks=args.max_tasks)
|
||||
stats = generate_sft_dataset(
|
||||
args.out,
|
||||
tasks=tasks,
|
||||
tools=tools,
|
||||
rollout_fn=rollout_fn,
|
||||
verify_fn=gold_coverage_verify,
|
||||
samples_per_task=args.samples_per_task,
|
||||
max_keep_per_task=args.max_keep_per_task,
|
||||
reward_fn=lambda r: -r.cost_usd, # cheapest-correct gets highest reward
|
||||
)
|
||||
print(json.dumps(stats, indent=2))
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,109 @@
|
||||
#!/usr/bin/env python
|
||||
"""Interactive REPL to chat with a trained orchestrator checkpoint.
|
||||
|
||||
Reuses the SAME routing loop the eval uses (OrchestratorBackend.generate_full),
|
||||
so what you see here is exactly how it behaves on the benchmarks: it reads your
|
||||
question, decides whether to answer itself / call a tool / delegate to an expert
|
||||
model, runs that rollout, and prints the final answer + a routing summary.
|
||||
|
||||
Prereqs:
|
||||
1. Serve the checkpoint you want on a vLLM port (see --endpoint). e.g. 8k:
|
||||
PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True CUDA_VISIBLE_DEVICES=0 \
|
||||
.venv/bin/python -m vllm.entrypoints.openai.api_server \
|
||||
--model "$OJ_WORK/qwen_eval_ckpts/8k_served" \
|
||||
--served-model-name sft-qwen-8k --port 8020 \
|
||||
--gpu-memory-utilization 0.88 --trust-remote-code --max-model-len 32768 \
|
||||
--enforce-eager --enable-auto-tool-choice --tool-call-parser qwen3_xml
|
||||
2. source .env (cloud experts GPT-5.5 / Opus need the API keys)
|
||||
|
||||
Usage:
|
||||
.venv/bin/python scripts/orchestrator/chat_orchestrator.py \
|
||||
--endpoint http://localhost:8020/v1 --model sft-qwen-8k
|
||||
|
||||
# point local expert tiers at their own vLLMs if you have them up (optional):
|
||||
# --local-endpoint Qwen/Qwen3.6-27B=http://localhost:8002/v1
|
||||
"""
|
||||
import argparse
|
||||
import sys
|
||||
import time
|
||||
|
||||
|
||||
def _parse_args(argv=None):
|
||||
p = argparse.ArgumentParser(description=__doc__,
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter)
|
||||
p.add_argument("--endpoint", default="http://localhost:8020/v1",
|
||||
help="vLLM endpoint serving the orchestrator checkpoint.")
|
||||
p.add_argument("--model", default="sft-qwen-8k",
|
||||
help="Served-model-name of the checkpoint.")
|
||||
p.add_argument("--api-key", default="EMPTY", help="API key for the endpoint (local vLLM = EMPTY).")
|
||||
p.add_argument("--temperature", type=float, default=0.0)
|
||||
p.add_argument("--max-turns", type=int, default=8, help="Same default as the eval.")
|
||||
p.add_argument("--max-tokens", type=int, default=3000,
|
||||
help="Keep high: the model reasons a lot before it emits the delegation.")
|
||||
p.add_argument("--local-endpoint", action="append", default=[],
|
||||
help="MODEL_ID=URL for a local expert tier. Repeatable.")
|
||||
return p.parse_args(argv)
|
||||
|
||||
|
||||
def main(argv=None):
|
||||
args = _parse_args(argv)
|
||||
|
||||
local_endpoints = {}
|
||||
for pair in args.local_endpoint:
|
||||
if "=" not in pair:
|
||||
raise SystemExit(f"--local-endpoint expects MODEL_ID=URL, got: {pair!r}")
|
||||
mid, url = pair.split("=", 1)
|
||||
local_endpoints[mid.strip()] = url.strip()
|
||||
|
||||
from openjarvis.learning.intelligence.orchestrator.eval_backend import (
|
||||
OrchestratorBackend,
|
||||
)
|
||||
|
||||
backend = OrchestratorBackend(
|
||||
orchestrator_endpoint=args.endpoint,
|
||||
orchestrator_model=args.model,
|
||||
api_key=args.api_key,
|
||||
local_endpoints=local_endpoints,
|
||||
max_turns=args.max_turns,
|
||||
temperature=args.temperature,
|
||||
)
|
||||
|
||||
print(f"orchestrator: {args.model} @ {args.endpoint} "
|
||||
f"(temp={args.temperature}, max_turns={args.max_turns})")
|
||||
print("type a question and hit enter. Ctrl-D or 'quit' to exit.\n")
|
||||
|
||||
try:
|
||||
while True:
|
||||
try:
|
||||
prompt = input("you> ").strip()
|
||||
except EOFError:
|
||||
print()
|
||||
break
|
||||
if not prompt:
|
||||
continue
|
||||
if prompt.lower() in {"quit", "exit"}:
|
||||
break
|
||||
|
||||
started = time.time()
|
||||
full = backend.generate_full(
|
||||
prompt, model=args.model, temperature=args.temperature,
|
||||
max_tokens=args.max_tokens,
|
||||
)
|
||||
elapsed = time.time() - started
|
||||
|
||||
if full.get("error"):
|
||||
print(f"\n[ERROR] {full['error']}\n", file=sys.stderr)
|
||||
continue
|
||||
|
||||
print(f"\norch> {full.get('content', '').strip()}\n")
|
||||
print(f" [turns={full.get('turn_count', '?')} "
|
||||
f"tool_calls={full.get('tool_calls', '?')} "
|
||||
f"cost=${full.get('cost_usd', 0.0):.4f} "
|
||||
f"{elapsed:.1f}s]\n")
|
||||
finally:
|
||||
backend.close()
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -301,8 +301,29 @@ def main(argv: Optional[List[str]] = None) -> int:
|
||||
finally:
|
||||
backend.close()
|
||||
|
||||
# Resolve the EXACT model behind the served alias (e.g. "gemma-sft" ->
|
||||
# the real id + checkpoint path vLLM reports), so results are self-identifying
|
||||
# instead of an opaque label. Best-effort: never fail the summary over it.
|
||||
served_id, served_path = args.orchestrator_model, None
|
||||
try:
|
||||
import urllib.request
|
||||
req = urllib.request.Request(
|
||||
args.orchestrator_endpoint.rstrip("/") + "/models",
|
||||
headers={"Authorization": f"Bearer {args.orchestrator_api_key}"},
|
||||
)
|
||||
with urllib.request.urlopen(req, timeout=10) as resp:
|
||||
data = json.load(resp).get("data", [])
|
||||
if data:
|
||||
served_id = data[0].get("id", served_id)
|
||||
# vLLM reports the loaded checkpoint path in `root` (or `parent`).
|
||||
served_path = data[0].get("root") or data[0].get("parent")
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
combined = {
|
||||
"orchestrator_model": args.orchestrator_model,
|
||||
"orchestrator_served_id": served_id,
|
||||
"orchestrator_model_path": served_path,
|
||||
"orchestrator_endpoint": args.orchestrator_endpoint,
|
||||
"judge_model": args.judge_model,
|
||||
"judge_engine": args.judge_engine,
|
||||
|
||||
@@ -0,0 +1,239 @@
|
||||
#!/usr/bin/env python
|
||||
"""Render graded orchestrator-eval JSONL records into clean, readable .txt files.
|
||||
|
||||
Companion to ``format_sft_sample.py`` (same banner/indent visual style), but
|
||||
for the *scored* eval outputs written by ``eval_orchestrator.py``:
|
||||
|
||||
results/orch-eval-<model>-<tranche>-<suite>/<benchmark>_orchestrator.jsonl
|
||||
|
||||
Each record has: record_id, benchmark, model, model_answer (the model's
|
||||
FINAL_ANSWER string), is_correct, score, latency_seconds, cost_usd, error, and
|
||||
scoring_metadata. The full step-by-step routing trace is NOT saved — only the
|
||||
final answer + gold + score — so this renderer cannot show intermediate steps.
|
||||
|
||||
The QUESTION text is not in the result JSONL; it's loaded from the benchmark
|
||||
dataset by ``record_id`` (via ``openjarvis.evals.cli._build_dataset``). If the
|
||||
dataset can't be loaded / the id doesn't match, we render everything else and
|
||||
show "(question unavailable)".
|
||||
|
||||
Gold answer lives in ``scoring_metadata`` (shape varies by scorer):
|
||||
* MCQ (mmlu-pro, supergpqa): ``reference_letter``.
|
||||
* LLM-judge (gaia): parsed from ``raw_judge_output`` (`gold target "..."`),
|
||||
plus the judge's ``reasoning``.
|
||||
* anything else: dumped verbatim under SCORING (raw).
|
||||
|
||||
Usage:
|
||||
.venv/bin/python scripts/orchestrator/format_eval_sample.py \
|
||||
--input results/orch-eval-gemma-2k-full/gaia_orchestrator.jsonl --n 2
|
||||
|
||||
... --input <file> --lines 1,5,42 # specific 1-indexed lines
|
||||
... --input <file> --all # one .txt per record
|
||||
... --input <file> --all --only-wrong # only incorrect samples
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import re
|
||||
from pathlib import Path
|
||||
|
||||
WIDTH = 80
|
||||
|
||||
# Friendly aliases -> the registry keys _build_dataset expects (mirrors
|
||||
# eval_orchestrator.BENCHMARK_ALIASES for the ones that write .jsonl here).
|
||||
BENCHMARK_ALIASES = {
|
||||
"mmlu_pro": "mmlu-pro",
|
||||
"mmlu-pro": "mmlu-pro",
|
||||
"gaia": "gaia",
|
||||
"taubench": "taubench",
|
||||
"supergpqa": "supergpqa",
|
||||
"terminalbench_v2_1": "terminalbench-v2.1",
|
||||
"terminalbench-v2.1": "terminalbench-v2.1",
|
||||
}
|
||||
|
||||
|
||||
def _banner(label: str) -> str:
|
||||
"""Full-width headline: ``━━━ LABEL ━━━━━━…``."""
|
||||
prefix = f"━━━ {label} "
|
||||
fill = "━" * max(3, WIDTH - len(prefix))
|
||||
return f"\n{prefix}{fill}\n"
|
||||
|
||||
|
||||
def _indent(text: str, pad: str = " ") -> str:
|
||||
text = (text or "").rstrip()
|
||||
return "\n".join(pad + ln if ln.strip() else ln for ln in text.splitlines())
|
||||
|
||||
|
||||
def _normalize_benchmark(name: str) -> str:
|
||||
key = (name or "").strip()
|
||||
return BENCHMARK_ALIASES.get(key, BENCHMARK_ALIASES.get(key.lower(), key))
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Question map: record_id -> problem text, loaded from the benchmark dataset.
|
||||
# ---------------------------------------------------------------------------
|
||||
def build_question_map(benchmark: str, n: int, seed: int = 42) -> dict:
|
||||
"""Best-effort {record_id: problem}. Returns {} if the dataset can't load."""
|
||||
try:
|
||||
from openjarvis.evals.cli import _build_dataset
|
||||
|
||||
ds = _build_dataset(_normalize_benchmark(benchmark))
|
||||
ds.load(max_samples=n, seed=seed)
|
||||
return {r.record_id: r.problem for r in ds.iter_records()}
|
||||
except Exception as exc: # noqa: BLE001 - never fail the render over this
|
||||
print(f" [warn] could not load dataset for {benchmark!r}: "
|
||||
f"{type(exc).__name__}: {exc}")
|
||||
return {}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Gold / judge parsing out of scoring_metadata.
|
||||
# ---------------------------------------------------------------------------
|
||||
_GOLD_TARGET_RE = re.compile(r'gold target\s*"([^"]*)"', re.IGNORECASE)
|
||||
_JUDGE_REASONING_RE = re.compile(
|
||||
r"reasoning:\s*(.*?)(?:\n\s*correct:|\Z)", re.IGNORECASE | re.DOTALL
|
||||
)
|
||||
|
||||
|
||||
def parse_scoring(meta: dict) -> dict:
|
||||
"""Return {gold, judge, kind, clean}. ``clean`` False => dump raw block.
|
||||
|
||||
kind is one of 'mcq' | 'judge' | 'unknown'.
|
||||
"""
|
||||
if not isinstance(meta, dict):
|
||||
return {"gold": None, "judge": None, "kind": "unknown", "clean": False}
|
||||
|
||||
# MCQ scorer (mmlu-pro / supergpqa).
|
||||
if "reference_letter" in meta:
|
||||
return {
|
||||
"gold": meta.get("reference_letter"),
|
||||
"judge": None,
|
||||
"kind": "mcq",
|
||||
"clean": True,
|
||||
}
|
||||
|
||||
# LLM-judge scorer (gaia).
|
||||
raw = meta.get("raw_judge_output")
|
||||
if raw:
|
||||
gold_m = _GOLD_TARGET_RE.search(raw)
|
||||
reason_m = _JUDGE_REASONING_RE.search(raw)
|
||||
return {
|
||||
"gold": gold_m.group(1) if gold_m else None,
|
||||
"judge": (reason_m.group(1).strip() if reason_m else raw.strip()),
|
||||
"kind": "judge",
|
||||
"clean": True,
|
||||
}
|
||||
|
||||
# Unknown shape (e.g. {"reason": "no_choice_letter_extracted"}).
|
||||
return {"gold": None, "judge": None, "kind": "unknown", "clean": False}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Render one record.
|
||||
# ---------------------------------------------------------------------------
|
||||
def format_record(rec: dict, question: str | None) -> str:
|
||||
parsed = parse_scoring(rec.get("scoring_metadata"))
|
||||
is_correct = rec.get("is_correct")
|
||||
verdict = "CORRECT" if is_correct else ("INCORRECT" if is_correct is False else "UNSCORED")
|
||||
|
||||
def _fmt(v, fmt):
|
||||
try:
|
||||
return fmt.format(v)
|
||||
except (ValueError, TypeError):
|
||||
return str(v)
|
||||
|
||||
top = " · ".join([
|
||||
str(rec.get("record_id", "?")),
|
||||
str(rec.get("benchmark", "?")),
|
||||
verdict,
|
||||
f"score {_fmt(rec.get('score'), '{:.3f}')}",
|
||||
f"lat {_fmt(rec.get('latency_seconds'), '{:.1f}')}s",
|
||||
f"cost ${_fmt(rec.get('cost_usd'), '{:.4f}')}",
|
||||
])
|
||||
parts = [top]
|
||||
|
||||
err = rec.get("error")
|
||||
if err:
|
||||
parts.append(f" error: {err}")
|
||||
|
||||
parts.append(_banner("QUESTION"))
|
||||
parts.append(_indent(question if question else "(question unavailable)"))
|
||||
|
||||
parts.append(_banner("MODEL ANSWER"))
|
||||
parts.append(_indent(str(rec.get("model_answer") or "(empty)")))
|
||||
|
||||
parts.append(_banner("GOLD"))
|
||||
gold = parsed["gold"]
|
||||
parts.append(_indent(str(gold) if gold not in (None, "") else "(gold unavailable — see SCORING below)"))
|
||||
|
||||
if parsed["kind"] == "judge" and parsed["judge"]:
|
||||
parts.append(_banner("JUDGE"))
|
||||
parts.append(_indent(parsed["judge"]))
|
||||
|
||||
if not parsed["clean"]:
|
||||
parts.append(_banner("SCORING (raw)"))
|
||||
parts.append(_indent(json.dumps(rec.get("scoring_metadata"), indent=2, default=str)))
|
||||
|
||||
return "\n".join(parts).rstrip() + "\n"
|
||||
|
||||
|
||||
def main(argv=None):
|
||||
p = argparse.ArgumentParser(
|
||||
description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter
|
||||
)
|
||||
p.add_argument("--input", required=True, help="Graded *_orchestrator.jsonl file.")
|
||||
p.add_argument("--out-dir", default="results/formatted", help="Where the .txt files go.")
|
||||
p.add_argument("--seed", type=int, default=42, help="Subset seed used at eval time.")
|
||||
p.add_argument("--only-wrong", action="store_true",
|
||||
help="Render only incorrect/unscored samples.")
|
||||
g = p.add_mutually_exclusive_group()
|
||||
g.add_argument("--n", type=int, default=1, help="Format the first N records.")
|
||||
g.add_argument("--lines", help="Comma-separated 1-indexed line numbers.")
|
||||
g.add_argument("--all", action="store_true", help="Format every record.")
|
||||
args = p.parse_args(argv)
|
||||
|
||||
src = Path(args.input)
|
||||
out_dir = Path(args.out_dir)
|
||||
out_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
records = [json.loads(l) for l in src.open() if l.strip()]
|
||||
if not records:
|
||||
print("no records in input")
|
||||
return 1
|
||||
|
||||
# Pick indices first (so we only load the dataset once, sized to the file).
|
||||
if args.lines:
|
||||
idxs = [int(x) - 1 for x in args.lines.split(",") if x.strip()]
|
||||
elif args.all:
|
||||
idxs = list(range(len(records)))
|
||||
else:
|
||||
idxs = list(range(min(args.n, len(records))))
|
||||
idxs = [i for i in idxs if 0 <= i < len(records)]
|
||||
|
||||
if args.only_wrong:
|
||||
idxs = [i for i in idxs if not records[i].get("is_correct")]
|
||||
|
||||
benchmark = _normalize_benchmark(records[0].get("benchmark", src.stem.split("_")[0]))
|
||||
# Load the whole subset so any selected id resolves (the eval-time subset is
|
||||
# the first len(records) of seed=42).
|
||||
qmap = build_question_map(benchmark, n=len(records), seed=args.seed)
|
||||
|
||||
written = []
|
||||
for i in idxs:
|
||||
rec = records[i]
|
||||
rid = str(rec.get("record_id", i))
|
||||
bench = rec.get("benchmark", benchmark)
|
||||
fname = re.sub(r"[^A-Za-z0-9_.-]", "_", f"{bench}__{rid}")[:120] + ".txt"
|
||||
out = out_dir / fname
|
||||
out.write_text(format_record(rec, qmap.get(rid)))
|
||||
written.append(out)
|
||||
|
||||
for w in written:
|
||||
print(w)
|
||||
print(f"\nwrote {len(written)} file(s) to {out_dir}/"
|
||||
+ (" (dataset unavailable — questions omitted)" if not qmap else ""))
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,120 @@
|
||||
#!/usr/bin/env python
|
||||
"""Carve train / holdout / overfit100 splits from a clean orchestrator-SFT pool.
|
||||
|
||||
Replaces make_tranches.py + make_gemma_tranches.py, which carved 1k/2k/4k/8k
|
||||
data-scaling tranches from the June pool. That lineage is dead: the tranches it
|
||||
produced trained checkpoints that evaluated at base ~= 1k ~= 2k (the pool taught
|
||||
the orchestrator to re-derive answers itself instead of routing), and the pool
|
||||
was deleted. See data/DELETED_ERA1_MANIFEST.md.
|
||||
|
||||
Naming is uniform: ``{name}_orch_{split}_{date}.jsonl`` with name in
|
||||
{qwen, gemma, pooled}. Pass several pools to merge-and-restratify into `pooled`.
|
||||
|
||||
The holdout is domain-stratified so val-loss is leak-free, and `overfit100` is a
|
||||
strict prefix of train (it is a memorisation sanity check — the model must be
|
||||
able to fit 100 rows it *has* seen, else the format/masking is broken).
|
||||
Deterministic (seed 42).
|
||||
|
||||
# one pool -> qwen_orch_{train,holdout,overfit100}_0711.jsonl
|
||||
python scripts/orchestrator/make_splits.py --name qwen \
|
||||
--pool data/sft_variants/qwen_orch_clean_0711.jsonl
|
||||
|
||||
# merge both -> pooled_orch_{train,holdout,overfit100}_0711.jsonl
|
||||
python scripts/orchestrator/make_splits.py --name pooled \
|
||||
--pool data/sft_variants/qwen_orch_clean_0711.jsonl \
|
||||
--pool data/sft_variants/gemma_orch_clean_0711.jsonl
|
||||
"""
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import random
|
||||
import sys
|
||||
from collections import defaultdict
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
|
||||
OUT = Path("data/sft_variants")
|
||||
SEED = 42
|
||||
OVERFIT_N = 100
|
||||
# Orchestrator that generated each pool — stamped onto every row so provenance
|
||||
# survives a merge (the filename encodes it too, but the field is cheap insurance
|
||||
# and is what the `pooled` splits rely on to stay attributable).
|
||||
ORCH_MODEL = {"qwen": "qwen3.5-9b", "gemma": "gemma-4-26b"}
|
||||
|
||||
|
||||
def main() -> int:
|
||||
ap = argparse.ArgumentParser(description=__doc__,
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter)
|
||||
ap.add_argument("--name", required=True,
|
||||
help="split family: qwen | gemma | pooled (drives the filename)")
|
||||
ap.add_argument("--pool", action="append", required=True, metavar="PATH",
|
||||
help="clean pool jsonl; repeat to merge (use with --name pooled)")
|
||||
ap.add_argument("--holdout-frac", type=float, default=0.15,
|
||||
help="fraction held out, domain-stratified (default 0.15)")
|
||||
ap.add_argument("--date", default=datetime.now().strftime("%m%d"),
|
||||
help="date tag in the filename (default: today, MMDD)")
|
||||
ap.add_argument("--out", type=Path, default=OUT)
|
||||
args = ap.parse_args()
|
||||
|
||||
rows = []
|
||||
for p in args.pool:
|
||||
path = Path(p)
|
||||
if not path.exists():
|
||||
print(f"!! pool not found: {path}", file=sys.stderr)
|
||||
return 1
|
||||
# Infer the source orchestrator from the filename so merged pools stay
|
||||
# attributable; don't clobber a stamp the pool already carries.
|
||||
stem = path.name.split("_", 1)[0]
|
||||
loaded = [json.loads(line) for line in path.open() if line.strip()]
|
||||
for r in loaded:
|
||||
r.setdefault("orchestrator_model", ORCH_MODEL.get(stem, stem))
|
||||
rows.extend(loaded)
|
||||
print(f"loaded {len(loaded):>5} rows from {path}")
|
||||
print(f"pool total: {len(rows)}")
|
||||
|
||||
# Stratified holdout: per-domain, proportional to domain size.
|
||||
by_dom = defaultdict(list)
|
||||
for i, r in enumerate(rows):
|
||||
by_dom[r.get("domain", "misc")].append(i)
|
||||
rng = random.Random(SEED)
|
||||
holdout_idx: set = set()
|
||||
for dom, idxs in sorted(by_dom.items()):
|
||||
k = round(args.holdout_frac * len(idxs))
|
||||
pick = rng.sample(idxs, min(k, len(idxs)))
|
||||
holdout_idx.update(pick)
|
||||
print(f" {dom:<9} pool={len(idxs):>4} holdout={len(pick)}")
|
||||
|
||||
holdout = [rows[i] for i in sorted(holdout_idx)]
|
||||
train = [r for i, r in enumerate(rows) if i not in holdout_idx] # order preserved
|
||||
overfit = train[:OVERFIT_N]
|
||||
print(f"train={len(train)} holdout={len(holdout)} overfit={len(overfit)}")
|
||||
|
||||
args.out.mkdir(parents=True, exist_ok=True)
|
||||
written = {}
|
||||
for split, data in (("train", train), ("holdout", holdout),
|
||||
(f"overfit{OVERFIT_N}", overfit)):
|
||||
f = args.out / f"{args.name}_orch_{split}_{args.date}.jsonl"
|
||||
f.write_text("".join(json.dumps(r) + "\n" for r in data))
|
||||
written[split] = f
|
||||
print(f"wrote {f} ({len(data)})")
|
||||
|
||||
# ---- auto-upload train + holdout to Braintrust ----
|
||||
# Gated by OJ_BRAINTRUST_AUTOUPLOAD (default ON); a total no-op / never raises
|
||||
# if the key/pkg is missing or the upload errors (see upload_to_braintrust).
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
||||
try:
|
||||
from upload_to_braintrust import autoupload
|
||||
|
||||
autoupload(
|
||||
[f"{written['train']}={args.name}_train_{args.date}",
|
||||
f"{written['holdout']}={args.name}_holdout_{args.date}"],
|
||||
run_label=os.getenv("OJ_RUN_LABEL", f"{args.name}_splits_{args.date}"),
|
||||
description=f"{args.name} orchestrator-SFT splits (make_splits.py)",
|
||||
)
|
||||
except Exception as exc: # telemetry must never break the data pipeline
|
||||
print(f"[braintrust] autoupload hook skipped ({exc})")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,290 @@
|
||||
#!/usr/bin/env python
|
||||
"""Render an orchestrator SFT JSONL into human-viewable companions.
|
||||
|
||||
The trainer-facing ``*.jsonl`` packs every trajectory onto one line with all
|
||||
newlines escaped, which is unreadable by eye. This writes two siblings next to
|
||||
it so you can actually inspect what the orchestrator did:
|
||||
|
||||
* ``<name>.pretty.json`` — the same records as an indented JSON array.
|
||||
* ``<name>.txt`` — a clean plain-text transcript. The shared system
|
||||
prompt + tool catalog (identical across every record) is printed ONCE at the
|
||||
top; each trajectory then shows just its turns: the user question, the
|
||||
assistant's reasoning/answer, tool calls rendered as ``-> name({args})``, and
|
||||
tool results truncated so a single web-search dump doesn't bury the trace.
|
||||
|
||||
Usage:
|
||||
.venv/bin/python scripts/orchestrator/render_sft_data.py <path.jsonl>
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import html as _htmllib
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import List
|
||||
|
||||
# Tool observations (web_search dumps, code stdout, file reads) can be huge; cap
|
||||
# them in the transcript so the actual orchestration stays legible. The full
|
||||
# content is always in the .jsonl / .pretty.json.
|
||||
_OBS_CAP = 1200
|
||||
_RULE = "-" * 80
|
||||
_TOOL_CALL_RE = re.compile(r"<tool_call>\s*(\{.*?\})\s*</tool_call>", re.DOTALL)
|
||||
|
||||
|
||||
def _parse_tool_calls(content: str):
|
||||
"""Parse the model's ``<tool_call>{json}</tool_call>`` tags into a list of
|
||||
parsed ``[(name, arguments_dict), ...]`` calls, de-duplicating the
|
||||
double-emit the template produces (each call shows up twice)."""
|
||||
seen = []
|
||||
for m in _TOOL_CALL_RE.finditer(content):
|
||||
try:
|
||||
call = json.loads(m.group(1))
|
||||
item = (call.get("name"), call.get("arguments", {}))
|
||||
except Exception:
|
||||
item = (None, {"_raw": m.group(1)})
|
||||
key = json.dumps(item, ensure_ascii=False, sort_keys=True, default=str)
|
||||
if not seen or seen[-1][0] != key: # collapse the adjacent duplicate
|
||||
seen.append((key, item))
|
||||
return [it for _, it in seen]
|
||||
|
||||
|
||||
def _call_label(name, anon_map) -> str:
|
||||
"""Display a call name; if it's an anonymized expert label, append the real
|
||||
model it maps to (from the record's ``metrics.anon_map``)."""
|
||||
real = (anon_map or {}).get(name)
|
||||
return f"{name} -> {real}" if real else f"{name}"
|
||||
|
||||
|
||||
def _fmt_tool_calls(content: str, anon_map=None) -> str:
|
||||
"""Plain-text tool calls with real (un-escaped) newlines in each arg value."""
|
||||
blocks = []
|
||||
for name, args in _parse_tool_calls(content):
|
||||
lines = [f"-> {_call_label(name, anon_map)}"]
|
||||
if isinstance(args, dict):
|
||||
for k, v in args.items():
|
||||
val = v if isinstance(v, str) else json.dumps(v, ensure_ascii=False, indent=2)
|
||||
indented = "\n".join(" " + ln for ln in val.splitlines())
|
||||
lines.append(f" {k}:\n{indented}")
|
||||
blocks.append("\n".join(lines))
|
||||
return "\n".join(blocks)
|
||||
|
||||
|
||||
def _clip(text: str, cap: int) -> str:
|
||||
text = text or ""
|
||||
if len(text) <= cap:
|
||||
return text
|
||||
return f"{text[:cap]}\n ... [truncated {len(text) - cap} chars]"
|
||||
|
||||
|
||||
def _render_turn(turn: dict, anon_map=None) -> str:
|
||||
role = str(turn.get("role", "?")).lower()
|
||||
content = turn.get("content", "") or ""
|
||||
|
||||
if role == "user":
|
||||
return f"USER:\n{content.strip()}"
|
||||
|
||||
if role in ("tool", "function"):
|
||||
name = turn.get("name", "tool")
|
||||
return f"TOOL [{name}]:\n{_clip(content.strip(), _OBS_CAP)}"
|
||||
|
||||
if role == "assistant":
|
||||
calls = _fmt_tool_calls(content, anon_map)
|
||||
# Strip the raw <tool_call> tags out of the prose so it isn't shown twice.
|
||||
prose = _TOOL_CALL_RE.sub("", content).strip()
|
||||
parts = []
|
||||
if prose:
|
||||
parts.append(f"ASSISTANT:\n{prose}")
|
||||
if calls:
|
||||
parts.append(f"ASSISTANT calls:\n{calls}")
|
||||
return "\n".join(parts) if parts else "ASSISTANT: (empty)"
|
||||
|
||||
# system handled separately (printed once); fall through for anything else
|
||||
return f"{role.upper()}:\n{content.strip()}"
|
||||
|
||||
|
||||
def _transcript(record: dict, index: int) -> str:
|
||||
m = record.get("metrics", {}) or {}
|
||||
correct = record.get("correct")
|
||||
kept = record.get("kept")
|
||||
flags = []
|
||||
if correct is not None:
|
||||
flags.append("correct" if correct else "WRONG")
|
||||
if kept is not None and kept:
|
||||
flags.append("kept")
|
||||
head = (
|
||||
f"### [{index}] task={record.get('task_id')} domain={record.get('domain')}"
|
||||
+ (f" {' '.join(flags)}" if flags else "")
|
||||
+ f"\n cost=${m.get('cost_usd')} tokens={m.get('tokens')} "
|
||||
f"tool_calls={m.get('num_tool_calls')} turns={m.get('num_turns')} "
|
||||
f"reward={record.get('reward')}"
|
||||
)
|
||||
anon_map = m.get("anon_map")
|
||||
body = [
|
||||
_render_turn(t, anon_map)
|
||||
for t in record.get("conversations", [])
|
||||
if str(t.get("role", "")).lower() != "system"
|
||||
]
|
||||
return head + "\n\n" + "\n\n".join(body)
|
||||
|
||||
|
||||
def _system_header(records: List[dict]) -> str:
|
||||
"""The system message is identical across records — render it once."""
|
||||
for r in records:
|
||||
for t in r.get("conversations", []):
|
||||
if str(t.get("role", "")).lower() == "system":
|
||||
return (
|
||||
"=" * 80 + "\nSHARED SYSTEM PROMPT + TOOL CATALOG "
|
||||
"(identical for every record below)\n" + "=" * 80
|
||||
+ f"\n{t.get('content', '').strip()}\n"
|
||||
)
|
||||
return ""
|
||||
|
||||
|
||||
_HTML_CSS = """
|
||||
body{font:14px/1.5 system-ui,sans-serif;margin:0;background:#f5f5f7;color:#1d1d1f}
|
||||
header{position:sticky;top:0;background:#1d1d1f;color:#fff;padding:12px 20px;z-index:9}
|
||||
header b{font-size:16px}
|
||||
.rec{background:#fff;margin:14px 20px;border-radius:10px;box-shadow:0 1px 3px #0002;overflow:hidden}
|
||||
.rec>summary{cursor:pointer;padding:10px 16px;font-weight:600;list-style:none;display:flex;gap:14px;align-items:center;flex-wrap:wrap}
|
||||
.rec>summary::-webkit-details-marker{display:none}
|
||||
.rec>summary:hover{background:#fafafa}
|
||||
.body{padding:6px 16px 16px}
|
||||
.turn{margin:10px 0;padding:10px 14px;border-radius:8px;white-space:pre-wrap;word-break:break-word}
|
||||
.user{background:#e8f0fe}
|
||||
.assistant{background:#f1f8e9}
|
||||
.call{background:#fff3e0}
|
||||
.call .cname{font-family:ui-monospace,monospace;font-weight:700;color:#b25b00;margin-bottom:6px}
|
||||
.call .cname .real{background:#b25b00;color:#fff;padding:1px 7px;border-radius:10px;font-size:12px;margin-left:4px}
|
||||
.call .arg{margin:6px 0 0 14px}
|
||||
.call .arg b{font-family:ui-monospace,monospace;color:#7a4500}
|
||||
.call .arg pre{white-space:pre-wrap;word-break:break-word;margin:3px 0 0;padding:8px 10px;background:#fff;border:1px solid #f0d9b8;border-radius:6px;font-size:13px}
|
||||
.tool details{background:#fafafa;border:1px solid #eee;border-radius:8px;padding:6px 10px;margin:10px 0}
|
||||
.tool summary{cursor:pointer;color:#555;font-weight:600}
|
||||
.tool pre{white-space:pre-wrap;word-break:break-word;margin:8px 0 0;font-size:13px}
|
||||
.role{font-size:11px;text-transform:uppercase;letter-spacing:.5px;color:#888;margin-bottom:4px}
|
||||
.pill{font-size:12px;padding:2px 8px;border-radius:10px;font-weight:600}
|
||||
.ok{background:#d7f5dd;color:#1a7f37}.wrong{background:#ffe0e0;color:#c0392b}
|
||||
.kept{background:#fff0c2;color:#9a6700}.dom{background:#eee;color:#444}
|
||||
.meta{font-weight:400;color:#888;font-size:12px}
|
||||
#sys details{margin:14px 20px;background:#fff;border-radius:10px;padding:10px 16px}
|
||||
#sys pre{white-space:pre-wrap;word-break:break-word;font-size:12px;color:#444}
|
||||
"""
|
||||
|
||||
|
||||
def _esc(s: str) -> str:
|
||||
return _htmllib.escape(s or "")
|
||||
|
||||
|
||||
def _html_turns(record: dict) -> str:
|
||||
anon_map = (record.get("metrics", {}) or {}).get("anon_map") or {}
|
||||
out = []
|
||||
for t in record.get("conversations", []):
|
||||
role = str(t.get("role", "")).lower()
|
||||
content = t.get("content", "") or ""
|
||||
if role == "system":
|
||||
continue
|
||||
if role == "user":
|
||||
out.append(f'<div class="turn user"><div class="role">user</div>{_esc(content.strip())}</div>')
|
||||
elif role in ("tool", "function"):
|
||||
name = _esc(t.get("name", "tool"))
|
||||
out.append(
|
||||
f'<div class="tool"><details><summary>tool result · {name} '
|
||||
f'({len(content)} chars)</summary><pre>{_esc(content.strip())}</pre></details></div>'
|
||||
)
|
||||
elif role == "assistant":
|
||||
prose = _TOOL_CALL_RE.sub("", content).strip()
|
||||
if prose:
|
||||
out.append(f'<div class="turn assistant"><div class="role">assistant</div>{_esc(prose)}</div>')
|
||||
for name, args in _parse_tool_calls(content):
|
||||
real = anon_map.get(name)
|
||||
label = (f'{_esc(str(name))} <span class="real">→ {_esc(str(real))}</span>'
|
||||
if real else _esc(str(name)))
|
||||
rows = [f'<div class="cname">→ {label}</div>']
|
||||
if isinstance(args, dict):
|
||||
for k, v in args.items():
|
||||
val = v if isinstance(v, str) else json.dumps(v, ensure_ascii=False, indent=2)
|
||||
rows.append(f'<div class="arg"><b>{_esc(str(k))}</b><pre>{_esc(val)}</pre></div>')
|
||||
out.append(f'<div class="turn call"><div class="role">tool call</div>{"".join(rows)}</div>')
|
||||
return "".join(out)
|
||||
|
||||
|
||||
def _html_record(record: dict, index: int) -> str:
|
||||
m = record.get("metrics", {}) or {}
|
||||
pills = [f'<span class="pill dom">{_esc(str(record.get("domain")))}</span>']
|
||||
correct = record.get("correct")
|
||||
if correct is not None:
|
||||
pills.append('<span class="pill ok">correct</span>' if correct
|
||||
else '<span class="pill wrong">wrong</span>')
|
||||
if record.get("kept"):
|
||||
pills.append('<span class="pill kept">kept</span>')
|
||||
meta = (f'<span class="meta">cost ${m.get("cost_usd")} · {m.get("tokens")} tok · '
|
||||
f'{m.get("num_tool_calls")} calls · {m.get("num_turns")} turns</span>')
|
||||
summary = (f'<span>#{index}</span><span>{_esc(str(record.get("task_id")))}</span>'
|
||||
+ "".join(pills) + meta)
|
||||
return (f'<details class="rec"><summary>{summary}</summary>'
|
||||
f'<div class="body">{_html_turns(record)}</div></details>')
|
||||
|
||||
|
||||
def _html_doc(records: List[dict], title: str) -> str:
|
||||
sys_txt = ""
|
||||
for r in records:
|
||||
for t in r.get("conversations", []):
|
||||
if str(t.get("role", "")).lower() == "system":
|
||||
sys_txt = t.get("content", "")
|
||||
break
|
||||
if sys_txt:
|
||||
break
|
||||
recs_html = "".join(_html_record(r, i + 1) for i, r in enumerate(records))
|
||||
return (
|
||||
f"<!doctype html><html><head><meta charset='utf-8'><title>{_esc(title)}</title>"
|
||||
f"<style>{_HTML_CSS}</style></head><body>"
|
||||
f"<header><b>{_esc(title)}</b> {len(records)} records "
|
||||
f"· click a row to expand</header>"
|
||||
f"<div id='sys'><details><summary style='cursor:pointer;font-weight:600;padding:4px'>"
|
||||
f"Shared system prompt + tool catalog (same for all)</summary>"
|
||||
f"<pre>{_esc(sys_txt.strip())}</pre></details></div>"
|
||||
f"{recs_html}</body></html>"
|
||||
)
|
||||
|
||||
|
||||
def render(jsonl_path: str) -> dict:
|
||||
src = Path(jsonl_path)
|
||||
records: List[dict] = []
|
||||
with src.open() as fh:
|
||||
for line in fh:
|
||||
line = line.strip()
|
||||
if line:
|
||||
records.append(json.loads(line))
|
||||
|
||||
pretty = src.with_suffix(".pretty.json")
|
||||
pretty.write_text(json.dumps(records, indent=2, ensure_ascii=False))
|
||||
|
||||
txt = src.with_suffix(".txt")
|
||||
banner = (
|
||||
f"ORCHESTRATOR SFT TRANSCRIPT — {src.name}\n{len(records)} records\n\n"
|
||||
)
|
||||
sep = f"\n\n{_RULE}\n\n"
|
||||
parts = [banner + _system_header(records)]
|
||||
parts.append(sep.join(_transcript(r, i + 1) for i, r in enumerate(records)))
|
||||
txt.write_text("\n\n".join(parts))
|
||||
|
||||
htmlp = src.with_suffix(".html")
|
||||
htmlp.write_text(_html_doc(records, src.name))
|
||||
|
||||
return {"records": len(records), "pretty": str(pretty),
|
||||
"txt": str(txt), "html": str(htmlp)}
|
||||
|
||||
|
||||
def main(argv: List[str]) -> int:
|
||||
if not argv:
|
||||
print(__doc__)
|
||||
return 2
|
||||
info = render(argv[0])
|
||||
print(json.dumps(info, indent=2))
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main(sys.argv[1:]))
|
||||
@@ -0,0 +1,338 @@
|
||||
#!/usr/bin/env python
|
||||
"""Full-parameter SFT for the orchestrator policy via FSDP (multi-GPU).
|
||||
|
||||
Launch with accelerate so the model + optimizer shard across GPUs (an 8-9B full
|
||||
fine-tune won't fit one L40S otherwise):
|
||||
|
||||
accelerate launch --config_file <fsdp.yaml> \
|
||||
scripts/orchestrator/run_sft_fsdp.py \
|
||||
--data data/runs/<...>/data.jsonl [--data <more> ...] \
|
||||
--variant correct --model Qwen/Qwen3.5-9B \
|
||||
--out checkpoints/sft_correct --epochs 3 --batch-size 8 --grad-accum 8 \
|
||||
--wandb-project orchestrator-sft
|
||||
|
||||
--variant selects which trajectories to train on:
|
||||
all -> every record
|
||||
correct -> only records the verifier marked correct
|
||||
correct_routed -> only correct records that delegated to a model expert
|
||||
|
||||
By default (--require-clean) records explicitly marked clean=False by the clean
|
||||
gate (bloated / garbled / unrouted) are also dropped; rows missing the flag
|
||||
(legacy data) are kept. Pass --no-require-clean to skip this filter.
|
||||
|
||||
Reuses the conversation->tokens + assistant-only masking from sft_tokenize.py.
|
||||
On these no-NVLink L40S, the accelerate/FSDP NCCL env (NCCL_P2P_DISABLE=1 etc.)
|
||||
must be set by the launcher.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import math
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
# reuse the data builder from the LoRA launcher
|
||||
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
||||
from sft_tokenize import build_example # noqa: E402
|
||||
|
||||
MODEL_EXPERTS = {"gpt_5_5", "claude_opus_4_8", "qwen3_5_9b", "qwen3_6_27b_fp8",
|
||||
"qwen3_5_122b_a10b_fp8", "qwen3_5_397b_a17b_fp8"}
|
||||
|
||||
|
||||
def record_is_correct(r: dict) -> bool:
|
||||
c = r.get("correct")
|
||||
if c is None:
|
||||
c = r.get("metrics", {}).get("correct")
|
||||
return bool(c)
|
||||
|
||||
|
||||
def record_routed_to_expert(r: dict) -> bool:
|
||||
"""True if the trajectory called a model expert (resolving anon labels)."""
|
||||
amap = r.get("metrics", {}).get("anon_map", {})
|
||||
names = re.findall(r'"name"\s*:\s*"([a-z0-9_]+)"', json.dumps(r.get("conversations", [])))
|
||||
for n in names:
|
||||
real = amap.get(n, n)
|
||||
if real in MODEL_EXPERTS:
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def record_clean_flag(r: dict):
|
||||
"""The record's `clean` flag: True/False, or None if absent (legacy data).
|
||||
|
||||
None means the clean gate never ran on this row, so we can't judge it -> we
|
||||
keep it (legacy behavior) rather than silently dropping older datasets.
|
||||
"""
|
||||
c = r.get("clean")
|
||||
if c is None:
|
||||
c = r.get("metrics", {}).get("clean")
|
||||
return c
|
||||
|
||||
|
||||
def select(records, variant: str, require_clean: bool = True):
|
||||
"""Filter by --variant, then (default) drop records explicitly marked unclean.
|
||||
|
||||
require_clean drops rows whose `clean` flag is False. Rows missing the flag
|
||||
(legacy) are kept regardless, matching pre-clean-gate behavior.
|
||||
"""
|
||||
if variant == "all":
|
||||
base = list(records)
|
||||
elif variant == "correct":
|
||||
base = [r for r in records if record_is_correct(r)]
|
||||
elif variant == "correct_routed":
|
||||
base = [r for r in records if record_is_correct(r) and record_routed_to_expert(r)]
|
||||
else:
|
||||
raise ValueError(f"unknown variant {variant}")
|
||||
if not require_clean:
|
||||
return base
|
||||
return [r for r in base if record_clean_flag(r) is not False]
|
||||
|
||||
|
||||
def main() -> int:
|
||||
p = argparse.ArgumentParser()
|
||||
p.add_argument("--data", action="append", required=True, help="JSONL(s) (repeatable)")
|
||||
p.add_argument("--val-data", default=None,
|
||||
help="Held-out JSONL for val-loss (excluded from --data). Eval'd per epoch.")
|
||||
p.add_argument("--variant", choices=["all", "correct", "correct_routed"], default="correct")
|
||||
p.add_argument("--require-clean", action=argparse.BooleanOptionalAction, default=True,
|
||||
help="Drop records whose `clean` flag is False (bloated / garbled / "
|
||||
"unrouted). Rows missing the flag (legacy data) are kept. "
|
||||
"Use --no-require-clean to disable.")
|
||||
p.add_argument("--model", default="Qwen/Qwen3.5-9B")
|
||||
p.add_argument("--out", required=True)
|
||||
p.add_argument("--epochs", type=float, default=3.0)
|
||||
p.add_argument("--batch-size", type=int, default=8)
|
||||
p.add_argument("--grad-accum", type=int, default=8)
|
||||
p.add_argument("--lr", type=float, default=1e-5)
|
||||
p.add_argument("--max-seq", type=int, default=8192)
|
||||
p.add_argument("--supervise-last-only", action="store_true",
|
||||
help="Legacy: supervise ONLY the final assistant turn. Default "
|
||||
"(off) supervises every assistant turn incl. routing.")
|
||||
p.add_argument("--warmup-ratio", type=float, default=0.03)
|
||||
p.add_argument("--seed", type=int, default=42)
|
||||
p.add_argument("--wandb-project", default="orchestrator-sft")
|
||||
p.add_argument("--wandb-name", default="")
|
||||
args = p.parse_args()
|
||||
|
||||
import random
|
||||
from datetime import timedelta
|
||||
import torch
|
||||
from torch.utils.data import DataLoader
|
||||
from accelerate import Accelerator
|
||||
from accelerate.utils import set_seed, InitProcessGroupKwargs
|
||||
from transformers import AutoModelForCausalLM, AutoTokenizer
|
||||
|
||||
set_seed(args.seed)
|
||||
# 30-min collective timeout: a slow first-step shard gather / graph build on a
|
||||
# contended node was tripping the default 10-min NCCL watchdog (job 3892 died
|
||||
# at step 1). This only delays a *real* hang's crash; it fixes spurious ones.
|
||||
pg_timeout_min = int(os.environ.get("SFT_PG_TIMEOUT_MIN", "30"))
|
||||
accelerator = Accelerator(
|
||||
gradient_accumulation_steps=args.grad_accum,
|
||||
kwargs_handlers=[InitProcessGroupKwargs(timeout=timedelta(minutes=pg_timeout_min))])
|
||||
is_main = accelerator.is_main_process
|
||||
|
||||
def log(m):
|
||||
if is_main:
|
||||
print(f"[fsdp-sft] {time.strftime('%H:%M:%S')} {m}", flush=True)
|
||||
|
||||
tok = AutoTokenizer.from_pretrained(args.model, trust_remote_code=True)
|
||||
if tok.pad_token_id is None:
|
||||
tok.pad_token = tok.eos_token
|
||||
|
||||
# ---- data ----
|
||||
records = []
|
||||
for f in args.data:
|
||||
records += [json.loads(l) for l in open(f) if l.strip()]
|
||||
sel = select(records, args.variant, require_clean=args.require_clean)
|
||||
log(f"variant={args.variant}: {len(sel)}/{len(records)} records")
|
||||
if args.require_clean:
|
||||
pre_clean = select(records, args.variant, require_clean=False)
|
||||
n_unclean = len(pre_clean) - len(sel)
|
||||
log(f"require_clean=True: dropped {n_unclean} unclean rows "
|
||||
f"({len(pre_clean)}->{len(sel)}); use --no-require-clean to keep them")
|
||||
else:
|
||||
log("require_clean=False: NOT filtering on `clean` flag")
|
||||
examples = []
|
||||
for r in sel:
|
||||
ex = build_example(tok, r.get("conversations", []), args.max_seq,
|
||||
supervise_all_turns=not args.supervise_last_only)
|
||||
if ex:
|
||||
examples.append(ex)
|
||||
log(f"built {len(examples)} training examples")
|
||||
if not examples:
|
||||
log("no examples; abort")
|
||||
return 1
|
||||
random.Random(args.seed).shuffle(examples)
|
||||
|
||||
def collate(batch):
|
||||
maxlen = max(len(b["input_ids"]) for b in batch)
|
||||
ii, lab, am = [], [], []
|
||||
for b in batch:
|
||||
pad = maxlen - len(b["input_ids"])
|
||||
ii.append(b["input_ids"] + [tok.pad_token_id] * pad)
|
||||
lab.append(b["labels"] + [-100] * pad)
|
||||
am.append([1] * len(b["input_ids"]) + [0] * pad)
|
||||
return (torch.tensor(ii), torch.tensor(lab), torch.tensor(am))
|
||||
|
||||
dl = DataLoader(examples, batch_size=args.batch_size, shuffle=True, collate_fn=collate)
|
||||
|
||||
# ---- held-out val set (leak-free; excluded from --data upstream) ----
|
||||
val_dl = None
|
||||
if args.val_data:
|
||||
vrecs = [json.loads(l) for l in open(args.val_data) if l.strip()]
|
||||
vex = []
|
||||
for r in vrecs:
|
||||
ex = build_example(tok, r.get("conversations", []), args.max_seq,
|
||||
supervise_all_turns=not args.supervise_last_only)
|
||||
if ex:
|
||||
vex.append(ex)
|
||||
log(f"val: {len(vex)} examples from {args.val_data}")
|
||||
if vex:
|
||||
val_dl = DataLoader(vex, batch_size=args.batch_size, shuffle=False, collate_fn=collate)
|
||||
|
||||
model = AutoModelForCausalLM.from_pretrained(
|
||||
args.model, dtype=torch.bfloat16, trust_remote_code=True,
|
||||
attn_implementation="sdpa")
|
||||
if os.environ.get("SFT_NO_GRAD_CKPT") == "1":
|
||||
log("gradient checkpointing DISABLED (SFT_NO_GRAD_CKPT=1)")
|
||||
else:
|
||||
model.gradient_checkpointing_enable(gradient_checkpointing_kwargs={"use_reentrant": False})
|
||||
model.config.use_cache = False
|
||||
opt = torch.optim.AdamW(model.parameters(), lr=args.lr, weight_decay=0.0)
|
||||
|
||||
model, opt, dl = accelerator.prepare(model, opt, dl)
|
||||
if val_dl is not None:
|
||||
val_dl = accelerator.prepare(val_dl)
|
||||
|
||||
@torch.no_grad()
|
||||
def evaluate():
|
||||
# Corpus-level mean token loss over the holdout. FSDP needs every rank to
|
||||
# run the forward collectively, so each rank evals its shard and we reduce
|
||||
# token-weighted sums (not a mean-of-batch-means, which would mis-weight).
|
||||
model.eval()
|
||||
tot_loss = torch.zeros((), device=accelerator.device)
|
||||
tot_tok = torch.zeros((), device=accelerator.device)
|
||||
for ii, lab, am in val_dl:
|
||||
out = model(input_ids=ii, attention_mask=am, labels=lab)
|
||||
ntok = (lab != -100).sum()
|
||||
tot_loss += out.loss.detach() * ntok
|
||||
tot_tok += ntok
|
||||
tot_loss = accelerator.reduce(tot_loss, reduction="sum")
|
||||
tot_tok = accelerator.reduce(tot_tok, reduction="sum")
|
||||
model.train()
|
||||
return (tot_loss / tot_tok.clamp(min=1)).item()
|
||||
|
||||
steps_per_epoch = math.ceil(len(dl) / args.grad_accum)
|
||||
total_steps = max(1, int(steps_per_epoch * args.epochs))
|
||||
warmup = max(1, int(total_steps * args.warmup_ratio))
|
||||
|
||||
def lr_at(s):
|
||||
if s < warmup:
|
||||
return s / warmup
|
||||
return 0.5 * (1 + math.cos(math.pi * (s - warmup) / max(1, total_steps - warmup)))
|
||||
|
||||
use_wandb = False
|
||||
if is_main:
|
||||
try:
|
||||
import wandb
|
||||
base_name = args.wandb_name or f"{Path(args.model).name}-{args.variant}-fsdp"
|
||||
run_name = f"{base_name}-{time.strftime('%m%d-%H%M', time.gmtime())}"
|
||||
wandb.init(project=args.wandb_project,
|
||||
name=run_name,
|
||||
config=vars(args) | {"n_examples": len(examples), "total_steps": total_steps})
|
||||
use_wandb = True
|
||||
except Exception as e:
|
||||
log(f"wandb off ({e})")
|
||||
|
||||
log(f"total_steps={total_steps} steps/epoch={steps_per_epoch} warmup={warmup}")
|
||||
|
||||
def _save_ckpt(out_path):
|
||||
# Full-model checkpoint (gathers FSDP shards to rank0). Called per-epoch
|
||||
# so a long run leaves usable partial models + lets us eval intermediate
|
||||
# checkpoints (e.g. epoch1/epoch2) for the data-scaling sweep.
|
||||
accelerator.wait_for_everyone()
|
||||
unwrapped = accelerator.unwrap_model(model)
|
||||
p = Path(out_path)
|
||||
if is_main:
|
||||
p.mkdir(parents=True, exist_ok=True)
|
||||
tok.save_pretrained(str(p))
|
||||
unwrapped.save_pretrained(
|
||||
str(p), is_main_process=is_main,
|
||||
save_function=accelerator.save,
|
||||
state_dict=accelerator.get_state_dict(model))
|
||||
if is_main:
|
||||
log(f"checkpoint saved -> {p}")
|
||||
|
||||
model.train()
|
||||
gstep = 0
|
||||
t0 = time.time()
|
||||
if val_dl is not None:
|
||||
vloss = evaluate()
|
||||
log(f"val/loss (baseline, step 0) = {vloss:.4f}")
|
||||
if use_wandb:
|
||||
wandb.log({"val/loss": vloss}, step=0)
|
||||
for epoch in range(math.ceil(args.epochs)):
|
||||
for ii, lab, am in dl:
|
||||
with accelerator.accumulate(model):
|
||||
out = model(input_ids=ii, attention_mask=am, labels=lab)
|
||||
accelerator.backward(out.loss)
|
||||
if accelerator.sync_gradients:
|
||||
lr = args.lr * lr_at(gstep)
|
||||
for g in opt.param_groups:
|
||||
g["lr"] = lr
|
||||
accelerator.clip_grad_norm_(model.parameters(), 1.0)
|
||||
opt.step()
|
||||
opt.zero_grad()
|
||||
if accelerator.sync_gradients:
|
||||
gstep += 1
|
||||
# accelerator.gather() is a COLLECTIVE — EVERY rank must call it.
|
||||
# Guarding it behind `is_main` made only rank0 run the [1]-float
|
||||
# all-gather while ranks 1-3 marched on to the next step's FSDP
|
||||
# param all-gather, desyncing the process group and hanging the job
|
||||
# at the step1->step2 boundary (NCCL collective-shape mismatch).
|
||||
loss = accelerator.gather(out.loss.detach()).mean().item()
|
||||
if is_main:
|
||||
log(f"step {gstep}/{total_steps} loss={loss:.4f} lr={lr:.2e}")
|
||||
if use_wandb:
|
||||
wandb.log({"train/loss": loss, "train/lr": lr}, step=gstep)
|
||||
if gstep >= total_steps:
|
||||
break
|
||||
_save_ckpt(Path(args.out) / f"epoch{epoch + 1}")
|
||||
if val_dl is not None:
|
||||
vloss = evaluate()
|
||||
log(f"val/loss (epoch {epoch + 1}) = {vloss:.4f}")
|
||||
if use_wandb:
|
||||
wandb.log({"val/loss": vloss, "epoch": epoch + 1}, step=gstep)
|
||||
if gstep >= total_steps:
|
||||
break
|
||||
|
||||
accelerator.wait_for_everyone()
|
||||
log("saving full model...")
|
||||
unwrapped = accelerator.unwrap_model(model)
|
||||
out_dir = Path(args.out)
|
||||
if is_main:
|
||||
out_dir.mkdir(parents=True, exist_ok=True)
|
||||
tok.save_pretrained(str(out_dir))
|
||||
unwrapped.save_pretrained(
|
||||
str(out_dir), is_main_process=is_main,
|
||||
save_function=accelerator.save,
|
||||
state_dict=accelerator.get_state_dict(model))
|
||||
if is_main:
|
||||
log(f"saved -> {out_dir}")
|
||||
if use_wandb:
|
||||
try:
|
||||
wandb.finish()
|
||||
except Exception:
|
||||
pass
|
||||
print("FSDP_SFT_DONE " + json.dumps({"variant": args.variant, "steps": gstep,
|
||||
"wall_s": round(time.time() - t0, 1), "out": str(out_dir)}))
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,13 @@
|
||||
compute_environment: LOCAL_MACHINE
|
||||
distributed_type: FSDP
|
||||
mixed_precision: bf16
|
||||
num_machines: 1
|
||||
num_processes: 4
|
||||
fsdp_config:
|
||||
fsdp_sharding_strategy: FULL_SHARD
|
||||
fsdp_auto_wrap_policy: SIZE_BASED_WRAP
|
||||
fsdp_min_num_params: 100000000
|
||||
fsdp_state_dict_type: FULL_STATE_DICT
|
||||
fsdp_use_orig_params: true
|
||||
fsdp_cpu_ram_efficient_loading: false
|
||||
fsdp_sync_module_states: false
|
||||
@@ -0,0 +1,148 @@
|
||||
#!/usr/bin/env python
|
||||
"""Conversation->tokens + assistant-only masking shared by the SFT trainers.
|
||||
|
||||
Extracted from the old LoRA trainer so ``run_sft_fsdp.py`` (full-parameter FSDP,
|
||||
the path we actually use) doesn't depend on it. ``build_example`` tokenizes one
|
||||
``conversations`` record and returns ``input_ids`` + ``labels`` with everything
|
||||
but the supervised assistant turns masked to -100.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
|
||||
def normalize_messages(conversations: List[Dict[str, Any]]) -> List[Dict[str, str]]:
|
||||
"""Map raw conversation turns to {role, content} with role in
|
||||
system/user/assistant. tool turns fold into user (matches the native
|
||||
OrchestratorSFTDataset fallback)."""
|
||||
msgs: List[Dict[str, str]] = []
|
||||
for turn in conversations:
|
||||
role = turn.get("role") or turn.get("from", "")
|
||||
content = turn.get("content") or turn.get("value", "") or ""
|
||||
if role in ("human", "user"):
|
||||
role = "user"
|
||||
elif role in ("gpt", "assistant"):
|
||||
role = "assistant"
|
||||
elif role == "tool":
|
||||
content = f"[Tool '{turn.get('name', 'tool')}' returned]: {content}"
|
||||
role = "user"
|
||||
if role in ("system", "user", "assistant"):
|
||||
msgs.append({"role": role, "content": str(content)})
|
||||
return msgs
|
||||
|
||||
|
||||
def _lcp_len(a: List[int], b: List[int]) -> int:
|
||||
"""Length of the longest common prefix of two token-id lists."""
|
||||
n = 0
|
||||
for x, y in zip(a, b):
|
||||
if x != y:
|
||||
break
|
||||
n += 1
|
||||
return n
|
||||
|
||||
|
||||
def _chatml_assistant_spans(ids: List[int], tok) -> Optional[List[tuple]]:
|
||||
"""``(start, end)`` token spans of each assistant turn's content in a
|
||||
ChatML-rendered sequence, located by ``<|im_start|>assistant ... <|im_end|>``
|
||||
markers (the ``<|im_end|>`` is included so the model learns to stop).
|
||||
|
||||
Operates on the FINAL token stream, so it is correct even when the chat
|
||||
template strips prior-turn ``<think>`` reasoning from history (Qwen3.x does),
|
||||
which breaks any incremental re-rendering / prefix approach. Returns None for
|
||||
non-ChatML templates (e.g. gemma) so the caller can fall back."""
|
||||
im_start = tok.convert_tokens_to_ids("<|im_start|>")
|
||||
im_end = tok.convert_tokens_to_ids("<|im_end|>")
|
||||
asst = tok.convert_tokens_to_ids("assistant")
|
||||
unk = tok.unk_token_id
|
||||
if im_start is None or im_end is None or im_start == unk or im_end == unk:
|
||||
return None
|
||||
nl = tok.convert_tokens_to_ids("Ċ") # the '\n' after the role header
|
||||
spans: List[tuple] = []
|
||||
i, n = 0, len(ids)
|
||||
while i < n:
|
||||
if ids[i] == im_start and i + 1 < n and ids[i + 1] == asst:
|
||||
j = i + 2
|
||||
if j < n and ids[j] == nl:
|
||||
j += 1
|
||||
k = j
|
||||
while k < n and ids[k] != im_end:
|
||||
k += 1
|
||||
end = min(k + 1, n)
|
||||
if end > j:
|
||||
spans.append((j, end))
|
||||
i = end
|
||||
else:
|
||||
i += 1
|
||||
return spans
|
||||
|
||||
|
||||
def build_example(
|
||||
tok,
|
||||
conversations,
|
||||
max_seq: int,
|
||||
supervise_all_turns: bool = True,
|
||||
) -> Optional[Dict[str, List[int]]]:
|
||||
"""Tokenize one record -> input_ids + labels.
|
||||
|
||||
``supervise_all_turns=True`` (default): supervise EVERY assistant turn — the
|
||||
intermediate routing ``<tool_call>`` turns AND the final answer; only
|
||||
system/user/tool tokens are masked. This teaches the model to actually route,
|
||||
not just synthesize the last answer given someone else's tool calls.
|
||||
|
||||
``supervise_all_turns=False`` (legacy): supervise only the final assistant
|
||||
turn; everything before it is masked to -100.
|
||||
|
||||
Assistant spans are found by the ChatML turn markers on the rendered stream
|
||||
(robust to Qwen3's prior-turn think-stripping). For non-ChatML templates
|
||||
(gemma) we fall back to the proven last-turn longest-common-prefix boundary;
|
||||
all-turns supervision there needs per-template markers and is not yet wired,
|
||||
so it degrades to last-turn. Records whose last turn isn't assistant, that
|
||||
supervise nothing, or whose final turn alone overflows max_seq are skipped."""
|
||||
msgs = normalize_messages(conversations)
|
||||
if len(msgs) < 2 or msgs[-1]["role"] != "assistant":
|
||||
return None
|
||||
try:
|
||||
full_text = tok.apply_chat_template(msgs, tokenize=False,
|
||||
add_generation_prompt=False)
|
||||
full = tok(full_text, add_special_tokens=False)["input_ids"]
|
||||
except Exception:
|
||||
return None
|
||||
if not full:
|
||||
return None
|
||||
|
||||
spans = _chatml_assistant_spans(full, tok)
|
||||
if spans:
|
||||
selected = spans if supervise_all_turns else spans[-1:]
|
||||
labels = [-100] * len(full)
|
||||
for (s, e) in selected:
|
||||
for k in range(s, e):
|
||||
labels[k] = full[k]
|
||||
last_start = selected[-1][0]
|
||||
else:
|
||||
# Non-ChatML template: proven last-turn boundary via longest-common-prefix
|
||||
# (robust to templates whose add_generation_prompt emits extra preamble).
|
||||
try:
|
||||
prompt_text = tok.apply_chat_template(msgs[:-1], tokenize=False,
|
||||
add_generation_prompt=True)
|
||||
prompt = tok(prompt_text, add_special_tokens=False)["input_ids"]
|
||||
except Exception:
|
||||
return None
|
||||
boundary = _lcp_len(prompt, full)
|
||||
if boundary >= len(full):
|
||||
return None
|
||||
labels = list(full)
|
||||
for i in range(min(boundary, len(labels))):
|
||||
labels[i] = -100
|
||||
last_start = boundary
|
||||
|
||||
if all(v == -100 for v in labels): # nothing left to supervise
|
||||
return None
|
||||
|
||||
# Keep the final assistant turn if too long: left-truncate from the front.
|
||||
if len(full) > max_seq:
|
||||
if len(full) - last_start >= max_seq: # final answer alone overflows -> drop
|
||||
return None
|
||||
cut = len(full) - max_seq
|
||||
full = full[cut:]
|
||||
labels = labels[cut:]
|
||||
return {"input_ids": full, "labels": labels}
|
||||
@@ -0,0 +1,361 @@
|
||||
#!/usr/bin/env python
|
||||
"""Upload orchestrator-SFT JSONL datasets to Braintrust so they're browsable/
|
||||
filterable in the UI (by area / difficulty / dataset / routed-model / correct).
|
||||
|
||||
Datasets land in the SAME Braintrust project as the rollout TRACES (the
|
||||
``research`` project) so a run's data + traces sit side by side. The target is
|
||||
resolved from ``OJ_BRAINTRUST_PROJECT_ID`` (default: the research project id);
|
||||
``--project`` overrides by NAME if you want a different project.
|
||||
|
||||
Each record -> a Braintrust dataset row:
|
||||
input = the problem/question
|
||||
expected = the FINAL_ANSWER value
|
||||
tags = [gen_model, domain, correct|incorrect, clean|dirty, kept|dropped]
|
||||
metadata = domain, area, difficulty, dataset, subsector, task_id, correct,
|
||||
clean, kept, reward, gen_model, orchestrator_model, num_tool_calls,
|
||||
n_turns, routed_models (real names)
|
||||
(the full conversation is kept under metadata.conversation for inspection)
|
||||
|
||||
The dataset itself carries run-level metadata (run_label, specific gen_model,
|
||||
git sha, config knobs, counts, domain distribution) so future experiments are
|
||||
distinguishable in the UI.
|
||||
|
||||
Usage (CLI):
|
||||
.venv/bin/python scripts/orchestrator/upload_to_braintrust.py \
|
||||
data/sft_variants/qwen_orch_train_0707.jsonl=qwen_train_0707 \
|
||||
data/sft_variants/qwen_orch_holdout_0707.jsonl=qwen_holdout_0707
|
||||
|
||||
# name is optional; defaults to {model_short}_{split}_{date}
|
||||
.venv/bin/python scripts/orchestrator/upload_to_braintrust.py \
|
||||
data/sft_variants/qwen_orch_holdout_0707.jsonl
|
||||
|
||||
Programmatic (used by the pipeline auto-upload hook in make_splits.py):
|
||||
from upload_to_braintrust import autoupload
|
||||
autoupload(["data/sft_variants/qwen_orch_train_0707.jsonl=qwen_train_0707"],
|
||||
run_label="...")
|
||||
"""
|
||||
import argparse
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
import subprocess
|
||||
from collections import Counter
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# The `research` project — datasets go here, same place the rollout traces land.
|
||||
DEFAULT_PROJECT_ID = "c707124e-ce9d-4187-ad11-f49f19f777ad"
|
||||
|
||||
_FA = re.compile(r"(?im)FINAL[_\s]?ANSWER\s*:?")
|
||||
|
||||
|
||||
def _ordered_turns(convs, anon_map, orchestrator):
|
||||
"""Rebuild each turn as role/model/content, in that display order.
|
||||
|
||||
Braintrust sorts object keys alphabetically on write, which would push the
|
||||
(very long) `content` to the top. We prefix the keys (`1_role`, `2_model`,
|
||||
`3_content`) so they sort into role -> model -> content and the content
|
||||
renders last. `model` is the model behind the turn:
|
||||
* assistant turns -> the orchestrator model
|
||||
* tool turns -> the expert model that answered (de-anonymized)
|
||||
* system/user -> None
|
||||
"""
|
||||
out = []
|
||||
for c in convs:
|
||||
role = c.get("role")
|
||||
if role == "tool":
|
||||
model = anon_map.get(c.get("name")) or c.get("name")
|
||||
elif role == "assistant":
|
||||
model = orchestrator
|
||||
else:
|
||||
model = None
|
||||
out.append({"1_role": role, "2_model": model, "3_content": c.get("content")})
|
||||
return out
|
||||
|
||||
|
||||
def _dataset_desc(dataset):
|
||||
"""Fallback dataset blurb for records generated before the field existed."""
|
||||
try:
|
||||
from openjarvis.learning.intelligence.orchestrator.sft_data.reject_sample import (
|
||||
dataset_description,
|
||||
)
|
||||
return dataset_description(dataset)
|
||||
except Exception:
|
||||
return ""
|
||||
|
||||
|
||||
def _git_sha():
|
||||
try:
|
||||
return subprocess.check_output(
|
||||
["git", "rev-parse", "--short", "HEAD"],
|
||||
cwd=Path(__file__).resolve().parent, stderr=subprocess.DEVNULL,
|
||||
).decode().strip() or None
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
|
||||
def _model_short(rows, path):
|
||||
"""Short family tag for the default dataset name (qwen / gemma / ...)."""
|
||||
stem = Path(path).stem.lower()
|
||||
for fam in ("qwen", "gemma"):
|
||||
if stem.startswith(fam) or fam in stem:
|
||||
return fam
|
||||
gm = next((r.get("gen_model") for r in rows if r.get("gen_model")), "") or ""
|
||||
gm = gm.lower()
|
||||
for fam in ("qwen", "gemma"):
|
||||
if fam in gm:
|
||||
return fam
|
||||
return "orch"
|
||||
|
||||
|
||||
def _split_of(path):
|
||||
stem = Path(path).stem.lower()
|
||||
for s in ("holdout", "train", "clean", "overfit", "partial", "8k", "4k", "2k", "1k"):
|
||||
if s in stem:
|
||||
return s
|
||||
return "data"
|
||||
|
||||
|
||||
def _default_name(rows, path):
|
||||
return f"{_model_short(rows, path)}_{_split_of(path)}_{datetime.now():%m%d}"
|
||||
|
||||
|
||||
def _config_from_env():
|
||||
"""Config knobs from the run env (set by build_orchestrator_sft.py). Omit unset."""
|
||||
cfg = {}
|
||||
for env_key, meta_key, cast in [
|
||||
("OJ_CFG_TEMPERATURE", "temperature", float),
|
||||
("OJ_CFG_MAX_TURNS", "max_turns", int),
|
||||
("OJ_CFG_ANONYMIZE", "anonymize", lambda v: v.strip().lower() in ("1", "true", "yes")),
|
||||
("OJ_CFG_REJECTION_ONLY", "rejection_only", lambda v: v.strip().lower() in ("1", "true", "yes")),
|
||||
]:
|
||||
v = os.getenv(env_key)
|
||||
if v not in (None, ""):
|
||||
try:
|
||||
cfg[meta_key] = cast(v)
|
||||
except Exception:
|
||||
cfg[meta_key] = v
|
||||
return cfg
|
||||
|
||||
|
||||
def _row(rec):
|
||||
convs = rec.get("conversations", [])
|
||||
user = next((c["content"] for c in convs if c["role"] == "user"), "")
|
||||
question = re.sub(r"^\s*Problem:\s*", "", user, count=1).strip()
|
||||
asst = [c["content"] for c in convs if c["role"] == "assistant"]
|
||||
fa = asst[-1] if asst else ""
|
||||
m = list(_FA.finditer(fa))
|
||||
model_answer = fa[m[-1].end():].strip() if m else fa.strip()
|
||||
# Gold reference the verifier graded against (stamped at generation time as
|
||||
# `gold_answer`). `expected` in Braintrust is the GOLD so the UI shows
|
||||
# gold-vs-model; the model's own answer goes to metadata.model_answer.
|
||||
gold = rec.get("gold_answer") or None
|
||||
has_gold = gold is not None
|
||||
am = (rec.get("metrics", {}) or {}).get("anon_map", {}) or {}
|
||||
routed = []
|
||||
for c in convs:
|
||||
if c["role"] == "assistant":
|
||||
for t in re.findall(r"<tool_call>(\{.*?\})</tool_call>", c["content"], re.DOTALL):
|
||||
try:
|
||||
real = am.get(json.loads(t).get("name"))
|
||||
if real:
|
||||
routed.append(real)
|
||||
except Exception:
|
||||
pass
|
||||
metrics = rec.get("metrics", {}) or {}
|
||||
gen_model = rec.get("gen_model")
|
||||
domain = rec.get("domain")
|
||||
correct = rec.get("correct")
|
||||
clean = rec.get("clean")
|
||||
kept = rec.get("kept")
|
||||
tags = [t for t in [
|
||||
gen_model,
|
||||
f"domain:{domain}" if domain else None,
|
||||
"correct" if correct else "incorrect",
|
||||
"clean" if clean else "dirty",
|
||||
"kept" if kept else "dropped",
|
||||
"gold" if has_gold else "no_gold",
|
||||
] if t]
|
||||
return {
|
||||
"input": question,
|
||||
"expected": gold,
|
||||
"tags": tags,
|
||||
"metadata": {
|
||||
"gold_answer": gold,
|
||||
"model_answer": model_answer,
|
||||
"has_gold": has_gold,
|
||||
"domain": domain,
|
||||
"area": rec.get("area"),
|
||||
"difficulty": rec.get("difficulty") or None,
|
||||
"dataset": rec.get("dataset"),
|
||||
"dataset_description": rec.get("dataset_description") or _dataset_desc(rec.get("dataset")),
|
||||
"subsector": rec.get("subsector"),
|
||||
"task_id": rec.get("task_id"),
|
||||
"correct": correct,
|
||||
"clean": clean,
|
||||
"kept": kept,
|
||||
"reward": rec.get("reward"),
|
||||
"gen_model": gen_model,
|
||||
"orchestrator_model": rec.get("orchestrator_model"),
|
||||
"num_tool_calls": metrics.get("num_tool_calls"),
|
||||
"n_turns": metrics.get("num_turns"),
|
||||
"routed_models": routed,
|
||||
"conversation": _ordered_turns(convs, am, rec.get("orchestrator_model")),
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def _dist(rows, key):
|
||||
"""{value: count} for a record field, most-common first, None/'' dropped."""
|
||||
c = Counter(r.get(key) for r in rows if (r.get(key) or "") != "")
|
||||
return {str(k): v for k, v in sorted(c.items(), key=lambda x: -x[1])}
|
||||
|
||||
|
||||
def _routed_dist(rows):
|
||||
"""Distribution of real (de-anonymized) expert models routed to."""
|
||||
c = Counter()
|
||||
for r in rows:
|
||||
am = (r.get("metrics", {}) or {}).get("anon_map", {}) or {}
|
||||
for cc in r.get("conversations", []):
|
||||
if cc.get("role") != "assistant":
|
||||
continue
|
||||
for t in re.findall(r"<tool_call>(\{.*?\})</tool_call>", cc.get("content", ""), re.DOTALL):
|
||||
try:
|
||||
real = am.get(json.loads(t).get("name"))
|
||||
if real:
|
||||
c[real] += 1
|
||||
except Exception:
|
||||
pass
|
||||
return {k: v for k, v in sorted(c.items(), key=lambda x: -x[1])}
|
||||
|
||||
|
||||
def _avg(rows, *path):
|
||||
vals = []
|
||||
for r in rows:
|
||||
v = r.get("metrics", {}) or {}
|
||||
for p in path:
|
||||
v = (v or {}).get(p) if isinstance(v, dict) else None
|
||||
if isinstance(v, (int, float)):
|
||||
vals.append(v)
|
||||
return round(sum(vals) / len(vals), 2) if vals else None
|
||||
|
||||
|
||||
def _dataset_metadata(rows, path, run_label, gen_model):
|
||||
n = len(rows)
|
||||
meta = {
|
||||
"model": _model_short(rows, path),
|
||||
"split": _split_of(path),
|
||||
"task": "orchestrator-SFT unified-tool routing traces",
|
||||
"date": "2026-07-07",
|
||||
"run_label": run_label,
|
||||
"gen_model": gen_model,
|
||||
"git_sha": _git_sha(),
|
||||
"source_file": str(path),
|
||||
"source_datasets": ["GeneralThought-430K-filtered", "OpenThoughts3-1.2M"],
|
||||
"task_mix": "balanced (GeneralThought + OpenThoughts code/math/science)",
|
||||
"filter": "correct + clean (rejects file-write echoes, garble, "
|
||||
"reasoning-degeneration, truncated tails, essays/markdown dumps)",
|
||||
"leak_free": "train/holdout disjoint by task_id",
|
||||
"n_total": n,
|
||||
"n_correct": sum(1 for r in rows if r.get("correct")),
|
||||
"n_clean": sum(1 for r in rows if r.get("clean")),
|
||||
"n_kept": sum(1 for r in rows if r.get("kept")),
|
||||
"n_with_gold": sum(1 for r in rows if (r.get("gold_answer") or "").strip()),
|
||||
"avg_tool_calls": _avg(rows, "num_tool_calls"),
|
||||
"avg_turns": _avg(rows, "num_turns"),
|
||||
"area_distribution": _dist(rows, "area"),
|
||||
"difficulty_distribution": _dist(rows, "difficulty"),
|
||||
"dataset_distribution": _dist(rows, "dataset"),
|
||||
"routed_model_distribution": _routed_dist(rows),
|
||||
}
|
||||
cfg = _config_from_env()
|
||||
if cfg:
|
||||
meta["config"] = cfg
|
||||
return {k: v for k, v in meta.items() if v is not None}
|
||||
|
||||
|
||||
def upload_dataset(path, name=None, *, project_id=None, project=None,
|
||||
run_label=None, gen_model=None, description=""):
|
||||
"""Upload one JSONL file as a Braintrust dataset. Returns (name, n_rows, url).
|
||||
|
||||
Target project: ``project`` (name) if given, else ``project_id`` /
|
||||
``OJ_BRAINTRUST_PROJECT_ID`` / the research default.
|
||||
"""
|
||||
import braintrust
|
||||
|
||||
rows = [json.loads(l) for l in open(path) if l.strip()]
|
||||
name = name or _default_name(rows, path)
|
||||
gen_model = gen_model or os.getenv("OJ_GEN_MODEL") or \
|
||||
next((r.get("gen_model") for r in rows if r.get("gen_model")), None)
|
||||
run_label = run_label or os.getenv("OJ_RUN_LABEL") or name
|
||||
|
||||
init_kwargs = {"name": name, "description": description or None,
|
||||
"metadata": _dataset_metadata(rows, path, run_label, gen_model)}
|
||||
if project:
|
||||
init_kwargs["project"] = project
|
||||
else:
|
||||
init_kwargs["project_id"] = project_id or os.getenv(
|
||||
"OJ_BRAINTRUST_PROJECT_ID", DEFAULT_PROJECT_ID)
|
||||
|
||||
ds = braintrust.init_dataset(**init_kwargs)
|
||||
for r in rows:
|
||||
ds.insert(**_row(r))
|
||||
ds.flush()
|
||||
summ = ds.summarize()
|
||||
url = getattr(summ, "dataset_url", None) or (project or init_kwargs.get("project_id"))
|
||||
return name, len(rows), url
|
||||
|
||||
|
||||
def autoupload(specs, *, run_label=None, gen_model=None, description=""):
|
||||
"""No-op-safe wrapper for pipeline hooks. Honors OJ_BRAINTRUST_AUTOUPLOAD
|
||||
(default ON) and never raises: any failure (missing key/pkg, network) is
|
||||
logged and swallowed so the data pipeline is never broken by telemetry."""
|
||||
if os.getenv("OJ_BRAINTRUST_AUTOUPLOAD", "1").strip().lower() in ("0", "false", "no", "off", ""):
|
||||
logger.info("[braintrust] autoupload disabled (OJ_BRAINTRUST_AUTOUPLOAD)")
|
||||
return
|
||||
if not os.getenv("BRAINTRUST_API_KEY"):
|
||||
logger.info("[braintrust] autoupload skipped — BRAINTRUST_API_KEY unset")
|
||||
return
|
||||
try:
|
||||
import braintrust # noqa: F401
|
||||
except Exception:
|
||||
logger.info("[braintrust] autoupload skipped — braintrust not installed")
|
||||
return
|
||||
for spec in specs:
|
||||
path, _, name = spec.partition("=")
|
||||
try:
|
||||
nm, n, url = upload_dataset(path, name or None, run_label=run_label,
|
||||
gen_model=gen_model, description=description)
|
||||
logger.info("[braintrust] uploaded %s: %d rows -> %s", nm, n, url)
|
||||
print(f"[braintrust] uploaded {nm}: {n} rows -> {url}")
|
||||
except Exception as exc:
|
||||
logger.warning("[braintrust] autoupload FAILED for %s (%s) — continuing", path, exc)
|
||||
print(f"[braintrust] autoupload FAILED for {path} ({exc}) — pipeline unaffected")
|
||||
|
||||
|
||||
def main():
|
||||
logging.basicConfig(level=logging.INFO, format="%(message)s")
|
||||
p = argparse.ArgumentParser()
|
||||
p.add_argument("--project", default=None,
|
||||
help="Target project by NAME (overrides OJ_BRAINTRUST_PROJECT_ID).")
|
||||
p.add_argument("--project-id", default=None,
|
||||
help="Target project by id (default: OJ_BRAINTRUST_PROJECT_ID or research).")
|
||||
p.add_argument("--run-label", default=None)
|
||||
p.add_argument("--gen-model", default=None, help="Specific gen model id override.")
|
||||
p.add_argument("--description", default="")
|
||||
p.add_argument("specs", nargs="+", help="path.jsonl[=dataset_name]")
|
||||
args = p.parse_args()
|
||||
for spec in args.specs:
|
||||
path, _, name = spec.partition("=")
|
||||
nm, n, url = upload_dataset(
|
||||
path, name or None, project_id=args.project_id, project=args.project,
|
||||
run_label=args.run_label, gen_model=args.gen_model,
|
||||
description=args.description)
|
||||
print(f"[braintrust] {nm}: {n} rows -> {url}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,90 @@
|
||||
#!/bin/bash
|
||||
# Serve a trained gemma-4-26B-A4B SFT checkpoint (orchestrator) on 2xL40S (mkt1),
|
||||
# then run eval_orchestrator over benchmarks. Self-contained: serve -> health -> eval -> teardown.
|
||||
# Usage: sbatch eval_orch_gemma_matx.sbatch <1k|2k> <benchmarks> <N> <tag>
|
||||
#SBATCH --job-name=eval_gemma
|
||||
#SBATCH --account=mkt
|
||||
#SBATCH --partition=mkt
|
||||
#SBATCH -w mkt1
|
||||
#SBATCH --gres=gpu:2
|
||||
#SBATCH --cpus-per-gpu=10
|
||||
#SBATCH --mem=200G
|
||||
#SBATCH -t 24:00:00
|
||||
#SBATCH --output=logs/eval_gemma_matx_%j.log
|
||||
|
||||
set -uo pipefail
|
||||
SIZE=${1:?usage: <1k|2k> <benchmarks> <N> <tag>}
|
||||
BENCHES=${2:-gaia}
|
||||
N=${3:-2}
|
||||
TAG=${4:-smoke}
|
||||
PORT=8123
|
||||
|
||||
# Paths are resolved relative to the repo, or overridden by env:
|
||||
# OJ_ROOT repo checkout (default: two levels up from this script)
|
||||
# OJ_WORK working-state dir (default: $HOME/.openjarvis)
|
||||
# OJ_CKPT_ROOT trained SFT checkpoints (default: $OJ_WORK/lambda_ckpts)
|
||||
OJ_ROOT=${OJ_ROOT:-$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)}
|
||||
OJ_WORK=${OJ_WORK:-$HOME/.openjarvis}
|
||||
OJ_CKPT_ROOT=${OJ_CKPT_ROOT:-$OJ_WORK/lambda_ckpts}
|
||||
|
||||
cd "$OJ_ROOT"
|
||||
set -a
|
||||
source .env
|
||||
[ -f "$OJ_WORK/eval_key_override.sh" ] && source "$OJ_WORK/eval_key_override.sh"
|
||||
set +a
|
||||
export HF_HOME=${HF_HOME:?set HF_HOME to a cache dir with room for the weights}
|
||||
export PYTHONPATH="$PWD/src"
|
||||
# judge/expert cloud engine needs a config with an [engine.cloud] section
|
||||
export OPENJARVIS_CONFIG=${OPENJARVIS_CONFIG:?set OPENJARVIS_CONFIG to a cell config.toml}
|
||||
source .venv/bin/activate
|
||||
|
||||
if [ "$SIZE" = "base" ]; then
|
||||
CKPT=google/gemma-4-26B-A4B-it # un-fine-tuned base orchestrator (from HF cache)
|
||||
else
|
||||
CKPT=$OJ_CKPT_ROOT/sft_gemma_${SIZE}_bf16/epoch3
|
||||
fi
|
||||
OUT=results/orch-eval-gemma-${SIZE}-${TAG}
|
||||
mkdir -p "$OUT" logs
|
||||
echo "=== [$(date)] eval_gemma SIZE=$SIZE BENCHES=$BENCHES N=$N TAG=$TAG on $(hostname) ==="
|
||||
nvidia-smi --query-gpu=index,name,memory.total --format=csv,noheader
|
||||
|
||||
# ---- serve: TP=2 bf16 + MANDATORY no-NVLink L40S flags (else NCCL hangs) ----
|
||||
echo "=== [$(date)] launching vLLM (TP=2) serve of $CKPT ==="
|
||||
NCCL_P2P_DISABLE=1 NCCL_CUMEM_HOST_ENABLE=0 VLLM_WORKER_MULTIPROC_METHOD=spawn \
|
||||
PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True \
|
||||
python -m vllm.entrypoints.openai.api_server \
|
||||
--model "$CKPT" --served-model-name gemma-sft \
|
||||
--port $PORT --tensor-parallel-size 2 --dtype bfloat16 \
|
||||
--gpu-memory-utilization 0.90 --trust-remote-code \
|
||||
--max-model-len 32768 --enforce-eager --disable-custom-all-reduce \
|
||||
--enable-auto-tool-choice --tool-call-parser gemma4 \
|
||||
--max-num-batched-tokens 8192 --safetensors-load-strategy eager \
|
||||
--enable-prefix-caching > logs/serve_gemma_${SIZE}_${SLURM_JOB_ID}.log 2>&1 &
|
||||
SERVE_PID=$!
|
||||
echo "serve pid=$SERVE_PID log=logs/serve_gemma_${SIZE}_${SLURM_JOB_ID}.log"
|
||||
cleanup(){ echo "=== [$(date)] teardown kill $SERVE_PID ==="; kill $SERVE_PID 2>/dev/null; sleep 5; kill -9 $SERVE_PID 2>/dev/null; }
|
||||
trap cleanup EXIT
|
||||
|
||||
# ---- health poll (up to 25 min; TP + 52GB load is slow on L40S) ----
|
||||
echo "=== [$(date)] waiting for /v1/models ==="
|
||||
READY=0
|
||||
for i in $(seq 1 150); do
|
||||
if ! kill -0 $SERVE_PID 2>/dev/null; then echo "!! serve died during startup"; tail -40 logs/serve_gemma_${SIZE}_${SLURM_JOB_ID}.log; exit 3; fi
|
||||
if curl -sf http://localhost:$PORT/v1/models 2>/dev/null | grep -q gemma-sft; then READY=1; echo "=== [$(date)] READY after ${i}0s ==="; break; fi
|
||||
sleep 10
|
||||
done
|
||||
[ $READY -eq 1 ] || { echo "!! not ready in 25min"; tail -40 logs/serve_gemma_${SIZE}_${SLURM_JOB_ID}.log; exit 4; }
|
||||
|
||||
# ---- eval ----
|
||||
echo "=== [$(date)] running eval_orchestrator benches=$BENCHES n=$N ==="
|
||||
python scripts/orchestrator/eval_orchestrator.py \
|
||||
--benchmarks "$BENCHES" --n "$N" --seed 42 \
|
||||
--orchestrator-endpoint http://localhost:$PORT/v1 \
|
||||
--orchestrator-model gemma-sft --orchestrator-api-key EMPTY \
|
||||
--judge-model gemini-2.5-flash --judge-engine cloud \
|
||||
--max-workers 4 --max-turns 8 \
|
||||
--output-dir "$OUT"
|
||||
RC=$?
|
||||
echo "=== [$(date)] eval exit=$RC summary=$OUT/summary.json ==="
|
||||
echo "EVAL_GEMMA_DONE size=$SIZE tag=$TAG rc=$RC"
|
||||
exit $RC
|
||||
@@ -0,0 +1,25 @@
|
||||
compute_environment: LOCAL_MACHINE
|
||||
distributed_type: FSDP
|
||||
downcast_bf16: 'no'
|
||||
machine_rank: 0
|
||||
main_process_ip: null
|
||||
main_process_port: null
|
||||
main_training_function: main
|
||||
mixed_precision: bf16
|
||||
num_machines: 1
|
||||
num_processes: 4
|
||||
rdzv_backend: static
|
||||
same_network: true
|
||||
use_cpu: false
|
||||
fsdp_config:
|
||||
fsdp_version: 1
|
||||
fsdp_auto_wrap_policy: SIZE_BASED_WRAP
|
||||
fsdp_min_num_params: 100000000
|
||||
fsdp_sharding_strategy: FULL_SHARD
|
||||
fsdp_state_dict_type: FULL_STATE_DICT
|
||||
fsdp_offload_params: false
|
||||
fsdp_backward_prefetch: BACKWARD_PRE
|
||||
fsdp_forward_prefetch: false
|
||||
fsdp_use_orig_params: true
|
||||
fsdp_cpu_ram_efficient_loading: true
|
||||
fsdp_sync_module_states: true
|
||||
@@ -0,0 +1,25 @@
|
||||
compute_environment: LOCAL_MACHINE
|
||||
distributed_type: FSDP
|
||||
downcast_bf16: 'no'
|
||||
machine_rank: 0
|
||||
main_process_ip: null
|
||||
main_process_port: null
|
||||
main_training_function: main
|
||||
mixed_precision: bf16
|
||||
num_machines: 1
|
||||
num_processes: 8
|
||||
rdzv_backend: static
|
||||
same_network: true
|
||||
use_cpu: false
|
||||
fsdp_config:
|
||||
fsdp_version: 1
|
||||
fsdp_auto_wrap_policy: SIZE_BASED_WRAP
|
||||
fsdp_min_num_params: 100000000
|
||||
fsdp_sharding_strategy: FULL_SHARD
|
||||
fsdp_state_dict_type: FULL_STATE_DICT
|
||||
fsdp_offload_params: false
|
||||
fsdp_backward_prefetch: BACKWARD_PRE
|
||||
fsdp_forward_prefetch: false
|
||||
fsdp_use_orig_params: true
|
||||
fsdp_cpu_ram_efficient_loading: true
|
||||
fsdp_sync_module_states: true
|
||||
@@ -445,8 +445,8 @@ class LocalCloudAgent(BaseAgent):
|
||||
tools: Optional[list] = None,
|
||||
tool_choice: Optional[dict] = None,
|
||||
output_config: Optional[dict] = None,
|
||||
timeout: float = 600.0,
|
||||
max_retries: int = 12,
|
||||
timeout: float = 60.0,
|
||||
max_retries: int = 2,
|
||||
trace_role: str = "cloud",
|
||||
) -> Tuple[str, int, int, int]:
|
||||
"""Single Anthropic call. Returns (text, p_tok, c_tok, n_web_searches).
|
||||
@@ -536,7 +536,7 @@ class LocalCloudAgent(BaseAgent):
|
||||
response_format: Optional[dict] = None,
|
||||
tools: Optional[list] = None,
|
||||
tool_choice: Optional[Any] = None,
|
||||
timeout: float = 600.0,
|
||||
timeout: float = 60.0,
|
||||
trace_role: str = "cloud",
|
||||
) -> Tuple[str, int, int]:
|
||||
"""Single OpenAI call. Returns (text, p_tok, c_tok). Trace-captured;
|
||||
@@ -604,7 +604,7 @@ class LocalCloudAgent(BaseAgent):
|
||||
system: Optional[str] = None,
|
||||
max_tokens: int = 4096,
|
||||
temperature: float = 0.0,
|
||||
timeout: float = 600.0,
|
||||
timeout: float = 60.0,
|
||||
trace_role: str = "cloud",
|
||||
extra_body: Optional[Dict[str, Any]] = None,
|
||||
) -> Tuple[str, int, int]:
|
||||
|
||||
@@ -34,6 +34,14 @@ PRICES: dict[str, tuple[float, float]] = {
|
||||
"qwen/qwen-2.5-coder-32b-instruct": (0.08, 0.18),
|
||||
"qwen/qwen3-32b": (0.10, 0.30),
|
||||
"meta-llama/llama-3.3-70b-instruct": (0.13, 0.39),
|
||||
# OpenRouter slugs for the orchestrator's local-OSS class when routed via
|
||||
# OpenRouter instead of self-hosted vLLM. ESTIMATES — no public list price
|
||||
# exists yet for these Qwen3.5/3.6 builds; scaled by active-param size.
|
||||
# VERIFY against openrouter.ai before trusting the cost-aware reward numbers.
|
||||
"qwen/qwen3.5-9b": (0.05, 0.10),
|
||||
"qwen/qwen3.6-27b": (0.10, 0.30),
|
||||
"qwen/qwen3.5-122b-a10b": (0.20, 0.60),
|
||||
"qwen/qwen3.5-397b-a17b": (0.40, 1.20),
|
||||
}
|
||||
|
||||
# Models whose API rejects an explicit `temperature` param — callers should
|
||||
@@ -42,6 +50,10 @@ NO_TEMP_PREFIXES: tuple[str, ...] = (
|
||||
"claude-opus-4-7",
|
||||
"claude-sonnet-4-7",
|
||||
"claude-haiku-4-7",
|
||||
# 4-8 family also rejects `temperature` ("deprecated for this model").
|
||||
"claude-opus-4-8",
|
||||
"claude-sonnet-4-8",
|
||||
"claude-haiku-4-8",
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -23,6 +23,7 @@ circular import.
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import random
|
||||
import re
|
||||
from dataclasses import dataclass, field
|
||||
@@ -42,15 +43,15 @@ VALID_KINDS = (KIND_MODEL, KIND_WEB_SEARCH, KIND_LOCAL_SEARCH, KIND_CODE, KIND_T
|
||||
# Flat-catalog category label, surfaced in each tool's spec so the orchestrator
|
||||
# can tell tool types apart without us imposing any hierarchy (the menu stays
|
||||
# flat — this is just a tag).
|
||||
CATEGORY_GENERALIST = "generalist_model"
|
||||
CATEGORY_SPECIALIZED = "specialized_model"
|
||||
CATEGORY_BASIC = "basic_tool"
|
||||
CATEGORY_SMALL_GENERALIST = "small_generalist"
|
||||
CATEGORY_STRONG_GENERALIST = "strong_generalist"
|
||||
# Two-model-class taxonomy for the orchestrator catalog (the specialized /
|
||||
# small-vs-strong generalist tiers are superseded).
|
||||
# Two-model-class taxonomy for the orchestrator catalog — the only model tiers.
|
||||
# (The legacy default_catalog still uses CATEGORY_GENERALIST/SPECIALIZED below.)
|
||||
CATEGORY_CLOUD_FRONTIER = "cloud_frontier"
|
||||
CATEGORY_LOCAL_OSS = "local_open_source"
|
||||
# Legacy tiers kept only for default_catalog (build_unified_sft.py); not used by
|
||||
# orchestrator_catalog. Superseded by the two-class taxonomy above.
|
||||
CATEGORY_GENERALIST = "generalist_model"
|
||||
CATEGORY_SPECIALIZED = "specialized_model"
|
||||
|
||||
# Backend dispatch types understood by ``toolorchestra._call_worker`` (plus the
|
||||
# ``openjarvis-tool`` bridge, dispatched in ``unified.make_dispatch`` via the
|
||||
@@ -81,11 +82,14 @@ class ExpertTool:
|
||||
price_in: float = 0.0
|
||||
price_out: float = 0.0
|
||||
latency_s: float = 5.0
|
||||
category: str = "" # generalist_model | specialized_model | basic_tool
|
||||
category: str = "" # cloud_frontier | local_open_source | basic_tool
|
||||
# Optional explicit JSON-schema for the tool's arguments. Set for bridged
|
||||
# real OpenJarvis tools (``openjarvis-tool`` backend) whose params don't fit
|
||||
# the fixed kind-based schemas; takes precedence over the kind default.
|
||||
param_schema: Optional[dict] = None
|
||||
# When True, ``description()`` omits the price/latency line — used by the
|
||||
# anonymized catalog so the orchestrator can't route on cost/identity.
|
||||
hide_cost: bool = False
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
if self.kind not in VALID_KINDS:
|
||||
@@ -141,6 +145,8 @@ class ExpertTool:
|
||||
|
||||
def description(self) -> str:
|
||||
"""Full description incl. the price/latency line (paper bakes this in)."""
|
||||
if self.hide_cost:
|
||||
return self.summary
|
||||
if self.kind == KIND_MODEL:
|
||||
cost_line = (
|
||||
f" Pricing: ${self.price_in:.2f}/1M input, "
|
||||
@@ -181,6 +187,103 @@ def _tool_name(model: str) -> str:
|
||||
return safe or "local_model"
|
||||
|
||||
|
||||
_ANON_ALPHABET = "abcdefghijklmnopqrstuvwxyz0123456789"
|
||||
|
||||
# Rough size/scale hint per model — a capability signal WITHOUT the brand name,
|
||||
# so the orchestrator can route big-vs-small on task difficulty but can't route on
|
||||
# a proprietary name/tier prior. Closed models have no public param count, so they
|
||||
# get a scale tier; open models get their param count (else parsed from the name).
|
||||
_MODEL_SIZE = {
|
||||
# The real orchestrator_catalog: 2 cloud frontier + 4 local OSS Qwen.
|
||||
"gpt-5.5": "frontier-scale",
|
||||
"claude-opus-4-8": "frontier-scale",
|
||||
"Qwen/Qwen3.5-9B": "~9B params",
|
||||
"Qwen/Qwen3.6-27B-FP8": "~27B params",
|
||||
"Qwen/Qwen3.5-122B-A10B-FP8": "~122B total, ~10B active (MoE)",
|
||||
"Qwen/Qwen3.5-397B-A17B-FP8": "~397B total, ~17B active (MoE)",
|
||||
# OpenRouter slugs (the tool.model id when these route through OpenRouter).
|
||||
"qwen/qwen3.5-122b-a10b": "~122B total, ~10B active (MoE)",
|
||||
"qwen/qwen3.5-397b-a17b": "~397B total, ~17B active (MoE)",
|
||||
}
|
||||
|
||||
|
||||
def _size_hint(model: str) -> str:
|
||||
"""Anonymized scale descriptor for an expert (no brand). Falls back to a
|
||||
param count parsed from the model id (``...-9b`` -> ``~9B params``)."""
|
||||
if model in _MODEL_SIZE:
|
||||
return _MODEL_SIZE[model]
|
||||
m = re.search(r"(\d+(?:\.\d+)?)\s*b\b", model.lower())
|
||||
return f"~{m.group(1)}B params" if m else "unspecified scale"
|
||||
|
||||
|
||||
def _class_hint(tool) -> str:
|
||||
"""Which of the two model classes this expert belongs to (anonymized)."""
|
||||
cat = getattr(tool, "category", "")
|
||||
if cat == CATEGORY_CLOUD_FRONTIER:
|
||||
return "cloud frontier"
|
||||
if cat == CATEGORY_LOCAL_OSS:
|
||||
return "local open-source"
|
||||
return ""
|
||||
|
||||
|
||||
def _cost_tier_hint(tool) -> str:
|
||||
"""Coarse cost tier (not exact pricing — that's the bias we anonymize away).
|
||||
Shown so the policy learns the strong-but-expensive vs cheap tradeoff instead
|
||||
of always delegating to the biggest model. Driven by class: cloud frontier is
|
||||
expensive, self-hosted open-source is cheap."""
|
||||
cat = getattr(tool, "category", "")
|
||||
if cat == CATEGORY_CLOUD_FRONTIER:
|
||||
return "expensive"
|
||||
if cat == CATEGORY_LOCAL_OSS:
|
||||
return "cheap"
|
||||
if getattr(tool, "backend_type", "") == "vllm" or (tool.price_out or 0) <= 0:
|
||||
return "cheap"
|
||||
po = tool.price_out
|
||||
return "cheap" if po < 5 else "moderate" if po < 20 else "expensive"
|
||||
|
||||
|
||||
def anonymize_tools(tools, rng):
|
||||
"""Strip model identity for unbiased routing data.
|
||||
|
||||
Each MODEL expert is replaced with an opaque random label
|
||||
(``expert_<4 rand>``), a uniform description, a uniform category and no
|
||||
price/latency line — so the orchestrator cannot route on a model's name,
|
||||
position, cost, or tier (all of which we found dominate the choice). The full
|
||||
list is shuffled to kill position bias. Basic tools (calculator, web_search,
|
||||
…) keep their real names — the policy must still know what they do.
|
||||
|
||||
Returns ``(anon_tools, anon_to_real)`` where ``anon_to_real`` maps each opaque
|
||||
label back to the real tool name (for offline analysis only; never shown to
|
||||
the model). ``.model`` is preserved on each tool so dispatch still reaches the
|
||||
right backend. Pass a fresh ``rng`` per rollout so labels don't stabilise.
|
||||
"""
|
||||
from dataclasses import replace
|
||||
|
||||
anon_to_real: Dict[str, str] = {}
|
||||
experts: List[ExpertTool] = []
|
||||
basics: List[ExpertTool] = []
|
||||
for t in tools:
|
||||
if t.kind == KIND_MODEL:
|
||||
tag = "model_" + "".join(rng.choice(_ANON_ALPHABET) for _ in range(4))
|
||||
while tag in anon_to_real:
|
||||
tag = "model_" + "".join(rng.choice(_ANON_ALPHABET) for _ in range(4))
|
||||
anon_to_real[tag] = t.name
|
||||
bits = [b for b in (_class_hint(t), _size_hint(t.model), _cost_tier_hint(t)) if b]
|
||||
experts.append(replace(
|
||||
t, name=tag,
|
||||
summary="Another model — " + ", ".join(bits) + ". Send it a sub-question.",
|
||||
category="model", hide_cost=True,
|
||||
))
|
||||
else:
|
||||
basics.append(t)
|
||||
# Shuffle WITHIN the experts to kill per-expert position bias, but keep all
|
||||
# experts as a block on top and the basic tools underneath — a clean split
|
||||
# (experts first) rather than experts and utilities interleaved.
|
||||
rng.shuffle(experts)
|
||||
out = experts + basics
|
||||
return out, anon_to_real
|
||||
|
||||
|
||||
# Default catalog: the paper's tool categories, mapped onto the models/tools
|
||||
# OpenJarvis can actually call. One named tool per model (faithful §3.1).
|
||||
def default_catalog(
|
||||
@@ -414,62 +517,149 @@ def _openjarvis_basic_tools() -> List[ExpertTool]:
|
||||
|
||||
|
||||
# Local-Cloud Hybrid orchestrator catalog. The menu is two model classes — cloud
|
||||
# frontier models and locally-served open-source models — plus the basic tools
|
||||
# (web search, code interpreter, and the bridged real OpenJarvis tools). The
|
||||
# orchestrator model itself is NOT in the catalog.
|
||||
# frontier models and open-source models — plus the basic tools (web search, code
|
||||
# interpreter, and the bridged real OpenJarvis tools). The orchestrator model
|
||||
# itself is NOT in the catalog.
|
||||
#
|
||||
# Local model ids -> vLLM base_url come from ``local_endpoints``; unmapped ids
|
||||
# get ``base_url=None`` (tool still listed, dispatch wires the endpoint later).
|
||||
# Routing default is OpenRouter for every model tool, so the catalog works with
|
||||
# no self-hosted servers. A model is routed to local vLLM instead when its id is
|
||||
# in ``local_endpoints`` or ``model_backends`` overrides it. The
|
||||
# orchestrator-visible tool *name* is always derived from the canonical id, so
|
||||
# routing can change (vLLM <-> OpenRouter <-> native API) without shifting the
|
||||
# menu the policy was trained on.
|
||||
_LOCAL_OSS_MODELS = (
|
||||
"Qwen/Qwen3.5-9B",
|
||||
"Qwen/Qwen3.6-27B-FP8",
|
||||
"Qwen/Qwen3.5-122B-A10B-FP8",
|
||||
"Qwen/Qwen3.5-397B-A17B-FP8",
|
||||
# (canonical_id, openrouter_slug)
|
||||
("Qwen/Qwen3.5-9B", "qwen/qwen3.5-9b"),
|
||||
("Qwen/Qwen3.6-27B-FP8", "qwen/qwen3.6-27b"),
|
||||
("Qwen/Qwen3.5-122B-A10B-FP8", "qwen/qwen3.5-122b-a10b"),
|
||||
("Qwen/Qwen3.5-397B-A17B-FP8", "qwen/qwen3.5-397b-a17b"),
|
||||
)
|
||||
|
||||
_CLOUD_FRONTIER_MODELS = (
|
||||
# (model, backend, summary, latency_s)
|
||||
("gpt-5.5", "openai",
|
||||
"Cloud frontier generalist (GPT-5.5). Strongest reasoning across domains.", 30.0),
|
||||
("claude-opus-4-8", "anthropic",
|
||||
"Cloud frontier generalist (Claude Opus 4.8). Strong long-horizon reasoning.", 26.0),
|
||||
# (canonical, native_backend, openrouter_slug, summary, latency_s)
|
||||
# Neutral, uniform summaries (no capability ranking) so the orchestrator
|
||||
# doesn't just pick whichever model is labelled "strongest" — routing should
|
||||
# be learned from the reward, not hand-labelled here.
|
||||
("gpt-5.5", "openai", "openai/gpt-5.5",
|
||||
"Expert model (GPT-5.5).", 30.0),
|
||||
("claude-opus-4-8", "anthropic", "anthropic/claude-opus-4.8",
|
||||
"Expert model (Claude Opus 4.8).", 26.0),
|
||||
)
|
||||
|
||||
|
||||
def _model_tool(
|
||||
canonical: str,
|
||||
*,
|
||||
native_backend: str,
|
||||
or_slug: str,
|
||||
summary: str,
|
||||
lat: float,
|
||||
category: str,
|
||||
local_endpoints: Dict[str, str],
|
||||
model_backends: Dict[str, str],
|
||||
openrouter_slugs: Dict[str, str],
|
||||
) -> ExpertTool:
|
||||
"""Build one model tool, resolving its backend.
|
||||
|
||||
Backend precedence: explicit ``model_backends[canonical]`` > vLLM if the model
|
||||
has a ``local_endpoints`` entry > the model's NATIVE provider (openai /
|
||||
anthropic / gemini) when it has one > OpenRouter. Frontier models thus hit
|
||||
their first-party API by default (OpenRouter's gpt-5.5 was returning 400s);
|
||||
OSS Qwen experts (native_backend="vllm") fall through to OpenRouter.
|
||||
"""
|
||||
backend = (
|
||||
model_backends.get(canonical)
|
||||
or ("vllm" if canonical in local_endpoints else None)
|
||||
or (native_backend if native_backend in ("openai", "anthropic", "gemini")
|
||||
else "openrouter")
|
||||
)
|
||||
name = _tool_name(canonical)
|
||||
if backend == "vllm":
|
||||
# Self-hosted: free per the cost model.
|
||||
return ExpertTool(
|
||||
name=name, kind=KIND_MODEL, backend_type="vllm", summary=summary,
|
||||
model=canonical, base_url=local_endpoints.get(canonical),
|
||||
price_in=0.0, price_out=0.0, latency_s=lat, category=category,
|
||||
)
|
||||
if backend == "openrouter":
|
||||
slug = openrouter_slugs.get(canonical, or_slug)
|
||||
pi, po = _price(slug)
|
||||
if (pi, po) == (0.0, 0.0): # fall back to the canonical id's price
|
||||
pi, po = _price(canonical)
|
||||
return ExpertTool(
|
||||
name=name, kind=KIND_MODEL, backend_type="openrouter", summary=summary,
|
||||
model=slug, base_url=None, price_in=pi, price_out=po,
|
||||
latency_s=lat, category=category,
|
||||
)
|
||||
# native provider API (openai / anthropic / gemini)
|
||||
pi, po = _price(canonical)
|
||||
return ExpertTool(
|
||||
name=name, kind=KIND_MODEL, backend_type=backend, summary=summary,
|
||||
model=canonical, price_in=pi, price_out=po, latency_s=lat, category=category,
|
||||
)
|
||||
|
||||
|
||||
def orchestrator_catalog(
|
||||
*,
|
||||
local_endpoints: Optional[Dict[str, str]] = None,
|
||||
model_backends: Optional[Dict[str, str]] = None,
|
||||
openrouter_slugs: Optional[Dict[str, str]] = None,
|
||||
include_tools: bool = True,
|
||||
) -> List[ExpertTool]:
|
||||
"""Return the orchestrator's tool catalog: two model classes + basic tools.
|
||||
|
||||
``local_endpoints`` maps a local model id (e.g. ``"Qwen/Qwen3.5-9B"``) to its
|
||||
vLLM ``base_url``; ids not present get ``base_url=None``. ``include_tools``
|
||||
(default True) appends the basic tools — web search, code interpreter, and the
|
||||
bridged real OpenJarvis tools (calculator, shell_exec, file_read, file_write,
|
||||
http_request).
|
||||
Routing for the model tools defaults to **OpenRouter** (so the catalog works
|
||||
with no self-hosted servers). Overrides, in precedence order:
|
||||
|
||||
* ``model_backends`` maps a canonical model id -> ``"vllm" | "openrouter" |
|
||||
"openai" | "anthropic" | "gemini"`` to force that model's backend.
|
||||
* ``local_endpoints`` maps a canonical id (e.g. ``"Qwen/Qwen3.5-9B"``) to a
|
||||
vLLM ``base_url``; a model present here routes to vLLM (free) unless
|
||||
``model_backends`` says otherwise.
|
||||
* ``openrouter_slugs`` overrides the per-model OpenRouter slug used when a
|
||||
model routes through OpenRouter.
|
||||
|
||||
``include_tools`` (default True) appends the basic tools — web search, code
|
||||
interpreter, and the bridged real OpenJarvis tools (calculator, shell_exec,
|
||||
file_read, file_write, http_request).
|
||||
"""
|
||||
local_endpoints = local_endpoints or {}
|
||||
model_backends = model_backends or {}
|
||||
openrouter_slugs = openrouter_slugs or {}
|
||||
cat: List[ExpertTool] = []
|
||||
|
||||
# Env-gated expert exclusion: OJ_EXCLUDE_EXPERTS is a comma-separated list of
|
||||
# case-insensitive substrings matched against a model's canonical id. Any
|
||||
# match is skipped from the catalog. Used to temporarily drop unreliable
|
||||
# experts (e.g. OpenRouter giants during a provider outage) without editing
|
||||
# the registry — unset the var to restore them.
|
||||
_excl = {s.strip().lower() for s in os.environ.get("OJ_EXCLUDE_EXPERTS", "").split(",") if s.strip()}
|
||||
def _excluded(canonical: str) -> bool:
|
||||
c = canonical.lower()
|
||||
return any(x in c for x in _excl)
|
||||
|
||||
# ---- cloud frontier models ----
|
||||
for model, backend, summary, lat in _CLOUD_FRONTIER_MODELS:
|
||||
pi, po = _price(model)
|
||||
cat.append(ExpertTool(
|
||||
name=_tool_name(model), kind=KIND_MODEL, backend_type=backend,
|
||||
summary=summary, model=model, price_in=pi, price_out=po,
|
||||
latency_s=lat, category=CATEGORY_CLOUD_FRONTIER,
|
||||
for canonical, native_backend, or_slug, summary, lat in _CLOUD_FRONTIER_MODELS:
|
||||
if _excluded(canonical):
|
||||
continue
|
||||
cat.append(_model_tool(
|
||||
canonical, native_backend=native_backend, or_slug=or_slug,
|
||||
summary=summary, lat=lat, category=CATEGORY_CLOUD_FRONTIER,
|
||||
local_endpoints=local_endpoints, model_backends=model_backends,
|
||||
openrouter_slugs=openrouter_slugs,
|
||||
))
|
||||
|
||||
# ---- local open-source models (served via vLLM, price 0/0) ----
|
||||
for model in _LOCAL_OSS_MODELS:
|
||||
cat.append(ExpertTool(
|
||||
name=_tool_name(model), kind=KIND_MODEL, backend_type="vllm",
|
||||
summary=(f"Locally-served open-source model ({model}). Cheap and "
|
||||
"private; no per-token cost."),
|
||||
model=model, base_url=local_endpoints.get(model),
|
||||
price_in=0.0, price_out=0.0, latency_s=4.0,
|
||||
category=CATEGORY_LOCAL_OSS,
|
||||
# ---- open-source models (OpenRouter by default; vLLM when an endpoint or
|
||||
# a model_backends override is supplied) ----
|
||||
for canonical, or_slug in _LOCAL_OSS_MODELS:
|
||||
if _excluded(canonical):
|
||||
continue
|
||||
cat.append(_model_tool(
|
||||
canonical, native_backend="vllm", or_slug=or_slug,
|
||||
summary=f"Expert model ({canonical}).",
|
||||
lat=4.0, category=CATEGORY_LOCAL_OSS,
|
||||
local_endpoints=local_endpoints, model_backends=model_backends,
|
||||
openrouter_slugs=openrouter_slugs,
|
||||
))
|
||||
|
||||
if include_tools:
|
||||
@@ -576,9 +766,7 @@ __all__ = [
|
||||
"CATEGORY_CLOUD_FRONTIER",
|
||||
"CATEGORY_GENERALIST",
|
||||
"CATEGORY_LOCAL_OSS",
|
||||
"CATEGORY_SMALL_GENERALIST",
|
||||
"CATEGORY_SPECIALIZED",
|
||||
"CATEGORY_STRONG_GENERALIST",
|
||||
"ExpertTool",
|
||||
"KIND_CODE",
|
||||
"KIND_LOCAL_SEARCH",
|
||||
|
||||
@@ -13,6 +13,27 @@ _TOOL_CALL_TAG_RE = re.compile(
|
||||
r"<tool_call>\s*(\{.*?\})\s*</tool_call>", re.DOTALL
|
||||
)
|
||||
|
||||
# Fallback for a known failure mode: the SFT'd orchestrator often emits its
|
||||
# delegation as ``\boxed{expert_ab12: <sub-question>}`` (math-data habit bleeding
|
||||
# into routing) instead of a real ``<tool_call>`` JSON block. The name before the
|
||||
# colon is a valid (anonymized) tool label; the rest is the sub-question/args.
|
||||
#
|
||||
# Real outputs use several shapes, all handled here:
|
||||
# \boxed{web_search: <query>} -> name, args
|
||||
# \boxed{web_search query: <query>} -> the word "query" is dropped
|
||||
# \boxed{file_read, path: <path>} -> the ", <key>" hint sets the arg key
|
||||
# We require a ``name <optional , key | query> : <args>`` structure, so a genuine
|
||||
# answer like ``\boxed{10}``, ``\boxed{b,e}`` or ``\boxed{The answer is: 42}``
|
||||
# (first token isn't followed by a bare ``:`` / ``, key:`` / ``query:``) is left
|
||||
# alone. ``group("key")`` is the explicit arg key when the model wrote ``, key:``.
|
||||
_BOXED_DELEGATION_RE = re.compile(
|
||||
r"\\boxed\{\s*([A-Za-z_][\w-]*)\s*"
|
||||
r"(?:,\s*(?P<key>\w+)\s*)?" # optional ", key" hint (e.g. file_read, path:)
|
||||
r"(?:query\s*)?" # optional literal "query" word before the colon
|
||||
r":\s*(.+?)\s*\}\s*$",
|
||||
re.DOTALL,
|
||||
)
|
||||
|
||||
|
||||
def _parse_rl_tool_call(content: str, sdk_tool_calls: Any) -> Optional[Dict[str, Any]]:
|
||||
"""Return ``{"name": str, "arguments": dict}`` or None.
|
||||
@@ -37,19 +58,23 @@ def _parse_rl_tool_call(content: str, sdk_tool_calls: Any) -> Optional[Dict[str,
|
||||
if not isinstance(content, str):
|
||||
return None
|
||||
m = _TOOL_CALL_TAG_RE.search(content)
|
||||
if not m:
|
||||
return None
|
||||
try:
|
||||
obj = json.loads(m.group(1))
|
||||
except json.JSONDecodeError:
|
||||
return None
|
||||
if not isinstance(obj, dict):
|
||||
return None
|
||||
name = obj.get("name")
|
||||
args = obj.get("arguments", {})
|
||||
if not isinstance(name, str) or not isinstance(args, dict):
|
||||
return None
|
||||
return {"name": name, "arguments": args}
|
||||
if m:
|
||||
try:
|
||||
obj = json.loads(m.group(1))
|
||||
except json.JSONDecodeError:
|
||||
obj = None
|
||||
if isinstance(obj, dict):
|
||||
name = obj.get("name")
|
||||
args = obj.get("arguments", {})
|
||||
if isinstance(name, str) and isinstance(args, dict):
|
||||
return {"name": name, "arguments": args}
|
||||
# \boxed{name[, key][ query]: <args>} fallback (see _BOXED_DELEGATION_RE).
|
||||
bm = _BOXED_DELEGATION_RE.search(content.strip())
|
||||
if bm:
|
||||
arg_val = bm.groups()[-1].strip()
|
||||
key = bm.group("key") or "input"
|
||||
return {"name": bm.group(1), "arguments": {key: arg_val}}
|
||||
return None
|
||||
|
||||
|
||||
def _build_pool_block(workers: List[Dict[str, Any]]) -> str:
|
||||
|
||||
@@ -9,24 +9,55 @@ final answer) or ``max_turns`` is hit.
|
||||
The loop is parameterized over two injected callables so it is pure control flow
|
||||
(no network) and unit-testable with fakes — the agent supplies real ones:
|
||||
|
||||
* ``call_orchestrator(system, user, tool_specs) -> (text, tool_calls, p_tok, c_tok)``
|
||||
where ``tool_calls`` is a list of ``(name, arguments)`` (possibly empty).
|
||||
* ``call_orchestrator(messages, tool_specs) -> (text, tool_calls, p_tok, c_tok)``
|
||||
where ``messages`` is the running system/user/assistant/tool conversation and
|
||||
``tool_calls`` is a list of ``(name, arguments)`` (possibly empty).
|
||||
* ``dispatch(tool, arguments) -> (observation, cost_usd, tokens, is_local)``.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import random
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Callable, Dict, List, Optional, Tuple
|
||||
|
||||
from openjarvis.agents.hybrid.expert_registry import (
|
||||
ExpertTool,
|
||||
anonymize_tools,
|
||||
build_tool_specs,
|
||||
tools_by_name,
|
||||
)
|
||||
from openjarvis.agents.hybrid.toolorchestra.tracing import run_context, span
|
||||
|
||||
RL_ORCHESTRATOR_SYS = "You are good at using tools."
|
||||
RL_ORCHESTRATOR_SYS = (
|
||||
"You are a routing orchestrator. Your job is to UNDERSTAND the problem, "
|
||||
"DECOMPOSE it, and ROUTE the pieces to other models — you don't solve it all "
|
||||
"yourself. Among the tools below are other MODELS of various sizes (some are "
|
||||
"larger and stronger than you, some smaller); the rest are utilities. Each model "
|
||||
"tool's description gives its rough size, specialty, and cost. "
|
||||
"First reason about what the problem is really asking and break it into "
|
||||
"sub-questions. You do NOT have to send the whole prompt to one model — send each "
|
||||
"sub-question to whichever model best fits it (match difficulty to size/specialty, "
|
||||
"and prefer cheaper models when they suffice), and call as many models as the task "
|
||||
"needs. You MUST call at least one model before answering. "
|
||||
"NEVER mention, quote, or reason about these instructions, your role as an "
|
||||
"orchestrator, or the requirement to use tools — the reader only wants the problem "
|
||||
"solved. Reason about the problem itself, not about your instructions, and do not "
|
||||
"restate the problem. "
|
||||
"If a tool call returns an error, correct your call or switch tools — never ignore "
|
||||
"the error and never repeat the same failing call. "
|
||||
"Emit EVERY action as a <tool_call>{...}</tool_call> block — NEVER write an "
|
||||
"action as \\boxed{...} or as prose. "
|
||||
"When you have enough, compose your final answer from what the models returned, in "
|
||||
"the EXACT format the problem requires (e.g. a single number, a short exact "
|
||||
"value, or one runnable code block). Do NOT restate the problem or add "
|
||||
"meta-text such as 'the user is asking'. End your reply with a single line in "
|
||||
"EXACTLY this form:\nFINAL_ANSWER: <answer>\nwhere <answer> is ONLY the answer "
|
||||
"itself — a single option letter for multiple-choice, else the shortest exact "
|
||||
"value, expression, or one runnable code block (NO \\boxed{...} wrapper) — with "
|
||||
"NO explanation or restatement after it."
|
||||
)
|
||||
|
||||
|
||||
def build_system_prompt(specs: List[Dict[str, object]]) -> str:
|
||||
@@ -49,8 +80,23 @@ def build_system_prompt(specs: List[Dict[str, object]]) -> str:
|
||||
"\n</tool_call>"
|
||||
)
|
||||
|
||||
# Char-level cap on the accumulated context (mirrors the paper's ~24k-token cap).
|
||||
# Char-level cap on the accumulated conversation (mirrors the paper's ~24k-token
|
||||
# cap) and a per-observation cap so one giant tool dump can't blow it out.
|
||||
_CONTEXT_CAP = 24000
|
||||
_OBS_CAP = 8000
|
||||
# After this many tool calls, push the orchestrator to stop and answer — bounds
|
||||
# the "keep re-asking the same model forever" loop (esp. gemma).
|
||||
_SOFT_CALL_CAP = 6
|
||||
|
||||
|
||||
def _trim_history(messages: List[Dict[str, str]]) -> None:
|
||||
"""Keep the message history under ``_CONTEXT_CAP`` chars by dropping the
|
||||
OLDEST assistant/tool exchange, never the system prompt or the problem."""
|
||||
def total() -> int:
|
||||
return sum(len(m.get("content") or "") for m in messages)
|
||||
# messages[0]=system, messages[1]=user problem — always keep those two.
|
||||
while total() > _CONTEXT_CAP and len(messages) > 4:
|
||||
del messages[2:4]
|
||||
|
||||
|
||||
@dataclass
|
||||
@@ -71,6 +117,16 @@ class UnifiedRollout:
|
||||
tokens: int = 0
|
||||
num_tool_calls: int = 0
|
||||
parse_failures: int = 0
|
||||
# When experts were anonymized for this rollout, maps the opaque label the
|
||||
# policy saw (e.g. ``expert_a3f9``) back to the real tool name (``gpt_5_5``).
|
||||
anon_map: Optional[Dict[str, str]] = None
|
||||
# The EXACT tool specs the policy saw this rollout (anonymized when
|
||||
# ``anonymize=True``). Serialization MUST build the saved system prompt from
|
||||
# these — not from the real registry — or the prompt's ``<tools>`` block ends
|
||||
# up with real model names+pricing while the assistant ``<tool_call>`` tags
|
||||
# use the anon labels, which both breaks SFT (calls a tool absent from its
|
||||
# list) and re-injects the name/cost bias anonymization removed.
|
||||
tool_specs: Optional[List[Dict[str, object]]] = None
|
||||
|
||||
def tool_calls(self) -> List[Tuple[str, Dict[str, object]]]:
|
||||
return [(t.tool_name, t.arguments) for t in self.turns if t.tool_name]
|
||||
@@ -85,7 +141,7 @@ def _tool_prompt(tool: ExpertTool, arguments: Dict[str, object], question: str)
|
||||
return question
|
||||
|
||||
|
||||
def run_unified_rollout(
|
||||
def _run_unified_rollout_inner(
|
||||
question: str,
|
||||
tools: List[ExpertTool],
|
||||
*,
|
||||
@@ -93,43 +149,90 @@ def run_unified_rollout(
|
||||
dispatch: Callable[[ExpertTool, Dict[str, object]], Tuple[str, float, int, bool]],
|
||||
max_turns: int = 50,
|
||||
system: str = RL_ORCHESTRATOR_SYS,
|
||||
anonymize: bool = False,
|
||||
) -> UnifiedRollout:
|
||||
"""Drive the faithful unified-tool rollout for one task."""
|
||||
"""Drive the faithful unified-tool rollout for one task.
|
||||
|
||||
``anonymize``: replace each model expert with an opaque random label, a
|
||||
uniform description and no cost line, shuffled — so the policy can't route on
|
||||
a model's name/position/cost (which we found dominate the choice). The
|
||||
anon->real mapping is returned on the rollout for offline analysis.
|
||||
"""
|
||||
anon_map: Optional[Dict[str, str]] = None
|
||||
if anonymize:
|
||||
tools, anon_map = anonymize_tools(tools, random.Random())
|
||||
specs = build_tool_specs(tools)
|
||||
by_name = tools_by_name(tools)
|
||||
|
||||
context = ""
|
||||
# Proper multi-turn conversation (NOT a flattened user blob): the orchestrator
|
||||
# sees its own calls as `assistant` turns and each observation as a `tool`
|
||||
# turn, so it can tell a tool result from user input and doesn't re-derive the
|
||||
# whole problem every turn. This mirrors the serialized SFT format exactly.
|
||||
messages: List[Dict[str, str]] = [
|
||||
{"role": "system", "content": system},
|
||||
{"role": "user", "content": f"Problem: {question}"},
|
||||
]
|
||||
turns: List[UnifiedTurn] = []
|
||||
cost = 0.0
|
||||
tokens = 0
|
||||
n_tool_calls = 0
|
||||
parse_failures = 0
|
||||
nudges = 0
|
||||
empty_input_nudges = 0
|
||||
final_answer = ""
|
||||
|
||||
for _ in range(max_turns):
|
||||
user = (
|
||||
f"Problem: {question}\n\n{context or '(no context yet)'}\n\n"
|
||||
"Choose an appropriate tool, or answer directly if you have enough."
|
||||
)
|
||||
text, tool_calls, p_tok, c_tok = call_orchestrator(system, user, specs)
|
||||
text, tool_calls, p_tok, c_tok = call_orchestrator(messages, specs)
|
||||
tokens += int(p_tok) + int(c_tok)
|
||||
|
||||
if not tool_calls:
|
||||
# Enforce the "MUST delegate to a model" rule: if the orchestrator
|
||||
# tries to answer before ANY tool call, nudge it to route instead of
|
||||
# accepting a solve-it-yourself answer (which reject-sampling drops).
|
||||
if n_tool_calls == 0 and nudges < 2:
|
||||
nudges += 1
|
||||
messages.append({"role": "assistant", "content": text or ""})
|
||||
messages.append({"role": "user", "content": (
|
||||
"Make progress by delegating a concrete sub-question to one of the "
|
||||
"models now via a <tool_call>.")})
|
||||
continue
|
||||
# No tool call -> the orchestrator is answering. Terminate.
|
||||
final_answer = (text or "").strip()
|
||||
turns.append(UnifiedTurn(reasoning=text or "", tool_name=None))
|
||||
messages.append({"role": "assistant", "content": text or ""})
|
||||
break
|
||||
|
||||
name, arguments = tool_calls[0]
|
||||
if name not in by_name:
|
||||
parse_failures += 1
|
||||
context = (context + f"\n[invalid tool {name!r} — choose from the list]")[-_CONTEXT_CAP:]
|
||||
messages.append({"role": "assistant", "content": text or ""})
|
||||
messages.append({"role": "user",
|
||||
"content": f"[invalid tool {name!r} — choose one from the provided tool list]"})
|
||||
if parse_failures >= 2:
|
||||
final_answer = (text or "").strip()
|
||||
break
|
||||
continue
|
||||
|
||||
tool = by_name[name]
|
||||
# Empty-input guard: the small orchestrator often emits a <tool_call> with
|
||||
# a blank/missing input on harder multi-turn tasks. Dispatching it returns
|
||||
# a "no input provided" error observation that poisons the whole (often
|
||||
# otherwise-correct) trajectory — the dominant clean-yield killer on hard
|
||||
# tasks. Instead, nudge the model to resend WITH input and drop the
|
||||
# malformed turn (don't record it), so it never enters the training data.
|
||||
if tool.kind == "model" or tool.backend_type != "openjarvis-tool":
|
||||
_has_input = any(
|
||||
isinstance(arguments.get(k), str) and arguments.get(k).strip()
|
||||
for k in ("input", "query", "code")
|
||||
)
|
||||
if not _has_input and empty_input_nudges < 3:
|
||||
empty_input_nudges += 1
|
||||
messages.append({"role": "assistant", "content": text or ""})
|
||||
messages.append({"role": "user", "content": (
|
||||
f"Your call to {name} had an empty 'input'. Resend the "
|
||||
"<tool_call> with a non-empty 'input' field containing the "
|
||||
"concrete sub-question to delegate.")})
|
||||
continue
|
||||
obs, dcost, dtok, _is_local = dispatch(tool, arguments)
|
||||
cost += float(dcost)
|
||||
tokens += int(dtok)
|
||||
@@ -138,7 +241,22 @@ def run_unified_rollout(
|
||||
reasoning=text or "", tool_name=name, arguments=dict(arguments),
|
||||
observation=obs,
|
||||
))
|
||||
context = (context + f"\n[{name}] {obs}")[-_CONTEXT_CAP:]
|
||||
# The model's own action as an assistant turn (the <tool_call> tag in
|
||||
# content, matching the SFT serialization), then the observation as a
|
||||
# distinct `tool` turn the model reads as a tool response.
|
||||
call_content = text or ""
|
||||
if "<tool_call>" not in call_content:
|
||||
call_content = (call_content + "\n" + tool_call_tag(name, arguments)).strip()
|
||||
messages.append({"role": "assistant", "content": call_content})
|
||||
obs_text = obs or ""
|
||||
if len(obs_text) > _OBS_CAP:
|
||||
obs_text = obs_text[:_OBS_CAP] + "\n…[truncated]"
|
||||
messages.append({"role": "tool", "name": name, "content": obs_text})
|
||||
if n_tool_calls >= _SOFT_CALL_CAP:
|
||||
messages.append({"role": "user", "content": (
|
||||
"You now have enough information from the models. Do NOT call any "
|
||||
"more tools — reply with your FINAL_ANSWER line only.")})
|
||||
_trim_history(messages)
|
||||
else:
|
||||
# Hit max_turns with no explicit answer: use the last observation/text.
|
||||
final_answer = (turns[-1].observation or turns[-1].reasoning).strip() if turns else ""
|
||||
@@ -146,9 +264,51 @@ def run_unified_rollout(
|
||||
return UnifiedRollout(
|
||||
turns=turns, final_answer=final_answer, cost_usd=cost, tokens=tokens,
|
||||
num_tool_calls=n_tool_calls, parse_failures=parse_failures,
|
||||
anon_map=anon_map, tool_specs=specs,
|
||||
)
|
||||
|
||||
|
||||
def run_unified_rollout(
|
||||
question: str,
|
||||
tools: List[ExpertTool],
|
||||
*,
|
||||
call_orchestrator: Callable[..., Tuple[str, List[Tuple[str, Dict[str, object]]], int, int]],
|
||||
dispatch: Callable[[ExpertTool, Dict[str, object]], Tuple[str, float, int, bool]],
|
||||
max_turns: int = 50,
|
||||
system: str = RL_ORCHESTRATOR_SYS,
|
||||
anonymize: bool = False,
|
||||
) -> UnifiedRollout:
|
||||
"""One ToolOrchestra rollout = one Braintrust trace. Opens the root span,
|
||||
runs the loop (orchestrator turns + expert/tool dispatches nest inside as
|
||||
child spans), then logs the final answer + cost/tokens. No-op when tracing
|
||||
is disabled (see :mod:`.tracing`)."""
|
||||
_run_meta, _run_tags = run_context()
|
||||
_fields = dict(
|
||||
input={"question": question},
|
||||
metadata={"max_turns": max_turns, "anonymize": anonymize,
|
||||
"n_tools": len(tools), **_run_meta},
|
||||
)
|
||||
if _run_tags:
|
||||
_fields["tags"] = _run_tags
|
||||
with span("toolorchestra.rollout", span_type="task", **_fields) as _s:
|
||||
roll = _run_unified_rollout_inner(
|
||||
question, tools, call_orchestrator=call_orchestrator, dispatch=dispatch,
|
||||
max_turns=max_turns, system=system, anonymize=anonymize,
|
||||
)
|
||||
_s.log(
|
||||
output=roll.final_answer,
|
||||
metrics={"cost_usd": roll.cost_usd, "tokens": roll.tokens,
|
||||
"num_tool_calls": roll.num_tool_calls,
|
||||
"parse_failures": roll.parse_failures},
|
||||
metadata={"n_experts_available": len(roll.anon_map or {}),
|
||||
"answered": bool(roll.final_answer),
|
||||
# label -> real model, so every anonymized route span in this
|
||||
# trace can be decoded back to the model that actually ran.
|
||||
"anon_map": roll.anon_map or {}},
|
||||
)
|
||||
return roll
|
||||
|
||||
|
||||
def tool_call_tag(name: str, arguments: Dict[str, object]) -> str:
|
||||
"""Render a tool call as the ``<tool_call>{...}</tool_call>`` text the model emits."""
|
||||
return f"<tool_call>{json.dumps({'name': name, 'arguments': arguments})}</tool_call>"
|
||||
|
||||
@@ -0,0 +1,161 @@
|
||||
"""Braintrust telemetry for the ToolOrchestra rollout.
|
||||
|
||||
Each ``run_unified_rollout`` becomes ONE Braintrust trace (root span); the
|
||||
orchestrator turns and every expert/tool dispatch nest inside it, and the
|
||||
underlying OpenAI/Anthropic calls (wrapped clients) nest one level deeper — so
|
||||
you see the full routing tree with inputs/outputs/tokens/cost per node.
|
||||
|
||||
On by default. Degrades to a total no-op (never raises, never changes behavior)
|
||||
when: ``OJ_BRAINTRUST=0``, the ``braintrust`` package isn't installed, or
|
||||
``BRAINTRUST_API_KEY`` is unset. Concurrency: rollouts run one-per-thread
|
||||
(ThreadPoolExecutor); threads don't inherit contextvars, so each rollout opens
|
||||
its own independent root trace and same-thread child calls nest correctly.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import contextlib
|
||||
import logging
|
||||
import os
|
||||
from typing import Any, Optional
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_STATE: dict = {"resolved": False, "enabled": False, "bt": None}
|
||||
|
||||
|
||||
def _truthy(v: str) -> bool:
|
||||
return v.strip().lower() not in ("0", "false", "no", "off", "")
|
||||
|
||||
|
||||
def _resolve() -> bool:
|
||||
"""Lazily decide whether tracing is active and init the logger once."""
|
||||
if _STATE["resolved"]:
|
||||
return _STATE["enabled"]
|
||||
_STATE["resolved"] = True
|
||||
if not _truthy(os.getenv("OJ_BRAINTRUST", "1")): # on by default
|
||||
return False
|
||||
if not os.getenv("BRAINTRUST_API_KEY"):
|
||||
logger.info("braintrust on-by-default but BRAINTRUST_API_KEY unset — tracing disabled")
|
||||
return False
|
||||
try:
|
||||
import braintrust as _bt
|
||||
|
||||
proj_id = os.getenv("OJ_BRAINTRUST_PROJECT_ID")
|
||||
if proj_id:
|
||||
_bt.init_logger(project_id=proj_id)
|
||||
else:
|
||||
_bt.init_logger(project=os.getenv("OJ_BRAINTRUST_PROJECT", "toolorchestra"))
|
||||
_STATE["bt"] = _bt
|
||||
_STATE["enabled"] = True
|
||||
logger.info("braintrust tracing ENABLED (%s)",
|
||||
f"project_id={proj_id}" if proj_id
|
||||
else f"project={os.getenv('OJ_BRAINTRUST_PROJECT', 'toolorchestra')}")
|
||||
except Exception as exc: # missing pkg / bad key / init failure — never crash
|
||||
logger.warning("braintrust init failed (%s) — tracing disabled", exc)
|
||||
return _STATE["enabled"]
|
||||
|
||||
|
||||
def enabled() -> bool:
|
||||
return _resolve()
|
||||
|
||||
|
||||
def run_context() -> tuple[dict, list]:
|
||||
"""Run-level metadata + tags for the ROOT rollout span, sourced from env
|
||||
(set by the generation driver). No-op-safe: returns ({}, []) if nothing is
|
||||
set, and never raises. Env keys:
|
||||
|
||||
- ``OJ_RUN_LABEL`` — human run label (also stamped on the uploaded dataset).
|
||||
- ``OJ_GEN_MODEL`` — SPECIFIC gen model id (e.g. ``Qwen/Qwen3.5-9B``).
|
||||
- ``OJ_RUN_STAGE`` — ``prod``|``smoke`` (inferred from the label if unset).
|
||||
- ``OJ_CFG_*`` — config knobs (temperature/max_turns/anonymize/...).
|
||||
"""
|
||||
import datetime
|
||||
|
||||
meta: dict = {}
|
||||
tags: list = []
|
||||
try:
|
||||
label = os.getenv("OJ_RUN_LABEL")
|
||||
gen_model = os.getenv("OJ_GEN_MODEL")
|
||||
stage = os.getenv("OJ_RUN_STAGE")
|
||||
if not stage and label:
|
||||
stage = "smoke" if "smoke" in label.lower() else "prod"
|
||||
if label:
|
||||
meta["run_label"] = label
|
||||
if gen_model:
|
||||
meta["gen_model"] = gen_model
|
||||
if stage:
|
||||
meta["stage"] = stage
|
||||
cfg = {}
|
||||
for env_key, key in (("OJ_CFG_TEMPERATURE", "temperature"),
|
||||
("OJ_CFG_MAX_TURNS", "max_turns"),
|
||||
("OJ_CFG_ANONYMIZE", "anonymize"),
|
||||
("OJ_CFG_REJECTION_ONLY", "rejection_only")):
|
||||
v = os.getenv(env_key)
|
||||
if v not in (None, ""):
|
||||
cfg[key] = v
|
||||
if cfg:
|
||||
meta["config"] = cfg
|
||||
date = datetime.date.today().isoformat()
|
||||
tags = [t for t in (gen_model, stage, date) if t]
|
||||
except Exception: # never let telemetry enrichment break a rollout
|
||||
return {}, []
|
||||
return meta, tags
|
||||
|
||||
|
||||
def wrap_client(client: Any) -> Any:
|
||||
"""Wrap an OpenAI/Anthropic client so its calls auto-log under the current
|
||||
span. Pass-through (unchanged client) when tracing is off or on any error —
|
||||
so this is always safe to call at client construction."""
|
||||
if not _resolve():
|
||||
return client
|
||||
try:
|
||||
bt = _STATE["bt"]
|
||||
mod = type(client).__module__.lower()
|
||||
if "anthropic" in mod:
|
||||
return bt.wrap_anthropic(client)
|
||||
return bt.wrap_openai(client)
|
||||
except Exception as exc:
|
||||
logger.warning("braintrust wrap_client failed (%s) — using raw client", exc)
|
||||
return client
|
||||
|
||||
|
||||
class _NullSpan:
|
||||
"""No-op span used when tracing is disabled (keeps call sites branch-free)."""
|
||||
|
||||
def log(self, **_kw: Any) -> None:
|
||||
pass
|
||||
|
||||
def __enter__(self) -> "_NullSpan":
|
||||
return self
|
||||
|
||||
def __exit__(self, *_a: Any) -> bool:
|
||||
return False
|
||||
|
||||
|
||||
@contextlib.contextmanager
|
||||
def span(name: str, *, span_type: str = "task", **fields: Any):
|
||||
"""Open a Braintrust span (or a no-op). Use as::
|
||||
|
||||
with span("toolorchestra.rollout", input=...) as s:
|
||||
...
|
||||
s.log(output=..., metrics=..., metadata=...)
|
||||
|
||||
``fields`` (input/metadata/...) are forwarded to ``start_span``.
|
||||
"""
|
||||
if not _resolve():
|
||||
yield _NullSpan()
|
||||
return
|
||||
# Guard only span CREATION (so a Braintrust hiccup degrades to a no-op). Do
|
||||
# NOT wrap the yielded body in try/except: an exception from the rollout would
|
||||
# be thrown into this generator, caught here, and masked by a second yield
|
||||
# ("generator didn't stop after throw()"). Let the body's own exceptions
|
||||
# propagate through the `with` untouched — Braintrust records + re-raises.
|
||||
try:
|
||||
cm = _STATE["bt"].start_span(name=name, type=span_type, **fields)
|
||||
except Exception as exc: # span creation failed — run trace-less, never break
|
||||
logger.warning("braintrust start_span(%s) failed (%s) — continuing untraced", name, exc)
|
||||
yield _NullSpan()
|
||||
return
|
||||
with cm as s:
|
||||
yield s
|
||||
@@ -11,16 +11,25 @@ network or keys.
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import threading
|
||||
from typing import Any, Callable, Dict, List, Optional, Tuple
|
||||
|
||||
# Guards the lazy, one-time build of the shared ToolExecutor in
|
||||
# ``_dispatch_openjarvis_tool``: with concurrent rollouts (parallel rejection
|
||||
# sampling) several threads can reach the build at once and would each instantiate
|
||||
# the full tool registry. The lock makes it build-once.
|
||||
_EXECUTOR_BUILD_LOCK = threading.Lock()
|
||||
|
||||
from openjarvis.agents.hybrid._prices import cost as _model_cost
|
||||
from openjarvis.agents.hybrid.expert_registry import ExpertTool, to_worker_dict
|
||||
from openjarvis.agents.hybrid.toolorchestra.parsing import _parse_rl_tool_call
|
||||
from openjarvis.agents.hybrid.toolorchestra.rollout import (
|
||||
UnifiedRollout,
|
||||
build_system_prompt,
|
||||
run_unified_rollout,
|
||||
)
|
||||
from openjarvis.agents.hybrid.toolorchestra.workers import _call_worker
|
||||
from openjarvis.agents.hybrid.toolorchestra.tracing import span
|
||||
|
||||
|
||||
def make_call_orchestrator(
|
||||
@@ -31,24 +40,68 @@ def make_call_orchestrator(
|
||||
temperature: float = 1.0,
|
||||
max_tokens: int = 4096,
|
||||
timeout: float = 600.0,
|
||||
native_tools: bool = True,
|
||||
frequency_penalty: float = 0.2,
|
||||
presence_penalty: float = 0.0,
|
||||
repetition_penalty: float = 1.15,
|
||||
) -> Callable[..., Tuple[str, List[Tuple[str, Dict[str, Any]]], int, int]]:
|
||||
"""Teacher-orchestrator caller. ``base_url=None`` → OpenAI cloud; set it to a
|
||||
vLLM endpoint (with ``api_key="EMPTY"``) to drive a local served teacher.
|
||||
|
||||
``native_tools`` picks the tool-calling convention, and the two modes MUST NOT
|
||||
be mixed on the same model:
|
||||
|
||||
* ``True`` (default — DATA GENERATION with a *base* model): pass the OpenAI
|
||||
``tools=`` param so the server's native tool template + parser (gemma4 /
|
||||
qwen3_xml / hermes) drive the base model, which only emits reliable tool
|
||||
calls that way. The rollout captures ``(name, arguments)`` and the serializer
|
||||
re-renders them into the canonical JSON ``<tool_call>`` form regardless.
|
||||
* ``False`` (SERVING a *fine-tuned* model): bake the ``<tools>`` block + JSON
|
||||
``<tool_call>`` format into the system prompt (exactly what the serializer
|
||||
trained on) and DROP ``tools=``. If ``tools=`` is set here, the served chat
|
||||
template injects its own XML ``<function=>`` instructions, which conflict
|
||||
with the JSON format the model learned -> it falls back to ``\\boxed{}`` and
|
||||
never routes. ``_parse_rl_tool_call`` scrapes the JSON tag from raw text.
|
||||
"""
|
||||
|
||||
def call_orchestrator(system: str, user: str, specs: List[Dict[str, Any]]):
|
||||
from openai import OpenAI
|
||||
# Create the client ONCE and reuse it for every turn. Creating a fresh OpenAI
|
||||
# client per call (as before) leaks an httpx connection pool each time -> under
|
||||
# heavy parallel generation, sockets pile up in CLOSE-WAIT, the run degrades and
|
||||
# has to be restarted. The orchestrator is the highest-frequency call, so reusing
|
||||
# one client here removes the dominant leak. httpx clients are thread-safe.
|
||||
from openai import OpenAI
|
||||
|
||||
client = OpenAI(base_url=base_url, api_key=api_key or "EMPTY", timeout=timeout)
|
||||
from openjarvis.agents.hybrid.toolorchestra.tracing import wrap_client
|
||||
|
||||
client = wrap_client(OpenAI(base_url=base_url, api_key=api_key or "EMPTY", timeout=timeout))
|
||||
|
||||
def call_orchestrator(messages: List[Dict[str, Any]], specs: List[Dict[str, Any]]):
|
||||
# ``messages`` is the full running conversation (system/user/assistant/tool)
|
||||
# built by run_unified_rollout — pass it through so the model sees its own
|
||||
# prior calls + tool responses, not a flattened user blob.
|
||||
if native_tools:
|
||||
# Base-model generation: native tool template drives the tool calls.
|
||||
send = messages
|
||||
kwargs = {"tools": specs} if specs else {}
|
||||
else:
|
||||
# Fine-tuned serving: JSON format is baked into the system prompt and
|
||||
# tools= is dropped (see make_call_orchestrator docstring).
|
||||
send = list(messages)
|
||||
if specs and send and send[0].get("role") == "system":
|
||||
send[0] = {"role": "system", "content": build_system_prompt(specs)}
|
||||
kwargs = {}
|
||||
# repetition_penalty is a vLLM extra (not OpenAI-native); only send it to a
|
||||
# local vLLM endpoint (base_url set), never to the cloud frontier APIs.
|
||||
if base_url and repetition_penalty and repetition_penalty != 1.0:
|
||||
kwargs.setdefault("extra_body", {})["repetition_penalty"] = repetition_penalty
|
||||
resp = client.chat.completions.create(
|
||||
model=model,
|
||||
messages=[
|
||||
{"role": "system", "content": system},
|
||||
{"role": "user", "content": user},
|
||||
],
|
||||
messages=send,
|
||||
temperature=temperature,
|
||||
max_tokens=max_tokens,
|
||||
tools=specs,
|
||||
frequency_penalty=frequency_penalty,
|
||||
presence_penalty=presence_penalty,
|
||||
**kwargs,
|
||||
)
|
||||
msg = resp.choices[0].message
|
||||
text = msg.content or ""
|
||||
@@ -78,26 +131,34 @@ def _dispatch_openjarvis_tool(
|
||||
try:
|
||||
executor = executor_holder.get("executor")
|
||||
if executor is None:
|
||||
from openjarvis.core.registry import ToolRegistry
|
||||
from openjarvis.tools._stubs import ToolExecutor
|
||||
with _EXECUTOR_BUILD_LOCK:
|
||||
# Re-check inside the lock: another thread may have built it while
|
||||
# we waited.
|
||||
executor = executor_holder.get("executor")
|
||||
if executor is None:
|
||||
from openjarvis.core.registry import ToolRegistry
|
||||
from openjarvis.tools._stubs import ToolExecutor
|
||||
|
||||
instances = []
|
||||
for name in ToolRegistry.keys():
|
||||
entry = ToolRegistry.get(name)
|
||||
try:
|
||||
instances.append(entry() if isinstance(entry, type) else entry)
|
||||
except Exception:
|
||||
continue
|
||||
# Auto-approve confirmation-gated tools (shell_exec, git_commit, ...):
|
||||
# rollouts/eval run headless in a sandbox, so there is no human to
|
||||
# confirm. Without this, those tools return a "requires confirmation"
|
||||
# error instead of executing — which would silently break TerminalBench.
|
||||
executor = ToolExecutor(
|
||||
instances,
|
||||
interactive=True,
|
||||
confirm_callback=lambda _prompt: True,
|
||||
)
|
||||
executor_holder["executor"] = executor
|
||||
instances = []
|
||||
for name in ToolRegistry.keys():
|
||||
entry = ToolRegistry.get(name)
|
||||
try:
|
||||
instances.append(
|
||||
entry() if isinstance(entry, type) else entry
|
||||
)
|
||||
except Exception:
|
||||
continue
|
||||
# Auto-approve confirmation-gated tools (shell_exec,
|
||||
# git_commit, ...): rollouts/eval run headless in a sandbox, so
|
||||
# there is no human to confirm. Without this, those tools return
|
||||
# a "requires confirmation" error instead of executing — which
|
||||
# would silently break TerminalBench.
|
||||
executor = ToolExecutor(
|
||||
instances,
|
||||
interactive=True,
|
||||
confirm_callback=lambda _prompt: True,
|
||||
)
|
||||
executor_holder["executor"] = executor
|
||||
|
||||
from openjarvis.core.types import ToolCall
|
||||
|
||||
@@ -126,7 +187,7 @@ def make_dispatch(
|
||||
cfg = cfg or {}
|
||||
executor_holder: Dict[str, Any] = {}
|
||||
|
||||
def dispatch(tool: ExpertTool, arguments: Dict[str, Any]):
|
||||
def _dispatch_inner(tool: ExpertTool, arguments: Dict[str, Any]):
|
||||
# Bridged real OpenJarvis tools run through the ToolExecutor, not
|
||||
# the model/worker path.
|
||||
if tool.backend_type == "openjarvis-tool":
|
||||
@@ -139,10 +200,32 @@ def make_dispatch(
|
||||
if isinstance(val, str) and val.strip():
|
||||
prompt = val
|
||||
break
|
||||
if not prompt.strip():
|
||||
# The orchestrator emitted a tool call with no/empty input. Don't hit
|
||||
# the API (it 400s on empty content) — return a usable error so the
|
||||
# rollout keeps going instead of dropping.
|
||||
return (f"[{tool.name}: no input provided — supply a non-empty "
|
||||
"'input' to delegate]", 0.0, 0, True)
|
||||
text, p, c, is_local, extra_cost, _n = _call_worker(worker, prompt, cfg)
|
||||
usd = (0.0 if is_local else _model_cost(str(tool.model), p, c)) + float(extra_cost)
|
||||
return text, usd, int(p) + int(c), bool(is_local)
|
||||
|
||||
def dispatch(tool: ExpertTool, arguments: Dict[str, Any]):
|
||||
# One span per route so the trace tree shows which expert/tool was called,
|
||||
# with the delegated input, the returned observation, and cost/tokens. The
|
||||
# wrapped OpenAI/Anthropic call (if any) nests one level deeper.
|
||||
# Span is titled with the REAL model that actually ran (``tool.model``) so
|
||||
# the trace reads clearly even under anonymization; ``anon_label`` in the
|
||||
# metadata records the opaque label the orchestrator actually saw/chose.
|
||||
real_model = str(tool.model)
|
||||
with span(f"route:{real_model}", span_type="tool", input=arguments,
|
||||
metadata={"real_model": real_model, "anon_label": tool.name,
|
||||
"backend": tool.backend_type}) as s:
|
||||
obs, usd, toks, is_local = _dispatch_inner(tool, arguments)
|
||||
s.log(output=obs, metrics={"cost_usd": usd, "tokens": toks},
|
||||
metadata={"is_local": is_local})
|
||||
return obs, usd, toks, is_local
|
||||
|
||||
return dispatch
|
||||
|
||||
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, List, Optional, Tuple
|
||||
|
||||
@@ -285,7 +286,7 @@ def _call_worker(
|
||||
) -> Tuple[str, int, int, bool, float, int]:
|
||||
"""Returns (text, p_tok, c_tok, is_local, extra_cost, n_web_searches)."""
|
||||
wtype = worker.get("type", "openai")
|
||||
max_tok = int(cfg.get("worker_max_tokens", 4096))
|
||||
max_tok = int(cfg.get("worker_max_tokens") or os.environ.get("OJ_WORKER_MAX_TOKENS", "4096"))
|
||||
temp = float(cfg.get("worker_temperature", 0.2))
|
||||
|
||||
if wtype == "vllm":
|
||||
|
||||
@@ -13,6 +13,7 @@ Supports two modes:
|
||||
from __future__ import annotations
|
||||
|
||||
import concurrent.futures
|
||||
import json
|
||||
import re
|
||||
from typing import Any, List, Optional
|
||||
|
||||
@@ -201,6 +202,70 @@ class OrchestratorAgent(ToolUsingAgent):
|
||||
|
||||
return result
|
||||
|
||||
@staticmethod
|
||||
def _boxed_tool_call(content, openai_tools, call_id):
|
||||
r"""Salvage a tool call the model wrote as ``\boxed{tool: query}``.
|
||||
|
||||
Some SFT'd checkpoints (Qwen reasoning-mode habit) emit their action
|
||||
as a boxed string instead of a real ``<tool_call>`` — e.g.
|
||||
``\boxed{web_search: latest news}`` or ``\boxed{web_search query: ...}``.
|
||||
The engine's tool-call parser sees no structured call, so without this
|
||||
the agent would grade the half-finished thought as its final answer and
|
||||
never actually act. Here we recover ``(tool, args)`` and build a real
|
||||
``ToolCall`` keyed on the tool's primary parameter. Returns ``None`` when
|
||||
the boxed content isn't a recognisable tool action (leaving genuine
|
||||
boxed final answers untouched).
|
||||
"""
|
||||
if not openai_tools or "\\boxed" not in content:
|
||||
return None
|
||||
|
||||
# tool name -> primary parameter (first required, else first property)
|
||||
param_of: dict[str, str] = {}
|
||||
for t in openai_tools:
|
||||
fn = t.get("function", t)
|
||||
name = fn.get("name")
|
||||
if not name:
|
||||
continue
|
||||
params = fn.get("parameters") or {}
|
||||
props = list((params.get("properties") or {}).keys())
|
||||
req = params.get("required") or []
|
||||
param_of[name] = req[0] if req else (props[0] if props else "query")
|
||||
|
||||
# The action is the LAST boxed expression in the response.
|
||||
boxed = re.findall(r"\\boxed\{([^{}]*)\}", content)
|
||||
if not boxed:
|
||||
return None
|
||||
inner = boxed[-1].strip()
|
||||
|
||||
# Match a leading tool name (longest first to avoid prefix collisions),
|
||||
# then strip the separator between name and args. Handles
|
||||
# ``tool: q`` / ``tool query: q`` / ``tool q`` / ``tool(q)``.
|
||||
for name in sorted(param_of, key=len, reverse=True):
|
||||
m = re.match(
|
||||
r"^\s*" + re.escape(name) + r"\b(.*)$",
|
||||
inner,
|
||||
re.DOTALL | re.IGNORECASE,
|
||||
)
|
||||
if not m:
|
||||
continue
|
||||
rest = m.group(1).strip()
|
||||
if rest.startswith("(") and rest.endswith(")"):
|
||||
rest = rest[1:-1] # tool(args) wrapper -> drop matching parens
|
||||
else:
|
||||
# optional "query" word, then a ":" / "-" separator
|
||||
rest = re.sub(
|
||||
r"^\s*(?:query\s*)?[:\-]\s*", "", rest, flags=re.IGNORECASE
|
||||
)
|
||||
arg = rest.strip().strip("\"'").strip()
|
||||
if not arg:
|
||||
return None
|
||||
return ToolCall(
|
||||
id=call_id,
|
||||
name=name,
|
||||
arguments=json.dumps({param_of[name]: arg}),
|
||||
)
|
||||
return None
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Function-calling mode (original behaviour)
|
||||
# ------------------------------------------------------------------
|
||||
@@ -245,8 +310,31 @@ class OrchestratorAgent(ToolUsingAgent):
|
||||
content = result.get("content", "")
|
||||
raw_tool_calls = result.get("tool_calls", [])
|
||||
|
||||
# No tool calls -> check continuation, then final answer
|
||||
# No tool calls -> try to salvage a \boxed{tool: query} action the
|
||||
# model wrote instead of a real <tool_call>; else final answer.
|
||||
if not raw_tool_calls:
|
||||
boxed_call = self._boxed_tool_call(
|
||||
content, openai_tools, f"boxed_{turns}"
|
||||
)
|
||||
if boxed_call is not None:
|
||||
messages.append(
|
||||
Message(
|
||||
role=Role.ASSISTANT,
|
||||
content=content,
|
||||
tool_calls=[boxed_call],
|
||||
)
|
||||
)
|
||||
tool_result = self._executor.execute(boxed_call)
|
||||
all_tool_results.append(tool_result)
|
||||
messages.append(
|
||||
Message(
|
||||
role=Role.TOOL,
|
||||
content=tool_result.content,
|
||||
tool_call_id=boxed_call.id,
|
||||
name=boxed_call.name,
|
||||
)
|
||||
)
|
||||
continue
|
||||
content = self._check_continuation(result, messages)
|
||||
content = self._strip_think_tags(content)
|
||||
self._emit_turn_end(turns=turns, content_length=len(content))
|
||||
|
||||
@@ -2,12 +2,44 @@
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import random
|
||||
import time
|
||||
from abc import ABC, abstractmethod
|
||||
from typing import Any, Dict, Optional, Tuple
|
||||
|
||||
from openjarvis.evals.core.backend import InferenceBackend
|
||||
from openjarvis.evals.core.types import EvalRecord
|
||||
|
||||
LOGGER = logging.getLogger(__name__)
|
||||
|
||||
# Transient failures worth retrying the judge call on. A single rate-limit
|
||||
# (429) used to fall straight through to `llm_fallback_error` and score the
|
||||
# sample WRONG — e.g. 93/100 GAIA judge calls 429'd on one run and zeroed the
|
||||
# whole bench. Retry with exponential backoff so a busy judge endpoint doesn't
|
||||
# get mis-scored as a wrong answer.
|
||||
_JUDGE_MAX_RETRIES = 6
|
||||
_JUDGE_BASE_DELAY_S = 2.0
|
||||
_JUDGE_MAX_DELAY_S = 60.0
|
||||
_RETRYABLE_MARKERS = (
|
||||
"429",
|
||||
"rate_limit",
|
||||
"rate limit",
|
||||
"overloaded",
|
||||
"timeout",
|
||||
"timed out",
|
||||
"503",
|
||||
"502",
|
||||
"500",
|
||||
"connection",
|
||||
"temporarily unavailable",
|
||||
)
|
||||
|
||||
|
||||
def _is_retryable_judge_error(exc: Exception) -> bool:
|
||||
msg = str(exc).lower()
|
||||
return any(marker in msg for marker in _RETRYABLE_MARKERS)
|
||||
|
||||
|
||||
class Scorer(ABC):
|
||||
"""Base class for all scorers."""
|
||||
@@ -42,14 +74,36 @@ class LLMJudgeScorer(Scorer):
|
||||
temperature: float = 0.0,
|
||||
max_tokens: int = 2048,
|
||||
) -> str:
|
||||
"""Send a prompt to the judge LLM and return the response text."""
|
||||
return self._judge_backend.generate(
|
||||
prompt,
|
||||
model=self._judge_model,
|
||||
system=system,
|
||||
temperature=temperature,
|
||||
max_tokens=max_tokens,
|
||||
)
|
||||
"""Send a prompt to the judge LLM and return the response text.
|
||||
|
||||
Retries transient failures (429 / rate-limit / 5xx / timeout) with
|
||||
exponential backoff + jitter so a busy judge endpoint doesn't get
|
||||
mis-scored as a wrong answer. Non-retryable errors, or exhaustion of
|
||||
the retry budget, re-raise to the caller (which records the failure).
|
||||
"""
|
||||
last_exc: Optional[Exception] = None
|
||||
for attempt in range(_JUDGE_MAX_RETRIES):
|
||||
try:
|
||||
return self._judge_backend.generate(
|
||||
prompt,
|
||||
model=self._judge_model,
|
||||
system=system,
|
||||
temperature=temperature,
|
||||
max_tokens=max_tokens,
|
||||
)
|
||||
except Exception as exc: # noqa: BLE001 - re-raised below
|
||||
last_exc = exc
|
||||
if attempt == _JUDGE_MAX_RETRIES - 1 or not _is_retryable_judge_error(exc):
|
||||
raise
|
||||
delay = min(_JUDGE_BASE_DELAY_S * (2 ** attempt), _JUDGE_MAX_DELAY_S)
|
||||
delay += random.uniform(0.0, delay * 0.25) # jitter
|
||||
LOGGER.warning(
|
||||
"judge call failed (attempt %d/%d): %s — retrying in %.1fs",
|
||||
attempt + 1, _JUDGE_MAX_RETRIES, exc, delay,
|
||||
)
|
||||
time.sleep(delay)
|
||||
# Unreachable (loop either returns or raises), but keeps type-checkers happy.
|
||||
raise last_exc # type: ignore[misc]
|
||||
|
||||
|
||||
__all__ = ["LLMJudgeScorer", "Scorer"]
|
||||
|
||||
@@ -140,7 +140,14 @@ class GAIAScorer(LLMJudgeScorer):
|
||||
if structured_match:
|
||||
is_correct = structured_match.group(1).lower() == "yes"
|
||||
else:
|
||||
is_correct = "CORRECT" in raw.upper() and "INCORRECT" not in raw.upper()
|
||||
up = raw.upper()
|
||||
# "not correct" / "is not correct" contain "CORRECT" and lack
|
||||
# "INCORRECT" — guard the negations explicitly or they read True.
|
||||
is_correct = (
|
||||
"CORRECT" in up
|
||||
and "INCORRECT" not in up
|
||||
and "NOT CORRECT" not in up
|
||||
)
|
||||
|
||||
meta: Dict[str, Any] = {
|
||||
"match_type": "llm_fallback",
|
||||
|
||||
@@ -27,6 +27,25 @@ class MMLUProScorer(LLMJudgeScorer):
|
||||
return "".join(chr(ord("A") + i) for i in range(n))
|
||||
return "ABCDEFGHIJ"
|
||||
|
||||
def _extract_answer_direct(
|
||||
self,
|
||||
model_answer: str,
|
||||
valid_letters: str,
|
||||
) -> Optional[str]:
|
||||
"""High-confidence, unambiguous letter straight from the output.
|
||||
|
||||
Only an explicit ``\\boxed{X}`` counts here. We deliberately do NOT regex
|
||||
prose like "the answer is X": the letters "I" and "A" are real English
|
||||
words, so "FINAL_ANSWER: I need ..." would grab "I" — the exact bug that
|
||||
mislabels answers. Semantic extraction is left to the judge below.
|
||||
"""
|
||||
if not model_answer:
|
||||
return None
|
||||
for cand in reversed(re.findall(r"\\boxed\{\s*([A-Za-z])\s*\}", model_answer)):
|
||||
if cand.upper() in valid_letters:
|
||||
return cand.upper()
|
||||
return None
|
||||
|
||||
def _extract_answer_with_llm(
|
||||
self,
|
||||
problem: str,
|
||||
@@ -56,10 +75,13 @@ class MMLUProScorer(LLMJudgeScorer):
|
||||
)
|
||||
|
||||
extracted = raw_response.strip().upper()
|
||||
if "NONE" in extracted:
|
||||
return None
|
||||
|
||||
# Handle "The answer is: A" etc.
|
||||
# Accept only an ISOLATED letter — never the first cap of a word like
|
||||
# "It"/"None". Prefer an explicit "the answer is X" phrasing.
|
||||
answer_match = re.search(
|
||||
r"(?:THE ANSWER IS:?\s*)?([A-Z])",
|
||||
r"(?:THE ANSWER IS:?\s*)?\b([A-Z])\b(?![A-Za-z'])",
|
||||
extracted,
|
||||
re.IGNORECASE,
|
||||
)
|
||||
@@ -86,11 +108,17 @@ class MMLUProScorer(LLMJudgeScorer):
|
||||
|
||||
valid_letters = self._valid_letters_from_options(record.metadata)
|
||||
|
||||
candidate = self._extract_answer_with_llm(
|
||||
record.problem,
|
||||
model_answer,
|
||||
valid_letters,
|
||||
)
|
||||
# Parse straight from the output first (fast, deterministic, no "I" bug);
|
||||
# only fall back to the judge when no isolated letter is present.
|
||||
method = "direct"
|
||||
candidate = self._extract_answer_direct(model_answer, valid_letters)
|
||||
if not candidate:
|
||||
method = "llm"
|
||||
candidate = self._extract_answer_with_llm(
|
||||
record.problem,
|
||||
model_answer,
|
||||
valid_letters,
|
||||
)
|
||||
if not candidate:
|
||||
return None, {"reason": "no_choice_letter_extracted"}
|
||||
|
||||
@@ -99,6 +127,7 @@ class MMLUProScorer(LLMJudgeScorer):
|
||||
"reference_letter": ref,
|
||||
"candidate_letter": candidate,
|
||||
"valid_letters": valid_letters,
|
||||
"extract_method": method,
|
||||
}
|
||||
return is_correct, meta
|
||||
|
||||
|
||||
@@ -27,6 +27,25 @@ class SuperGPQAScorer(LLMJudgeScorer):
|
||||
return "".join(chr(ord("A") + i) for i in range(n))
|
||||
return "ABCD"
|
||||
|
||||
def _extract_answer_direct(
|
||||
self,
|
||||
model_answer: str,
|
||||
valid_letters: str,
|
||||
) -> Optional[str]:
|
||||
"""High-confidence, unambiguous letter straight from the output.
|
||||
|
||||
Only an explicit ``\\boxed{X}`` counts here. We deliberately do NOT regex
|
||||
prose like "the answer is X": the letters "I" and "A" are real English
|
||||
words, so "FINAL_ANSWER: I need ..." would grab "I" — the exact bug that
|
||||
mislabels answers. Semantic extraction is left to the judge below.
|
||||
"""
|
||||
if not model_answer:
|
||||
return None
|
||||
for cand in reversed(re.findall(r"\\boxed\{\s*([A-Za-z])\s*\}", model_answer)):
|
||||
if cand.upper() in valid_letters:
|
||||
return cand.upper()
|
||||
return None
|
||||
|
||||
def _extract_answer_with_llm(
|
||||
self,
|
||||
problem: str,
|
||||
@@ -56,10 +75,13 @@ class SuperGPQAScorer(LLMJudgeScorer):
|
||||
)
|
||||
|
||||
extracted = raw_response.strip().upper()
|
||||
if "NONE" in extracted:
|
||||
return None
|
||||
|
||||
# Handle "The answer is: A" etc.
|
||||
# Accept only an ISOLATED letter — never the first cap of a word like
|
||||
# "It"/"None". Prefer an explicit "the answer is X" phrasing.
|
||||
answer_match = re.search(
|
||||
r"(?:THE ANSWER IS:?\s*)?([A-Z])",
|
||||
r"(?:THE ANSWER IS:?\s*)?\b([A-Z])\b(?![A-Za-z'])",
|
||||
extracted,
|
||||
re.IGNORECASE,
|
||||
)
|
||||
@@ -86,11 +108,17 @@ class SuperGPQAScorer(LLMJudgeScorer):
|
||||
|
||||
valid_letters = self._valid_letters_from_options(record.metadata)
|
||||
|
||||
candidate = self._extract_answer_with_llm(
|
||||
record.problem,
|
||||
model_answer,
|
||||
valid_letters,
|
||||
)
|
||||
# Parse straight from the output first (fast, deterministic, no "I" bug);
|
||||
# only fall back to the judge when no isolated letter is present.
|
||||
method = "direct"
|
||||
candidate = self._extract_answer_direct(model_answer, valid_letters)
|
||||
if not candidate:
|
||||
method = "llm"
|
||||
candidate = self._extract_answer_with_llm(
|
||||
record.problem,
|
||||
model_answer,
|
||||
valid_letters,
|
||||
)
|
||||
if not candidate:
|
||||
return None, {"reason": "no_choice_letter_extracted"}
|
||||
|
||||
@@ -99,6 +127,7 @@ class SuperGPQAScorer(LLMJudgeScorer):
|
||||
"reference_letter": ref,
|
||||
"candidate_letter": candidate,
|
||||
"valid_letters": valid_letters,
|
||||
"extract_method": method,
|
||||
}
|
||||
return is_correct, meta
|
||||
|
||||
|
||||
-54
@@ -1,54 +0,0 @@
|
||||
# Orchestrator GRPO — Gemma-4-12B (cost-aware, lambda-swept).
|
||||
#
|
||||
# Trajectory-level GRPO over the UNIFIED tool-use rollout (the same faithful
|
||||
# <tool_call> multi-turn path the SFT data is built from). For each prompt we
|
||||
# sample group_size rollouts with the *policy* as orchestrator
|
||||
# (run_unified_rollout), verify each rollout's final_answer against the gold
|
||||
# answer, score with the cost-aware reward (r = +1/-1, R -= lam * cost /
|
||||
# cost_max), normalise advantages within the group, and PPO-clip the
|
||||
# trajectory log-prob update (summed over assistant-masked tokens only).
|
||||
#
|
||||
# Batch math: prompts_per_step * group_size = 256 * 8 = 2048 rollouts per
|
||||
# step. grpo_max_prompts / prompts_per_step = 20000 / 256 ~= 78 batches
|
||||
# per epoch.
|
||||
|
||||
[model]
|
||||
model_name = "google/gemma-4-12B-it"
|
||||
max_prompt_length = 24000
|
||||
max_response_length = 8768
|
||||
|
||||
[training]
|
||||
num_epochs = 5
|
||||
learning_rate = 1e-6
|
||||
max_grad_norm = 1.0
|
||||
gradient_checkpointing = true
|
||||
|
||||
[grpo]
|
||||
# group_size = number of trajectories sampled per prompt.
|
||||
num_samples_per_prompt = 8
|
||||
temperature = 1.0
|
||||
kl_coef = 0.0001
|
||||
clip_ratio = 0.2
|
||||
# Per-rollout tool-use turn cap (one assistant turn per loop iteration).
|
||||
max_turns = 10
|
||||
|
||||
[data]
|
||||
grpo_max_prompts = 20000 # pooled GRPO prompts (cached on the trainer)
|
||||
prompts_per_step = 256 # prompts per _grpo_step (256 * 8 = 2048 traj/step)
|
||||
|
||||
[reward]
|
||||
# Cost-aware reward: R = base(+1/-1) - lam * (cost_usd / cost_max).
|
||||
# `lam` is SWEPT across runs to trace the accuracy/cost Pareto frontier,
|
||||
# e.g. [0.0, 0.05, 0.1, 0.2, 0.4]. The value below is the default run.
|
||||
lam = 0.0
|
||||
cost_max = 0.10
|
||||
|
||||
# Optional: map local OSS model ids -> served vLLM base_url so the local-model
|
||||
# tools in the orchestrator catalog dispatch to a real endpoint. Omit to leave
|
||||
# them unbacked (tool still listed).
|
||||
[local_endpoints]
|
||||
# "Qwen/Qwen3.5-9B" = "http://localhost:8001/v1"
|
||||
|
||||
[checkpoint]
|
||||
checkpoint_dir = "checkpoints/orchestrator_grpo_gemma_4_12b"
|
||||
save_every_n_epochs = 1
|
||||
-44
@@ -1,44 +0,0 @@
|
||||
# Orchestrator GRPO — Qwen3-8B (cost-aware, lambda-swept).
|
||||
#
|
||||
# Fields map onto OrchestratorGRPOConfig. GRPO uses group-relative
|
||||
# advantages (no critic): for each prompt, sample group_size trajectories,
|
||||
# verify against the gold answer, score with the cost-aware reward
|
||||
# (r = +1/-1, R -= lam * cost / cost_max), normalise advantages within the
|
||||
# group, and PPO-clip the policy update.
|
||||
#
|
||||
# Batch math: prompts_per_step * group_size = 256 * 8 = 2048 trajectories
|
||||
# per step. grpo_max_prompts / prompts_per_step = 20000 / 256 ~= 78 batches
|
||||
# per epoch.
|
||||
|
||||
[model]
|
||||
model_name = "Qwen/Qwen3-8B"
|
||||
max_prompt_length = 24000
|
||||
max_response_length = 8768
|
||||
|
||||
[training]
|
||||
num_epochs = 5
|
||||
learning_rate = 1e-6
|
||||
max_grad_norm = 1.0
|
||||
gradient_checkpointing = true
|
||||
|
||||
[grpo]
|
||||
# group_size = number of trajectories sampled per prompt.
|
||||
num_samples_per_prompt = 8
|
||||
temperature = 1.0
|
||||
kl_coef = 0.0001
|
||||
clip_ratio = 0.2
|
||||
|
||||
[data]
|
||||
grpo_max_prompts = 20000 # pooled GRPO prompts (cached on the trainer)
|
||||
prompts_per_step = 256 # prompts per _grpo_step (256 * 8 = 2048 traj/step)
|
||||
|
||||
[reward]
|
||||
# Cost-aware reward: R = base(+1/-1) - lam * (cost_usd / cost_max).
|
||||
# `lam` is SWEPT across runs to trace the accuracy/cost Pareto frontier,
|
||||
# e.g. [0.0, 0.05, 0.1, 0.2, 0.4]. The value below is the default run.
|
||||
lam = 0.0
|
||||
cost_max = 0.10
|
||||
|
||||
[checkpoint]
|
||||
checkpoint_dir = "checkpoints/orchestrator_grpo"
|
||||
save_every_n_epochs = 1
|
||||
-54
@@ -1,54 +0,0 @@
|
||||
# Orchestrator GRPO — Qwen3.5-9B (cost-aware, lambda-swept).
|
||||
#
|
||||
# Trajectory-level GRPO over the UNIFIED tool-use rollout (the same faithful
|
||||
# <tool_call> multi-turn path the SFT data is built from). For each prompt we
|
||||
# sample group_size rollouts with the *policy* as orchestrator
|
||||
# (run_unified_rollout), verify each rollout's final_answer against the gold
|
||||
# answer, score with the cost-aware reward (r = +1/-1, R -= lam * cost /
|
||||
# cost_max), normalise advantages within the group, and PPO-clip the
|
||||
# trajectory log-prob update (summed over assistant-masked tokens only).
|
||||
#
|
||||
# Batch math: prompts_per_step * group_size = 256 * 8 = 2048 rollouts per
|
||||
# step. grpo_max_prompts / prompts_per_step = 20000 / 256 ~= 78 batches
|
||||
# per epoch.
|
||||
|
||||
[model]
|
||||
model_name = "Qwen/Qwen3.5-9B"
|
||||
max_prompt_length = 24000
|
||||
max_response_length = 8768
|
||||
|
||||
[training]
|
||||
num_epochs = 5
|
||||
learning_rate = 1e-6
|
||||
max_grad_norm = 1.0
|
||||
gradient_checkpointing = true
|
||||
|
||||
[grpo]
|
||||
# group_size = number of trajectories sampled per prompt.
|
||||
num_samples_per_prompt = 8
|
||||
temperature = 1.0
|
||||
kl_coef = 0.0001
|
||||
clip_ratio = 0.2
|
||||
# Per-rollout tool-use turn cap (one assistant turn per loop iteration).
|
||||
max_turns = 10
|
||||
|
||||
[data]
|
||||
grpo_max_prompts = 20000 # pooled GRPO prompts (cached on the trainer)
|
||||
prompts_per_step = 256 # prompts per _grpo_step (256 * 8 = 2048 traj/step)
|
||||
|
||||
[reward]
|
||||
# Cost-aware reward: R = base(+1/-1) - lam * (cost_usd / cost_max).
|
||||
# `lam` is SWEPT across runs to trace the accuracy/cost Pareto frontier,
|
||||
# e.g. [0.0, 0.05, 0.1, 0.2, 0.4]. The value below is the default run.
|
||||
lam = 0.0
|
||||
cost_max = 0.10
|
||||
|
||||
# Optional: map local OSS model ids -> served vLLM base_url so the local-model
|
||||
# tools in the orchestrator catalog dispatch to a real endpoint. Omit to leave
|
||||
# them unbacked (tool still listed).
|
||||
[local_endpoints]
|
||||
# "Qwen/Qwen3.5-9B" = "http://localhost:8001/v1"
|
||||
|
||||
[checkpoint]
|
||||
checkpoint_dir = "checkpoints/orchestrator_grpo_qwen3.5_9b"
|
||||
save_every_n_epochs = 1
|
||||
@@ -18,9 +18,28 @@ inside ``call_orchestrator`` at generate time (lazy import in ``unified.py``).
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
import time
|
||||
from typing import Any, Dict, Optional
|
||||
|
||||
# The served fine-tuned model sometimes over-emits the answer marker, e.g.
|
||||
# "FINAL_ANSWER: FINAL_ANSWER: 42" — a doubled prefix that breaks answer
|
||||
# extraction and auto-scores the sample 0. Collapse any run of FINAL_ANSWER
|
||||
# markers to a single one, keeping the text after the LAST marker as the answer.
|
||||
_FA_MARK = re.compile(r"(?im)FINAL[_\s]?ANSWER\s*:?")
|
||||
|
||||
|
||||
def _clean_final_answer(text: str) -> str:
|
||||
t = (text or "").strip()
|
||||
marks = list(_FA_MARK.finditer(t))
|
||||
if not marks:
|
||||
return t
|
||||
answer = t[marks[-1].end():].strip()
|
||||
return f"FINAL_ANSWER: {answer}" if answer else t
|
||||
|
||||
|
||||
_OBS_CAP = 4000 # cap persisted observations so the eval JSONL stays readable
|
||||
|
||||
from openjarvis.agents.hybrid.expert_registry import orchestrator_catalog
|
||||
from openjarvis.agents.hybrid.toolorchestra import rollout as _rollout_mod
|
||||
from openjarvis.agents.hybrid.toolorchestra.unified import (
|
||||
@@ -138,6 +157,9 @@ class OrchestratorBackend(InferenceBackend):
|
||||
api_key=self.api_key,
|
||||
temperature=temp,
|
||||
max_tokens=max_tokens,
|
||||
# Serving a fine-tuned model: JSON <tool_call> format in the system
|
||||
# prompt, no tools= param (matches how it was trained/serialized).
|
||||
native_tools=False,
|
||||
)
|
||||
dispatch = make_dispatch(self._dispatch_cfg)
|
||||
rollout = _rollout_mod.run_unified_rollout(
|
||||
@@ -146,6 +168,10 @@ class OrchestratorBackend(InferenceBackend):
|
||||
call_orchestrator=call_orch,
|
||||
dispatch=dispatch,
|
||||
max_turns=self.max_turns,
|
||||
# Match TRAINING: the model was fine-tuned on anonymized expert
|
||||
# labels (expert_xxxx), so it only picks valid labels / routes
|
||||
# correctly when it sees the same anonymized names at inference.
|
||||
anonymize=True,
|
||||
)
|
||||
except Exception as exc: # noqa: BLE001 - surface as a failed sample
|
||||
return {
|
||||
@@ -162,8 +188,9 @@ class OrchestratorBackend(InferenceBackend):
|
||||
|
||||
latency = time.time() - started
|
||||
total_tokens = int(getattr(rollout, "tokens", 0) or 0)
|
||||
anon_map = getattr(rollout, "anon_map", None) or {}
|
||||
return {
|
||||
"content": rollout.final_answer or "",
|
||||
"content": _clean_final_answer(rollout.final_answer or ""),
|
||||
# The rollout reports a single combined token count; we surface it
|
||||
# as completion_tokens so it still flows into total-token telemetry.
|
||||
"usage": {
|
||||
@@ -181,6 +208,23 @@ class OrchestratorBackend(InferenceBackend):
|
||||
"tool_calls": [
|
||||
{"name": n, "arguments": a} for n, a in rollout.tool_calls()
|
||||
],
|
||||
# Full step-by-step trace so eval samples are inspectable
|
||||
# (format_eval_sample.py renders this). Observations capped.
|
||||
"anon_map": anon_map,
|
||||
"turns": [
|
||||
{
|
||||
"reasoning": t.reasoning or "",
|
||||
"tool_name": t.tool_name,
|
||||
"real_model": anon_map.get(t.tool_name) if t.tool_name else None,
|
||||
"arguments": t.arguments,
|
||||
"observation": (
|
||||
(t.observation or "")[:_OBS_CAP]
|
||||
+ ("…[truncated]" if t.observation and len(t.observation) > _OBS_CAP else "")
|
||||
),
|
||||
}
|
||||
for t in (getattr(rollout, "turns", []) or [])
|
||||
],
|
||||
"final_answer": _clean_final_answer(rollout.final_answer or ""),
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
@@ -6,23 +6,8 @@ doesn't require a separate critic model — instead, it uses
|
||||
trajectories, compute rewards, normalise within the group, and update
|
||||
the policy to increase the probability of better solutions.
|
||||
|
||||
**Trajectory-level GRPO over the unified tool-use rollout.** Generation
|
||||
flows through :func:`run_unified_rollout` (the SAME faithful
|
||||
``<tool_call>`` multi-turn loop our SFT data is built from), NOT the old
|
||||
single-turn ``THOUGHT/TOOL/INPUT`` environment path. The policy *is* the
|
||||
orchestrator: each turn we tokenize ``system+user`` with the model's chat
|
||||
template, generate ONE assistant turn, parse any
|
||||
``<tool_call>{...}</tool_call>``, and record the exact
|
||||
``(input_ids, generated_ids)`` for that turn. Once a rollout finishes, the
|
||||
recorded turns are stitched into one trajectory token sequence with an
|
||||
*assistant mask* (1 only on policy-generated tokens) so the policy gradient
|
||||
is computed solely over the tokens the model actually authored, under both
|
||||
the current policy (with grad) and the frozen reference (no grad).
|
||||
|
||||
All ``torch``/``transformers`` imports are guarded so the module can be
|
||||
imported without GPU dependencies; the rollout + log-prob paths are
|
||||
monkeypatchable so ``_grpo_step``/``_train_epoch`` are CPU-smoke-testable
|
||||
with fakes (``policy.model is None``).
|
||||
imported without GPU dependencies.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -32,7 +17,7 @@ import logging
|
||||
import shutil
|
||||
from dataclasses import asdict, dataclass, field
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, List, Optional, Tuple
|
||||
from typing import Any, Dict, List
|
||||
|
||||
# Optional imports -----------------------------------------------------------
|
||||
try:
|
||||
@@ -83,24 +68,11 @@ class OrchestratorGRPOConfig:
|
||||
max_grad_norm: float = 1.0
|
||||
|
||||
# GRPO specific
|
||||
# ``num_samples_per_prompt`` is the GRPO *group size*: N trajectories
|
||||
# sampled per prompt for group-relative advantages.
|
||||
num_samples_per_prompt: int = 8
|
||||
temperature: float = 1.0
|
||||
kl_coef: float = 0.0001
|
||||
clip_ratio: float = 0.2
|
||||
|
||||
# Data / batching
|
||||
# How many prompts to pool from the GRPO dataset (cached on the trainer).
|
||||
grpo_max_prompts: int = 20000
|
||||
# How many prompts go into one ``_grpo_step`` (a batch / step). With
|
||||
# group_size N this is prompts_per_step * N trajectories per step.
|
||||
prompts_per_step: int = 256
|
||||
|
||||
# Cost-aware reward (r=+1/-1, R -= lam * cost / cost_max). ``lam`` is swept.
|
||||
lam: float = 0.0
|
||||
cost_max: float = 0.10
|
||||
|
||||
# Environment
|
||||
available_tools: List[str] = field(
|
||||
default_factory=lambda: [
|
||||
@@ -111,11 +83,6 @@ class OrchestratorGRPOConfig:
|
||||
]
|
||||
)
|
||||
max_turns: int = 10
|
||||
# Optional local OSS model endpoints, mapping full model id (e.g.
|
||||
# "Qwen/Qwen3.5-9B") -> vLLM base_url. Passed to ``orchestrator_catalog``
|
||||
# so the local-model tools dispatch to a served endpoint; omitted when
|
||||
# unset (tool still listed, just unbacked).
|
||||
local_endpoints: Dict[str, str] = field(default_factory=dict)
|
||||
|
||||
# Checkpoint
|
||||
checkpoint_dir: str = "checkpoints/orchestrator_grpo"
|
||||
@@ -128,24 +95,6 @@ class OrchestratorGRPOConfig:
|
||||
use_8bit_optimizer: bool = False
|
||||
|
||||
|
||||
@dataclass
|
||||
class RolloutRecord:
|
||||
"""One sampled unified-rollout trajectory, with the tokens + assistant mask
|
||||
needed to compute its policy log-prob.
|
||||
|
||||
``token_ids`` is the full stitched trajectory (prompt / observation tokens
|
||||
interleaved with policy-generated tokens over turns). ``assistant_mask``
|
||||
is parallel: 1 only on policy-generated positions. ``advantage`` is filled
|
||||
in after group-relative normalisation.
|
||||
"""
|
||||
|
||||
final_answer: str
|
||||
cost_usd: float
|
||||
token_ids: List[int] = field(default_factory=list)
|
||||
assistant_mask: List[int] = field(default_factory=list)
|
||||
advantage: float = 0.0
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Trainer
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -162,37 +111,12 @@ class OrchestratorGRPOTrainer:
|
||||
self.device = None
|
||||
self.global_step = 0
|
||||
|
||||
# Cost-aware reward (r=+1/-1, R -= lam * cost / cost_max). ``lam``
|
||||
# is swept across runs to trace the accuracy/cost Pareto frontier.
|
||||
from openjarvis.learning.intelligence.orchestrator.reward import (
|
||||
CostAwareReward,
|
||||
)
|
||||
|
||||
self._reward = CostAwareReward(lam=config.lam, cost_max=config.cost_max)
|
||||
|
||||
# Lazily-loaded, cached GRPO prompt pool (list[Task]).
|
||||
self._prompts: List[Any] | None = None
|
||||
# Lazily-built, cached unified tool catalog (list[ExpertTool]).
|
||||
self._tool_catalog: Optional[List[Any]] = None
|
||||
|
||||
if HAS_TORCH and torch is not None:
|
||||
self.device = _select_torch_device()
|
||||
|
||||
self._init_model()
|
||||
self._init_optimizer()
|
||||
|
||||
# -- Data ----------------------------------------------------------------
|
||||
|
||||
def _load_prompts(self) -> List[Any]:
|
||||
"""Load (and cache) the GRPO prompt pool — a ``list[Task]``."""
|
||||
if self._prompts is None:
|
||||
from openjarvis.learning.intelligence.orchestrator.sft_data.datasets import ( # noqa: E501
|
||||
load_grpo_prompts,
|
||||
)
|
||||
|
||||
self._prompts = load_grpo_prompts(n=self.config.grpo_max_prompts)
|
||||
return self._prompts
|
||||
|
||||
# -- Initialisation ------------------------------------------------------
|
||||
|
||||
def _init_model(self) -> None:
|
||||
@@ -264,270 +188,82 @@ class OrchestratorGRPOTrainer:
|
||||
self._save_checkpoint(epoch)
|
||||
|
||||
def _train_epoch(self, epoch: int) -> Dict[str, float]:
|
||||
"""Iterate the GRPO prompt pool in chunks and run ``_grpo_step``.
|
||||
|
||||
Prompts are loaded once via :meth:`_load_prompts` (cached on the
|
||||
trainer) and chunked into batches of ``config.prompts_per_step``.
|
||||
Each chunk is one ``_grpo_step`` call (group_size trajectories per
|
||||
prompt). Real model work happens inside ``_grpo_step`` and is
|
||||
guarded there; ``_train_epoch`` itself does no torch work, so it
|
||||
can be smoke-tested on CPU by monkeypatching ``_grpo_step``.
|
||||
"""
|
||||
if HAS_TORCH and self.policy.model is not None:
|
||||
self.policy.model.train()
|
||||
|
||||
prompts = self._load_prompts()
|
||||
step_size = max(int(self.config.prompts_per_step), 1)
|
||||
if self.policy.model is None:
|
||||
return {"epoch": epoch, "loss": 0.0, "reward": 0.0}
|
||||
|
||||
self.policy.model.train()
|
||||
total_loss = 0.0
|
||||
total_reward = 0.0
|
||||
total_accuracy = 0.0
|
||||
n_prompts = 0
|
||||
num_batches = 0
|
||||
|
||||
for start in range(0, len(prompts), step_size):
|
||||
chunk = prompts[start : start + step_size]
|
||||
if not chunk:
|
||||
continue
|
||||
metrics = self._grpo_step(chunk)
|
||||
loss = float(metrics.get("loss", 0.0))
|
||||
reward = float(metrics.get("reward", 0.0))
|
||||
accuracy = float(metrics.get("accuracy", 0.0))
|
||||
n = int(metrics.get("n_prompts", len(chunk)))
|
||||
# In a real implementation, iterate over a task dataset.
|
||||
# Here we provide the skeleton; actual data loading is
|
||||
# trainer-specific.
|
||||
self.global_step += 1
|
||||
num_batches = max(num_batches, 1)
|
||||
|
||||
total_loss += loss * n
|
||||
total_reward += reward * n
|
||||
total_accuracy += accuracy * n
|
||||
n_prompts += n
|
||||
num_batches += 1
|
||||
self.global_step += 1
|
||||
avg_loss = total_loss / num_batches if num_batches > 0 else 0.0
|
||||
avg_reward = total_reward / num_batches if num_batches > 0 else 0.0
|
||||
return {"epoch": epoch, "loss": avg_loss, "reward": avg_reward}
|
||||
|
||||
denom = n_prompts if n_prompts > 0 else 1
|
||||
return {
|
||||
"epoch": epoch,
|
||||
"loss": total_loss / denom,
|
||||
"reward": total_reward / denom,
|
||||
"accuracy": total_accuracy / denom,
|
||||
"n_prompts": n_prompts,
|
||||
"n_batches": num_batches,
|
||||
}
|
||||
def _grpo_step(
|
||||
self,
|
||||
prompts: List[str],
|
||||
ground_truths: List[str],
|
||||
) -> tuple:
|
||||
"""Perform one GRPO training step.
|
||||
|
||||
# -- Tool catalog --------------------------------------------------------
|
||||
|
||||
def _tools(self) -> List[Any]:
|
||||
"""The orchestrator's unified tool catalog (cached).
|
||||
|
||||
Two model classes (cloud frontier + local OSS) plus the basic tools,
|
||||
exactly what the rollout / SFT data condition on. ``local_endpoints``
|
||||
wires the local-model tools to their served vLLM endpoints.
|
||||
"""
|
||||
if getattr(self, "_tool_catalog", None) is None:
|
||||
from openjarvis.agents.hybrid.expert_registry import (
|
||||
orchestrator_catalog,
|
||||
)
|
||||
|
||||
self._tool_catalog = orchestrator_catalog(
|
||||
local_endpoints=dict(self.config.local_endpoints) or None,
|
||||
)
|
||||
return self._tool_catalog
|
||||
|
||||
# -- Rollout / trajectory collection -------------------------------------
|
||||
|
||||
def _rollout_group(self, task: Any) -> List["RolloutRecord"]:
|
||||
"""Sample ``group_size`` rollouts for one task through the unified loop.
|
||||
|
||||
Each rollout is driven by a *policy* ``call_orchestrator`` (the HF
|
||||
model being trained generating one assistant turn at a time) and the
|
||||
real :func:`make_dispatch` tool executor. Per turn we capture the exact
|
||||
``(input_ids, generated_ids)`` so the trajectory token sequence +
|
||||
assistant mask can be rebuilt for the log-prob pass.
|
||||
|
||||
Returns one :class:`RolloutRecord` per sampled rollout. Overridable /
|
||||
monkeypatchable in tests so ``_grpo_step`` runs with fakes when
|
||||
``policy.model is None``.
|
||||
"""
|
||||
from openjarvis.agents.hybrid.toolorchestra.rollout import (
|
||||
build_system_prompt,
|
||||
run_unified_rollout,
|
||||
)
|
||||
from openjarvis.agents.hybrid.toolorchestra.unified import make_dispatch
|
||||
from openjarvis.agents.hybrid.expert_registry import build_tool_specs
|
||||
|
||||
tools = self._tools()
|
||||
system = build_system_prompt(build_tool_specs(tools))
|
||||
dispatch = make_dispatch({})
|
||||
|
||||
records: List[RolloutRecord] = []
|
||||
for _ in range(self.config.num_samples_per_prompt):
|
||||
turn_buffer: List[Tuple[List[int], List[int]]] = []
|
||||
call_orchestrator = self._make_policy_caller(turn_buffer)
|
||||
rollout = run_unified_rollout(
|
||||
task.instruction,
|
||||
tools,
|
||||
call_orchestrator=call_orchestrator,
|
||||
dispatch=dispatch,
|
||||
max_turns=self.config.max_turns,
|
||||
system=system,
|
||||
)
|
||||
token_ids, assistant_mask = self._stitch_trajectory(turn_buffer)
|
||||
records.append(
|
||||
RolloutRecord(
|
||||
final_answer=rollout.final_answer,
|
||||
cost_usd=float(rollout.cost_usd),
|
||||
token_ids=token_ids,
|
||||
assistant_mask=assistant_mask,
|
||||
)
|
||||
)
|
||||
return records
|
||||
|
||||
def _make_policy_caller(self, turn_buffer):
|
||||
"""Build a policy-driven ``call_orchestrator(system, user, specs)``.
|
||||
|
||||
Tokenizes ``system+user`` with the model's chat template, generates ONE
|
||||
assistant turn with the current policy, parses any
|
||||
``<tool_call>{...}</tool_call>``, appends the per-turn
|
||||
``(input_ids, generated_ids)`` to ``turn_buffer`` (for the later
|
||||
log-prob pass), and returns
|
||||
``(text, tool_calls, prompt_tokens, completion_tokens)`` as the unified
|
||||
rollout expects.
|
||||
"""
|
||||
from openjarvis.agents.hybrid.toolorchestra.parsing import (
|
||||
_parse_rl_tool_call,
|
||||
)
|
||||
|
||||
tok = self.policy.tokenizer
|
||||
|
||||
def call_orchestrator(system: str, user: str, specs):
|
||||
messages = [
|
||||
{"role": "system", "content": system},
|
||||
{"role": "user", "content": user},
|
||||
]
|
||||
input_ids = tok.apply_chat_template(
|
||||
messages, add_generation_prompt=True, tokenize=True
|
||||
)
|
||||
inputs = {
|
||||
"input_ids": torch.tensor([input_ids], device=self.device),
|
||||
}
|
||||
input_len = len(input_ids)
|
||||
max_new = min(self.config.max_response_length, 32000 - input_len - 100)
|
||||
max_new = max(min(max_new, 2048), 16)
|
||||
|
||||
with torch.no_grad():
|
||||
out = self.policy.model.generate(
|
||||
**inputs,
|
||||
max_new_tokens=max_new,
|
||||
temperature=self.config.temperature,
|
||||
do_sample=True,
|
||||
)
|
||||
gen_ids = out[0][input_len:].tolist()
|
||||
text = tok.decode(gen_ids, skip_special_tokens=True)
|
||||
|
||||
# Record exact tokens for the trajectory log-prob pass.
|
||||
turn_buffer.append((list(input_ids), list(gen_ids)))
|
||||
|
||||
parsed = _parse_rl_tool_call(text, None)
|
||||
tool_calls = (
|
||||
[(parsed["name"], parsed["arguments"])] if parsed else []
|
||||
)
|
||||
return text, tool_calls, int(input_len), int(len(gen_ids))
|
||||
|
||||
return call_orchestrator
|
||||
|
||||
def _stitch_trajectory(
|
||||
self, turn_buffer: List[Tuple[List[int], List[int]]]
|
||||
) -> Tuple[List[int], List[int]]:
|
||||
"""Concatenate per-turn tokens into one trajectory + assistant mask.
|
||||
|
||||
For turn ``i`` the buffer holds ``(prompt_ids_i, gen_ids_i)`` where
|
||||
``prompt_ids_i`` already contains all prior turns' assistant text and
|
||||
the tool observations re-rendered into the chat by the rollout loop
|
||||
(the rollout rebuilds ``user`` each turn from the running context). So
|
||||
the faithful trajectory token stream is, for each turn, the *new*
|
||||
prefix tokens (mask 0 — prompt / observation / chat-template tokens)
|
||||
followed by the generated assistant tokens (mask 1).
|
||||
|
||||
We reconstruct it by diffing successive prompts: turn ``i`` contributes
|
||||
the prompt tokens that weren't already emitted, then its generated
|
||||
tokens. This keeps the assistant mask exact (1 only on policy-generated
|
||||
positions) without re-tokenizing the whole transcript.
|
||||
"""
|
||||
token_ids: List[int] = []
|
||||
mask: List[int] = []
|
||||
emitted = 0 # length of token_ids already produced
|
||||
for prompt_ids, gen_ids in turn_buffer:
|
||||
# New prompt-side tokens for this turn (everything past what we've
|
||||
# already emitted). These are observation / chat-template tokens →
|
||||
# mask 0 (not policy-authored).
|
||||
new_prompt = prompt_ids[emitted:] if len(prompt_ids) > emitted else []
|
||||
token_ids.extend(new_prompt)
|
||||
mask.extend([0] * len(new_prompt))
|
||||
# Generated assistant tokens → mask 1.
|
||||
token_ids.extend(gen_ids)
|
||||
mask.extend([1] * len(gen_ids))
|
||||
emitted = len(token_ids)
|
||||
return token_ids, mask
|
||||
|
||||
def _grpo_step(self, tasks: List[Any]) -> Dict[str, float]:
|
||||
"""Perform one GRPO training step over a batch of ``Task`` objects.
|
||||
|
||||
For each task:
|
||||
1. Sample N candidate trajectories (``group_size`` =
|
||||
``num_samples_per_prompt``) through the unified tool-use rollout —
|
||||
the SAME ``<tool_call>`` multi-turn path the SFT data uses, driven
|
||||
by the *policy* as orchestrator (see :meth:`_rollout_group`).
|
||||
2. Verify each rollout's ``final_answer`` against the gold answer
|
||||
(:func:`verify_answer`) and score with :class:`CostAwareReward`
|
||||
over an :class:`Episode` carrying ``.correct`` and
|
||||
``.total_cost_usd``.
|
||||
3. Normalise advantages within the group (mean/std).
|
||||
4. Compute the trajectory-level clipped policy gradient + KL penalty.
|
||||
The trajectory log-prob is the sum of per-token log-probs over the
|
||||
*assistant-masked* positions (:meth:`_trajectory_logprob`).
|
||||
For each prompt:
|
||||
1. Generate N candidate trajectories.
|
||||
2. Compute reward for each.
|
||||
3. Normalise advantages within the group.
|
||||
4. Compute clipped policy gradient + KL penalty.
|
||||
5. Backward + clip + step.
|
||||
|
||||
Returns a metrics dict with ``loss``, ``reward``, ``accuracy``,
|
||||
and ``n_prompts``.
|
||||
Returns ``(loss_value, avg_reward)``.
|
||||
"""
|
||||
if self.policy.model is None or not HAS_TORCH:
|
||||
raise RuntimeError("Cannot train without PyTorch and model.")
|
||||
|
||||
self.policy.model.train()
|
||||
|
||||
from openjarvis.learning.intelligence.orchestrator.sft_data.verify import (
|
||||
verify_answer,
|
||||
all_prompts: list[str] = []
|
||||
all_responses: list[str] = []
|
||||
all_advantages: list[float] = []
|
||||
all_rewards: list[float] = []
|
||||
|
||||
from openjarvis.learning.intelligence.orchestrator.reward import (
|
||||
MultiObjectiveReward,
|
||||
Normalizers,
|
||||
RewardWeights,
|
||||
)
|
||||
from openjarvis.learning.intelligence.orchestrator.types import (
|
||||
Episode,
|
||||
)
|
||||
|
||||
reward_fn = self._reward
|
||||
reward_fn = MultiObjectiveReward(RewardWeights(), Normalizers())
|
||||
|
||||
# (token_ids, assistant_mask, advantage) per trajectory, plus running
|
||||
# reward / accuracy stats.
|
||||
traj_records: list[RolloutRecord] = []
|
||||
all_advantages: list[float] = []
|
||||
all_rewards: list[float] = []
|
||||
all_correct: list[float] = []
|
||||
|
||||
for task in tasks:
|
||||
records = self._rollout_group(task)
|
||||
for prompt, gt in zip(prompts, ground_truths):
|
||||
group_rewards: list[float] = []
|
||||
group_responses: list[str] = []
|
||||
|
||||
for rec in records:
|
||||
correct = verify_answer(task, rec.final_answer)
|
||||
for _ in range(self.config.num_samples_per_prompt):
|
||||
response, _log_probs = self._generate_with_log_probs(prompt)
|
||||
|
||||
# Build a minimal episode for reward computation
|
||||
episode = Episode(
|
||||
task_id=getattr(task, "task_id", ""),
|
||||
initial_prompt=task.instruction,
|
||||
ground_truth=getattr(task, "answer", ""),
|
||||
final_answer=rec.final_answer,
|
||||
correct=bool(correct),
|
||||
total_cost_usd=rec.cost_usd,
|
||||
task_id="grpo",
|
||||
initial_prompt=prompt,
|
||||
ground_truth=gt,
|
||||
final_answer=response,
|
||||
correct=(response.strip() == gt.strip()),
|
||||
)
|
||||
reward = reward_fn.compute(episode)
|
||||
group_rewards.append(reward)
|
||||
all_correct.append(1.0 if correct else 0.0)
|
||||
|
||||
# Group-relative advantages (mean/std within the group).
|
||||
group_rewards.append(reward)
|
||||
group_responses.append(response)
|
||||
|
||||
# Group-relative advantages
|
||||
mean_r = sum(group_rewards) / len(group_rewards)
|
||||
std_r = (
|
||||
sum((r - mean_r) ** 2 for r in group_rewards) / len(group_rewards)
|
||||
@@ -537,25 +273,21 @@ class OrchestratorGRPOTrainer:
|
||||
else:
|
||||
advantages = [0.0] * len(group_rewards)
|
||||
|
||||
for rec, adv, rew in zip(records, advantages, group_rewards):
|
||||
rec.advantage = adv
|
||||
traj_records.append(rec)
|
||||
for resp, adv, rew in zip(group_responses, advantages, group_rewards):
|
||||
all_prompts.append(prompt)
|
||||
all_responses.append(resp)
|
||||
all_advantages.append(adv)
|
||||
all_rewards.append(rew)
|
||||
|
||||
# Trajectory-level policy gradient loss.
|
||||
# Policy gradient loss
|
||||
total_loss = torch.tensor(0.0, device=self.device, requires_grad=True)
|
||||
|
||||
for rec in traj_records:
|
||||
if not rec.token_ids or sum(rec.assistant_mask) == 0:
|
||||
continue
|
||||
current_lp = self._trajectory_logprob(
|
||||
rec.token_ids, rec.assistant_mask, ref=False
|
||||
)
|
||||
for prompt, response, advantage in zip(
|
||||
all_prompts, all_responses, all_advantages
|
||||
):
|
||||
current_lp = self._compute_log_probs(prompt, response)
|
||||
with torch.no_grad():
|
||||
ref_lp = self._trajectory_logprob(
|
||||
rec.token_ids, rec.assistant_mask, ref=True
|
||||
)
|
||||
ref_lp = self._compute_log_probs_ref(prompt, response)
|
||||
|
||||
log_ratio = current_lp - ref_lp
|
||||
ratio = torch.exp(log_ratio)
|
||||
@@ -564,25 +296,16 @@ class OrchestratorGRPOTrainer:
|
||||
clip = self.config.clip_ratio
|
||||
clipped = torch.clamp(ratio, 1 - clip, 1 + clip)
|
||||
|
||||
policy_loss = -torch.min(ratio * rec.advantage, clipped * rec.advantage)
|
||||
policy_loss = -torch.min(ratio * advantage, clipped * advantage)
|
||||
kl = (ratio - 1) - log_ratio
|
||||
total_loss = total_loss + policy_loss + self.config.kl_coef * kl
|
||||
|
||||
n_prompts = len(tasks)
|
||||
avg_reward = sum(all_rewards) / len(all_rewards) if all_rewards else 0.0
|
||||
avg_acc = sum(all_correct) / len(all_correct) if all_correct else 0.0
|
||||
|
||||
n_traj = max(len(traj_records), 1)
|
||||
avg_loss = total_loss / n_traj
|
||||
avg_loss = total_loss / max(len(all_prompts), 1)
|
||||
loss_val = avg_loss.item()
|
||||
|
||||
if torch.isnan(avg_loss) or torch.isinf(avg_loss):
|
||||
return {
|
||||
"loss": 0.0,
|
||||
"reward": float(avg_reward),
|
||||
"accuracy": float(avg_acc),
|
||||
"n_prompts": n_prompts,
|
||||
}
|
||||
avg_reward = sum(all_rewards) / len(all_rewards) if all_rewards else 0.0
|
||||
return 0.0, float(avg_reward)
|
||||
|
||||
self.optimizer.zero_grad()
|
||||
avg_loss.backward()
|
||||
@@ -591,68 +314,107 @@ class OrchestratorGRPOTrainer:
|
||||
for param in self.policy.model.parameters():
|
||||
if param.grad is not None and torch.isnan(param.grad).any():
|
||||
self.optimizer.zero_grad()
|
||||
return {
|
||||
"loss": float(loss_val),
|
||||
"reward": float(avg_reward),
|
||||
"accuracy": float(avg_acc),
|
||||
"n_prompts": n_prompts,
|
||||
}
|
||||
avg_reward = sum(all_rewards) / len(all_rewards) if all_rewards else 0.0
|
||||
return float(loss_val), float(avg_reward)
|
||||
|
||||
torch.nn.utils.clip_grad_norm_(
|
||||
self.policy.model.parameters(), self.config.max_grad_norm
|
||||
)
|
||||
self.optimizer.step()
|
||||
|
||||
return {
|
||||
"loss": float(loss_val),
|
||||
"reward": float(avg_reward),
|
||||
"accuracy": float(avg_acc),
|
||||
"n_prompts": n_prompts,
|
||||
}
|
||||
avg_reward = sum(all_rewards) / len(all_rewards) if all_rewards else 0.0
|
||||
return float(loss_val), float(avg_reward)
|
||||
|
||||
# -- Trajectory log-prob -------------------------------------------------
|
||||
# -- Generation / log-prob helpers ---------------------------------------
|
||||
|
||||
def _trajectory_logprob(
|
||||
self,
|
||||
token_ids: List[int],
|
||||
assistant_mask: List[int],
|
||||
*,
|
||||
ref: bool,
|
||||
) -> "torch.Tensor":
|
||||
"""Sum of per-token log-probs over the assistant-masked positions.
|
||||
def _generate_with_log_probs(self, prompt: str) -> "tuple[str, Any]":
|
||||
"""Generate a response and return ``(text, log_probs)``."""
|
||||
inputs = self.policy.tokenizer(
|
||||
prompt,
|
||||
return_tensors="pt",
|
||||
truncation=True,
|
||||
max_length=min(self.config.max_prompt_length, 16000),
|
||||
).to(self.device)
|
||||
|
||||
``token_ids`` is the full trajectory (prompt + observation + assistant
|
||||
tokens interleaved over turns); ``assistant_mask[i] == 1`` iff token
|
||||
``i`` was generated by the policy. We run the model once over the whole
|
||||
sequence and accumulate ``log p(token_{i} | token_{<i})`` only where the
|
||||
*predicted* token (position ``i``) is masked 1.
|
||||
input_len = inputs.input_ids.shape[1]
|
||||
max_new = min(self.config.max_response_length, 32000 - input_len - 100)
|
||||
max_new = max(min(max_new, 2048), 128)
|
||||
|
||||
``ref=True`` → frozen reference model, no grad; else current policy with
|
||||
grad. Mirrors the single-turn log-prob math the old code used, lifted to
|
||||
the masked multi-turn trajectory.
|
||||
"""
|
||||
model = self.ref_policy.model if ref else self.policy.model
|
||||
ids = torch.tensor([token_ids], device=self.device)
|
||||
with torch.no_grad():
|
||||
outputs = self.policy.model.generate(
|
||||
**inputs,
|
||||
max_new_tokens=max_new,
|
||||
temperature=self.config.temperature,
|
||||
do_sample=True,
|
||||
output_scores=True,
|
||||
return_dict_in_generate=True,
|
||||
)
|
||||
|
||||
if ref:
|
||||
ctx = torch.no_grad()
|
||||
else:
|
||||
ctx = torch.enable_grad()
|
||||
with ctx:
|
||||
logits = model(input_ids=ids).logits
|
||||
generated_ids = outputs.sequences[0][input_len:]
|
||||
if len(generated_ids) == 0:
|
||||
return "", torch.tensor(0.0, device=self.device)
|
||||
|
||||
text = self.policy.tokenizer.decode(generated_ids, skip_special_tokens=True)
|
||||
|
||||
log_probs = []
|
||||
for token_id, logits in zip(generated_ids, outputs.scores):
|
||||
probs = F.softmax(logits[0], dim=-1)
|
||||
tid = token_id.item()
|
||||
if 0 <= tid < probs.shape[0]:
|
||||
log_probs.append(torch.log(probs[tid] + 1e-10))
|
||||
|
||||
total_lp = (
|
||||
torch.stack(log_probs).sum()
|
||||
if log_probs
|
||||
else torch.tensor(0.0, device=self.device)
|
||||
)
|
||||
return text, total_lp
|
||||
|
||||
def _compute_log_probs(self, prompt: str, response: str) -> "torch.Tensor":
|
||||
"""Log-probs of *response* given *prompt* under current policy."""
|
||||
full = prompt + response
|
||||
inputs = self.policy.tokenizer(full, return_tensors="pt", truncation=True).to(
|
||||
self.device
|
||||
)
|
||||
prompt_inputs = self.policy.tokenizer(
|
||||
prompt, return_tensors="pt", truncation=True
|
||||
).to(self.device)
|
||||
|
||||
with torch.enable_grad():
|
||||
logits = self.policy.model(**inputs).logits
|
||||
|
||||
start = len(prompt_inputs.input_ids[0])
|
||||
end = len(inputs.input_ids[0])
|
||||
|
||||
lps = []
|
||||
# Predict token i from logits at position i-1; include it iff token i is
|
||||
# an assistant-generated token.
|
||||
for i in range(1, len(token_ids)):
|
||||
if assistant_mask[i] != 1:
|
||||
continue
|
||||
lp = F.log_softmax(logits[0, i - 1, :], dim=-1)[token_ids[i]]
|
||||
for i in range(start, end - 1):
|
||||
lp = F.log_softmax(logits[0, i, :], dim=-1)[inputs.input_ids[0, i + 1]]
|
||||
lps.append(lp)
|
||||
|
||||
if not lps:
|
||||
return torch.tensor(0.0, device=self.device)
|
||||
return torch.stack(lps).sum()
|
||||
return torch.stack(lps).sum() if lps else torch.tensor(0.0)
|
||||
|
||||
def _compute_log_probs_ref(self, prompt: str, response: str) -> "torch.Tensor":
|
||||
"""Log-probs under the frozen reference policy (no grad)."""
|
||||
full = prompt + response
|
||||
inputs = self.ref_policy.tokenizer(
|
||||
full, return_tensors="pt", truncation=True
|
||||
).to(self.device)
|
||||
prompt_inputs = self.ref_policy.tokenizer(
|
||||
prompt, return_tensors="pt", truncation=True
|
||||
).to(self.device)
|
||||
|
||||
with torch.no_grad():
|
||||
logits = self.ref_policy.model(**inputs).logits
|
||||
|
||||
start = len(prompt_inputs.input_ids[0])
|
||||
end = len(inputs.input_ids[0])
|
||||
|
||||
lps = []
|
||||
for i in range(start, end - 1):
|
||||
lp = F.log_softmax(logits[0, i, :], dim=-1)[inputs.input_ids[0, i + 1]]
|
||||
lps.append(lp)
|
||||
|
||||
return torch.stack(lps).sum() if lps else torch.tensor(0.0)
|
||||
|
||||
# -- Checkpointing -------------------------------------------------------
|
||||
|
||||
@@ -726,5 +488,4 @@ _ensure_registered()
|
||||
__all__ = [
|
||||
"OrchestratorGRPOConfig",
|
||||
"OrchestratorGRPOTrainer",
|
||||
"RolloutRecord",
|
||||
]
|
||||
|
||||
@@ -38,7 +38,10 @@ class Task:
|
||||
task_id: str
|
||||
question: str
|
||||
answer: str
|
||||
domain: str
|
||||
domain: str # coarse area: math / code / science / medical / chat / misc
|
||||
difficulty: str = "" # OpenThoughts difficulty tier (GeneralThought has none)
|
||||
dataset: str = "" # source dataset: GeneralThought / OpenThoughts3
|
||||
subsector: str = "" # fine source: NuminaMath / NHSQA / TACO / glaive / ...
|
||||
|
||||
@property
|
||||
def instruction(self) -> str:
|
||||
@@ -90,6 +93,9 @@ def _normalize_generalthought(row: Dict[str, Any], *, index: int = 0) -> Optiona
|
||||
question=question,
|
||||
answer=answer,
|
||||
domain=_generalthought_domain(row),
|
||||
difficulty=str(row.get("difficulty") or "").strip(), # usually absent for GT
|
||||
dataset="GeneralThought",
|
||||
subsector=str(row.get("question_source") or "").strip(),
|
||||
)
|
||||
|
||||
|
||||
@@ -98,12 +104,15 @@ def load_generalthought(
|
||||
n: int = 2000,
|
||||
seed: int = 42,
|
||||
source: Optional[Iterable[Dict[str, Any]]] = None,
|
||||
buffer: Optional[int] = None,
|
||||
) -> Iterator[Task]:
|
||||
"""Yield up to ``n`` GeneralThought tasks, randomly mixed across categories.
|
||||
|
||||
``source`` overrides the HF stream with an iterable of raw row dicts (tests).
|
||||
We over-stream a buffer and shuffle it so the yielded tasks are a random mix
|
||||
of the source's categories rather than a single leading shard.
|
||||
of the source's categories rather than a single leading shard. ``buffer``
|
||||
overrides the shuffle-buffer size; pass a small value (e.g. for smoke runs)
|
||||
to avoid streaming the default 6000-row floor.
|
||||
"""
|
||||
if source is None:
|
||||
from datasets import load_dataset # lazy: optional dep / network
|
||||
@@ -113,7 +122,7 @@ def load_generalthought(
|
||||
rng = random.Random(seed)
|
||||
# Buffer a generous multiple of n so the shuffle actually mixes categories,
|
||||
# but stay bounded for the multi-hundred-K streams.
|
||||
buf_cap = max(n * 6, 6000)
|
||||
buf_cap = buffer if buffer is not None else max(n * 6, 6000)
|
||||
buf: List[Task] = []
|
||||
for i, row in enumerate(source):
|
||||
task = _normalize_generalthought(dict(row), index=i)
|
||||
@@ -186,7 +195,29 @@ def _normalize_openthoughts(row: Dict[str, Any], *, index: int = 0) -> Optional[
|
||||
return None
|
||||
domain = str(row.get("domain") or "unknown").strip().lower() or "unknown"
|
||||
task_id = str(row.get("id") or row.get("source") or f"openthoughts-{index}") + f"-{index}"
|
||||
return Task(task_id=task_id, question=question, answer=answer, domain=domain)
|
||||
return Task(
|
||||
task_id=task_id,
|
||||
question=question,
|
||||
answer=answer,
|
||||
domain=domain,
|
||||
difficulty=str(row.get("difficulty") or "").strip(),
|
||||
dataset="OpenThoughts3",
|
||||
subsector=str(row.get("source") or "").strip(),
|
||||
)
|
||||
|
||||
|
||||
# The 1.2M set ships as 120 uniform parquet shards with no domain in the file
|
||||
# names, but the rows are front-ordered by domain. Probing one row per shard puts
|
||||
# code in shards ~0-30, math ~32-104, science ~105-119. Streaming the single
|
||||
# combined split therefore has to read the entire code (+ math) prefix before it
|
||||
# reaches math/science — minutes of wasted I/O. Instead we stream each domain from
|
||||
# shards *inside* its own region. A few shards (~10K rows each) cover any quota.
|
||||
_OT_SHARD_FMT = "data/train-{:05d}-of-00120.parquet"
|
||||
_OT_DOMAIN_SHARDS: Dict[str, List[int]] = {
|
||||
"code": [0, 1, 2, 3],
|
||||
"math": [45, 46, 47, 48],
|
||||
"science": [112, 113, 114, 115, 116, 117, 118, 119],
|
||||
}
|
||||
|
||||
|
||||
def load_openthoughts(
|
||||
@@ -199,26 +230,43 @@ def load_openthoughts(
|
||||
) -> Iterator[Task]:
|
||||
"""Yield OpenThoughts3 tasks balanced across code / math / science.
|
||||
|
||||
The HF stream is sharded by domain (all-code first), so we route rows into
|
||||
per-domain buffers and stop once every quota is met. ``source`` overrides the
|
||||
stream with raw row dicts (tests).
|
||||
``source`` overrides the stream with raw row dicts (tests): rows are routed
|
||||
into per-domain buffers, stopping once every quota is met. With no override we
|
||||
stream each domain from its own shard region (see ``_OT_DOMAIN_SHARDS``) so a
|
||||
balanced pull never has to read the entire front-ordered code/math prefix.
|
||||
"""
|
||||
if source is None:
|
||||
from datasets import load_dataset # lazy: optional dep / network
|
||||
|
||||
source = load_dataset(OPENTHOUGHTS_ID, split="train", streaming=True)
|
||||
|
||||
quotas = {"code": n_code, "math": n_math, "science": n_science}
|
||||
bufs: Dict[str, List[Task]] = {k: [] for k in quotas}
|
||||
for i, row in enumerate(source):
|
||||
task = _normalize_openthoughts(dict(row), index=i)
|
||||
if task is None:
|
||||
continue
|
||||
dom = task.domain
|
||||
if dom in bufs and len(bufs[dom]) < quotas[dom]:
|
||||
bufs[dom].append(task)
|
||||
if all(len(bufs[k]) >= quotas[k] for k in quotas):
|
||||
break
|
||||
|
||||
if source is not None:
|
||||
for i, row in enumerate(source):
|
||||
task = _normalize_openthoughts(dict(row), index=i)
|
||||
if task is None:
|
||||
continue
|
||||
dom = task.domain
|
||||
if dom in bufs and len(bufs[dom]) < quotas[dom]:
|
||||
bufs[dom].append(task)
|
||||
if all(len(bufs[k]) >= quotas[k] for k in quotas):
|
||||
break
|
||||
else:
|
||||
from datasets import load_dataset # lazy: optional dep / network
|
||||
|
||||
idx = 0
|
||||
for dom, quota in quotas.items():
|
||||
if quota <= 0:
|
||||
continue
|
||||
files = [_OT_SHARD_FMT.format(s) for s in _OT_DOMAIN_SHARDS[dom]]
|
||||
ds = load_dataset(
|
||||
OPENTHOUGHTS_ID, data_files=files, split="train", streaming=True
|
||||
)
|
||||
for row in ds:
|
||||
task = _normalize_openthoughts(dict(row), index=idx)
|
||||
idx += 1
|
||||
if task is None or task.domain != dom:
|
||||
continue
|
||||
bufs[dom].append(task)
|
||||
if len(bufs[dom]) >= quota:
|
||||
break
|
||||
|
||||
rng = random.Random(seed)
|
||||
out: List[Task] = []
|
||||
@@ -232,8 +280,33 @@ def load_openthoughts(
|
||||
# --- combined SFT / GRPO sets -------------------------------------------------
|
||||
|
||||
|
||||
def load_sft_tasks(*, seed: int = 42) -> List[Task]:
|
||||
"""The 8K cold-start SFT set: 2K GeneralThought + 2K code + 2K math + 2K science."""
|
||||
def load_sft_tasks(
|
||||
*, seed: int = 42, cap: Optional[int] = None, balanced: bool = True
|
||||
) -> List[Task]:
|
||||
"""The 8K cold-start SFT set: 2K GeneralThought + 2K code + 2K math + 2K science.
|
||||
|
||||
``cap`` (smoke runs): when set, draw ~``cap`` tasks total.
|
||||
- default (``balanced=False``): GeneralThought only with a small stream
|
||||
buffer — fastest, but the domain mix is whatever GeneralThought yields
|
||||
(math/code/medical/chat), so a given sample can skew (e.g. all-medical).
|
||||
- ``balanced=True``: ~cap/4 from each of GeneralThought + OpenThoughts
|
||||
code/math/science, so the smoke is representative of the real run. Now
|
||||
cheap because OpenThoughts streams from per-domain shards (no front-prefix).
|
||||
"""
|
||||
if cap is not None and cap > 0:
|
||||
if balanced:
|
||||
per = max(cap // 4, 1)
|
||||
tasks = list(load_generalthought(n=per, seed=seed, buffer=max(per * 6, 64)))
|
||||
tasks.extend(
|
||||
load_openthoughts(n_code=per, n_math=per, n_science=per, seed=seed)
|
||||
)
|
||||
rng = random.Random(seed)
|
||||
rng.shuffle(tasks)
|
||||
return tasks[:cap]
|
||||
buf = max(cap * 4, 64)
|
||||
tasks = list(load_generalthought(n=cap, seed=seed, buffer=buf))
|
||||
random.Random(seed).shuffle(tasks)
|
||||
return tasks[:cap]
|
||||
tasks: List[Task] = []
|
||||
tasks.extend(load_generalthought(n=2000, seed=seed))
|
||||
tasks.extend(load_openthoughts(n_code=2000, n_math=2000, n_science=2000, seed=seed))
|
||||
|
||||
@@ -19,6 +19,7 @@ from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
from collections import Counter
|
||||
from pathlib import Path
|
||||
from typing import Callable, Iterable, List, Optional
|
||||
@@ -28,6 +29,8 @@ from typing import Any
|
||||
from openjarvis.agents.hybrid.expert_registry import ExpertTool
|
||||
from openjarvis.agents.hybrid.toolorchestra.rollout import UnifiedRollout
|
||||
from openjarvis.learning.intelligence.orchestrator.sft_data.unified_serialize import (
|
||||
_CONTROL_TOKEN_RE,
|
||||
_strip_control_tokens,
|
||||
trajectory_to_record,
|
||||
)
|
||||
|
||||
@@ -42,6 +45,177 @@ TaskLike = Any
|
||||
RolloutFn = Callable[[TaskLike], Optional[UnifiedRollout]]
|
||||
VerifyFn = Callable[[TaskLike, UnifiedRollout], bool]
|
||||
|
||||
# Human-readable blurb for each source dataset, stamped onto every record as
|
||||
# ``dataset_description`` so a row is self-describing (what corpus the question
|
||||
# came from) without a lookup. Keyed on the record's ``dataset`` value.
|
||||
DATASET_DESCRIPTIONS = {
|
||||
"GeneralThought": (
|
||||
"natolambert/GeneralThought-430K-filtered — filtered reasoning Q&A from "
|
||||
"gr.inc; each item has a question and a short reference (gold) answer "
|
||||
"distilled from DeepSeek-R1 traces."
|
||||
),
|
||||
"OpenThoughts3": (
|
||||
"open-thoughts/OpenThoughts3-1.2M — code/math/science reasoning tasks; "
|
||||
"the gold answer is extracted from the boxed final solution."
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
def dataset_description(dataset: str) -> str:
|
||||
"""Blurb for a source dataset name (empty string if unknown)."""
|
||||
return DATASET_DESCRIPTIONS.get((dataset or "").strip(), "")
|
||||
|
||||
|
||||
# Markers of a broken expert/observation that must never become a training
|
||||
# target (empty turns, tracebacks, truncated tool errors, dead-provider strings).
|
||||
_ERR_MARKERS = (
|
||||
"Traceback (most recent call last)",
|
||||
"insufficient_quota",
|
||||
"rate limit",
|
||||
"RateLimitError",
|
||||
"[invalid tool",
|
||||
"Error code: 4",
|
||||
"Error code: 5",
|
||||
"InternalServerError",
|
||||
"ConnectionError",
|
||||
"timed out",
|
||||
# A tool call that reached dispatch with no/empty input, or a bridged-tool
|
||||
# crash — the call was malformed. The trajectory routed garbage; drop it.
|
||||
"no input provided",
|
||||
"[openjarvis-tool error",
|
||||
)
|
||||
|
||||
# A final answer cut off mid-expression ends on a dangling connective
|
||||
# (``a=1, b=``, ``result = ``, ``compute (``) — the decode hit the token cap
|
||||
# before finishing. A real distilled answer never ends this way.
|
||||
_TRUNCATED_TAIL_RE = re.compile(r"[=,(\[{+\-*/]\s*$")
|
||||
|
||||
|
||||
def _target_is_clean(roll: UnifiedRollout) -> bool:
|
||||
"""Structural gate on whether a trajectory is fit to be an SFT *target*.
|
||||
|
||||
The judge decides semantic correctness; this catches the ~10-12% of records
|
||||
whose targets are empty / a Traceback / a truncated tool error / unrouted —
|
||||
garbage the model must never be trained to imitate. Requirements:
|
||||
* non-empty final answer, no error marker, balanced ``<think>`` (not
|
||||
truncated mid-thought);
|
||||
* at least one model-expert tool call (it actually *routed*);
|
||||
* no error-marker / empty observation in the kept trajectory.
|
||||
"""
|
||||
fa = (roll.final_answer or "").strip()
|
||||
if not fa:
|
||||
return False
|
||||
if any(m in fa for m in _ERR_MARKERS):
|
||||
return False
|
||||
if fa.count("<think>") != fa.count("</think>"): # unbalanced think tags (truncated OR stray </think>)
|
||||
return False
|
||||
if _TRUNCATED_TAIL_RE.search(fa): # truncated mid-expression (e.g. "a=1, b=")
|
||||
return False
|
||||
# Malformed final: a proper final answer is plain text, NOT a stray/broken
|
||||
# <tool_call> tag (the model's answer leaking inside a tool call that failed
|
||||
# to parse). (\boxed{} is NOT rejected here — the serializer de-boxes it, so
|
||||
# an otherwise-good boxed answer is salvaged rather than thrown away.)
|
||||
if "<tool_call>" in fa or "</tool_call>" in fa:
|
||||
return False
|
||||
# Control-token backstop (defense in depth). The serializer strips leaked
|
||||
# control/special tokens (<|im_end|>, <|tool_call>, <start_of_turn>, …) to
|
||||
# salvage good answers, so we mirror that strip and gate on the RESULT:
|
||||
# * strips to empty -> the answer WAS nothing but control tokens -> drop.
|
||||
# * a token survives the strip -> something malformed the stripper can't
|
||||
# safely delete mid-answer -> drop rather than train on it.
|
||||
# A clean answer (or one the stripper fully salvages) sails through.
|
||||
fa_stripped = _strip_control_tokens(fa)
|
||||
if not fa_stripped:
|
||||
return False
|
||||
if _CONTROL_TOKEN_RE.search(fa_stripped):
|
||||
return False
|
||||
# Degenerate repetition: the same substantive line emitted many times (the
|
||||
# small-model decode loop). Reject rather than train on it.
|
||||
_lines = [ln.strip() for ln in fa.split("\n") if len(ln.strip()) > 30]
|
||||
if _lines and Counter(_lines).most_common(1)[0][1] >= 4:
|
||||
return False
|
||||
# Word-salad run-on: a giant unbroken line (no newline) is the other decode
|
||||
# collapse — the model spraying novel tokens instead of repeating. Reject.
|
||||
if any(len(ln) > 2000 for ln in fa.split("\n")):
|
||||
return False
|
||||
# Markdown-table dump: the model wrote a formatted table/essay instead of the
|
||||
# short exact value the format demands. A ``|---|`` separator row is the tell.
|
||||
if re.search(r"\|\s*:?-{2,}", fa):
|
||||
return False
|
||||
# Essay-style final: the format wants a distilled answer, not a multi-section
|
||||
# writeup. Check the answer AFTER the FINAL_ANSWER marker (reasoning before it
|
||||
# is fine). NOTE: length/bold limits relaxed for the BALANCED/harder mix —
|
||||
# code & multi-step math answers are legitimately longer and use **bold** for
|
||||
# the key result, so the old 700-char + any-bold rejects were dropping ~half
|
||||
# the correct hard-task answers. Keep the STRUCTURAL essay signals (many
|
||||
# numbered sections, markdown headers, tables) which still catch real essays.
|
||||
_marks = list(re.finditer(r"(?im)FINAL[_\s]?ANSWER\s*:?", fa))
|
||||
_ans = fa[_marks[-1].end():].strip() if _marks else fa
|
||||
if len(_ans) > 2000: # was 700 — too tight for code/math
|
||||
return False
|
||||
# Tool-status echo as the final answer: on code tasks the orchestrator
|
||||
# sometimes ends with a file_write/shell status line ("Successfully wrote to
|
||||
# /tmp/x.py") instead of the actual answer — a non-answer that slips the
|
||||
# length/format checks. Reject. (Audit: ~code/math trajectories where the
|
||||
# trace collapsed but the row was still scored correct.)
|
||||
if re.search(r"(?i)\b(successfully (wrote|created|saved|executed|ran)|written to /|"
|
||||
r"file (written|saved|created)|no further actions?)\b", _ans):
|
||||
return False
|
||||
if len(re.findall(r"(?m)^\s*\d+\.\s", _ans)) >= 6: # many numbered sections = essay (was 4)
|
||||
return False
|
||||
if re.search(r"(?m)^\s*#{1,6}\s", _ans): # markdown headers = essay
|
||||
return False
|
||||
# Garbled / shouty final: a multi-word answer that's mostly UPPERCASE, or has
|
||||
# an absurdly long merged all-letter token, is decode garble — the audit found
|
||||
# e.g. "RABES PEPTETANUS BOOSTERS" (for "rabies PEP, tetanus boosters") passing.
|
||||
if len(_ans) > 20:
|
||||
_alpha = [c for c in _ans if c.isalpha()]
|
||||
if (_alpha and sum(c.isupper() for c in _alpha) / len(_alpha) > 0.7
|
||||
and len(_ans.split()) >= 3):
|
||||
return False
|
||||
if any(len(w) > 25 and w.isalpha() for w in _ans.split()):
|
||||
return False
|
||||
# Reject runaway final answers — a distilled answer, not a multi-KB essay or
|
||||
# a verbatim dump of an expert observation. (Audit: 279 finals >4k chars, the
|
||||
# worst a 409k-char wordlist; ~363 were prefix-identical to a tool obs.)
|
||||
if len(fa) > 8000:
|
||||
return False
|
||||
fa_head = re.sub(r"\s+", " ", fa[:200]).strip()
|
||||
for t in roll.turns:
|
||||
if t.observation:
|
||||
obs_head = re.sub(r"\s+", " ", t.observation[:200]).strip()
|
||||
if fa_head and fa_head == obs_head and len(fa) > 200: # long final = copied tool dump (short relays are legit)
|
||||
return False
|
||||
# "routed" = delegated to a model EXPERT (not just a utility like web_search).
|
||||
# When anonymized, expert calls are the anon labels in anon_map; otherwise
|
||||
# fall back to "made any tool call".
|
||||
expert_names = set(roll.anon_map or {})
|
||||
called = {name for name, _ in roll.tool_calls()}
|
||||
if expert_names:
|
||||
if not (called & expert_names):
|
||||
return False
|
||||
elif roll.num_tool_calls < 1:
|
||||
return False
|
||||
for t in roll.turns:
|
||||
if t.tool_name is not None:
|
||||
obs = (t.observation or "").strip()
|
||||
if not obs or any(m in obs for m in _ERR_MARKERS):
|
||||
return False
|
||||
# Reasoning-side decode collapse: an intermediate turn whose reasoning is a
|
||||
# giant unbroken line, or is dominated by exotic unicode / emoji spam, is
|
||||
# garbage even when the final answer looks fine (the answer-only guards above
|
||||
# miss it). The audit found a trajectory with a thousands-char symbol blob in
|
||||
# its reasoning that scored correct and slipped through.
|
||||
for t in roll.turns:
|
||||
r = t.reasoning or ""
|
||||
if any(len(ln) > 2000 for ln in r.split("\n")):
|
||||
return False
|
||||
if len(r) > 200:
|
||||
weird = sum(1 for ch in r if ord(ch) > 0x2100 and not ch.isalnum())
|
||||
if weird / len(r) > 0.10: # >10% exotic-unicode/emoji = decode spam
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def gold_coverage_verify(task: TaskLike, rollout: UnifiedRollout) -> bool:
|
||||
"""Dependency-free proxy verifier: trajectory must (a) produce a non-empty
|
||||
@@ -59,6 +233,43 @@ def gold_coverage_verify(task: TaskLike, rollout: UnifiedRollout) -> bool:
|
||||
return gold.issubset(called)
|
||||
|
||||
|
||||
def _process_task(
|
||||
task: TaskLike,
|
||||
*,
|
||||
rollout_fn: RolloutFn,
|
||||
verify_fn: VerifyFn,
|
||||
samples_per_task: int,
|
||||
max_keep_per_task: int,
|
||||
stop_at_keep: bool = False,
|
||||
keep_all: bool = False,
|
||||
) -> List[tuple[UnifiedRollout, bool]]:
|
||||
"""One task's rejection-sampling unit of work: roll out ``samples_per_task``
|
||||
times and verify each. Returns ``(rollout, is_correct)`` for **every** sample
|
||||
(the caller decides what to write). Runs in a worker thread, so the only
|
||||
shared state it touches is the injected fns (each rollout builds its own
|
||||
OpenAI client; the tool executor is built under a lock).
|
||||
|
||||
``stop_at_keep``: short-circuit as soon as ``max_keep_per_task`` passing
|
||||
trajectories are found, instead of always exhausting ``samples_per_task``.
|
||||
Big throughput win (no wasted rollouts on already-solved tasks), but it
|
||||
trades away the cheapest-of-N cost optimisation (we keep the first passers,
|
||||
not the cheapest). Ignored when ``keep_all`` is set (we want every sample).
|
||||
Default off preserves the original semantics."""
|
||||
results: List[tuple[UnifiedRollout, bool]] = []
|
||||
n_correct = 0
|
||||
for _ in range(samples_per_task):
|
||||
roll = rollout_fn(task)
|
||||
if roll is None:
|
||||
continue
|
||||
ok = bool(verify_fn(task, roll))
|
||||
results.append((roll, ok))
|
||||
if ok:
|
||||
n_correct += 1
|
||||
if stop_at_keep and not keep_all and n_correct >= max_keep_per_task:
|
||||
break
|
||||
return results
|
||||
|
||||
|
||||
def generate_sft_dataset(
|
||||
out_path: str,
|
||||
*,
|
||||
@@ -69,55 +280,153 @@ def generate_sft_dataset(
|
||||
samples_per_task: int = 4,
|
||||
max_keep_per_task: int = 1,
|
||||
reward_fn: Optional[Callable[[UnifiedRollout], float]] = None,
|
||||
concurrency: int = 1,
|
||||
stop_at_keep: bool = False,
|
||||
keep_all: bool = False,
|
||||
record_extra: Optional[dict] = None,
|
||||
) -> dict:
|
||||
"""Run rejection sampling over ``tasks`` and write the SFT JSONL.
|
||||
|
||||
``max_keep_per_task`` caps records kept per task; when >1 the cheapest
|
||||
passing trajectories are kept first. Returns stats + writes a ``.stats.json``.
|
||||
passing trajectories are kept first. ``concurrency`` rolls out that many tasks
|
||||
in parallel (each task issues its samples sequentially, so ~``concurrency``
|
||||
requests hit the served model at once) — set 1 for the original sequential
|
||||
behaviour. Records are written as each task finishes, so peak memory is bounded
|
||||
by the in-flight tasks, not the whole dataset. Returns stats + writes a
|
||||
``.stats.json``.
|
||||
|
||||
``keep_all``: write **every** rolled-out trajectory (correct and incorrect)
|
||||
rather than dropping the failures. Each record gets ``correct`` (verifier
|
||||
verdict) and ``kept`` (True on the cheapest-correct sample — the one the
|
||||
rejection sampler would have selected). This preserves the full sample set so
|
||||
you can compute accuracy / inspect failures; the SFT trainer should filter on
|
||||
``correct`` (or ``kept``). Default off = original drop-the-failures behaviour.
|
||||
"""
|
||||
out = Path(out_path)
|
||||
out.parent.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
seen = 0
|
||||
tasks = list(tasks)
|
||||
seen = len(tasks)
|
||||
written = 0
|
||||
dropped = 0
|
||||
correct_written = 0
|
||||
incorrect_written = 0
|
||||
dropped = 0 # tasks that produced no kept record
|
||||
tasks_solved = 0 # tasks with >=1 correct sample
|
||||
domain_counts: Counter[str] = Counter()
|
||||
|
||||
with out.open("w") as fh:
|
||||
for task in tasks:
|
||||
seen += 1
|
||||
passing: List[UnifiedRollout] = []
|
||||
for _ in range(samples_per_task):
|
||||
roll = rollout_fn(task)
|
||||
if roll is None:
|
||||
continue
|
||||
if verify_fn(task, roll):
|
||||
passing.append(roll)
|
||||
if not passing:
|
||||
def _work(task: TaskLike) -> tuple[TaskLike, List[tuple[UnifiedRollout, bool]]]:
|
||||
return task, _process_task(
|
||||
task,
|
||||
rollout_fn=rollout_fn,
|
||||
verify_fn=verify_fn,
|
||||
samples_per_task=samples_per_task,
|
||||
max_keep_per_task=max_keep_per_task,
|
||||
stop_at_keep=stop_at_keep,
|
||||
keep_all=keep_all,
|
||||
)
|
||||
|
||||
def _emit(fh, task: TaskLike, roll: UnifiedRollout, ok: bool, kept: bool) -> None:
|
||||
nonlocal written, correct_written, incorrect_written
|
||||
reward = reward_fn(roll) if reward_fn else 0.0
|
||||
record = trajectory_to_record(
|
||||
task.task_id, task.instruction, tools, roll,
|
||||
reward=reward, domain=task.domain,
|
||||
)
|
||||
record["correct"] = ok
|
||||
record["kept"] = kept
|
||||
record["clean"] = _target_is_clean(roll)
|
||||
# Task provenance so every record is self-describing: area (domain),
|
||||
# difficulty tier, source dataset, and fine subsector. Lets us slice
|
||||
# train/holdout/analysis by any of these later.
|
||||
record["area"] = task.domain
|
||||
record["difficulty"] = getattr(task, "difficulty", "") or ""
|
||||
record["dataset"] = getattr(task, "dataset", "") or ""
|
||||
record["dataset_description"] = dataset_description(record["dataset"])
|
||||
record["subsector"] = getattr(task, "subsector", "") or ""
|
||||
# Gold reference answer the verifier graded against. Persisted so every
|
||||
# record supports gold-vs-model inspection downstream (Braintrust, error
|
||||
# analysis) instead of only a bare `correct` boolean.
|
||||
record["gold_answer"] = getattr(task, "answer", "") or ""
|
||||
# Stamp provenance (which model/orchestrator generated this trajectory)
|
||||
# so records are self-identifying when pooled across model families.
|
||||
if record_extra:
|
||||
record.update(record_extra)
|
||||
fh.write(json.dumps(record) + "\n")
|
||||
fh.flush()
|
||||
written += 1
|
||||
correct_written += int(ok)
|
||||
incorrect_written += int(not ok)
|
||||
domain_counts[task.domain] += 1
|
||||
|
||||
def _write(fh, task: TaskLike, results: List[tuple[UnifiedRollout, bool]]) -> None:
|
||||
nonlocal dropped, tasks_solved
|
||||
# A trajectory is only eligible to be *kept* as a training target if the
|
||||
# judge passed it AND it's structurally clean (non-error, routed, not
|
||||
# truncated). Unclean-but-correct rollouts are still written in keep_all
|
||||
# mode (for analysis) but never marked kept.
|
||||
correct = [r for r in results if r[1] and _target_is_clean(r[0])]
|
||||
cheapest = min((r for r, _ in correct), key=lambda r: r.cost_usd, default=None)
|
||||
if correct:
|
||||
tasks_solved += 1
|
||||
if keep_all:
|
||||
# Write every sample; mark the cheapest-correct one as kept.
|
||||
if not results:
|
||||
dropped += 1
|
||||
continue
|
||||
# Keep cheapest-first.
|
||||
passing.sort(key=lambda r: r.cost_usd)
|
||||
for roll in passing[:max_keep_per_task]:
|
||||
reward = reward_fn(roll) if reward_fn else 0.0
|
||||
record = trajectory_to_record(
|
||||
task.task_id, task.instruction, tools, roll,
|
||||
reward=reward, domain=task.domain,
|
||||
)
|
||||
fh.write(json.dumps(record) + "\n")
|
||||
written += 1
|
||||
domain_counts[task.domain] += 1
|
||||
return
|
||||
for roll, ok in results:
|
||||
_emit(fh, task, roll, ok, kept=(roll is cheapest))
|
||||
else:
|
||||
# Original behaviour: keep only cheapest-correct, capped.
|
||||
if not correct:
|
||||
dropped += 1
|
||||
return
|
||||
for roll in sorted((r for r, _ in correct),
|
||||
key=lambda r: r.cost_usd)[:max_keep_per_task]:
|
||||
_emit(fh, task, roll, True, kept=(roll is cheapest))
|
||||
|
||||
with out.open("w") as fh:
|
||||
if concurrency <= 1:
|
||||
for task in tasks:
|
||||
_, results = _work(task)
|
||||
_write(fh, task, results)
|
||||
else:
|
||||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||
|
||||
done = 0
|
||||
with ThreadPoolExecutor(max_workers=concurrency) as pool:
|
||||
futures = {pool.submit(_work, t): t for t in tasks}
|
||||
for fut in as_completed(futures):
|
||||
task = futures[fut]
|
||||
try:
|
||||
_, results = fut.result()
|
||||
except Exception as exc: # a task's worker died; count + skip
|
||||
logger.warning("task %s failed: %s",
|
||||
getattr(task, "task_id", "?"), exc)
|
||||
results = []
|
||||
_write(fh, task, results)
|
||||
done += 1
|
||||
if done % 50 == 0 or done == seen:
|
||||
logger.info("rejection-sampling: %d/%d tasks done "
|
||||
"(%d written, %d solved, %d dropped)",
|
||||
done, seen, written, tasks_solved, dropped)
|
||||
|
||||
stats = {
|
||||
"out_path": str(out),
|
||||
"tasks_seen": seen,
|
||||
"records_written": written,
|
||||
"records_correct": correct_written,
|
||||
"records_incorrect": incorrect_written,
|
||||
"tasks_solved": tasks_solved,
|
||||
"tasks_dropped": dropped,
|
||||
"task_accuracy": round(tasks_solved / seen, 4) if seen else 0.0,
|
||||
"samples_per_task": samples_per_task,
|
||||
"keep_all": keep_all,
|
||||
"concurrency": concurrency,
|
||||
"domain_distribution": dict(domain_counts),
|
||||
}
|
||||
out.with_suffix(out.suffix + ".stats.json").write_text(json.dumps(stats, indent=2))
|
||||
logger.info("Wrote %d SFT records to %s", written, out)
|
||||
logger.info("Wrote %d SFT records to %s (%d correct, %d incorrect)",
|
||||
written, out, correct_written, incorrect_written)
|
||||
return stats
|
||||
|
||||
|
||||
|
||||
@@ -13,6 +13,7 @@ prompt), ``assistant`` (reasoning + a ``<tool_call>`` tag, or the final answer),
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import re
|
||||
from typing import Any, Dict, List
|
||||
|
||||
from openjarvis.agents.hybrid.expert_registry import ExpertTool, build_tool_specs
|
||||
@@ -23,9 +24,170 @@ from openjarvis.agents.hybrid.toolorchestra.rollout import (
|
||||
)
|
||||
|
||||
|
||||
def _system_prompt(tools: List[ExpertTool]) -> str:
|
||||
def _system_prompt(tools: List[ExpertTool], rollout: UnifiedRollout) -> str:
|
||||
# Faithful ToolOrchestra system prompt (paper's prepare_sft_data.py format).
|
||||
return build_system_prompt(build_tool_specs(tools))
|
||||
# Prefer the EXACT specs the policy saw this rollout (anonymized labels when
|
||||
# anonymize=True) so the saved <tools> block matches the assistant's
|
||||
# <tool_call> tags. Falling back to the real registry here is the bug that
|
||||
# leaked real model names+pricing into anonymized records.
|
||||
specs = rollout.tool_specs if rollout.tool_specs else build_tool_specs(tools)
|
||||
return build_system_prompt(specs)
|
||||
|
||||
|
||||
def _normalize_think(text: str) -> str:
|
||||
"""Ensure a reasoning block has matched tags. Qwen rollouts routinely yield a
|
||||
dangling ``</think>`` with NO opening ``<think>`` (the chat template consumes
|
||||
the opener in the prompt, so only the closer survives in the completion). Add
|
||||
the opener back so the trained turn is well-formed."""
|
||||
t = (text or "").lstrip()
|
||||
if "</think>" in t and "<think>" not in t:
|
||||
t = "<think>\n" + t
|
||||
return t
|
||||
|
||||
|
||||
def _debox(text: str) -> str:
|
||||
"""Unwrap every ``\\boxed{X}`` -> ``X`` (balanced braces). The format kills
|
||||
\\boxed everywhere, but small models keep emitting it in otherwise-correct
|
||||
answers — de-box to salvage them rather than reject the whole trajectory."""
|
||||
out, i = [], 0
|
||||
marker = r"\boxed{"
|
||||
while i < len(text):
|
||||
j = text.find(marker, i)
|
||||
if j == -1:
|
||||
out.append(text[i:])
|
||||
break
|
||||
out.append(text[i:j])
|
||||
k = j + len(marker)
|
||||
depth, inner = 1, []
|
||||
while k < len(text) and depth:
|
||||
c = text[k]
|
||||
if c == "{":
|
||||
depth += 1
|
||||
elif c == "}":
|
||||
depth -= 1
|
||||
if depth == 0:
|
||||
break
|
||||
inner.append(c)
|
||||
k += 1
|
||||
out.append("".join(inner))
|
||||
i = k + 1
|
||||
return "".join(out)
|
||||
|
||||
|
||||
# Reasoning that PAROTS the training constraint back ("I must call a model per
|
||||
# my instructions", "this is testing whether I need to route") — the model
|
||||
# reasoning about its own harness. Never a good target: drop the whole line.
|
||||
_LEAK_RE = re.compile(
|
||||
r"(?im)^.*\b(?:"
|
||||
r"the user (?:is|was|wants|has|had|needs|'s|would|will|asked|'ve asked|has asked)\b|"
|
||||
r"asked me to\b|" # echoes of an injected nudge turn
|
||||
r"make progress by delegat|"
|
||||
r"delegat\w+[^\n]*sub-?question|"
|
||||
r"(?:to |ask |send .{0,20}to )?one of the models\b|"
|
||||
r"instructions?|" # any self-reference to the rules
|
||||
r"requirement|"
|
||||
r"at least one (?:model|tool|expert)|" # "(call|invoke) at least one model"
|
||||
r"(?:call|invoke|route to|delegate to|use)\s+(?:a|one|at least one|the)\s+model\b|"
|
||||
r"route\s+(?:this|it|that)\b[^\n]{0,40}\bthrough\s+(?:a|one|the)\s+model\b|"
|
||||
r"without mentioning (?:any )?(?:tools?|reasoning|meta|steps)|"
|
||||
r"in the required format\b|"
|
||||
r"based on the (?:model|response|analysis|expert)|" # response-acknowledgment
|
||||
r"the model (?:provided|confirmed|gave|returned|correctly|analy\w+|respon\w+|said|indicated|identified)|"
|
||||
r"i (?:got|received|have) (?:a |the )?(?:comprehensive |clear |good |detailed )?answer\b|"
|
||||
r"now i can (?:confidently )?(?:give|provide|state|answer)|"
|
||||
r"perfect[!,]|great[!,]|excellent[!,]|" # narration exclamations
|
||||
r"as an orchestrator|"
|
||||
r"whose job is to (?:route|delegate|orchestrate)|"
|
||||
r"testing whether i (?:need|have|should|must)\b|"
|
||||
r"i(?:'m| am)? (?:required|instructed|supposed|expected) to\b"
|
||||
r")[^\n]*(?:\n|$)"
|
||||
)
|
||||
# A leading task-restatement opener ("The user is asking…") — the format
|
||||
# explicitly bans this meta-text; drop it when it opens the reasoning.
|
||||
_META_OPEN_RE = re.compile(
|
||||
r"(?is)^\s*(?:the user (?:is|was|wants|has|needs|would|'s)\b[^.\n]*[.\n]+\s*)+"
|
||||
)
|
||||
|
||||
|
||||
def _scrub_meta(text: str) -> str:
|
||||
"""Remove harness-leakage lines and a leading 'The user is asking…' meta
|
||||
opener from a reasoning block. Both are disallowed by the system prompt, so
|
||||
they must not survive into the supervised target. Preserves a leading
|
||||
``<think>`` marker so the block stays well-formed."""
|
||||
if not text:
|
||||
return text
|
||||
think = ""
|
||||
body = text
|
||||
m = re.match(r"(?is)^\s*(<think>)\s*", body)
|
||||
if m:
|
||||
think = "<think>\n"
|
||||
body = body[m.end():]
|
||||
body = _LEAK_RE.sub("", body)
|
||||
body = _META_OPEN_RE.sub("", body).lstrip()
|
||||
return (think + body).strip() if think else body.strip()
|
||||
|
||||
|
||||
# Control / special tokens that must never survive into a training target's
|
||||
# final answer. Three shapes, in order of alternation:
|
||||
# 1. closed pipe token — ``<|im_end|>``, ``<|im_start|>``, ``<|eot_id|>``,
|
||||
# ``<|end_of_turn|>``, ``<|"|>`` (the stray-quote leak).
|
||||
# 2. UNCLOSED pipe token — ``<|tool_call>`` (opened with ``<|`` but closed on a
|
||||
# bare ``>`` because the decode was cut before the second pipe).
|
||||
# 3. named angle special tokens — gemma ``<start_of_turn>`` / ``<end_of_turn>``
|
||||
# and the sentinel set ``<eos> <bos> <pad> <unk> <s> </s>``.
|
||||
# Deliberately narrow: only the ``<|...|>`` pipe form and this known name list
|
||||
# match, so legitimate ``<``/``>`` in math/code (``x < 3 and y > 2``) is left
|
||||
# untouched.
|
||||
_CONTROL_TOKEN_RE = re.compile(
|
||||
r"<\|[^>]*?\|>" # closed pipe token
|
||||
r"|<\|[^|>]*>" # unclosed pipe token
|
||||
r"|</?(?:start_of_turn|end_of_turn|eos|bos|pad|unk|s)>" # named angle tokens
|
||||
)
|
||||
|
||||
|
||||
def _strip_control_tokens(text: str) -> str:
|
||||
"""Delete residual control/special tokens from an answer so a good answer
|
||||
with a stray token is salvaged rather than dropped. Loops (bounded) because
|
||||
removing one match can expose a nested/overlapping leftover (e.g.
|
||||
``<|<|im_end|>|>``). Re-strips whitespace at the end."""
|
||||
if not text:
|
||||
return text
|
||||
out = text
|
||||
for _ in range(4):
|
||||
stripped = _CONTROL_TOKEN_RE.sub("", out)
|
||||
if stripped == out:
|
||||
break
|
||||
out = stripped
|
||||
return out.strip()
|
||||
|
||||
|
||||
def _final_answer_block(text: str) -> str:
|
||||
"""Render the final-answer assistant message with a single clean
|
||||
``FINAL_ANSWER: <value>`` line. De-boxes the answer, strips leaked control
|
||||
tokens, and normalizes whatever spelling the model used (``FINALANSWER``,
|
||||
``FINAL ANSWER``, no colon, …) to the exact tag. Keeps the think block at
|
||||
most once."""
|
||||
text = _debox(_normalize_think((text or "").strip()))
|
||||
# Normalize any final-answer marker spelling to the canonical form; take the
|
||||
# LAST one as the real answer (idempotent — never emits two tags).
|
||||
marks = list(re.finditer(r"(?im)FINAL[_\s]?ANSWER\s*:?", text))
|
||||
if marks:
|
||||
last = marks[-1]
|
||||
answer = _strip_control_tokens(text[last.end():].strip())
|
||||
# Final turn = the bare short answer only. Drop EVERYTHING before
|
||||
# FINAL_ANSWER — both the visible "Perfect! Based on the model's response…"
|
||||
# narration AND the final-turn <think>, which is just confirmatory
|
||||
# "both models agree… I've verified…" fluff. The routing reasoning already
|
||||
# lives in the earlier tool-call turns, so nothing of value is lost, and
|
||||
# this kills all final-turn narration deterministically (any wording).
|
||||
return f"FINAL_ANSWER: {answer}"
|
||||
if "</think>" in text:
|
||||
# No explicit marker: the answer is whatever follows the final </think>.
|
||||
# Drop the think block (same rationale as above) — keep only the answer.
|
||||
_, _, answer = text.rpartition("</think>")
|
||||
answer = _strip_control_tokens(answer.strip())
|
||||
return f"FINAL_ANSWER: {answer}" if answer else f"FINAL_ANSWER: {_strip_control_tokens(_scrub_meta(text))}"
|
||||
return f"FINAL_ANSWER: {_strip_control_tokens(text)}"
|
||||
|
||||
|
||||
def trajectory_to_record(
|
||||
@@ -39,21 +201,21 @@ def trajectory_to_record(
|
||||
) -> Dict[str, Any]:
|
||||
"""Convert a passing :class:`UnifiedRollout` into one SFT JSONL record."""
|
||||
conversations: List[Dict[str, str]] = [
|
||||
{"role": "system", "content": _system_prompt(tools)},
|
||||
{"role": "system", "content": _system_prompt(tools, rollout)},
|
||||
{"role": "user", "content": f"Problem: {question}"},
|
||||
]
|
||||
|
||||
for turn in rollout.turns:
|
||||
if turn.tool_name is None:
|
||||
# Final-answer turn.
|
||||
# Final-answer turn. turn.reasoning is the model's actual output for
|
||||
# this turn (which already contains the answer); render it once.
|
||||
conversations.append({
|
||||
"role": "assistant",
|
||||
"content": (turn.reasoning or "").rstrip()
|
||||
+ f"\nFINAL_ANSWER: {rollout.final_answer}",
|
||||
"content": _final_answer_block(turn.reasoning or rollout.final_answer),
|
||||
})
|
||||
continue
|
||||
tag = tool_call_tag(turn.tool_name, turn.arguments)
|
||||
reasoning = (turn.reasoning or "").rstrip()
|
||||
reasoning = _scrub_meta(_normalize_think((turn.reasoning or "").rstrip()))
|
||||
conversations.append({
|
||||
"role": "assistant",
|
||||
"content": (reasoning + "\n" + tag).strip(),
|
||||
@@ -68,7 +230,7 @@ def trajectory_to_record(
|
||||
if not rollout.turns or rollout.turns[-1].tool_name is not None:
|
||||
conversations.append({
|
||||
"role": "assistant",
|
||||
"content": f"FINAL_ANSWER: {rollout.final_answer}",
|
||||
"content": _final_answer_block(rollout.final_answer),
|
||||
})
|
||||
|
||||
return {
|
||||
@@ -81,6 +243,9 @@ def trajectory_to_record(
|
||||
"tokens": rollout.tokens,
|
||||
"num_tool_calls": rollout.num_tool_calls,
|
||||
"num_turns": len(rollout.turns),
|
||||
# Present only when experts were anonymized: opaque label -> real
|
||||
# tool name, so analysis can recover which model was actually picked.
|
||||
**({"anon_map": rollout.anon_map} if rollout.anon_map else {}),
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
@@ -29,6 +29,12 @@ from .datasets import Task
|
||||
|
||||
GEMINI_BASE_URL = "https://generativelanguage.googleapis.com/v1beta/openai/"
|
||||
GEMINI_MODEL = "gemini-2.5-flash"
|
||||
# LLM judge model. Switched from Gemini (free-tier rate limit stalled parallel gen)
|
||||
# to Anthropic Haiku: fast (~0.6s), direct PASS/FAIL, high rate limits.
|
||||
JUDGE_MODEL = "claude-haiku-4-5-20251001"
|
||||
# Reused across calls — creating a fresh Anthropic client per judge call leaks an
|
||||
# httpx connection pool (CLOSE-WAIT pileup under parallel generation). Thread-safe.
|
||||
_JUDGE_CLIENT = None
|
||||
|
||||
|
||||
# --- normalization (copied from hotpotqa.py — keep self-contained) -----------
|
||||
@@ -136,29 +142,11 @@ def _math_equal(prediction: str, gold: str) -> bool:
|
||||
if abs(pf - gf) <= 1e-6 * max(1.0, abs(gf)):
|
||||
return True
|
||||
|
||||
# Symbolic / exact equality via sympy when available.
|
||||
try:
|
||||
import sympy
|
||||
from sympy.parsing.latex import parse_latex # noqa: F401 (optional)
|
||||
|
||||
def _parse(x: str):
|
||||
try:
|
||||
return sympy.sympify(x.replace("^", "**"))
|
||||
except Exception:
|
||||
try:
|
||||
return sympy.parsing.latex.parse_latex(x)
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
pe, ge = _parse(pred), _parse(g)
|
||||
if pe is not None and ge is not None:
|
||||
try:
|
||||
if sympy.simplify(pe - ge) == 0:
|
||||
return True
|
||||
except Exception:
|
||||
pass
|
||||
except Exception:
|
||||
pass
|
||||
# NOTE: a sympy/parse_latex symbolic-equality block used to live here, but
|
||||
# sympy.simplify / antlr parse_latex can hang on pathological \boxed{} answers
|
||||
# while holding the GIL -> deadlocks the whole threaded rejection sampler
|
||||
# (all worker threads freeze in futex_wait, server goes idle, 0 records).
|
||||
# Symbolic equivalence is now deferred to the Gemini judge in verify_answer().
|
||||
|
||||
# Last resort: whole-word substring of the gold in the prediction.
|
||||
return _string_or_f1(prediction, g, f1_threshold=0.9)
|
||||
@@ -169,16 +157,24 @@ def _math_equal(prediction: str, gold: str) -> bool:
|
||||
|
||||
def _gemini_judge(task: Task, prediction: str) -> Optional[bool]:
|
||||
"""Ask Gemini PASS/FAIL. Returns None if unavailable (caller falls back)."""
|
||||
api_key = os.environ.get("GEMINI_API_KEY")
|
||||
api_key = os.environ.get("ANTHROPIC_API_KEY")
|
||||
if not api_key:
|
||||
return None
|
||||
try:
|
||||
from openai import OpenAI
|
||||
import anthropic
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
try:
|
||||
client = OpenAI(api_key=api_key, base_url=GEMINI_BASE_URL)
|
||||
# Judge = Anthropic Haiku (fast ~0.6s, direct PASS/FAIL, high rate limits).
|
||||
# Switched off the Gemini judge: its free-tier rate limit + SDK retry/backoff
|
||||
# blocked worker threads for minutes under parallel generation and stalled
|
||||
# the run. Fail-fast (no retries, short timeout): on error return None and the
|
||||
# caller falls back to string/f1.
|
||||
global _JUDGE_CLIENT
|
||||
if _JUDGE_CLIENT is None:
|
||||
_JUDGE_CLIENT = anthropic.Anthropic(api_key=api_key, max_retries=0, timeout=15)
|
||||
client = _JUDGE_CLIENT
|
||||
prompt = (
|
||||
"You are grading a candidate answer against a gold reference.\n"
|
||||
"Reply with exactly one word: PASS if the candidate is correct and "
|
||||
@@ -188,13 +184,12 @@ def _gemini_judge(task: Task, prediction: str) -> Optional[bool]:
|
||||
f"Candidate answer:\n{prediction}\n\n"
|
||||
"Verdict (PASS or FAIL):"
|
||||
)
|
||||
resp = client.chat.completions.create(
|
||||
model=GEMINI_MODEL,
|
||||
messages=[{"role": "user", "content": prompt}],
|
||||
temperature=0.0,
|
||||
resp = client.messages.create(
|
||||
model=JUDGE_MODEL,
|
||||
max_tokens=8,
|
||||
messages=[{"role": "user", "content": prompt}],
|
||||
)
|
||||
verdict = (resp.choices[0].message.content or "").strip().upper()
|
||||
verdict = (resp.content[0].text or "").strip().upper()
|
||||
if "PASS" in verdict:
|
||||
return True
|
||||
if "FAIL" in verdict:
|
||||
@@ -216,6 +211,10 @@ def verify_answer(task: Task, prediction: str) -> bool:
|
||||
domain = (task.domain or "").lower()
|
||||
|
||||
if domain == "math":
|
||||
# string + numeric + f1 only. sympy removed (GIL-deadlocked the threaded
|
||||
# sampler); Gemini judge NOT used here either (32 tasks x N samples of math
|
||||
# judge calls throttle Gemini and stall the whole run). Accept slightly lower
|
||||
# math yield for a fast, hang-free verifier.
|
||||
return _math_equal(pred, task.answer)
|
||||
|
||||
if domain == "code":
|
||||
|
||||
@@ -42,8 +42,23 @@ _MATH_FUNCS = {
|
||||
"sin": math.sin,
|
||||
"cos": math.cos,
|
||||
"tan": math.tan,
|
||||
"asin": math.asin,
|
||||
"acos": math.acos,
|
||||
"atan": math.atan,
|
||||
"atan2": math.atan2,
|
||||
"sinh": math.sinh,
|
||||
"cosh": math.cosh,
|
||||
"tanh": math.tanh,
|
||||
"radians": math.radians,
|
||||
"degrees": math.degrees,
|
||||
"exp": math.exp,
|
||||
"abs": abs,
|
||||
"fabs": math.fabs,
|
||||
"hypot": math.hypot,
|
||||
"factorial": math.factorial,
|
||||
"pi": math.pi,
|
||||
"e": math.e,
|
||||
"tau": math.tau,
|
||||
"ceil": math.ceil,
|
||||
"floor": math.floor,
|
||||
}
|
||||
|
||||
@@ -116,11 +116,31 @@ class WebSearchTool(BaseTool):
|
||||
return text
|
||||
|
||||
def _duckduckgo_search(self, query: str, max_results: int) -> str:
|
||||
"""Search using DuckDuckGo as fallback."""
|
||||
"""Search using DuckDuckGo as fallback.
|
||||
|
||||
ddgs queries several engines (yandex/yahoo/brave/...) that frequently
|
||||
hang; with no timeout this blocks the whole rollout. Cap each engine
|
||||
request (DDGS timeout) AND wrap the call in a hard wall-clock deadline so
|
||||
a flaky search fast-fails instead of stalling data generation.
|
||||
"""
|
||||
from concurrent.futures import ThreadPoolExecutor, TimeoutError as _FTO
|
||||
|
||||
from ddgs import DDGS
|
||||
|
||||
ddgs = DDGS()
|
||||
raw_results = list(ddgs.text(query, max_results=max_results))
|
||||
def _run():
|
||||
ddgs = DDGS(timeout=5)
|
||||
return list(ddgs.text(query, max_results=max_results))
|
||||
|
||||
# Don't use `with` — its shutdown(wait=True) blocks on the hung thread,
|
||||
# defeating the deadline. submit, wait up to 8s, then abandon the thread
|
||||
# (it finishes in the background harmlessly) and fast-fail.
|
||||
_ex = ThreadPoolExecutor(max_workers=1)
|
||||
try:
|
||||
raw_results = _ex.submit(_run).result(timeout=8)
|
||||
except Exception:
|
||||
_ex.shutdown(wait=False, cancel_futures=True)
|
||||
return "[web_search: no results (search backend timed out)]"
|
||||
_ex.shutdown(wait=False)
|
||||
results = []
|
||||
for r in raw_results:
|
||||
title = r.get("title", "Untitled")
|
||||
|
||||
@@ -0,0 +1,129 @@
|
||||
"""Unit coverage for ``anonymize_tools`` — the identity-stripping step that makes
|
||||
routing data unbiased. See ``expert_registry.anonymize_tools``.
|
||||
|
||||
The anonymizer takes the orchestrator catalog and replaces every MODEL expert
|
||||
with an opaque ``model_xxxx`` label, a uniform brand-free description, no
|
||||
price/latency line, and shuffles the expert block — so the policy can't route on
|
||||
a model's name, position, cost, or tier. Basic tools keep their real names.
|
||||
It returns ``(anon_tools, anon_to_real)``.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import random
|
||||
|
||||
from openjarvis.agents.hybrid.expert_registry import (
|
||||
KIND_MODEL,
|
||||
anonymize_tools,
|
||||
build_tool_specs,
|
||||
orchestrator_catalog,
|
||||
tools_by_name,
|
||||
)
|
||||
|
||||
# Real brand tokens that must never leak into the anonymized, model-facing specs.
|
||||
_BRANDS = ("gpt", "claude", "gemini", "qwen")
|
||||
|
||||
|
||||
def _model_names(cat):
|
||||
return [t.name for t in cat if t.kind == KIND_MODEL]
|
||||
|
||||
|
||||
def _basic_names(cat):
|
||||
return [t.name for t in cat if t.kind != KIND_MODEL]
|
||||
|
||||
|
||||
def test_no_brand_names_in_anonymized_specs():
|
||||
cat = orchestrator_catalog()
|
||||
anon, _ = anonymize_tools(cat, random.Random(0))
|
||||
specs = build_tool_specs(anon)
|
||||
# The whole model-facing payload (names + descriptions + categories) must be
|
||||
# brand-free. `.model` is preserved on the tool for dispatch but is NOT part
|
||||
# of the spec the orchestrator conditions on, so it's fine that it still holds
|
||||
# the real id.
|
||||
blob = json.dumps(specs).lower()
|
||||
for brand in _BRANDS:
|
||||
assert brand not in blob, f"brand {brand!r} leaked into anonymized specs"
|
||||
# And specifically the anonymized expert descriptions carry no brand.
|
||||
for t in anon:
|
||||
if t.name.startswith("model_"):
|
||||
assert not any(b in t.description().lower() for b in _BRANDS)
|
||||
|
||||
|
||||
def test_hide_cost_removes_price_line_from_descriptions():
|
||||
cat = orchestrator_catalog()
|
||||
# Sanity: the raw model tools DO surface a price line before anonymizing.
|
||||
raw_model = next(t for t in cat if t.kind == KIND_MODEL)
|
||||
assert "Pricing:" in raw_model.description()
|
||||
|
||||
anon, anon_to_real = anonymize_tools(cat, random.Random(1))
|
||||
for t in anon:
|
||||
if t.name in anon_to_real: # an anonymized model expert
|
||||
assert t.hide_cost is True
|
||||
desc = t.description()
|
||||
assert "Pricing:" not in desc
|
||||
assert "$" not in desc
|
||||
assert "/1M" not in desc
|
||||
assert "latency" not in desc.lower()
|
||||
|
||||
|
||||
def test_each_model_maps_to_opaque_label_and_round_trips():
|
||||
cat = orchestrator_catalog()
|
||||
orig_models = _model_names(cat)
|
||||
anon, anon_to_real = anonymize_tools(cat, random.Random(2))
|
||||
|
||||
# One opaque label per real model, all in the model_xxxx namespace.
|
||||
assert len(anon_to_real) == len(orig_models)
|
||||
assert all(lbl.startswith("model_") for lbl in anon_to_real)
|
||||
|
||||
# No collisions: labels unique, and each real model recovered exactly once.
|
||||
assert len(set(anon_to_real)) == len(anon_to_real)
|
||||
assert len(set(anon_to_real.values())) == len(anon_to_real)
|
||||
assert set(anon_to_real.values()) == set(orig_models)
|
||||
|
||||
# Round-trip: every anonymized expert's label resolves back to a real name,
|
||||
# and `.model` is preserved so dispatch still reaches the right backend.
|
||||
by_orig = tools_by_name(cat)
|
||||
for t in anon:
|
||||
if t.name.startswith("model_"):
|
||||
real = anon_to_real[t.name]
|
||||
assert real in by_orig
|
||||
assert t.model == by_orig[real].model # backend id untouched
|
||||
|
||||
# Basic tools keep their real names and are untouched by the label map.
|
||||
basics = _basic_names(cat)
|
||||
anon_names = {t.name for t in anon}
|
||||
assert set(basics) <= anon_names
|
||||
assert not (set(basics) & set(anon_to_real))
|
||||
|
||||
|
||||
def test_experts_block_on_top_then_basics():
|
||||
cat = orchestrator_catalog()
|
||||
anon, anon_to_real = anonymize_tools(cat, random.Random(3))
|
||||
n_models = len(anon_to_real)
|
||||
# Experts form a contiguous block at the front, basics underneath.
|
||||
assert all(t.name.startswith("model_") for t in anon[:n_models])
|
||||
assert all(not t.name.startswith("model_") for t in anon[n_models:])
|
||||
|
||||
|
||||
def test_labels_and_order_are_shuffled_not_identity():
|
||||
cat = orchestrator_catalog()
|
||||
orig_models = _model_names(cat)
|
||||
|
||||
orderings = set()
|
||||
labels_for_first = set()
|
||||
for seed in range(25):
|
||||
anon, anon_to_real = anonymize_tools(cat, random.Random(seed))
|
||||
# Real-model order as it appears in the anonymized expert block.
|
||||
order = tuple(anon_to_real[t.name] for t in anon if t.name.startswith("model_"))
|
||||
orderings.add(order)
|
||||
# Label assigned to the first catalog model varies across rngs.
|
||||
rev = {real: lbl for lbl, real in anon_to_real.items()}
|
||||
labels_for_first.add(rev[orig_models[0]])
|
||||
|
||||
# Not a fixed identity: across seeds the expert order actually varies...
|
||||
assert len(orderings) > 1
|
||||
# ...and at least one ordering differs from the input model order.
|
||||
assert any(order != tuple(orig_models) for order in orderings)
|
||||
# Labels are random per-rng, not a stable function of position.
|
||||
assert len(labels_for_first) > 1
|
||||
@@ -136,11 +136,14 @@ def test_orchestrator_catalog_two_model_classes_plus_basics():
|
||||
by = tools_by_name(cat)
|
||||
assert by["gpt_5_5"].category == CATEGORY_CLOUD_FRONTIER
|
||||
assert by["claude_opus_4_8"].category == CATEGORY_CLOUD_FRONTIER
|
||||
# Default routing for every model tool is OpenRouter (no servers required).
|
||||
for n in ("qwen3_5_9b", "qwen3_6_27b_fp8", "qwen3_5_122b_a10b_fp8",
|
||||
"qwen3_5_397b_a17b_fp8"):
|
||||
assert by[n].category == CATEGORY_LOCAL_OSS
|
||||
assert by[n].backend_type == "vllm"
|
||||
assert by[n].price_in == 0.0 and by[n].price_out == 0.0
|
||||
assert by[n].backend_type == "openrouter"
|
||||
assert by[n].base_url is None
|
||||
# OpenRouter routing carries the slug + a (estimated) per-token price.
|
||||
assert "/" in by[n].model and by[n].price_in > 0.0
|
||||
|
||||
|
||||
def test_orchestrator_catalog_categories_present():
|
||||
@@ -167,21 +170,54 @@ def test_orchestrator_specs_include_category_field():
|
||||
|
||||
|
||||
def test_orchestrator_local_models_get_base_url_when_provided():
|
||||
# An endpoint switches that model from the OpenRouter default to local vLLM.
|
||||
cat = orchestrator_catalog(
|
||||
local_endpoints={
|
||||
"Qwen/Qwen3.5-9B": "http://x/v1",
|
||||
"Qwen/Qwen3.6-27B-FP8": "http://y/v1",
|
||||
})
|
||||
by = tools_by_name(cat)
|
||||
assert by["qwen3_5_9b"].backend_type == "vllm"
|
||||
assert by["qwen3_5_9b"].base_url == "http://x/v1"
|
||||
assert by["qwen3_5_9b"].price_in == 0.0 and by["qwen3_5_9b"].price_out == 0.0
|
||||
assert by["qwen3_6_27b_fp8"].base_url == "http://y/v1"
|
||||
# Unmapped local model -> base_url None.
|
||||
# Unmapped local model -> stays on the OpenRouter default (base_url None).
|
||||
assert by["qwen3_5_122b_a10b_fp8"].backend_type == "openrouter"
|
||||
assert by["qwen3_5_122b_a10b_fp8"].base_url is None
|
||||
# Cloud frontier carries real pricing.
|
||||
assert by["claude_opus_4_8"].price_in > 0.0
|
||||
assert by["gpt_5_5"].price_in > 0.0
|
||||
|
||||
|
||||
def test_orchestrator_model_backends_override():
|
||||
# Force a cloud model onto OpenRouter and a local model onto vLLM explicitly.
|
||||
cat = orchestrator_catalog(
|
||||
model_backends={
|
||||
"claude-opus-4-8": "openrouter",
|
||||
"Qwen/Qwen3.5-397B-A17B-FP8": "vllm",
|
||||
},
|
||||
local_endpoints={"Qwen/Qwen3.5-397B-A17B-FP8": "http://z/v1"},
|
||||
)
|
||||
by = tools_by_name(cat)
|
||||
assert by["claude_opus_4_8"].backend_type == "openrouter"
|
||||
assert by["claude_opus_4_8"].model == "anthropic/claude-opus-4.8"
|
||||
# No override for gpt-5.5 -> it defaults to its NATIVE first-party API
|
||||
# (openai), not OpenRouter. Frontier models hit their native provider by
|
||||
# default; OpenRouter is only the fallback for OSS/local models or when
|
||||
# explicitly requested via model_backends.
|
||||
assert by["gpt_5_5"].backend_type == "openai" # native default
|
||||
assert by["gpt_5_5"].model == "gpt-5.5"
|
||||
assert by["qwen3_5_397b_a17b_fp8"].backend_type == "vllm"
|
||||
assert by["qwen3_5_397b_a17b_fp8"].base_url == "http://z/v1"
|
||||
|
||||
|
||||
def test_orchestrator_openrouter_slug_override():
|
||||
cat = orchestrator_catalog(
|
||||
openrouter_slugs={"Qwen/Qwen3.5-9B": "qwen/qwen3.5-9b-custom"})
|
||||
by = tools_by_name(cat)
|
||||
assert by["qwen3_5_9b"].model == "qwen/qwen3.5-9b-custom"
|
||||
|
||||
|
||||
def test_openjarvis_tool_bridges_real_tool_with_custom_schema():
|
||||
params = {
|
||||
"type": "object",
|
||||
|
||||
@@ -60,7 +60,9 @@ def test_run_unified_rollout_terminates_on_no_tool_call():
|
||||
]
|
||||
calls = iter(scripted)
|
||||
|
||||
def call_orch(system, user, specs):
|
||||
def call_orch(messages, specs):
|
||||
# Signature is now (messages, specs): the rollout drives a running
|
||||
# system/user/assistant/tool conversation, not a flattened (system, user).
|
||||
return next(calls)
|
||||
|
||||
def dispatch(tool, args):
|
||||
@@ -81,7 +83,11 @@ def test_serialize_record_shape_and_tool_call_tags():
|
||||
turns=[
|
||||
UnifiedTurn(reasoning="think", tool_name="qwen3_32b",
|
||||
arguments={"input": "q"}, observation="obs"),
|
||||
UnifiedTurn(reasoning="done", tool_name=None),
|
||||
# The final turn's reasoning IS the model's real final output (it
|
||||
# already contains the answer). The serializer now renders this turn
|
||||
# via _final_answer_block(turn.reasoning), not rollout.final_answer.
|
||||
UnifiedTurn(reasoning="The result is 42.\nFINAL_ANSWER: 42",
|
||||
tool_name=None),
|
||||
],
|
||||
final_answer="42", cost_usd=0.02, tokens=30, num_tool_calls=1,
|
||||
)
|
||||
@@ -131,7 +137,9 @@ def test_generate_sft_dataset_end_to_end(tmp_path):
|
||||
UnifiedTurn("", "refund", {"user": "8612"}, "ok"),
|
||||
UnifiedTurn("done", None),
|
||||
],
|
||||
final_answer="refunded $20.90", cost_usd=0.03,
|
||||
# num_tool_calls must reflect the two routed calls: the structural
|
||||
# _target_is_clean gate drops a trajectory with num_tool_calls < 1.
|
||||
final_answer="refunded $20.90", cost_usd=0.03, num_tool_calls=2,
|
||||
)
|
||||
return UnifiedRollout(turns=[UnifiedTurn("x", "cancel", {}, "ok")],
|
||||
final_answer="nope", cost_usd=0.05)
|
||||
|
||||
@@ -35,10 +35,14 @@ def test_math_numeric_float_match():
|
||||
assert verify_answer(_math("42"), "42.0") is True
|
||||
|
||||
|
||||
def test_math_symbolic_equivalence():
|
||||
# x^2 + 2x + 1 == (x+1)^2 via sympy (skips gracefully if sympy absent).
|
||||
def test_math_symbolic_equivalence_no_longer_verified_locally():
|
||||
# The sympy symbolic-equality block was removed from _math_equal: sympy.simplify
|
||||
# / parse_latex could hang on pathological \boxed{} answers while holding the GIL
|
||||
# and deadlock the threaded rejection sampler. The math domain now uses
|
||||
# string+numeric+f1 only (no judge call either), so a purely SYMBOLIC
|
||||
# equivalence that isn't string/numeric-equal is intentionally NOT recognized.
|
||||
t = _math("(x+1)^2")
|
||||
assert verify_answer(t, "x^2 + 2*x + 1") is True
|
||||
assert verify_answer(t, "x^2 + 2*x + 1") is False
|
||||
|
||||
|
||||
def test_empty_prediction_false():
|
||||
|
||||
@@ -18,7 +18,8 @@ from openjarvis.learning.intelligence.orchestrator.sft_data.reject_sample import
|
||||
)
|
||||
|
||||
|
||||
def _fake_tasks() -> list[Task]:
|
||||
def _fake_tasks(**_kwargs) -> list[Task]:
|
||||
# Accepts **kwargs so it can stand in for ``load_sft_tasks(cap=..., balanced=...)``.
|
||||
return [
|
||||
Task(task_id="t-math-1", question="What is 6 * 7?", answer="42", domain="math"),
|
||||
Task(task_id="t-code-1", question="Reverse 'ab'.", answer="ba", domain="code"),
|
||||
@@ -31,7 +32,7 @@ def _canned_rollout(task: Task) -> UnifiedRollout:
|
||||
turns=[
|
||||
UnifiedTurn(reasoning="let me compute", tool_name="code_interpreter",
|
||||
arguments={"code": "print(6*7)"}, observation="42"),
|
||||
UnifiedTurn(reasoning="that's the result", tool_name=None),
|
||||
UnifiedTurn(reasoning=f"The tool returned {task.answer}. FINAL_ANSWER: {task.answer}", tool_name=None),
|
||||
],
|
||||
final_answer=task.answer, cost_usd=0.01, tokens=20, num_tool_calls=1,
|
||||
)
|
||||
@@ -110,14 +111,19 @@ def test_driver_runs_end_to_end_with_fakes(tmp_path, monkeypatch):
|
||||
# make_call_orchestrator builds an OpenAI client lazily, so it never connects
|
||||
# here (run_unified_rollout is stubbed out).
|
||||
|
||||
out = tmp_path / "v1.jsonl"
|
||||
# The driver now treats --out as a LABEL and always writes to
|
||||
# data/runs/<stamp>_<label>/data.jsonl (relative to cwd). chdir into tmp_path
|
||||
# so the run folder is created under the test's temp dir, not the real repo.
|
||||
monkeypatch.chdir(tmp_path)
|
||||
rc = drv.main([
|
||||
"--out", str(out),
|
||||
"--out", "v1",
|
||||
"--samples-per-task", "1",
|
||||
"--max-tasks", "2",
|
||||
])
|
||||
assert rc == 0
|
||||
lines = out.read_text().strip().splitlines()
|
||||
produced = list((tmp_path / "data" / "runs").glob("*_v1/data.jsonl"))
|
||||
assert len(produced) == 1
|
||||
lines = produced[0].read_text().strip().splitlines()
|
||||
assert len(lines) == 2
|
||||
rec = json.loads(lines[0])
|
||||
assert rec["conversations"][0]["role"] == "system"
|
||||
|
||||
@@ -11,6 +11,7 @@ from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
|
||||
from openjarvis.agents.hybrid.toolorchestra.rollout import UnifiedTurn
|
||||
from openjarvis.learning.intelligence.orchestrator import eval_backend as eb
|
||||
from openjarvis.learning.intelligence.orchestrator.eval_backend import (
|
||||
OrchestratorBackend,
|
||||
@@ -26,7 +27,14 @@ class _CannedRollout:
|
||||
self.tokens = 42
|
||||
self.num_tool_calls = 1
|
||||
self.parse_failures = 0
|
||||
self.turns = [object()]
|
||||
self.anon_map = {}
|
||||
# generate_full now serializes a per-turn trace (reasoning / tool_name /
|
||||
# arguments / observation), so turns must be real UnifiedTurn-shaped
|
||||
# objects, not bare object() placeholders.
|
||||
self.turns = [
|
||||
UnifiedTurn(reasoning="let me search", tool_name="web_search",
|
||||
arguments={"query": "x"}, observation="result"),
|
||||
]
|
||||
|
||||
def tool_calls(self):
|
||||
return [("web_search", {"query": "x"})]
|
||||
|
||||
@@ -0,0 +1,112 @@
|
||||
"""Control-token sanitation in the SFT serializer + clean gate.
|
||||
|
||||
Regression guard for the leak audit: leaked control/special tokens
|
||||
(``<|im_end|>``, ``<|tool_call>``, gemma ``<start_of_turn>``/``<end_of_turn>``,
|
||||
``<|"|>`` …) used to survive into the supervised final answer because the clean
|
||||
gate only rejected the bare ``<tool_call>`` form. Defense in depth now:
|
||||
|
||||
1. ``_final_answer_block`` STRIPS residual control tokens so a good answer with a
|
||||
stray token is salvaged (not dropped).
|
||||
2. ``_target_is_clean`` REJECTS a rollout whose final answer is nothing but
|
||||
control tokens, or where a token survives the strip.
|
||||
|
||||
Both must leave legitimate ``<``/``>`` in math/code untouched.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
|
||||
from openjarvis.agents.hybrid.toolorchestra.rollout import UnifiedRollout, UnifiedTurn
|
||||
from openjarvis.learning.intelligence.orchestrator.sft_data.reject_sample import (
|
||||
_target_is_clean,
|
||||
)
|
||||
from openjarvis.learning.intelligence.orchestrator.sft_data.unified_serialize import (
|
||||
_CONTROL_TOKEN_RE,
|
||||
_final_answer_block,
|
||||
_strip_control_tokens,
|
||||
)
|
||||
|
||||
|
||||
# --- serializer strips control tokens, leaving a clean FINAL_ANSWER line ------
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"raw, expected_answer",
|
||||
[
|
||||
("FINAL_ANSWER: 42<end_of_turn>", "42"),
|
||||
("FINAL_ANSWER: 42<|end_of_turn|>", "42"),
|
||||
("FINAL_ANSWER: 42<|im_end|>", "42"),
|
||||
("FINAL_ANSWER: 42<|eot_id|>", "42"),
|
||||
("FINAL_ANSWER: The result<|tool_call>", "The result"),
|
||||
('FINAL_ANSWER: foo <|"|> bar', "foo bar"),
|
||||
("<start_of_turn>model", "model"),
|
||||
],
|
||||
)
|
||||
def test_final_answer_block_strips_tokens(raw: str, expected_answer: str) -> None:
|
||||
out = _final_answer_block(raw)
|
||||
assert out == f"FINAL_ANSWER: {expected_answer}"
|
||||
# No control token survives the serializer.
|
||||
assert not _CONTROL_TOKEN_RE.search(out), out
|
||||
|
||||
|
||||
def test_bare_im_end_only_becomes_empty_answer() -> None:
|
||||
# An answer that is nothing but a control token strips to empty; the clean
|
||||
# gate (below) is what rejects such a rollout.
|
||||
assert _final_answer_block("<|im_end|>") == "FINAL_ANSWER: "
|
||||
|
||||
|
||||
def test_math_and_code_angle_brackets_survive() -> None:
|
||||
# The narrow regex must NOT eat legitimate comparisons / generics.
|
||||
for good in [
|
||||
"FINAL_ANSWER: x < 3 and y > 2",
|
||||
"FINAL_ANSWER: List<int> and a<b>c",
|
||||
"FINAL_ANSWER: if a < b: return a > 0",
|
||||
]:
|
||||
out = _final_answer_block(good)
|
||||
assert out == good, out
|
||||
assert not _CONTROL_TOKEN_RE.search(out)
|
||||
|
||||
|
||||
def test_strip_helper_handles_nested_and_whitespace() -> None:
|
||||
assert _strip_control_tokens(" hi <|im_end|> ") == "hi"
|
||||
assert _strip_control_tokens("a<|im_start|>b<|im_end|>c") == "abc"
|
||||
|
||||
|
||||
# --- clean gate: salvage vs reject -------------------------------------------
|
||||
|
||||
def _roll(final_answer: str) -> UnifiedRollout:
|
||||
"""A minimal well-formed rollout (one expert call, clean obs) whose only
|
||||
variable is the final answer."""
|
||||
return UnifiedRollout(
|
||||
turns=[
|
||||
UnifiedTurn(reasoning="route it", tool_name="expert_a",
|
||||
arguments={"input": "q"}, observation="valid observation"),
|
||||
UnifiedTurn(reasoning=final_answer, tool_name=None),
|
||||
],
|
||||
final_answer=final_answer, cost_usd=0.01, tokens=10, num_tool_calls=1,
|
||||
anon_map={"expert_a": "gpt"},
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"final_answer, clean",
|
||||
[
|
||||
# Salvageable: a good answer with a trailing stray token stays clean
|
||||
# (the serializer strips the token from the emitted target).
|
||||
("42<end_of_turn>", True),
|
||||
("42<|end_of_turn|>", True),
|
||||
("The result<|tool_call>", True),
|
||||
('foo <|"|> bar', True),
|
||||
("<start_of_turn>model", True),
|
||||
("x < 3 and y > 2", True),
|
||||
("a normal plain answer", True),
|
||||
# Unsalvageable: the answer is nothing but a control token -> rejected.
|
||||
("<|im_end|>", False),
|
||||
("<|eot_id|>", False),
|
||||
("<end_of_turn>", False),
|
||||
# Bare tool-call tag: already rejected by the existing gate.
|
||||
("<tool_call>{}</tool_call>", False),
|
||||
],
|
||||
)
|
||||
def test_clean_gate_salvages_or_rejects(final_answer: str, clean: bool) -> None:
|
||||
assert _target_is_clean(_roll(final_answer)) is clean
|
||||
@@ -1,408 +0,0 @@
|
||||
"""Offline (no-GPU) tests for OrchestratorGRPOTrainer._train_epoch.
|
||||
|
||||
We never touch torch/model/network: ``load_grpo_prompts`` is monkeypatched
|
||||
to return a handful of fake Tasks, and ``_grpo_step`` is monkeypatched to
|
||||
return canned metrics. The test asserts ``_train_epoch`` chunks the prompt
|
||||
pool correctly and aggregates per-chunk metrics.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
|
||||
import pytest
|
||||
|
||||
import openjarvis.learning.intelligence.orchestrator.policy_model as policy_model_mod
|
||||
import openjarvis.learning.intelligence.orchestrator.sft_data.datasets as datasets
|
||||
from openjarvis.learning.intelligence.orchestrator.grpo_trainer import (
|
||||
OrchestratorGRPOConfig,
|
||||
OrchestratorGRPOTrainer,
|
||||
RolloutRecord,
|
||||
)
|
||||
from openjarvis.learning.intelligence.orchestrator.policy_model import (
|
||||
OrchestratorPolicyModel,
|
||||
)
|
||||
|
||||
|
||||
@dataclass
|
||||
class FakeTask:
|
||||
task_id: str
|
||||
question: str
|
||||
answer: str
|
||||
domain: str = "math"
|
||||
|
||||
@property
|
||||
def instruction(self) -> str:
|
||||
return self.question
|
||||
|
||||
|
||||
def _make_tasks(n: int):
|
||||
return [
|
||||
FakeTask(task_id=f"t{i}", question=f"q{i}", answer=str(i))
|
||||
for i in range(n)
|
||||
]
|
||||
|
||||
|
||||
def _build_trainer(monkeypatch, n_prompts: int, prompts_per_step: int):
|
||||
"""Build a trainer with no real model and patched data loading.
|
||||
|
||||
``from_pretrained`` is stubbed to a model-less policy so construction
|
||||
touches no GPU / network even when torch happens to be installed.
|
||||
"""
|
||||
tasks = _make_tasks(n_prompts)
|
||||
monkeypatch.setattr(datasets, "load_grpo_prompts", lambda *a, **k: tasks)
|
||||
monkeypatch.setattr(
|
||||
policy_model_mod.OrchestratorPolicyModel,
|
||||
"from_pretrained",
|
||||
classmethod(lambda cls, *a, **k: OrchestratorPolicyModel(model=None)),
|
||||
)
|
||||
|
||||
cfg = OrchestratorGRPOConfig(
|
||||
grpo_max_prompts=n_prompts,
|
||||
prompts_per_step=prompts_per_step,
|
||||
num_samples_per_prompt=2,
|
||||
)
|
||||
trainer = OrchestratorGRPOTrainer(cfg)
|
||||
return trainer, tasks
|
||||
|
||||
|
||||
def test_train_epoch_chunks_and_aggregates(monkeypatch):
|
||||
trainer, tasks = _build_trainer(monkeypatch, n_prompts=10, prompts_per_step=4)
|
||||
|
||||
calls = []
|
||||
|
||||
def fake_step(chunk):
|
||||
calls.append(len(chunk))
|
||||
return {
|
||||
"loss": 1.0,
|
||||
"reward": 0.5,
|
||||
"accuracy": 1.0,
|
||||
"n_prompts": len(chunk),
|
||||
}
|
||||
|
||||
monkeypatch.setattr(trainer, "_grpo_step", fake_step)
|
||||
|
||||
metrics = trainer._train_epoch(epoch=0)
|
||||
|
||||
# 10 prompts / 4 per step => chunks of [4, 4, 2]
|
||||
assert calls == [4, 4, 2]
|
||||
assert metrics["n_batches"] == 3
|
||||
assert metrics["n_prompts"] == 10
|
||||
# Weighted means: all chunks report the same per-prompt values.
|
||||
assert abs(metrics["loss"] - 1.0) < 1e-9
|
||||
assert abs(metrics["reward"] - 0.5) < 1e-9
|
||||
assert abs(metrics["accuracy"] - 1.0) < 1e-9
|
||||
assert metrics["epoch"] == 0
|
||||
|
||||
|
||||
def test_train_epoch_exact_division(monkeypatch):
|
||||
trainer, _ = _build_trainer(monkeypatch, n_prompts=8, prompts_per_step=4)
|
||||
chunks = []
|
||||
monkeypatch.setattr(
|
||||
trainer,
|
||||
"_grpo_step",
|
||||
lambda chunk: (
|
||||
chunks.append(len(chunk)),
|
||||
{"loss": 0.0, "reward": 0.0, "accuracy": 0.0, "n_prompts": len(chunk)},
|
||||
)[1],
|
||||
)
|
||||
metrics = trainer._train_epoch(epoch=2)
|
||||
assert chunks == [4, 4]
|
||||
assert metrics["n_batches"] == 2
|
||||
assert metrics["n_prompts"] == 8
|
||||
assert metrics["epoch"] == 2
|
||||
|
||||
|
||||
def test_train_epoch_weighted_mean(monkeypatch):
|
||||
trainer, _ = _build_trainer(monkeypatch, n_prompts=6, prompts_per_step=4)
|
||||
|
||||
# First chunk (4 prompts) reward=1.0, second chunk (2 prompts) reward=0.0
|
||||
seq = iter([1.0, 0.0])
|
||||
|
||||
def fake_step(chunk):
|
||||
return {
|
||||
"loss": 0.0,
|
||||
"reward": next(seq),
|
||||
"accuracy": 0.0,
|
||||
"n_prompts": len(chunk),
|
||||
}
|
||||
|
||||
monkeypatch.setattr(trainer, "_grpo_step", fake_step)
|
||||
metrics = trainer._train_epoch(epoch=0)
|
||||
# weighted: (1.0*4 + 0.0*2) / 6 = 0.6667
|
||||
assert abs(metrics["reward"] - (4.0 / 6.0)) < 1e-9
|
||||
|
||||
|
||||
def test_prompts_cached(monkeypatch):
|
||||
trainer, _ = _build_trainer(monkeypatch, n_prompts=4, prompts_per_step=4)
|
||||
n_loads = {"count": 0}
|
||||
|
||||
orig = datasets.load_grpo_prompts
|
||||
|
||||
def counting(*a, **k):
|
||||
n_loads["count"] += 1
|
||||
return orig(*a, **k)
|
||||
|
||||
monkeypatch.setattr(datasets, "load_grpo_prompts", counting)
|
||||
monkeypatch.setattr(
|
||||
trainer,
|
||||
"_grpo_step",
|
||||
lambda chunk: {"loss": 0.0, "reward": 0.0, "accuracy": 0.0, "n_prompts": len(chunk)},
|
||||
)
|
||||
trainer._train_epoch(epoch=0)
|
||||
trainer._train_epoch(epoch=1)
|
||||
# Loaded once, cached on the trainer thereafter.
|
||||
assert n_loads["count"] == 1
|
||||
|
||||
|
||||
def test_train_epoch_no_model_still_iterates(monkeypatch):
|
||||
"""Even with policy.model None (no GPU), epoch iterates via _grpo_step."""
|
||||
trainer, _ = _build_trainer(monkeypatch, n_prompts=5, prompts_per_step=2)
|
||||
assert trainer.policy.model is None # CPU/no-torch stub
|
||||
called = {"n": 0}
|
||||
|
||||
def fake_step(chunk):
|
||||
called["n"] += 1
|
||||
return {"loss": 0.0, "reward": 0.0, "accuracy": 0.0, "n_prompts": len(chunk)}
|
||||
|
||||
monkeypatch.setattr(trainer, "_grpo_step", fake_step)
|
||||
metrics = trainer._train_epoch(epoch=0)
|
||||
assert called["n"] == 3 # [2, 2, 1]
|
||||
assert metrics["n_prompts"] == 5
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# _grpo_step over the unified rollout (offline, fake model + patched paths)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
torch = pytest.importorskip("torch")
|
||||
|
||||
|
||||
class _FakeModel:
|
||||
"""Stand-in for the HF policy/ref model: trainable enough to backprop.
|
||||
|
||||
``parameters()`` returns no params so the NaN-grad scan and
|
||||
``clip_grad_norm_`` are no-ops; ``train()`` is a no-op. The real
|
||||
log-prob / generation paths are monkeypatched, so the model is never
|
||||
actually called.
|
||||
"""
|
||||
|
||||
def parameters(self):
|
||||
return iter(())
|
||||
|
||||
def train(self):
|
||||
return self
|
||||
|
||||
|
||||
class _FakeOptimizer:
|
||||
def zero_grad(self):
|
||||
pass
|
||||
|
||||
def step(self):
|
||||
pass
|
||||
|
||||
|
||||
def _build_step_trainer(monkeypatch, *, group_size=4):
|
||||
"""Trainer wired for a CPU ``_grpo_step`` run: fake model + optimizer,
|
||||
device='cpu', and the rollout/logprob paths left for the test to patch.
|
||||
"""
|
||||
tasks = _make_tasks(3)
|
||||
monkeypatch.setattr(datasets, "load_grpo_prompts", lambda *a, **k: tasks)
|
||||
monkeypatch.setattr(
|
||||
policy_model_mod.OrchestratorPolicyModel,
|
||||
"from_pretrained",
|
||||
classmethod(lambda cls, *a, **k: OrchestratorPolicyModel(model=None)),
|
||||
)
|
||||
cfg = OrchestratorGRPOConfig(
|
||||
grpo_max_prompts=3,
|
||||
prompts_per_step=3,
|
||||
num_samples_per_prompt=group_size,
|
||||
)
|
||||
trainer = OrchestratorGRPOTrainer(cfg)
|
||||
# Make it look like a real (CPU) training setup.
|
||||
trainer.policy.model = _FakeModel()
|
||||
trainer.ref_policy.model = _FakeModel()
|
||||
trainer.optimizer = _FakeOptimizer()
|
||||
trainer.device = torch.device("cpu")
|
||||
return trainer, tasks
|
||||
|
||||
|
||||
def test_grpo_step_uses_unified_rollout_not_old_env(monkeypatch):
|
||||
"""_grpo_step samples groups via the policy rollout caller, normalises
|
||||
advantages within the group, and never touches the old
|
||||
THOUGHT/TOOL/INPUT environment path."""
|
||||
|
||||
# Tripwire: the old environment generation path must NOT be invoked.
|
||||
import openjarvis.learning.intelligence.orchestrator.policy_model as pm
|
||||
|
||||
def _boom(*a, **k): # pragma: no cover - only fires on regression
|
||||
raise AssertionError(
|
||||
"GRPO generation hit the old THOUGHT/TOOL/INPUT path "
|
||||
"(policy_model._parse_output / predict_action)"
|
||||
)
|
||||
|
||||
monkeypatch.setattr(pm.OrchestratorPolicyModel, "_parse_output", _boom)
|
||||
monkeypatch.setattr(pm.OrchestratorPolicyModel, "predict_action", _boom)
|
||||
monkeypatch.setattr(pm.OrchestratorPolicyModel, "_build_prompt", _boom)
|
||||
|
||||
trainer, tasks = _build_step_trainer(monkeypatch, group_size=4)
|
||||
|
||||
# verify_answer: even-indexed rollouts correct, odd incorrect -> spread of
|
||||
# rewards inside each group so std > 0 and advantages get normalised.
|
||||
import openjarvis.learning.intelligence.orchestrator.sft_data.verify as verify_mod
|
||||
|
||||
seen_answers = []
|
||||
|
||||
def fake_verify(task, pred):
|
||||
seen_answers.append(pred)
|
||||
return pred.endswith("correct")
|
||||
|
||||
monkeypatch.setattr(verify_mod, "verify_answer", fake_verify)
|
||||
|
||||
def fake_rollout_group(task):
|
||||
recs = []
|
||||
for i in range(trainer.config.num_samples_per_prompt):
|
||||
ans = f"ans-{i}-correct" if i % 2 == 0 else f"ans-{i}-wrong"
|
||||
recs.append(
|
||||
RolloutRecord(
|
||||
final_answer=ans,
|
||||
cost_usd=0.01 * (i + 1),
|
||||
token_ids=[1, 2, 3, 4],
|
||||
assistant_mask=[0, 1, 0, 1],
|
||||
)
|
||||
)
|
||||
return recs
|
||||
|
||||
monkeypatch.setattr(trainer, "_rollout_group", fake_rollout_group)
|
||||
|
||||
def fake_logprob(token_ids, assistant_mask, *, ref):
|
||||
# token_ids/mask flow through unchanged from the records.
|
||||
assert token_ids == [1, 2, 3, 4]
|
||||
assert assistant_mask == [0, 1, 0, 1]
|
||||
# Distinct, grad-bearing values for current vs ref so the ratio is
|
||||
# finite and backward() builds a real graph.
|
||||
base = -1.0 if ref else -0.9
|
||||
return torch.tensor(base, requires_grad=not ref)
|
||||
|
||||
monkeypatch.setattr(trainer, "_trajectory_logprob", fake_logprob)
|
||||
|
||||
metrics = trainer._grpo_step(tasks)
|
||||
|
||||
# Metrics aggregate over the 3 tasks.
|
||||
assert metrics["n_prompts"] == 3
|
||||
# Half of each group is correct -> accuracy 0.5.
|
||||
assert abs(metrics["accuracy"] - 0.5) < 1e-9
|
||||
assert "loss" in metrics and "reward" in metrics
|
||||
# The unified-rollout caller produced our canned answers (proves we went
|
||||
# through the rollout path, not the old env).
|
||||
assert any(a.endswith("correct") for a in seen_answers)
|
||||
assert any(a.endswith("wrong") for a in seen_answers)
|
||||
|
||||
|
||||
def test_grpo_step_group_normalised_advantages(monkeypatch):
|
||||
"""Advantages within a group are mean-0 / std-1 (group-relative)."""
|
||||
trainer, tasks = _build_step_trainer(monkeypatch, group_size=4)
|
||||
|
||||
import openjarvis.learning.intelligence.orchestrator.sft_data.verify as verify_mod
|
||||
|
||||
monkeypatch.setattr(
|
||||
verify_mod, "verify_answer", lambda task, pred: pred == "good"
|
||||
)
|
||||
|
||||
all_groups = []
|
||||
|
||||
def fake_rollout_group(task):
|
||||
# 2 correct, 2 wrong, varied costs -> non-degenerate reward spread.
|
||||
answers = ["good", "good", "bad", "bad"]
|
||||
recs = [
|
||||
RolloutRecord(
|
||||
final_answer=answers[i],
|
||||
cost_usd=0.0,
|
||||
token_ids=[5, 6, 7],
|
||||
assistant_mask=[0, 1, 1],
|
||||
)
|
||||
for i in range(4)
|
||||
]
|
||||
all_groups.append(recs)
|
||||
return recs
|
||||
|
||||
monkeypatch.setattr(trainer, "_rollout_group", fake_rollout_group)
|
||||
|
||||
def fake_logprob(token_ids, assistant_mask, *, ref):
|
||||
return torch.tensor(0.0, requires_grad=not ref)
|
||||
|
||||
monkeypatch.setattr(trainer, "_trajectory_logprob", fake_logprob)
|
||||
|
||||
trainer._grpo_step(tasks)
|
||||
|
||||
# 3 tasks, each a group of 4. Read the advantages _grpo_step stamped onto
|
||||
# each record; every group must be group-normalised (mean ~0, std ~1).
|
||||
assert len(all_groups) == 3
|
||||
for recs in all_groups:
|
||||
group = [r.advantage for r in recs]
|
||||
assert len(group) == 4
|
||||
mean = sum(group) / 4
|
||||
assert abs(mean) < 1e-6
|
||||
var = sum((x - mean) ** 2 for x in group) / 4
|
||||
assert abs(var - 1.0) < 1e-6
|
||||
|
||||
|
||||
def test_stitch_trajectory_assistant_mask():
|
||||
"""Stitching per-turn (prompt_ids, gen_ids) yields a correct mask: 1 only
|
||||
on the policy-generated tokens, growing prompts diffed across turns."""
|
||||
trainer = OrchestratorGRPOTrainer.__new__(OrchestratorGRPOTrainer)
|
||||
# Turn 1: prompt [10,11], gen [20,21]. Turn 2 prompt is the full prior
|
||||
# transcript [10,11,20,21] + a new observation token [12], gen [22].
|
||||
turn_buffer = [
|
||||
([10, 11], [20, 21]),
|
||||
([10, 11, 20, 21, 12], [22]),
|
||||
]
|
||||
token_ids, mask = trainer._stitch_trajectory(turn_buffer)
|
||||
# Turn1: new prompt [10,11] mask0, gen [20,21] mask1.
|
||||
# Turn2: new prompt [12] mask0 (the [10,11,20,21] already emitted), gen [22] mask1.
|
||||
assert token_ids == [10, 11, 20, 21, 12, 22]
|
||||
assert mask == [0, 0, 1, 1, 0, 1]
|
||||
|
||||
|
||||
def test_trajectory_logprob_masks_to_assistant_positions(monkeypatch):
|
||||
"""_trajectory_logprob sums log-probs only over assistant-masked tokens,
|
||||
using logits at position i-1 to predict token i."""
|
||||
trainer = OrchestratorGRPOTrainer.__new__(OrchestratorGRPOTrainer)
|
||||
trainer.device = torch.device("cpu")
|
||||
|
||||
token_ids = [3, 7, 5, 9]
|
||||
assistant_mask = [0, 1, 0, 1] # tokens at idx 1 and 3 are assistant
|
||||
|
||||
vocab = 10
|
||||
seq = len(token_ids)
|
||||
# Deterministic logits so the expected sum is computable.
|
||||
logits = torch.zeros(1, seq, vocab)
|
||||
for i in range(seq):
|
||||
logits[0, i, :] = torch.arange(vocab, dtype=torch.float32) * 0.1 * (i + 1)
|
||||
|
||||
class _LogitModel:
|
||||
def __call__(self, input_ids=None):
|
||||
class _Out:
|
||||
pass
|
||||
|
||||
o = _Out()
|
||||
o.logits = logits
|
||||
return o
|
||||
|
||||
def parameters(self):
|
||||
return iter(())
|
||||
|
||||
trainer.policy = OrchestratorPolicyModel(model=_LogitModel())
|
||||
trainer.ref_policy = OrchestratorPolicyModel(model=_LogitModel())
|
||||
|
||||
out = trainer._trajectory_logprob(token_ids, assistant_mask, ref=True)
|
||||
|
||||
# Expected: log_softmax(logits[i-1])[token_i] summed over masked i in {1,3}.
|
||||
expected = 0.0
|
||||
for i in range(1, seq):
|
||||
if assistant_mask[i] != 1:
|
||||
continue
|
||||
lp = torch.log_softmax(logits[0, i - 1, :], dim=-1)[token_ids[i]]
|
||||
expected += lp.item()
|
||||
assert abs(out.item() - expected) < 1e-5
|
||||
Reference in New Issue
Block a user