From a689be0c69a97b8bde567010d872f0cfbfb47fe4 Mon Sep 17 00:00:00 2001 From: Avanika Narayan Date: Tue, 5 May 2026 19:28:50 -0700 Subject: [PATCH] fix(evals): clean up AI stack runtime harness (#320) Co-authored-by: krypticmouse --- scripts/experiments/a1_seed_feedback.py | 246 ------ .../generate_distillation_configs.py | 304 ------- scripts/experiments/m2_collect_results.py | 133 --- .../m2_create_distilled_configs.py | 307 ------- scripts/experiments/m2_run_distilled_evals.sh | 183 ---- scripts/experiments/m3_hill_climb.py | 581 ------------- .../run_distillation_experiments.sh | 307 ------- scripts/experiments/run_track_b_gepa_dspy.sh | 375 --------- scripts/experiments/run_track_d_lora_sft.sh | 778 ------------------ scripts/oauth_all.py | 40 +- scripts/smoke_framework_comparison.sh | 34 +- .../external/_runners/openclaw_runner.mjs | 40 +- src/openjarvis/evals/comparison/table_gen.py | 9 + src/openjarvis/evals/core/runner.py | 40 +- .../test_export_to_table_gen_roundtrip.py | 3 +- .../test_openclaw_runner_contract.py | 72 ++ .../test_runner_error_path_framework.py | 116 +++ tests/evals/comparison/test_table_gen.py | 18 + 18 files changed, 343 insertions(+), 3243 deletions(-) delete mode 100644 scripts/experiments/a1_seed_feedback.py delete mode 100644 scripts/experiments/generate_distillation_configs.py delete mode 100644 scripts/experiments/m2_collect_results.py delete mode 100644 scripts/experiments/m2_create_distilled_configs.py delete mode 100644 scripts/experiments/m2_run_distilled_evals.sh delete mode 100644 scripts/experiments/m3_hill_climb.py delete mode 100755 scripts/experiments/run_distillation_experiments.sh delete mode 100644 scripts/experiments/run_track_b_gepa_dspy.sh delete mode 100644 scripts/experiments/run_track_d_lora_sft.sh create mode 100644 tests/evals/comparison/test_openclaw_runner_contract.py diff --git a/scripts/experiments/a1_seed_feedback.py b/scripts/experiments/a1_seed_feedback.py deleted file mode 100644 index c6565d92..00000000 --- a/scripts/experiments/a1_seed_feedback.py +++ /dev/null @@ -1,246 +0,0 @@ -#!/usr/bin/env python3 -"""A1: Seed feedback on all traces in the local M1 traces.db. - -- Judges each unscored trace with Sonnet 4.6 using the calibration-validated prompt -- Parallelized via ThreadPoolExecutor (8 workers) for I/O-bound API calls -- Writes feedback to local traces.db via TraceStore.update_feedback -- Logs every call to a JSONL audit file -- Idempotent: skips traces that already have feedback (safe to re-run) -""" - -from __future__ import annotations - -import json -import os -import re -import sqlite3 -import sys -import threading -import time -from concurrent.futures import ThreadPoolExecutor, as_completed -from datetime import datetime -from pathlib import Path - -from openjarvis.core.types import Message, Role -from openjarvis.engine.cloud import CloudEngine -from openjarvis.traces.store import TraceStore - -HOME = Path(os.environ.get("OPENJARVIS_HOME", "/scratch/user/jonsaadfalcon/openjarvis-m1")) -DB = HOME / "traces.db" -LOG = HOME / "a1_feedback_log.jsonl" -MODEL = "claude-sonnet-4-6" -MAX_WORKERS = 8 - -JUDGE_PROMPT = """\ -You are evaluating whether an AI agent successfully completed its assigned task. - -Assign a SCORE from the set {{0.2, 0.4, 0.6, 0.8}} using this rubric: - -- 0.8 = Clean success. Task completed correctly. Minor stylistic issues don't affect correctness. -- 0.6 = Partial. Real progress made but the answer has real gaps — missed requirement, incomplete output, recovered from errors but final result imperfect. -- 0.4 = Poor. Some progress but the result is clearly incomplete, wrong, or the agent got mostly stuck. -- 0.2 = Failure. Agent crashed, got stuck in a loop, gave up, hit a budget/poll/token limit before finishing, or produced no usable result. - -IMPORTANT: Do not trust the agent's own self-assessment. Agents often narrate "I am stuck" or "I hit an error" — those are failure signals. Agents sometimes claim success when the actual output is incomplete — look at the concrete result, not the rhetoric. - -TASK QUERY (first 1200 chars): -<<< -{query} ->>> - -AGENT FINAL RESULT (first 2500 chars): -<<< -{result_head} ->>> -{tail_section} -Respond in EXACTLY this format, nothing else: -SCORE= -REASON= -""" - - -SCORE_RE = re.compile(r"SCORE\s*=\s*(0?\.[2468])", re.IGNORECASE) -REASON_RE = re.compile(r"REASON\s*=\s*(.+?)\s*$", re.IGNORECASE | re.DOTALL) - - -def build_prompt(query: str, result: str) -> str: - q = (query or "")[:1200] - head = (result or "")[:2500] - if result and len(result) > 3000: - tail = f"\nAGENT FINAL RESULT (last 500 chars):\n<<<\n{result[-500:]}\n>>>\n" - else: - tail = "" - return JUDGE_PROMPT.format(query=q, result_head=head, tail_section=tail) - - -def judge_one(ce: CloudEngine, trace_id: str, query: str, result: str) -> dict: - prompt = build_prompt(query, result) - t0 = time.time() - try: - resp = ce.generate( - messages=[Message(role=Role.USER, content=prompt)], - model=MODEL, - max_tokens=150, - temperature=0.0, - ) - content = resp.get("content", "") or "" - cost = resp.get("cost_usd", 0.0) or 0.0 - usage = resp.get("usage", {}) or {} - in_tok = usage.get("prompt_tokens", 0) or usage.get("input_tokens", 0) - out_tok = usage.get("completion_tokens", 0) or usage.get("output_tokens", 0) - - m = SCORE_RE.search(content) - score = float(m.group(1)) if m else None - mr = REASON_RE.search(content) - reason = mr.group(1).strip() if mr else "(parse failed)" - - return { - "trace_id": trace_id, - "score": score, - "reason": reason, - "raw": content, - "cost": cost, - "input_tokens": in_tok, - "output_tokens": out_tok, - "elapsed": time.time() - t0, - "judged_at": datetime.utcnow().isoformat(timespec="seconds") + "Z", - "error": None, - } - except Exception as e: - return { - "trace_id": trace_id, - "score": None, - "reason": None, - "raw": None, - "cost": 0.0, - "input_tokens": 0, - "output_tokens": 0, - "elapsed": time.time() - t0, - "judged_at": datetime.utcnow().isoformat(timespec="seconds") + "Z", - "error": f"{type(e).__name__}: {e}", - } - - -def main() -> int: - if not os.environ.get("ANTHROPIC_API_KEY"): - print("ERROR: ANTHROPIC_API_KEY not set", file=sys.stderr) - return 1 - - store = TraceStore(DB) - # Use the TraceStore's connection for the initial read - conn = store._conn - conn.row_factory = sqlite3.Row - - # Pull all traces that need scoring - rows = list(conn.execute( - "SELECT trace_id, query, result, agent, model FROM traces " - "WHERE feedback IS NULL" - )) - already_scored = conn.execute( - "SELECT COUNT(*) FROM traces WHERE feedback IS NOT NULL" - ).fetchone()[0] - - print(f"traces.db: {conn.execute('SELECT COUNT(*) FROM traces').fetchone()[0]} total") - print(f" already scored: {already_scored}") - print(f" to judge: {len(rows)}") - print(f"parallelism: {MAX_WORKERS} workers") - print(f"log: {LOG}") - - if not rows: - print("Nothing to do.") - return 0 - - ce = CloudEngine() - write_lock = threading.Lock() - log_lock = threading.Lock() - log_fp = open(LOG, "a", encoding="utf-8") - - # Write a header line marking this run - log_fp.write(json.dumps({ - "_run_started": datetime.utcnow().isoformat(timespec="seconds") + "Z", - "model": MODEL, - "workers": MAX_WORKERS, - "to_judge": len(rows), - "already_scored": already_scored, - }) + "\n") - log_fp.flush() - - total_cost = 0.0 - done = 0 - errors = 0 - score_counts: dict = {} - t_start = time.time() - - def on_result(res: dict) -> None: - nonlocal total_cost, done, errors - trace_id = res["trace_id"] - - # Write feedback to DB (if we got a valid score) - if res["score"] is not None and res["error"] is None: - with write_lock: - store.update_feedback(trace_id, res["score"]) - conn.commit() - else: - errors += 1 - - with log_lock: - log_fp.write(json.dumps(res) + "\n") - log_fp.flush() - - total_cost += res["cost"] or 0.0 - done += 1 - score_counts[res["score"]] = score_counts.get(res["score"], 0) + 1 - - # Progress every 50 or on error - if done % 50 == 0 or res["error"]: - elapsed = time.time() - t_start - rate = done / max(0.001, elapsed) - eta = (len(rows) - done) / max(0.001, rate) - tag = "ERR " if res["error"] else " " - print( - f"{tag}[{done:4}/{len(rows)}] score={res['score']} " - f"cost=${total_cost:.3f} " - f"rate={rate:.1f}/s eta={eta:.0f}s " - f"errors={errors}" - ) - if res["error"]: - print(f" ERROR on {trace_id[:12]}: {res['error']}") - - with ThreadPoolExecutor(max_workers=MAX_WORKERS) as pool: - futures = [ - pool.submit(judge_one, ce, r["trace_id"], r["query"] or "", r["result"] or "") - for r in rows - ] - for fut in as_completed(futures): - res = fut.result() - on_result(res) - - log_fp.close() - - elapsed = time.time() - t_start - print(f"\n{'='*60}") - print(f"A1 COMPLETE in {elapsed:.1f}s ({elapsed/60:.1f}min)") - print(f"Total cost: ${total_cost:.4f}") - print(f"Errors: {errors}/{len(rows)}") - print(f"Score distribution:") - for k in sorted(score_counts.keys(), key=lambda x: (x is None, x)): - v = score_counts[k] - pct = 100 * v / len(rows) - label = {0.2: "failure", 0.4: "poor", 0.6: "partial", 0.8: "clean", None: "ERROR"}.get(k, "?") - print(f" {k} ({label}): {v} ({pct:.1f}%)") - - # Verify by re-counting from DB - with_fb = conn.execute( - "SELECT COUNT(*) FROM traces WHERE feedback IS NOT NULL" - ).fetchone()[0] - above_gate = conn.execute( - "SELECT COUNT(*) FROM traces WHERE feedback >= 0.7" - ).fetchone()[0] - print(f"\nPost-A1 DB state:") - print(f" traces with feedback: {with_fb}") - print(f" traces passing 0.7 gate (eligible for personal benchmark): {above_gate}") - return 0 if errors == 0 else 2 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/scripts/experiments/generate_distillation_configs.py b/scripts/experiments/generate_distillation_configs.py deleted file mode 100644 index 1ef98274..00000000 --- a/scripts/experiments/generate_distillation_configs.py +++ /dev/null @@ -1,304 +0,0 @@ -#!/usr/bin/env python3 -"""Generate all distillation experiment TOML configs. - -Produces configs for 7 experiment axes × multiple settings. -Run: python scripts/experiments/generate_distillation_configs.py -""" - -from __future__ import annotations - -import itertools -from pathlib import Path - -CONFIGS_DIR = Path("src/openjarvis/evals/configs/distillation") - -# ── Teacher models ────────────────────────────────────────────────────────── -TEACHERS = { - "opus": { - "model": "claude-opus-4-6", - "engine": "cloud", - "provider": "anthropic", - }, - "gpt54": { - "model": "gpt-5.4", - "engine": "cloud", - "provider": "openai", - }, - "gemini": { - "model": "gemini-3.1-pro-preview", - "engine": "cloud", - "provider": "google", - }, - "qwen397b": { - "model": "Qwen/Qwen3.5-397B-A17B-FP8", - "engine": "vllm", - "provider": "local", - "note": "# Requires 8×H100, vLLM serve on port 8010", - }, -} - -# ── Student models ────────────────────────────────────────────────────────── -# Served via vLLM on this H100 node. 27B uses the FP8 weights that fit on a -# single H100; 2B and 9B use standard FP16. -STUDENTS = { - "2b": {"model": "Qwen/Qwen3.5-2B", "engine": "vllm", "port": 8000}, - "9b": {"model": "Qwen/Qwen3.5-9B", "engine": "vllm", "port": 8001}, - "27b": {"model": "Qwen/Qwen3.5-27B-FP8", "engine": "vllm", "port": 8002}, -} - -# ── Benchmarks ────────────────────────────────────────────────────────────── -BENCHMARKS = { - "pb": "pinchbench", - "tc15": "toolcall15", - "tb": "taubench", -} - -# ── Data configs ──────────────────────────────────────────────────────────── -DATA_CONFIGS = { - "C1": { - "desc": "Zero test data — external traces only (GeneralThought + ADP)", - "trace_source": "external", - "benchmark_queries_visible": False, - }, - "C2": { - "desc": "Test queries only — benchmark traces visible, answers hidden", - "trace_source": "benchmark", - "benchmark_queries_visible": True, - }, - "C3": { - "desc": "Test queries + external — both benchmark and external traces", - "trace_source": "both", - "benchmark_queries_visible": True, - }, -} - -# ── Budget presets ────────────────────────────────────────────────────────── -BUDGETS = { - "minimal": {"max_tool_calls": 5, "max_cost": 0.50}, - "standard": {"max_tool_calls": 15, "max_cost": 2.00}, - "thorough": {"max_tool_calls": 30, "max_cost": 5.00}, - "exhaustive": {"max_tool_calls": 50, "max_cost": 10.00}, -} - -# ── Gate presets ──────────────────────────────────────────────────────────── -GATES = { - "permissive": {"min_improvement": 0.0, "max_regression": 0.10}, - "standard": {"min_improvement": 0.0, "max_regression": 0.05}, - "strict": {"min_improvement": 0.02, "max_regression": 0.02}, - "none": {"min_improvement": -1.0, "max_regression": 1.0}, -} - -# ── Autonomy presets ──────────────────────────────────────────────────────── -AUTONOMY_MODES = ["auto", "tiered", "manual"] - - -def render_config( - *, - experiment: str, - teacher_key: str, - student_key: str, - benchmark_key: str, - data_config_key: str = "C2", - budget_key: str = "standard", - gate_key: str = "standard", - autonomy: str = "auto", - iterative_sessions: int = 1, -) -> str: - """Render a TOML config string.""" - teacher = TEACHERS[teacher_key] - student = STUDENTS[student_key] - benchmark = BENCHMARKS[benchmark_key] - data_cfg = DATA_CONFIGS[data_config_key] - budget = BUDGETS[budget_key] - gate = GATES[gate_key] - - note = teacher.get("note", "") - note_line = f"\n{note}" if note else "" - - return f"""\ -# Distillation Experiment Config -# Experiment: {experiment} -# Teacher: {teacher["model"]} ({teacher["provider"]}) -# Student: {student["model"]} ({student["engine"]}) -# Benchmark: {benchmark} -# Data config: {data_config_key} — {data_cfg["desc"]} -# Budget: {budget_key} ({budget["max_tool_calls"]} tool calls, ${budget["max_cost"]:.2f}) -# Gate: {gate_key} (min_improvement={gate["min_improvement"]}, max_regression={gate["max_regression"]}) -# Autonomy: {autonomy} -# Iterative sessions: {iterative_sessions} -{note_line} - -[intelligence] -default_model = "{student["model"]}" - -[engine] -default = "{student["engine"]}" - -[engine.vllm] -host = "http://localhost:{student.get("port", 8000)}" - -[learning.distillation] -enabled = true -autonomy_mode = "{autonomy}" -teacher_model = "{teacher["model"]}" -max_cost_per_session_usd = {budget["max_cost"]} -max_tool_calls_per_diagnosis = {budget["max_tool_calls"]} - -[learning.distillation.gate] -min_improvement = {gate["min_improvement"]} -max_regression = {gate["max_regression"]} -benchmark_subsample_size = 50 - -[learning.distillation.benchmark] -synthesis_feedback_threshold = 0.7 -max_benchmark_size = 200 - -[learning.distillation.experiment] -# Metadata for the experiment runner (not read by distillation itself) -experiment_id = "{experiment}" -teacher_key = "{teacher_key}" -student_key = "{student_key}" -benchmark = "{benchmark}" -data_config = "{data_config_key}" -trace_source = "{data_cfg["trace_source"]}" -benchmark_queries_visible = {str(data_cfg["benchmark_queries_visible"]).lower()} -budget_key = "{budget_key}" -gate_key = "{gate_key}" -iterative_sessions = {iterative_sessions} -""" - - -def write_config(subdir: str, filename: str, content: str) -> Path: - path = CONFIGS_DIR / subdir / filename - path.parent.mkdir(parents=True, exist_ok=True) - path.write_text(content, encoding="utf-8") - return path - - -def generate_all() -> int: - count = 0 - - # ── Exp 1a: Teacher Model Ablation ──────────────────────────────────── - # Fix: S-9b, B-standard, A-auto, G-standard, I-single - # Vary: teacher × benchmark × data_config - for teacher_key, bench_key, dc_key in itertools.product( - TEACHERS, BENCHMARKS, DATA_CONFIGS - ): - filename = f"{teacher_key}-9b-{bench_key}-{dc_key}.toml" - content = render_config( - experiment=f"exp1a-teacher/{teacher_key}-{bench_key}-{dc_key}", - teacher_key=teacher_key, - student_key="9b", - benchmark_key=bench_key, - data_config_key=dc_key, - ) - write_config("exp1a-teacher", filename, content) - count += 1 - - # ── Exp 1b: Budget Ablation ─────────────────────────────────────────── - # Fix: S-9b, T-sonnet(opus for quality), A-auto, G-standard, I-single - # Vary: budget × benchmark × data_config - for budget_key, bench_key, dc_key in itertools.product( - BUDGETS, BENCHMARKS, DATA_CONFIGS - ): - filename = f"{budget_key}-9b-{bench_key}-{dc_key}.toml" - content = render_config( - experiment=f"exp1b-budget/{budget_key}-{bench_key}-{dc_key}", - teacher_key="opus", - student_key="9b", - benchmark_key=bench_key, - data_config_key=dc_key, - budget_key=budget_key, - ) - write_config("exp1b-budget", filename, content) - count += 1 - - # ── Exp 1c: Student Model Scaling ───────────────────────────────────── - # Fix: T-opus, B-standard, A-auto, G-standard, I-single - # Vary: student × benchmark × data_config - for student_key, bench_key, dc_key in itertools.product( - STUDENTS, BENCHMARKS, DATA_CONFIGS - ): - filename = f"opus-{student_key}-{bench_key}-{dc_key}.toml" - content = render_config( - experiment=f"exp1c-student/opus-{student_key}-{bench_key}-{dc_key}", - teacher_key="opus", - student_key=student_key, - benchmark_key=bench_key, - data_config_key=dc_key, - ) - write_config("exp1c-student", filename, content) - count += 1 - - # ── Exp 2a: Gate Strictness ─────────────────────────────────────────── - # Fix: S-9b, T-opus, B-standard, A-auto, I-single - # Vary: gate × benchmark - for gate_key, bench_key in itertools.product(GATES, BENCHMARKS): - filename = f"{gate_key}-9b-{bench_key}.toml" - content = render_config( - experiment=f"exp2a-gate/{gate_key}-{bench_key}", - teacher_key="opus", - student_key="9b", - benchmark_key=bench_key, - gate_key=gate_key, - ) - write_config("exp2a-gate", filename, content) - count += 1 - - # ── Exp 2b: Autonomy Mode ──────────────────────────────────────────── - # Fix: S-9b, T-opus, B-standard, G-standard, I-single - # Vary: autonomy × benchmark - for autonomy, bench_key in itertools.product(AUTONOMY_MODES, BENCHMARKS): - filename = f"{autonomy}-9b-{bench_key}.toml" - content = render_config( - experiment=f"exp2b-autonomy/{autonomy}-{bench_key}", - teacher_key="opus", - student_key="9b", - benchmark_key=bench_key, - autonomy=autonomy, - ) - write_config("exp2b-autonomy", filename, content) - count += 1 - - # ── Exp 3a: Iterative Sessions ─────────────────────────────────────── - # Fix: S-9b, T-opus, B-standard, A-auto, G-standard - # Vary: number of chained sessions × benchmark - for n_sessions, bench_key in itertools.product([1, 3, 5], BENCHMARKS): - filename = f"iter{n_sessions}-9b-{bench_key}.toml" - content = render_config( - experiment=f"exp3a-iterative/iter{n_sessions}-{bench_key}", - teacher_key="opus", - student_key="9b", - benchmark_key=bench_key, - iterative_sessions=n_sessions, - ) - write_config("exp3a-iterative", filename, content) - count += 1 - - # ── Exp 3b: Cross-Benchmark Transfer ───────────────────────────────── - # Optimize using traces from benchmark X, eval on benchmark Y - for opt_bench, eval_bench in itertools.permutations(BENCHMARKS, 2): - filename = f"opt-{opt_bench}-eval-{eval_bench}-9b.toml" - content = render_config( - experiment=f"exp3b-transfer/opt-{opt_bench}-eval-{eval_bench}", - teacher_key="opus", - student_key="9b", - benchmark_key=opt_bench, # Traces from this benchmark - ) - # Add eval benchmark as metadata - content += f'\neval_benchmark = "{BENCHMARKS[eval_bench]}"\n' - write_config("exp3b-transfer", filename, content) - count += 1 - - return count - - -if __name__ == "__main__": - n = generate_all() - print(f"Generated {n} config files in {CONFIGS_DIR}/") - - # Print summary - for subdir in sorted(CONFIGS_DIR.iterdir()): - if subdir.is_dir(): - files = list(subdir.glob("*.toml")) - print(f" {subdir.name}/: {len(files)} configs") diff --git a/scripts/experiments/m2_collect_results.py b/scripts/experiments/m2_collect_results.py deleted file mode 100644 index 41319399..00000000 --- a/scripts/experiments/m2_collect_results.py +++ /dev/null @@ -1,133 +0,0 @@ -#!/usr/bin/env python3 -"""M2: Collect distilled eval results and produce comparison table. - -Reads .summary.json files from results/neurips-2026/{distilled,baselines}/ -and produces a before/after comparison against the Step 1 baseline numbers. -""" - -from __future__ import annotations - -import json -import sys -from pathlib import Path - -# Jon's Step 1 baselines — used when a local baseline result doesn't exist -STEP1_BASELINES = { - "2b": {"toolcall15": 33.3, "pinchbench": 69.6, "livecodebench": 5.6, "taubench": 70.0, "taubench-telecom": 0.0, "gaia": 0.0, "liveresearch": 0.0, "liveresearchbench": None}, - "9b": {"toolcall15": 46.7, "pinchbench": 95.7, "livecodebench": 17.6, "taubench": 85.0, "taubench-telecom": 80.0, "gaia": 38.0, "liveresearch": 75.0, "liveresearchbench": None}, - "27b": {"toolcall15": 40.0, "pinchbench": 65.2, "livecodebench": 20.0, "taubench": 75.0, "taubench-telecom": 75.0, "gaia": 48.0, "liveresearch": 66.7, "liveresearchbench": None}, -} - -# Which benchmarks go through the agent layer (where distillation edits actually apply) -AGENT_BENCHMARKS = {"pinchbench", "gaia", "liveresearch"} - -DISTILLED_ROOT = Path("results/neurips-2026/distilled") -BASELINE_ROOT = Path("results/neurips-2026/baselines") - - -def find_summary(root: Path, size: str, bench: str) -> Path | None: - """Find the summary JSON for a model × benchmark run.""" - # Expected path: root/qwen-{size}/{bench}/{bench}_Qwen-Qwen3.5-{size}.summary.json - candidates = list(root.glob(f"qwen-{size}/{bench}/*.summary.json")) - return candidates[0] if candidates else None - - -def load_accuracy(summary_path: Path) -> float | None: - """Extract overall accuracy from a summary.json file.""" - try: - d = json.loads(summary_path.read_text()) - # The summary has various shapes; try a few - for key in ["overall_accuracy", "accuracy", "overall_score"]: - if key in d: - return float(d[key]) * 100 if d[key] <= 1.0 else float(d[key]) - # Try nested - if "results" in d: - for r in d["results"]: - if "accuracy" in r: - return float(r["accuracy"]) * 100 if r["accuracy"] <= 1.0 else float(r["accuracy"]) - except Exception as e: - print(f" error reading {summary_path}: {e}", file=sys.stderr) - return None - - -def main() -> int: - print("=" * 100) - print("M2 Distilled vs Baseline Comparison") - print("=" * 100) - print() - print(f"{'Model':8} {'Benchmark':20} {'Baseline':>10} {'Distilled':>10} {'Delta':>10} {'Agent?':>10}") - print("-" * 100) - - benchmarks = [ - "toolcall15", "pinchbench", "livecodebench", - "taubench", "taubench-telecom", - "gaia", "liveresearch", "liveresearchbench", - ] - - summary_rows = [] - for size in ["2b", "9b", "27b"]: - for bench in benchmarks: - # Baseline: prefer local file (if run this session), fall back to Step 1 numbers - baseline_path = find_summary(BASELINE_ROOT, size, bench) - if baseline_path: - baseline = load_accuracy(baseline_path) - base_source = "local" - else: - baseline = STEP1_BASELINES[size].get(bench) - base_source = "step1" - - # Distilled: must be local from this session - distilled_path = find_summary(DISTILLED_ROOT, size, bench) - distilled = load_accuracy(distilled_path) if distilled_path else None - - # Format - b_str = f"{baseline:.1f}%" if baseline is not None else "—" - d_str = f"{distilled:.1f}%" if distilled is not None else "pending" - if baseline is not None and distilled is not None: - delta = distilled - baseline - d_sign = "+" if delta >= 0 else "" - delta_str = f"{d_sign}{delta:.1f}%" - else: - delta_str = "—" - agent = "AGENT" if bench in AGENT_BENCHMARKS else "direct" - - print(f"qwen-{size:4} {bench:20} {b_str:>10} {d_str:>10} {delta_str:>10} {agent:>10}") - summary_rows.append({ - "model": f"qwen-{size}", - "benchmark": bench, - "baseline": baseline, - "distilled": distilled, - "delta": distilled - baseline if (baseline is not None and distilled is not None) else None, - "agent_benchmark": bench in AGENT_BENCHMARKS, - }) - print() - - # Aggregate: agent vs direct benchmark deltas - print("=" * 100) - print("Aggregate deltas by benchmark type (paper finding)") - print("=" * 100) - agent_deltas = [r["delta"] for r in summary_rows if r["delta"] is not None and r["agent_benchmark"]] - direct_deltas = [r["delta"] for r in summary_rows if r["delta"] is not None and not r["agent_benchmark"]] - if agent_deltas: - mean_agent = sum(agent_deltas) / len(agent_deltas) - print(f"Agent benchmarks (PB, GAIA, DeepResearchBench): mean delta = {mean_agent:+.2f}% over {len(agent_deltas)} runs") - if direct_deltas: - mean_direct = sum(direct_deltas) / len(direct_deltas) - print(f"Direct benchmarks (TC15, TauB, TBTel, LRB, LCB): mean delta = {mean_direct:+.2f}% over {len(direct_deltas)} runs") - print() - - # Completion progress - expected = 24 - distilled_count = sum(1 for r in summary_rows if r["distilled"] is not None) - print(f"Distilled runs complete: {distilled_count}/{expected}") - - # Save JSON - out = Path("results/neurips-2026/distillation-m2/m2_comparison.json") - out.parent.mkdir(parents=True, exist_ok=True) - out.write_text(json.dumps(summary_rows, indent=2, default=str)) - print(f"Full data: {out}") - return 0 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/scripts/experiments/m2_create_distilled_configs.py b/scripts/experiments/m2_create_distilled_configs.py deleted file mode 100644 index a52e2eda..00000000 --- a/scripts/experiments/m2_create_distilled_configs.py +++ /dev/null @@ -1,307 +0,0 @@ -#!/usr/bin/env python3 -"""M2: Create distilled eval configs from M1 consensus edits. - -Generates 24 distilled configs (3 models × 8 benchmarks) by cloning -baseline configs and applying the consensus edits from M1. Also creates -4 missing baseline configs (livecodebench-qwen-9b, liveresearchbench-*). - -Usage: python scripts/experiments/m2_create_distilled_configs.py -""" - -from __future__ import annotations - -from pathlib import Path - -CONFIGS_DIR = Path("src/openjarvis/evals/configs") -M2_DIR = CONFIGS_DIR / "distillation" / "m2" - -# ── Consensus values from M1 (1,131 edits) ────────────────────────────── -DISTILLED_TEMP = 0.2 # 84/134 votes (agent benchmarks only) -DISTILLED_MAX_TURNS = 15 # 56/125 votes (close: 25 had 49) -REMOVE_TOOLS = {"shell_exec", "http_request"} # 13 + 6 votes - -# ── Model specs ────────────────────────────────────────────────────────── -MODELS = { - "2b": {"name": "Qwen/Qwen3.5-2B", "num_gpus": 1, "port": 8000}, - "9b": {"name": "Qwen/Qwen3.5-9B", "num_gpus": 1, "port": 8001}, - "27b": {"name": "Qwen/Qwen3.5-27B-FP8", "num_gpus": 1, "port": 8002}, -} - -# ── Benchmark specs ────────────────────────────────────────────────────── -# Each benchmark defines its baseline config and what changes in distilled. -BENCHMARKS = { - "toolcall15": { - "backend": "jarvis-direct", - "baseline_temp": 0.0, - "distilled_temp": 0.0, # CONTROL: no change for coding - "max_tokens": 4096, - "max_samples": None, - "judge_model": "gpt-5-mini-2025-08-07", - "judge_engine": "cloud", - "extra_benchmark_fields": {}, - }, - "pinchbench": { - "backend": "jarvis-agent", - "agent": "native_openhands", - "baseline_temp": 0.6, - "distilled_temp": DISTILLED_TEMP, - "max_tokens": 8192, - "max_samples": None, - "judge_model": "claude-opus-4-5", - "judge_engine": "cloud", - "baseline_tools": [ - "think", "file_read", "file_write", "web_search", "shell_exec", - "code_interpreter", "browser_navigate", "image_generate", - "calculator", "http_request", "pdf_extract", - ], - "extra_benchmark_fields": {}, - }, - "taubench": { - "backend": "jarvis-direct", - "baseline_temp": 0.7, - "distilled_temp": DISTILLED_TEMP, - "max_tokens": 4096, - "max_samples": 20, - "judge_model": "gpt-5-mini-2025-08-07", - "judge_engine": "cloud", - "extra_benchmark_fields": {"split": "airline,retail"}, - }, - "taubench-telecom": { - "benchmark_name": "taubench", # same benchmark, different split - "backend": "jarvis-direct", - "baseline_temp": 0.7, - "distilled_temp": DISTILLED_TEMP, - "max_tokens": 4096, - "max_samples": 20, - "judge_model": "gpt-5-mini-2025-08-07", - "judge_engine": "cloud", - "extra_benchmark_fields": {"split": "telecom"}, - }, - "gaia": { - "backend": "jarvis-agent", - "agent": "monitor_operative", - "baseline_temp": 0.6, - "distilled_temp": DISTILLED_TEMP, - "max_tokens": 8192, - "max_samples": 50, - "judge_model": "gpt-5-mini-2025-08-07", - "judge_engine": "cloud", - "baseline_tools": [ - "think", "calculator", "code_interpreter", "web_search", "file_read", - ], - "extra_benchmark_fields": {}, - }, - "liveresearch": { - "backend": "jarvis-agent", - "agent": "monitor_operative", - "baseline_temp": 0.6, - "distilled_temp": DISTILLED_TEMP, - "max_tokens": 16384, - "max_samples": 50, - "judge_model": "gpt-5-mini-2025-08-07", - "judge_engine": "cloud", - "baseline_tools": [ - "web_search", "file_read", "file_write", "code_interpreter", "think", - ], - "extra_benchmark_fields": {}, - }, - "liveresearchbench": { - "backend": "jarvis-direct", - "baseline_temp": 0.0, - "distilled_temp": 0.0, # CONTROL: reasoning benchmark - "max_tokens": 8192, - "max_samples": 50, - "judge_model": "gpt-5-mini-2025-08-07", - "judge_engine": "cloud", - "extra_benchmark_fields": {}, - }, - "livecodebench": { - "backend": "jarvis-direct", - "baseline_temp": 0.0, - "distilled_temp": 0.0, # CONTROL: coding benchmark - "max_tokens": 4096, - "max_samples": 20, - "judge_model": "gpt-5-mini-2025-08-07", - "judge_engine": "cloud", - "extra_benchmark_fields": {}, - }, -} - - -def render_config( - *, - comment: str, - meta_name: str, - description: str, - temperature: float, - max_tokens: int, - judge_model: str, - judge_engine: str, - output_dir: str, - model_name: str, - model_engine: str, - num_gpus: int, - benchmark_name: str, - backend: str, - agent: str | None = None, - tools: list[str] | None = None, - max_samples: int | None = None, - extra_benchmark: dict | None = None, - seed: int = 42, -) -> str: - lines = [f"# {comment}"] - lines.append(f'[meta]\nname = "{meta_name}"\ndescription = "{description}"\n') - lines.append(f"[defaults]\ntemperature = {temperature}\nmax_tokens = {max_tokens}\n") - lines.append(f'[judge]\nmodel = "{judge_model}"\ntemperature = 0.0') - if judge_engine: - lines.append(f'engine = "{judge_engine}"') - lines.append(f"max_tokens = 4096\n") - lines.append(f'[run]\nmax_workers = 1\noutput_dir = "{output_dir}"\nseed = {seed}\n') - lines.append(f'[[models]]\nname = "{model_name}"\nengine = "{model_engine}"\nnum_gpus = {num_gpus}\n') - lines.append(f'[[benchmarks]]\nname = "{benchmark_name}"\nbackend = "{backend}"') - if agent: - lines.append(f'agent = "{agent}"') - if max_samples: - lines.append(f"max_samples = {max_samples}") - if tools: - tools_str = ", ".join(f'"{t}"' for t in tools) - lines.append(f"tools = [{tools_str}]") - if extra_benchmark: - for k, v in extra_benchmark.items(): - if isinstance(v, str): - lines.append(f'{k} = "{v}"') - else: - lines.append(f"{k} = {v}") - lines.append("") - return "\n".join(lines) - - -def make_size_label(size: str) -> str: - return {"2b": "qwen-2b", "9b": "qwen-9b", "27b": "qwen-27b"}[size] - - -def generate_missing_baselines() -> int: - """Create baseline configs that don't exist yet.""" - count = 0 - - # livecodebench-qwen-9b (missing) - p = CONFIGS_DIR / "livecodebench-qwen-9b.toml" - if not p.exists(): - b = BENCHMARKS["livecodebench"] - m = MODELS["9b"] - p.write_text(render_config( - comment="LiveCodeBench eval: Qwen3.5-9B (vLLM, 1 GPU)", - meta_name="livecodebench-qwen-9b", - description="LiveCodeBench on Qwen/Qwen3.5-9B (vLLM, 1 GPU)", - temperature=b["baseline_temp"], - max_tokens=b["max_tokens"], - judge_model=b["judge_model"], - judge_engine=b["judge_engine"], - output_dir="results/neurips-2026/baselines/qwen-9b/livecodebench/", - model_name=m["name"], model_engine="vllm", num_gpus=m["num_gpus"], - benchmark_name="livecodebench", backend=b["backend"], - max_samples=b["max_samples"], - )) - count += 1 - print(f" created {p}") - - # liveresearchbench-qwen-{2b,9b,27b} - for size, m in MODELS.items(): - sl = make_size_label(size) - p = CONFIGS_DIR / f"liveresearchbench-{sl}.toml" - if not p.exists(): - b = BENCHMARKS["liveresearchbench"] - p.write_text(render_config( - comment=f"LiveResearchBench (Salesforce): Qwen3.5-{size.upper()} (vLLM)", - meta_name=f"liveresearchbench-{sl}", - description=f"LiveResearchBench on {m['name']} (vLLM)", - temperature=b["baseline_temp"], - max_tokens=b["max_tokens"], - judge_model=b["judge_model"], - judge_engine=b["judge_engine"], - output_dir=f"results/neurips-2026/baselines/{sl}/liveresearchbench/", - model_name=m["name"], model_engine="vllm", num_gpus=m["num_gpus"], - benchmark_name="liveresearchbench", backend=b["backend"], - max_samples=b["max_samples"], - )) - count += 1 - print(f" created {p}") - - return count - - -def generate_distilled_configs() -> int: - """Create distilled configs for all 24 model × benchmark combos.""" - M2_DIR.mkdir(parents=True, exist_ok=True) - count = 0 - - for size, model in MODELS.items(): - sl = make_size_label(size) - for bench_key, bench in BENCHMARKS.items(): - bench_name = bench.get("benchmark_name", bench_key) - is_agent = bench["backend"] == "jarvis-agent" - temp = bench["distilled_temp"] - - # Tool list: remove broken tools for agent benchmarks - tools = None - if is_agent and "baseline_tools" in bench: - tools = [t for t in bench["baseline_tools"] - if t not in REMOVE_TOOLS] - - fname = f"{bench_key}-{sl}-distilled.toml" - out_path = M2_DIR / fname - - # Determine what changed for the comment - changes = [] - if temp != bench["baseline_temp"]: - changes.append(f"temp {bench['baseline_temp']}→{temp}") - if tools and set(tools) != set(bench.get("baseline_tools", [])): - removed = set(bench.get("baseline_tools", [])) - set(tools) - changes.append(f"removed {removed}") - if is_agent: - changes.append(f"max_turns 10→{DISTILLED_MAX_TURNS} (via OPENJARVIS_CONFIG)") - change_str = "; ".join(changes) if changes else "CONTROL (no change)" - - out_path.write_text(render_config( - comment=f"M2 DISTILLED: {bench_key} × {model['name']} — {change_str}", - meta_name=f"{bench_key}-{sl}-distilled", - description=f"Distilled {bench_key} on {model['name']}", - temperature=temp, - max_tokens=bench["max_tokens"], - judge_model=bench["judge_model"], - judge_engine=bench["judge_engine"], - output_dir=f"results/neurips-2026/distilled/{sl}/{bench_key}/", - model_name=model["name"], model_engine="vllm", num_gpus=model["num_gpus"], - benchmark_name=bench_name, backend=bench["backend"], - agent=bench.get("agent"), - tools=tools, - max_samples=bench.get("max_samples"), - extra_benchmark=bench.get("extra_benchmark_fields"), - )) - count += 1 - - return count - - -if __name__ == "__main__": - print("=== Creating missing baseline configs ===") - n_base = generate_missing_baselines() - print(f"Created {n_base} missing baseline configs\n") - - print("=== Creating distilled M2 configs ===") - n_dist = generate_distilled_configs() - print(f"Created {n_dist} distilled configs in {M2_DIR}/\n") - - # Summary - print("=== Change matrix ===") - print(f"{'Benchmark':20} {'Backend':14} {'Temp Δ':12} {'Tool Δ':20} {'max_turns Δ':12}") - print("-" * 80) - for bk, b in BENCHMARKS.items(): - is_agent = b["backend"] == "jarvis-agent" - temp_change = f"{b['baseline_temp']}→{b['distilled_temp']}" if b["baseline_temp"] != b["distilled_temp"] else "—" - tool_change = "—" - if is_agent and "baseline_tools" in b: - removed = REMOVE_TOOLS & set(b.get("baseline_tools", [])) - tool_change = f"-{removed}" if removed else "—" - mt_change = f"10→{DISTILLED_MAX_TURNS}" if is_agent else "—" - print(f"{bk:20} {b['backend']:14} {temp_change:12} {str(tool_change):20} {mt_change:12}") diff --git a/scripts/experiments/m2_run_distilled_evals.sh b/scripts/experiments/m2_run_distilled_evals.sh deleted file mode 100644 index 9e17e48b..00000000 --- a/scripts/experiments/m2_run_distilled_evals.sh +++ /dev/null @@ -1,183 +0,0 @@ -#!/usr/bin/env bash -# ────────────────────────────────────────────────────────────────────────────── -# M2: Run distilled eval configs — resumable, agent-benchmarks-first ordering -# -# Assumes vLLM already running: 2B:8000, 9B:8001, 27B-FP8:8002 -# -# Usage: -# bash m2_run_distilled_evals.sh # all -# bash m2_run_distilled_evals.sh 9b # 9b only -# bash m2_run_distilled_evals.sh 9b gaia # 9b + gaia only -# ────────────────────────────────────────────────────────────────────────────── - -set -uo pipefail - -VENV=".venv/bin/python" -M2_CONFIGS="src/openjarvis/evals/configs/distillation/m2" -BASELINE_CONFIGS="src/openjarvis/evals/configs" -M2_HOME="/scratch/user/jonsaadfalcon/openjarvis-m2" -MODEL_FILTER=${1:-all} -BENCH_FILTER=${2:-all} -FORCE=${FORCE:-0} # set FORCE=1 to re-run completed configs - -RED='\033[0;31m'; GREEN='\033[0;32m'; YELLOW='\033[1;33m'; BLUE='\033[0;34m'; NC='\033[0m' -log() { echo -e "${BLUE}[m2]${NC} $*"; } -ok() { echo -e "${GREEN}[ OK ]${NC} $*"; } -warn() { echo -e "${YELLOW}[WARN]${NC} $*"; } -fail() { echo -e "${RED}[FAIL]${NC} $*"; } -skip() { echo -e "${YELLOW}[SKIP]${NC} $*"; } - -declare -A MODEL_PORT=( [2b]=8000 [9b]=8001 [27b]=8002 ) - -# Agent benchmarks FIRST (where distillation impact is expected) -AGENT_BENCHMARKS="pinchbench gaia liveresearch" -DIRECT_BENCHMARKS="toolcall15 taubench taubench-telecom livecodebench liveresearchbench" -ALL_BENCHMARKS="${AGENT_BENCHMARKS} ${DIRECT_BENCHMARKS}" - -check_vllm() { - for size in 2b 9b 27b; do - [ "$MODEL_FILTER" != "all" ] && [ "$MODEL_FILTER" != "$size" ] && continue - local port=${MODEL_PORT[$size]} - if ! curl -sf "http://localhost:${port}/v1/models" >/dev/null 2>&1; then - fail "vLLM ${size} not responding on port ${port}" - return 1 - fi - ok "vLLM ${size} healthy on port ${port}" - done -} - -# Check if a run already completed (summary.json exists AND has a real accuracy). -# A summary with errors=total_samples (like my earlier broken-routing tests) -# is treated as incomplete and re-run. -is_complete() { - local summary_path=$1 - [ -f "$summary_path" ] || return 1 - python3 -c " -import json, sys -try: - d = json.load(open('$summary_path')) - total = d.get('total_samples', 0) - errors = d.get('errors', 0) - scored = d.get('scored_samples', 0) - # Complete if at least some samples were scored successfully - sys.exit(0 if scored > 0 else 1) -except Exception: - sys.exit(1) -" 2>/dev/null -} - -run_eval() { - local config_path=$1 label=$2 size=$3 summary_path=$4 use_distilled=${5:-false} - - if [ "$FORCE" != "1" ] && is_complete "$summary_path"; then - skip "${label} [already complete]" - return 0 - fi - - local oj_config="${M2_HOME}/config-baseline-${size}.toml" - [ "$use_distilled" = "true" ] && oj_config="${M2_HOME}/config-${size}.toml" - - log "Running: ${label} [$(basename ${oj_config})]" - env OPENJARVIS_CONFIG="${oj_config}" ${VENV} -m openjarvis.evals run -c "${config_path}" 2>&1 - local rc=$? - if [ $rc -eq 0 ] && is_complete "$summary_path"; then - ok "Done: ${label}" - else - warn "Failed: ${label} (rc=$rc)" - fi -} - -# Derive the expected summary.json path for a given distilled config -summary_for_distilled() { - local bench=$1 size=$2 - # Output dir from the config template: results/neurips-2026/distilled/qwen-{size}/{bench}/ - # Summary file pattern: {bench}_Qwen-Qwen3.5-{size}.summary.json - local model_slug - if [ "$size" = "27b" ]; then model_slug="Qwen-Qwen3.5-27B-FP8" - else model_slug="Qwen-Qwen3.5-${size}"; fi - # Uppercase B for the model slug - model_slug=$(echo "$model_slug" | sed 's/-\([0-9][0-9]*\)b/-\1B/g') - # For taubench-telecom, the benchmark name in output is "taubench" not "taubench-telecom" - local bench_fname=$bench - [ "$bench" = "taubench-telecom" ] && bench_fname="taubench" - echo "results/neurips-2026/distilled/qwen-${size}/${bench}/${bench_fname}_${model_slug}.summary.json" -} - -summary_for_baseline() { - local bench=$1 size=$2 - local model_slug - if [ "$size" = "27b" ]; then model_slug="Qwen-Qwen3.5-27B-FP8" - else model_slug="Qwen-Qwen3.5-${size}"; fi - model_slug=$(echo "$model_slug" | sed 's/-\([0-9][0-9]*\)b/-\1B/g') - local bench_fname=$bench - [ "$bench" = "taubench-telecom" ] && bench_fname="taubench" - echo "results/neurips-2026/baselines/qwen-${size}/${bench}/${bench_fname}_${model_slug}.summary.json" -} - -log "M2 Distilled Eval Runner (resumable, agent-first)" -log "Model filter: ${MODEL_FILTER} Benchmark filter: ${BENCH_FILTER} FORCE=${FORCE}" - -check_vllm || exit 1 - -start_time=$(date +%s) - -# Phase B1: DISTILLED agent benchmarks (highest-priority: PinchBench, GAIA, DeepResearchBench) -log "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" -log "Phase B1: DISTILLED agent benchmarks (9 runs — the critical data)" -log "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" -for size in 2b 9b 27b; do - [ "$MODEL_FILTER" != "all" ] && [ "$MODEL_FILTER" != "$size" ] && continue - for bench in ${AGENT_BENCHMARKS}; do - [ "$BENCH_FILTER" != "all" ] && [ "$BENCH_FILTER" != "$bench" ] && continue - cfg="${M2_CONFIGS}/${bench}-qwen-${size}-distilled.toml" - sum=$(summary_for_distilled "$bench" "$size") - [ -f "$cfg" ] && run_eval "$cfg" "DISTILLED ${bench}-qwen-${size}" "${size}" "$sum" true - done -done - -# Phase B2: DISTILLED direct benchmarks (controls — should show minimal delta) -log "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" -log "Phase B2: DISTILLED direct benchmarks (15 runs — controls)" -log "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" -for size in 2b 9b 27b; do - [ "$MODEL_FILTER" != "all" ] && [ "$MODEL_FILTER" != "$size" ] && continue - for bench in ${DIRECT_BENCHMARKS}; do - [ "$BENCH_FILTER" != "all" ] && [ "$BENCH_FILTER" != "$bench" ] && continue - cfg="${M2_CONFIGS}/${bench}-qwen-${size}-distilled.toml" - sum=$(summary_for_distilled "$bench" "$size") - [ -f "$cfg" ] && run_eval "$cfg" "DISTILLED ${bench}-qwen-${size}" "${size}" "$sum" true - done -done - -# Phase A: LiveResearchBench baselines (last, since Step 1 baselines exist for other benchmarks) -log "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" -log "Phase A: LiveResearchBench baselines (3 runs — new benchmark only)" -log "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" -for size in 2b 9b 27b; do - [ "$MODEL_FILTER" != "all" ] && [ "$MODEL_FILTER" != "$size" ] && continue - [ "$BENCH_FILTER" != "all" ] && [ "$BENCH_FILTER" != "liveresearchbench" ] && continue - cfg="${BASELINE_CONFIGS}/liveresearchbench-qwen-${size}.toml" - sum=$(summary_for_baseline "liveresearchbench" "$size") - [ -f "$cfg" ] && run_eval "$cfg" "BASELINE liveresearchbench-qwen-${size}" "${size}" "$sum" false -done - -# Phase C: Spot-check 2 baselines against Jon's Step 1 numbers -log "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" -log "Phase C: Spot-check baselines (TC15-9B, GAIA-9B)" -log "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" -if [ "$MODEL_FILTER" = "all" ] || [ "$MODEL_FILTER" = "9b" ]; then - for bench in toolcall15 gaia; do - [ "$BENCH_FILTER" != "all" ] && [ "$BENCH_FILTER" != "$bench" ] && continue - cfg="${BASELINE_CONFIGS}/${bench}-qwen-9b.toml" - sum=$(summary_for_baseline "$bench" "9b") - [ -f "$cfg" ] && run_eval "$cfg" "SPOTCHECK ${bench}-qwen-9b" "9b" "$sum" false - done -fi - -end_time=$(date +%s) -elapsed=$((end_time - start_time)) - -log "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" -ok "M2 complete in ${elapsed}s ($(( elapsed / 3600 ))h $(( (elapsed % 3600) / 60 ))m)" -log "Distilled results: results/neurips-2026/distilled/" -log "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" diff --git a/scripts/experiments/m3_hill_climb.py b/scripts/experiments/m3_hill_climb.py deleted file mode 100644 index 3c785182..00000000 --- a/scripts/experiments/m3_hill_climb.py +++ /dev/null @@ -1,581 +0,0 @@ -#!/usr/bin/env python3 -"""M3: Empirical hill-climbing with an LLM proposer. - -Replaces M1's open-loop "aggregate consensus across sessions" with a closed -loop: - - For each target (student, benchmark, agent): - for round in 1..N: - edit = teacher.propose_one(history_with_measured_deltas) - score_new = eval_subsample(apply(edit)) - if score_new > current_score: accept - -Every proposal is empirically verified before the next is proposed, and the -teacher sees measured deltas (not just traces) in its context. - -Usage: - python scripts/experiments/m3_hill_climb.py \\ - --student 9b --benchmark liveresearch \\ - --rounds 4 --k-subsample 8 --k-final 30 -""" - -from __future__ import annotations - -import argparse -import json -import os -import re -import subprocess -import sys -import tempfile -import time -from dataclasses import asdict, dataclass, field -from pathlib import Path -from typing import Any - -from openjarvis.core.types import Message, Role -from openjarvis.engine.cloud import CloudEngine - - -# ═══════════════════════════════════════════════════════════════════════════ -# Config & constants -# ═══════════════════════════════════════════════════════════════════════════ - -STUDENT = { - "2b": {"name": "Qwen/Qwen3.5-2B", "port": 8000, "gpu": 4}, - "9b": {"name": "Qwen/Qwen3.5-9B", "port": 8001, "gpu": 5}, - "27b": {"name": "Qwen/Qwen3.5-27B-FP8", "port": 8002, "gpu": 6}, -} - -# Per-benchmark defaults (backend, baseline config) -BENCHMARK = { - "liveresearch": { - "backend": "jarvis-agent", - "agent": "monitor_operative", - "baseline_temp": 0.6, - "baseline_max_tokens": 16384, - "baseline_max_turns": 10, - "baseline_tools": ["web_search", "file_read", "file_write", - "code_interpreter", "think"], - "max_samples_final": 50, # final eval - "judge": "gpt-5-mini-2025-08-07", - }, - "gaia": { - "backend": "jarvis-agent", - "agent": "monitor_operative", - "baseline_temp": 0.6, - "baseline_max_tokens": 8192, - "baseline_max_turns": 10, - "baseline_tools": ["think", "calculator", "code_interpreter", - "web_search", "file_read"], - "max_samples_final": 50, - "judge": "gpt-5-mini-2025-08-07", - }, - "pinchbench": { - "backend": "jarvis-agent", - "agent": "native_openhands", - "baseline_temp": 0.6, - "baseline_max_tokens": 8192, - "baseline_max_turns": 10, - "baseline_tools": ["think", "file_read", "file_write", "web_search", - "shell_exec", "code_interpreter", "browser_navigate", - "image_generate", "calculator", "http_request", - "pdf_extract"], - "max_samples_final": 23, - "judge": "claude-opus-4-5", - }, -} - -AVAILABLE_TOOLS = [ - "think", "file_read", "file_write", "web_search", "shell_exec", - "code_interpreter", "browser_navigate", "image_generate", "calculator", - "http_request", "pdf_extract", "pdf_reader", "list_directory", -] - - -# ═══════════════════════════════════════════════════════════════════════════ -# Proposer (LLM) -# ═══════════════════════════════════════════════════════════════════════════ - - -PROPOSER_SYSTEM = """\ -You are optimizing an OpenJarvis agent configuration for maximum accuracy on \ -a benchmark. You propose ONE config edit per round. After each proposal, the \ -edit is applied and the benchmark is run on a subsample; you then see the \ -measured score delta and decide the next edit. - -Your job is to find the config that maximizes measured accuracy. - -EDIT GRAMMAR — return JSON with "op" and the parameter fields at top level: - -{"op": "", , "rationale": ""} - -Valid ops and their parameter fields: - -1. set_temperature: "value" (float, 0.0..1.0) -2. set_max_turns: "value" (int, 1..100) -3. set_max_tokens: "value" (int, 512..32768) -4. add_tool: "tool_name" (string; must be in AVAILABLE_TOOLS, not already active) -5. remove_tool: "tool_name" (string; must be in current tools) -6. noop: (no params; propose only if you believe no further edit will help) - -CONCRETE EXAMPLES: - {"op": "set_temperature", "value": 0.3, "rationale": "Reduce loop risk."} - {"op": "set_max_turns", "value": 20, "rationale": "More turns for research tasks."} - {"op": "add_tool", "tool_name": "pdf_extract", "rationale": "Tasks require PDF reading."} - {"op": "remove_tool", "tool_name": "shell_exec", "rationale": "Tool is broken in this env."} - {"op": "noop", "rationale": "Current config seems optimal."} - -EXPLORATION BIAS: - The config space has 5 distinct axes: temperature, max_turns, max_tokens, - tool additions (add_tool), tool removals (remove_tool). Before proposing a - second edit on an axis you've already tried, consider whether an untried - axis might reveal a larger gain. Tool-list edits (add/remove) often matter - more than numeric hyperparameters on benchmarks where the agent uses tools. - -Return ONLY the JSON object, no preamble, no code fences.""" - - -def build_user_prompt( - *, benchmark: str, student: str, agent: str, - baseline_score: float, current_score: float, current_config: dict, - edit_history: list[dict], available_tools: list[str], - sample_queries: list[str], -) -> str: - hist_lines = [] - for i, e in enumerate(edit_history, 1): - delta = e["score_after"] - e["score_before"] - status = "ACCEPTED" if e["accepted"] else "REJECTED" - hist_lines.append( - f" Round {i}: {json.dumps(e['edit'])} " - f"→ score {e['score_before']:.1f}% → {e['score_after']:.1f}% " - f"(Δ {delta:+.1f}, {status})" - ) - hist = "\n".join(hist_lines) if hist_lines else " (no edits yet — baseline is the starting point)" - - samples = "\n".join(f" - {q[:150]}..." for q in sample_queries[:3]) - - unused_tools = [t for t in available_tools if t not in current_config["tools"]] - - return f"""\ -TARGET: - student: {student} (vLLM-served Qwen3.5) - benchmark: {benchmark} - agent: {agent} (uses OpenAI-format structured tool calls) - -CURRENT CONFIG: - temperature = {current_config['temperature']} - max_turns = {current_config['max_turns']} - max_tokens = {current_config['max_tokens']} - tools = {current_config['tools']} - -TOOLS NOT CURRENTLY ACTIVE (available to add): - {unused_tools} - -BASELINE (unedited) SCORE: {baseline_score:.1f}% -CURRENT BEST SCORE: {current_score:.1f}% - -EDIT HISTORY (with measured deltas): -{hist} - -SAMPLE TASKS FROM THIS BENCHMARK: -{samples} - -Propose ONE edit that you predict will improve the measured accuracy. Consider \ -the edit history — do not repeat proposals that were rejected. If you believe \ -further edits will not help, propose noop. - -Return JSON only.""" - - -def call_proposer( - engine: CloudEngine, system: str, user: str, model: str = "claude-sonnet-4-6" -) -> dict: - resp = engine.generate( - messages=[ - Message(role=Role.SYSTEM, content=system), - Message(role=Role.USER, content=user), - ], - model=model, max_tokens=600, temperature=0.3, - ) - content = (resp.get("content") or "").strip() - # Be forgiving about code fences - if content.startswith("```"): - content = re.sub(r"^```(?:json)?\s*", "", content) - content = re.sub(r"\s*```\s*$", "", content) - # Find the JSON object - m = re.search(r"\{[\s\S]*\}", content) - if not m: - raise ValueError(f"No JSON object found in proposer output: {content[:200]}") - return json.loads(m.group(0)) - - -# ═══════════════════════════════════════════════════════════════════════════ -# Edit application & evaluation -# ═══════════════════════════════════════════════════════════════════════════ - - -@dataclass -class Config: - temperature: float - max_turns: int - max_tokens: int - tools: list[str] - - -def apply_edit(cfg: Config, edit: dict) -> Config: - """Apply an edit. Tolerant of both flat and nested (params) forms.""" - op = edit["op"] - # Merge top-level edit fields with params for flat-or-nested tolerance - p = {**edit.get("params", {}), **{k: v for k, v in edit.items() - if k not in ("op", "params", "rationale")}} - new = Config( - temperature=cfg.temperature, max_turns=cfg.max_turns, - max_tokens=cfg.max_tokens, tools=list(cfg.tools), - ) - if op == "noop": - return new - if op == "set_temperature": - new.temperature = float(p["value"]) - elif op == "set_max_turns": - new.max_turns = int(p["value"]) - elif op == "set_max_tokens": - new.max_tokens = int(p["value"]) - elif op == "add_tool": - tool = p["tool_name"] - if tool not in new.tools: - new.tools.append(tool) - elif op == "remove_tool": - tool = p["tool_name"] - new.tools = [t for t in new.tools if t != tool] - else: - raise ValueError(f"unknown edit op: {op}") - return new - - -def write_eval_toml( - *, bench: str, bench_spec: dict, student: dict, cfg: Config, - k_samples: int, output_dir: Path, -) -> Path: - agent_line = f'agent = "{bench_spec["agent"]}"' if bench_spec.get("agent") else "" - tools_str = "[" + ", ".join(f'"{t}"' for t in cfg.tools) + "]" - toml = f"""\ -[meta] -name = "m3-{bench}-{student['name'].replace('/', '-')}" -description = "M3 hill-climb round" - -[defaults] -temperature = {cfg.temperature} -max_tokens = {cfg.max_tokens} - -[judge] -model = "{bench_spec['judge']}" -temperature = 0.0 -max_tokens = 4096 -engine = "cloud" - -[run] -max_workers = 1 -output_dir = "{output_dir}" -seed = 42 - -[[models]] -name = "{student['name']}" -engine = "vllm" -num_gpus = 1 - -[[benchmarks]] -name = "{bench}" -backend = "{bench_spec['backend']}" -{agent_line} -max_samples = {k_samples} -tools = {tools_str} -""" - path = output_dir / "eval.toml" - output_dir.mkdir(parents=True, exist_ok=True) - path.write_text(toml) - return path - - -def write_openjarvis_config(home: Path, port: int, max_turns: int) -> Path: - p = home / "global-config.toml" - p.parent.mkdir(parents=True, exist_ok=True) - p.write_text(f"""\ -[agent] -max_turns = {max_turns} - -[engine] -default = "vllm" - -[engine.vllm] -host = "http://localhost:{port}" -""") - return p - - -def run_eval(eval_toml: Path, oj_config: Path) -> tuple[float, int, int]: - """Run one eval. Returns (accuracy_pct, scored, total).""" - env = {**os.environ, "OPENJARVIS_CONFIG": str(oj_config)} - result = subprocess.run( - [".venv/bin/python", "-m", "openjarvis.evals", "run", "-c", str(eval_toml)], - capture_output=True, text=True, env=env, timeout=7200, - ) - # Find the summary.json - out_dir = eval_toml.parent - sums = list(out_dir.glob("**/*.summary.json")) - if not sums: - print(f"[m3] WARNING: no summary.json in {out_dir}") - print(f"[m3] stderr tail: {result.stderr[-500:]}") - return 0.0, 0, 0 - d = json.loads(sums[0].read_text()) - acc = d.get("accuracy", 0.0) - acc_pct = acc * 100 if acc <= 1.0 else acc - return acc_pct, d.get("scored_samples", 0), d.get("total_samples", 0) - - -# ═══════════════════════════════════════════════════════════════════════════ -# Benchmark sample loader (for proposer context) -# ═══════════════════════════════════════════════════════════════════════════ - - -def load_sample_queries(bench: str, n: int = 3) -> list[str]: - try: - if bench == "liveresearch": - from openjarvis.evals.datasets.liveresearch import LiveResearchBenchDataset - ds = LiveResearchBenchDataset() - ds.load(max_samples=n) - elif bench == "pinchbench": - from openjarvis.evals.datasets.pinchbench import PinchBenchDataset - ds = PinchBenchDataset() - elif bench == "gaia": - from openjarvis.evals.datasets.gaia import GAIADataset - ds = GAIADataset() - if hasattr(ds, "load"): - ds.load(max_samples=n) - else: - return [] - return [r.problem for r in list(ds.iter_records())[:n]] - except Exception as e: - print(f"[m3] WARNING: could not load samples for {bench}: {e}") - return [] - - -# ═══════════════════════════════════════════════════════════════════════════ -# Main hill-climb loop -# ═══════════════════════════════════════════════════════════════════════════ - - -def hill_climb(args) -> dict: - bench_spec = BENCHMARK[args.benchmark] - student = STUDENT[args.student] - - # State dir (resumable) - base_dir = Path(args.out_dir) / f"{args.student}-{args.benchmark}" - base_dir.mkdir(parents=True, exist_ok=True) - state_path = base_dir / "state.json" - - # Load or initialize state - if state_path.exists() and not args.fresh: - state = json.loads(state_path.read_text()) - print(f"[m3] Resumed state from round {len(state['history'])}/{args.rounds}") - else: - baseline_cfg_dict = { - "temperature": bench_spec["baseline_temp"], - "max_turns": bench_spec["baseline_max_turns"], - "max_tokens": bench_spec["baseline_max_tokens"], - "tools": list(bench_spec["baseline_tools"]), - } - # ALWAYS measure the baseline today first (unless user provides --trust-baseline). - # This prevents anchoring to an unreproducible Step 1 number. - # We measure at k=k_final so the final delta is like-for-like. - if args.trust_baseline and args.baseline_score is not None: - measured_baseline = args.baseline_score - measured_baseline_k = None - print(f"[m3] Trusting provided baseline score: {measured_baseline:.1f}% " - f"(--trust-baseline set)") - else: - print(f"[m3] Measuring today's baseline with k={args.k_final} " - f"(matches k_final for clean like-for-like delta)...") - bl_dir = base_dir / "baseline_measure" - bl_toml = write_eval_toml( - bench=args.benchmark, bench_spec=bench_spec, student=student, - cfg=Config(**baseline_cfg_dict), - k_samples=args.k_final, output_dir=bl_dir, - ) - bl_oj = write_openjarvis_config(bl_dir, student["port"], baseline_cfg_dict["max_turns"]) - t0 = time.monotonic() - measured_baseline, bl_scored, bl_total = run_eval(bl_toml, bl_oj) - measured_baseline_k = bl_scored - print(f"[m3] Measured baseline: {measured_baseline:.1f}% " - f"({bl_scored}/{bl_total}) in {(time.monotonic() - t0)/60:.1f} min") - if args.baseline_score is not None: - print(f"[m3] (reference: --baseline-score was {args.baseline_score:.1f}%, " - f"drift = {measured_baseline - args.baseline_score:+.1f})") - - state = { - "args": vars(args), - "benchmark": args.benchmark, - "student": student["name"], - "agent": bench_spec["agent"], - "baseline_config": baseline_cfg_dict, - "baseline_score": measured_baseline, - "baseline_score_reference": args.baseline_score, - "current_config": dict(baseline_cfg_dict), - "current_score": measured_baseline, - "history": [], - } - - # Save state helper - def save(): - state_path.write_text(json.dumps(state, indent=2, default=str)) - - save() - - # Sample queries for proposer context - sample_queries = load_sample_queries(args.benchmark) - - engine = CloudEngine() - - # Hill-climb rounds - for round_num in range(len(state["history"]) + 1, args.rounds + 1): - print(f"\n{'═' * 70}") - print(f"[m3] Round {round_num}/{args.rounds}") - print(f"[m3] current_score = {state['current_score']:.1f}%") - print(f"[m3] current_config = {state['current_config']}") - - # Propose - user_prompt = build_user_prompt( - benchmark=args.benchmark, student=student["name"], - agent=bench_spec["agent"], - baseline_score=state["baseline_score"], - current_score=state["current_score"], - current_config=state["current_config"], - edit_history=state["history"], - available_tools=AVAILABLE_TOOLS, - sample_queries=sample_queries, - ) - try: - edit = call_proposer(engine, PROPOSER_SYSTEM, user_prompt, - model=args.proposer_model) - except Exception as e: - print(f"[m3] Proposer failed: {e}. Ending hill-climb.") - break - - print(f"[m3] Proposed: {json.dumps(edit)}") - - if edit.get("op") == "noop": - print(f"[m3] Teacher proposed noop; stopping.") - break - - # Apply and evaluate subsample - try: - candidate = apply_edit(Config(**state["current_config"]), edit) - except Exception as e: - print(f"[m3] apply_edit failed: {e}. Recording as malformed edit.") - # Record as a rejected malformed edit so teacher won't repeat - state["history"].append({ - "round": round_num, "edit": edit, - "config_after": None, - "score_before": state["current_score"], - "score_after": state["current_score"], # no change - "scored": 0, "total": 0, "elapsed_seconds": 0, - "accepted": False, - "error": f"malformed_edit: {e}", - }) - save() - continue - - round_dir = base_dir / f"round_{round_num}" - eval_toml = write_eval_toml( - bench=args.benchmark, bench_spec=bench_spec, student=student, - cfg=candidate, k_samples=args.k_subsample, - output_dir=round_dir, - ) - oj_cfg = write_openjarvis_config(round_dir, student["port"], candidate.max_turns) - - print(f"[m3] Running k={args.k_subsample} subsample eval...") - t0 = time.monotonic() - acc, scored, total = run_eval(eval_toml, oj_cfg) - elapsed = time.monotonic() - t0 - print(f"[m3] Subsample score: {acc:.1f}% ({scored}/{total}) in {elapsed/60:.1f} min") - - score_before = state["current_score"] - delta = acc - score_before - accepted = delta > args.accept_threshold - - # Record history - state["history"].append({ - "round": round_num, "edit": edit, - "config_after": asdict(candidate), - "score_before": score_before, "score_after": acc, - "scored": scored, "total": total, "elapsed_seconds": elapsed, - "accepted": accepted, - }) - - if accepted: - state["current_config"] = asdict(candidate) - state["current_score"] = acc - print(f"[m3] ACCEPTED (Δ={delta:+.1f})") - else: - print(f"[m3] REJECTED (Δ={delta:+.1f} ≤ {args.accept_threshold})") - - save() - - # Final eval with current config - print(f"\n{'═' * 70}") - print(f"[m3] Final eval with best config: {state['current_config']}") - final_dir = base_dir / "final" - final_cfg = Config(**state["current_config"]) - eval_toml = write_eval_toml( - bench=args.benchmark, bench_spec=bench_spec, student=student, - cfg=final_cfg, k_samples=args.k_final, - output_dir=final_dir, - ) - oj_cfg = write_openjarvis_config(final_dir, student["port"], final_cfg.max_turns) - - t0 = time.monotonic() - final_acc, final_scored, final_total = run_eval(eval_toml, oj_cfg) - elapsed = time.monotonic() - t0 - - state["final_score"] = final_acc - state["final_scored"] = final_scored - state["final_total"] = final_total - state["final_elapsed_seconds"] = elapsed - save() - - print(f"\n{'═' * 70}") - print(f"[m3] DONE") - print(f"[m3] baseline = {state['baseline_score']:.1f}%") - print(f"[m3] final = {final_acc:.1f}% ({final_scored}/{final_total}) in {elapsed/60:.1f} min") - print(f"[m3] Δ vs baseline = {final_acc - state['baseline_score']:+.1f}") - print(f"[m3] state: {state_path}") - - return state - - -def main() -> int: - ap = argparse.ArgumentParser() - ap.add_argument("--student", required=True, choices=list(STUDENT)) - ap.add_argument("--benchmark", required=True, choices=list(BENCHMARK)) - ap.add_argument("--rounds", type=int, default=4) - ap.add_argument("--k-subsample", type=int, default=8) - ap.add_argument("--k-final", type=int, default=30) - ap.add_argument("--baseline-score", type=float, default=None, - help="Optional reference baseline (shown alongside measured). " - "Hill-climb always measures today's baseline unless --trust-baseline.") - ap.add_argument("--trust-baseline", action="store_true", - help="Skip baseline re-measurement, trust --baseline-score.") - ap.add_argument("--accept-threshold", type=float, default=0.0, - help="Accept edit if score Δ > this (default: 0)") - ap.add_argument("--proposer-model", default="claude-sonnet-4-6") - ap.add_argument("--out-dir", default="results/neurips-2026/distillation-m3") - ap.add_argument("--fresh", action="store_true", - help="Overwrite existing state and start fresh") - args = ap.parse_args() - - result = hill_climb(args) - return 0 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/scripts/experiments/run_distillation_experiments.sh b/scripts/experiments/run_distillation_experiments.sh deleted file mode 100755 index 961db01c..00000000 --- a/scripts/experiments/run_distillation_experiments.sh +++ /dev/null @@ -1,307 +0,0 @@ -#!/usr/bin/env bash -# ────────────────────────────────────────────────────────────────────────────── -# Run distillation ablation experiments -# -# Prerequisites: -# - Ollama running with qwen3.5:{2b,9b,27b} -# - ANTHROPIC_API_KEY set (for Opus teacher) -# - OPENAI_API_KEY set (for GPT-5.4 teacher) -# - GOOGLE_API_KEY set (for Gemini teacher) -# - For Qwen-397B teacher: vLLM serving on port 8010 with 8×H100 -# - Traces seeded with feedback (run A1 blocker first) -# - jarvis learning init already run -# -# Usage: -# bash scripts/experiments/run_distillation_experiments.sh # Run all -# bash scripts/experiments/run_distillation_experiments.sh exp1a # Run Phase 1a only -# bash scripts/experiments/run_distillation_experiments.sh exp1a opus # Single config -# ────────────────────────────────────────────────────────────────────────────── - -set -euo pipefail - -CONFIGS_DIR="src/openjarvis/evals/configs/distillation" -RESULTS_DIR="results/neurips-2026/agent-optimization/distillation" -EXPERIMENT=${1:-all} -FILTER=${2:-} - -# ── Colors ─────────────────────────────────────────────────────────────────── -RED='\033[0;31m' -GREEN='\033[0;32m' -YELLOW='\033[1;33m' -BLUE='\033[0;34m' -NC='\033[0m' - -log() { echo -e "${BLUE}[distill]${NC} $*"; } -ok() { echo -e "${GREEN}[ OK ]${NC} $*"; } -warn() { echo -e "${YELLOW}[ WARN ]${NC} $*"; } -fail() { echo -e "${RED}[ FAIL ]${NC} $*"; } - -# ── Preflight checks ──────────────────────────────────────────────────────── -check_prereqs() { - log "Preflight checks..." - - # Check API keys - if [ -z "${ANTHROPIC_API_KEY:-}" ]; then - warn "ANTHROPIC_API_KEY not set — Opus teacher experiments will fail" - fi - if [ -z "${OPENAI_API_KEY:-}" ]; then - warn "OPENAI_API_KEY not set — GPT-5.4 teacher experiments will fail" - fi - if [ -z "${GOOGLE_API_KEY:-}" ]; then - warn "GOOGLE_API_KEY not set — Gemini teacher experiments will fail" - fi - - # Check Ollama - if ! ollama list &>/dev/null; then - fail "Ollama not running. Start it first." - exit 1 - fi - - # Check student models - for model in qwen3.5:2b qwen3.5:9b qwen3.5:27b; do - if ! ollama list 2>/dev/null | grep -q "$model"; then - warn "Model $model not found in Ollama. Pull with: ollama pull $model" - fi - done - - # Check distillation init - if [ ! -d "$HOME/.openjarvis/learning" ]; then - log "Running jarvis learning init..." - uv run jarvis learning init - fi - - ok "Preflight complete" -} - -# ── Run a single distillation session ──────────────────────────────────────── -run_session() { - local config_file=$1 - local experiment_name - experiment_name=$(basename "$(dirname "$config_file")") - local config_name - config_name=$(basename "${config_file%.toml}") - local output_dir="${RESULTS_DIR}/${experiment_name}/${config_name}" - - # Skip if already completed - if [ -f "${output_dir}/session/session.json" ]; then - ok "SKIP ${experiment_name}/${config_name} (already done)" - return 0 - fi - - log "──────────────────────────────────────────────────────" - log "Experiment: ${experiment_name}/${config_name}" - log "Config: ${config_file}" - log "Output: ${output_dir}" - log "──────────────────────────────────────────────────────" - - mkdir -p "${output_dir}" - - # Extract metadata from config - local teacher_model - teacher_model=$(grep 'teacher_model' "$config_file" | head -1 | sed 's/.*= *"\(.*\)"/\1/') - local student_model - student_model=$(grep 'default_model' "$config_file" | head -1 | sed 's/.*= *"\(.*\)"/\1/') - local benchmark - benchmark=$(grep '^benchmark ' "$config_file" | head -1 | sed 's/.*= *"\(.*\)"/\1/') - local data_config - data_config=$(grep 'data_config' "$config_file" | head -1 | sed 's/.*= *"\(.*\)"/\1/') - local iterative - iterative=$(grep 'iterative_sessions' "$config_file" | head -1 | sed 's/.*= *//') - - log "Teacher: ${teacher_model}" - log "Student: ${student_model}" - log "Data: ${data_config:-C2}" - log "Iter: ${iterative:-1}" - - # ── Step 1: Seed traces based on data config ───────────────────────── - # (In a full implementation, this would filter/prepare the TraceStore - # based on C1/C2/C3. For now we use whatever traces exist.) - - # ── Step 2: Run distillation session ───────────────────────────────── - local n_sessions=${iterative:-1} - local session_num=1 - local prev_session_id="" - - while [ "$session_num" -le "$n_sessions" ]; do - log "Session ${session_num}/${n_sessions}..." - - local session_output="${output_dir}/session_${session_num}" - mkdir -p "${session_output}" - - # Run the distillation session via Python - # (jarvis learning run doesn't support all config params yet, - # so we call the orchestrator directly) - uv run python << PYEOF > "${session_output}/run.log" 2>&1 || true -import json, os, shutil, sys -from pathlib import Path - -from openjarvis.engine.cloud import CloudEngine -from openjarvis.evals.backends.jarvis_direct import JarvisDirectBackend -from openjarvis.traces.store import TraceStore -from openjarvis.learning.distillation.checkpoint.store import CheckpointStore -from openjarvis.learning.distillation.models import AutonomyMode -from openjarvis.learning.distillation.orchestrator import DistillationOrchestrator -from openjarvis.learning.distillation.storage.session_store import SessionStore -from openjarvis.learning.distillation.student_runner import ( - VLLMStudentRunner, - build_benchmark_samples_from_traces, -) -from openjarvis.learning.distillation.triggers import OnDemandTrigger -from openjarvis.learning.optimize.feedback.judge import TraceJudge - -home = Path(os.environ.get("OPENJARVIS_HOME", str(Path.home() / ".openjarvis"))) - -# Read config params -teacher_model = "${teacher_model}" -student_model = "${student_model}" -autonomy = "auto" -max_cost = float("$(grep 'max_cost_per_session_usd' "$config_file" | head -1 | sed 's/.*= *//')") -max_tools = int("$(grep 'max_tool_calls_per_diagnosis' "$config_file" | head -1 | sed 's/.*= *//')") - -# Real student runner via vLLM -vllm_host = os.environ.get("VLLM_HOST", "http://localhost:8001") -student_runner = VLLMStudentRunner( - host=vllm_host, - model=student_model, -) - -# Real judge via cloud LLM -cloud_engine = CloudEngine() -judge_backend = JarvisDirectBackend(engine_key="cloud") -judge = TraceJudge(backend=judge_backend, model="gpt-5-mini-2025-08-07") - -# Build benchmark samples from existing traces -trace_store = TraceStore(home / "traces.db") -benchmark_samples = build_benchmark_samples_from_traces(trace_store, limit=50) - -orch = DistillationOrchestrator( - teacher_engine=cloud_engine, - teacher_model=teacher_model, - trace_store=trace_store, - benchmark_samples=benchmark_samples, - student_runner=student_runner, - judge=judge, - session_store=SessionStore(home / "learning" / "learning.db"), - checkpoint_store=CheckpointStore(home), - openjarvis_home=home, - autonomy_mode=AutonomyMode.AUTO, - scorer=None, - min_traces=10, - max_cost_usd=max_cost, - max_tool_calls=max_tools, -) -session = orch.run(OnDemandTrigger()) - -# Save results -result = { - "session_id": session.id, - "status": session.status.value, - "cost_usd": session.teacher_cost_usd, - "edits_total": len(session.edit_outcomes), - "edits_applied": len([o for o in session.edit_outcomes if o.status == "applied"]), - "edits_rejected": len([o for o in session.edit_outcomes if o.status == "rejected_by_gate"]), - "error": session.error, -} -Path("${session_output}/result.json").write_text(json.dumps(result, indent=2)) - -# Copy session artifacts -sd = home / "learning" / "sessions" / session.id -if sd.exists(): - shutil.copytree(sd, Path("${session_output}/artifacts"), dirs_exist_ok=True) - -print(json.dumps(result, indent=2)) -PYEOF - - # Check result - if [ -f "${session_output}/result.json" ]; then - local status - status=$(python3 -c "import json; print(json.load(open('${session_output}/result.json'))['status'])") - local cost - cost=$(python3 -c "import json; print(f\"\${json.load(open('${session_output}/result.json'))['cost_usd']:.4f}\")") - local applied - applied=$(python3 -c "import json; print(json.load(open('${session_output}/result.json'))['edits_applied'])") - - if [ "$status" = "completed" ]; then - ok "Session ${session_num}: status=${status}, cost=\$${cost}, applied=${applied}" - else - warn "Session ${session_num}: status=${status}, cost=\$${cost}" - fi - else - fail "Session ${session_num}: no result.json (check ${session_output}/run.log)" - fi - - session_num=$((session_num + 1)) - done - - ok "Done: ${experiment_name}/${config_name}" -} - -# ── Run experiment group ───────────────────────────────────────────────────── -run_experiment() { - local exp_dir=$1 - local exp_name - exp_name=$(basename "$exp_dir") - - log "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" - log "EXPERIMENT GROUP: ${exp_name}" - log "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" - - local count=0 - local total - total=$(ls "${exp_dir}"/*.toml 2>/dev/null | wc -l) - - for config in "${exp_dir}"/*.toml; do - [ -f "$config" ] || continue - - # Apply filter if specified - if [ -n "${FILTER}" ] && ! echo "$config" | grep -q "${FILTER}"; then - continue - fi - - count=$((count + 1)) - log "[${count}/${total}] $(basename "$config")" - run_session "$config" - done - - ok "Experiment group ${exp_name}: ${count} configs processed" -} - -# ── Main ───────────────────────────────────────────────────────────────────── -main() { - check_prereqs - - log "Starting distillation experiments" - log "Experiment filter: ${EXPERIMENT}" - log "Config filter: ${FILTER:-none}" - - local start_time - start_time=$(date +%s) - - if [ "$EXPERIMENT" = "all" ]; then - # Run in priority order - for exp in exp1a-teacher exp1b-budget exp1c-student \ - exp2a-gate exp2b-autonomy \ - exp3a-iterative exp3b-transfer; do - if [ -d "${CONFIGS_DIR}/${exp}" ]; then - run_experiment "${CONFIGS_DIR}/${exp}" - fi - done - elif [ -d "${CONFIGS_DIR}/${EXPERIMENT}" ]; then - run_experiment "${CONFIGS_DIR}/${EXPERIMENT}" - else - fail "Unknown experiment: ${EXPERIMENT}" - echo "Available: exp1a-teacher exp1b-budget exp1c-student exp2a-gate exp2b-autonomy exp3a-iterative exp3b-transfer" - exit 1 - fi - - local end_time - end_time=$(date +%s) - local elapsed=$((end_time - start_time)) - - log "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" - ok "All experiments complete in ${elapsed}s" - log "Results in: ${RESULTS_DIR}/" - log "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" -} - -main "$@" diff --git a/scripts/experiments/run_track_b_gepa_dspy.sh b/scripts/experiments/run_track_b_gepa_dspy.sh deleted file mode 100644 index a628be4a..00000000 --- a/scripts/experiments/run_track_b_gepa_dspy.sh +++ /dev/null @@ -1,375 +0,0 @@ -#!/usr/bin/env bash -# ============================================================================= -# Track B: GEPA/DSPy Agent Optimization -# NeurIPS 2026 — Agent Optimization Experiments -# -# Runs GEPA and DSPy BootstrapFewShot across: -# Models: qwen-9b, qwen-27b, qwen-35b -# Benchmarks: toolcall15, pinchbench, taubench -# -# Usage: -# bash scripts/experiments/run_track_b_gepa_dspy.sh -# bash scripts/experiments/run_track_b_gepa_dspy.sh --model qwen-9b --benchmark pinchbench -# bash scripts/experiments/run_track_b_gepa_dspy.sh --optimizer gepa -# bash scripts/experiments/run_track_b_gepa_dspy.sh --optimizer dspy -# -# Expected runtime: ~2-4 hours per GEPA run, ~1-2 hours per DSPy run -# Total wall-clock: ~12-18 hours (parallelized across GPUs) -# Estimated API cost: ~$90 GEPA + ~$90 DSPy = ~$180 total -# -# ============================================================================= -# vLLM Serving Commands (run these BEFORE this script on the GPU node) -# ============================================================================= -# -# GPU 0 — Qwen-9B (1x GPU): -# CUDA_VISIBLE_DEVICES=0 vllm serve Qwen/Qwen2.5-7B-Instruct \ -# --model Qwen/Qwen2.5-7B-Instruct \ -# --served-model-name qwen-9b \ -# --port 8001 --host 0.0.0.0 \ -# --max-model-len 32768 --gpu-memory-utilization 0.9 & -# -# GPU 1 — Qwen-27B (1-2x GPU): -# CUDA_VISIBLE_DEVICES=1,2 vllm serve Qwen/Qwen2.5-32B-Instruct \ -# --model Qwen/Qwen2.5-32B-Instruct \ -# --served-model-name qwen-27b \ -# --port 8002 --host 0.0.0.0 \ -# --tensor-parallel-size 2 \ -# --max-model-len 32768 --gpu-memory-utilization 0.9 & -# -# GPU 3 — Qwen-35B (1-2x GPU): -# CUDA_VISIBLE_DEVICES=3,4 vllm serve Qwen/Qwen2.5-72B-Instruct \ -# --model Qwen/Qwen2.5-72B-Instruct \ -# --served-model-name qwen-35b \ -# --port 8003 --host 0.0.0.0 \ -# --tensor-parallel-size 2 \ -# --max-model-len 32768 --gpu-memory-utilization 0.9 & -# -# Wait for all servers to be healthy: -# sleep 60 && curl -s http://localhost:8001/health && \ -# curl -s http://localhost:8002/health && \ -# curl -s http://localhost:8003/health -# -# ============================================================================= - -set -euo pipefail - -# --------------------------------------------------------------------------- -# Defaults -# --------------------------------------------------------------------------- -REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" -RESULTS_BASE="${REPO_ROOT}/results/neurips-2026/agent-optimization" -LOG_DIR="${REPO_ROOT}/results/neurips-2026/logs" -TIMESTAMP="$(date +%Y%m%d_%H%M%S)" - -ALL_MODELS=(qwen-9b qwen-27b qwen-35b) -ALL_BENCHMARKS=(toolcall15 pinchbench taubench) -ALL_OPTIMIZERS=(gepa dspy) - -# Override defaults with CLI flags -FILTER_MODEL="" -FILTER_BENCHMARK="" -FILTER_OPTIMIZER="" - -# GEPA settings -GEPA_TRIALS=20 -GEPA_MAX_SAMPLES=50 -GEPA_OPTIMIZER_MODEL="claude-sonnet-4-6" - -# DSPy settings -DSPY_OPTIMIZER="BootstrapFewShotWithRandomSearch" -DSPY_TEACHER_LM="claude-sonnet-4-6" - -# --------------------------------------------------------------------------- -# Parse CLI flags -# --------------------------------------------------------------------------- -while [[ $# -gt 0 ]]; do - case "$1" in - --model) - FILTER_MODEL="$2"; shift 2 ;; - --benchmark) - FILTER_BENCHMARK="$2"; shift 2 ;; - --optimizer) - FILTER_OPTIMIZER="$2"; shift 2 ;; - --gepa-trials) - GEPA_TRIALS="$2"; shift 2 ;; - --gepa-max-samples) - GEPA_MAX_SAMPLES="$2"; shift 2 ;; - --dspy-optimizer) - DSPY_OPTIMIZER="$2"; shift 2 ;; - -h|--help) - sed -n '2,30p' "$0" | grep '^#' | sed 's/^# \?//' - exit 0 ;; - *) - echo "Unknown flag: $1"; exit 1 ;; - esac -done - -# Apply filters -if [[ -n "$FILTER_MODEL" ]]; then - ALL_MODELS=("$FILTER_MODEL") -fi -if [[ -n "$FILTER_BENCHMARK" ]]; then - ALL_BENCHMARKS=("$FILTER_BENCHMARK") -fi -if [[ -n "$FILTER_OPTIMIZER" ]]; then - ALL_OPTIMIZERS=("$FILTER_OPTIMIZER") -fi - -# --------------------------------------------------------------------------- -# Logging helpers -# --------------------------------------------------------------------------- -mkdir -p "$LOG_DIR" -LOG_FILE="${LOG_DIR}/track_b_${TIMESTAMP}.log" - -log() { - local level="$1"; shift - local msg="[$(date '+%Y-%m-%d %H:%M:%S')] [$level] $*" - echo "$msg" - echo "$msg" >> "$LOG_FILE" -} - -log_info() { log "INFO " "$@"; } -log_ok() { log "OK " "$@"; } -log_warn() { log "WARN " "$@"; } -log_error() { log "ERROR" "$@"; } - -# --------------------------------------------------------------------------- -# Environment setup -# --------------------------------------------------------------------------- -setup_env() { - log_info "=== Track B: GEPA/DSPy Optimization ===" - log_info "Repo: $REPO_ROOT" - log_info "Log: $LOG_FILE" - log_info "Models: ${ALL_MODELS[*]}" - log_info "Benchmarks: ${ALL_BENCHMARKS[*]}" - log_info "Optimizers: ${ALL_OPTIMIZERS[*]}" - echo "" - - # Check we are in the repo root - if [[ ! -f "${REPO_ROOT}/pyproject.toml" ]]; then - log_error "pyproject.toml not found — is REPO_ROOT set correctly? ($REPO_ROOT)" - exit 1 - fi - - # Install/sync dependencies - log_info "Running uv sync..." - cd "$REPO_ROOT" - uv sync --extra dev 2>&1 | tail -5 - log_ok "uv sync complete" - - # Check required API keys - if [[ -z "${ANTHROPIC_API_KEY:-}" ]]; then - log_error "ANTHROPIC_API_KEY is not set. Required for the optimizer teacher model." - log_error " export ANTHROPIC_API_KEY=sk-ant-..." - exit 1 - fi - log_ok "ANTHROPIC_API_KEY is set" - - # Optional: OpenAI key (used if teacher_lm is an OpenAI model) - if [[ -z "${OPENAI_API_KEY:-}" ]]; then - log_warn "OPENAI_API_KEY not set (only needed if using OpenAI teacher models)" - fi - - echo "" - log_info "Model-to-port mapping (vLLM must be pre-started on these ports):" - log_info " qwen-9b -> http://localhost:8001" - log_info " qwen-27b -> http://localhost:8002" - log_info " qwen-35b -> http://localhost:8003" - echo "" -} - -# --------------------------------------------------------------------------- -# Model port lookup -# --------------------------------------------------------------------------- -model_port() { - case "$1" in - qwen-9b) echo 8001 ;; - qwen-27b) echo 8002 ;; - qwen-35b) echo 8003 ;; - *) - log_error "Unknown model: $1" - exit 1 ;; - esac -} - -# --------------------------------------------------------------------------- -# Health check: verify the vLLM server for a model is reachable -# --------------------------------------------------------------------------- -check_server_health() { - local model="$1" - local port - port="$(model_port "$model")" - local url="http://localhost:${port}/health" - if curl -sf "$url" > /dev/null 2>&1; then - log_ok "vLLM server for $model is healthy at port $port" - return 0 - else - log_error "vLLM server for $model NOT reachable at $url" - log_error "Start it with the vLLM commands in the script header." - return 1 - fi -} - -# --------------------------------------------------------------------------- -# GEPA optimization for one (model, benchmark) pair -# --------------------------------------------------------------------------- -run_gepa() { - local model="$1" - local bench="$2" - local port - port="$(model_port "$model")" - local out_dir="${RESULTS_BASE}/gepa/${model}/${bench}" - - log_info "--- GEPA: $model × $bench ---" - log_info " Output dir: $out_dir" - log_info " Trials: $GEPA_TRIALS Max-samples: $GEPA_MAX_SAMPLES" - mkdir -p "$out_dir" - - # Record start time - local t0 - t0="$(date +%s)" - - OPENAI_API_BASE="http://localhost:${port}/v1" \ - uv run jarvis optimize run \ - --benchmark "$bench" \ - --model "$model" \ - --optimizer-model "$GEPA_OPTIMIZER_MODEL" \ - --trials "$GEPA_TRIALS" \ - --max-samples "$GEPA_MAX_SAMPLES" \ - --output-dir "$out_dir" \ - 2>&1 | tee -a "$LOG_FILE" - - local exit_code=${PIPESTATUS[0]} - local t1 - t1="$(date +%s)" - local elapsed=$(( t1 - t0 )) - - if [[ $exit_code -eq 0 ]]; then - log_ok "GEPA $model/$bench done in ${elapsed}s" - else - log_error "GEPA $model/$bench FAILED (exit $exit_code) after ${elapsed}s" - return $exit_code - fi -} - -# --------------------------------------------------------------------------- -# DSPy optimization for one (model, benchmark) pair -# --------------------------------------------------------------------------- -run_dspy() { - local model="$1" - local bench="$2" - local port - port="$(model_port "$model")" - local out_dir="${RESULTS_BASE}/dspy/${model}/${bench}" - - log_info "--- DSPy: $model × $bench ---" - log_info " Output dir: $out_dir" - log_info " Teleprompter: $DSPY_OPTIMIZER Teacher: $DSPY_TEACHER_LM" - mkdir -p "$out_dir" - - local t0 - t0="$(date +%s)" - - OPENAI_API_BASE="http://localhost:${port}/v1" \ - uv run python - <&1 | tee -a "$LOG_FILE" -import sys -from openjarvis.learning.agents.dspy_optimizer import DSPyAgentOptimizer -from openjarvis.core.config import DSPyOptimizerConfig -from openjarvis.traces.store import TraceStore - -store = TraceStore() -config = DSPyOptimizerConfig( - optimizer="${DSPY_OPTIMIZER}", - teacher_lm="${DSPY_TEACHER_LM}", - config_dir="${out_dir}", - benchmark="${bench}", - agent_filter="${model}", -) -result = DSPyAgentOptimizer(config).optimize(store) -print(f"DSPy result for ${model}/${bench}: {result}") -if result.get("status") not in ("ok", "success", "done"): - sys.exit(1) -PYEOF - - local exit_code=${PIPESTATUS[0]} - local t1 - t1="$(date +%s)" - local elapsed=$(( t1 - t0 )) - - if [[ $exit_code -eq 0 ]]; then - log_ok "DSPy $model/$bench done in ${elapsed}s" - else - log_error "DSPy $model/$bench FAILED (exit $exit_code) after ${elapsed}s" - return $exit_code - fi -} - -# --------------------------------------------------------------------------- -# Summary: print result file locations -# --------------------------------------------------------------------------- -print_summary() { - echo "" - log_info "=== Track B Complete ===" - log_info "Results written to:" - for opt in "${ALL_OPTIMIZERS[@]}"; do - for model in "${ALL_MODELS[@]}"; do - for bench in "${ALL_BENCHMARKS[@]}"; do - local out_dir="${RESULTS_BASE}/${opt}/${model}/${bench}" - if [[ -d "$out_dir" ]]; then - log_ok " $opt/$model/$bench -> $out_dir" - else - log_warn " $opt/$model/$bench -> MISSING ($out_dir)" - fi - done - done - done - log_info "Full log: $LOG_FILE" -} - -# --------------------------------------------------------------------------- -# Main -# --------------------------------------------------------------------------- -main() { - setup_env - - local failed=0 - - # Pre-flight: check vLLM servers for all target models - log_info "Checking vLLM server health..." - for model in "${ALL_MODELS[@]}"; do - check_server_health "$model" || failed=$(( failed + 1 )) - done - if [[ $failed -gt 0 ]]; then - log_error "$failed vLLM server(s) not reachable. Start them first (see header)." - exit 1 - fi - echo "" - - # Run all requested (optimizer, model, benchmark) combinations - for opt in "${ALL_OPTIMIZERS[@]}"; do - log_info "==========================================" - log_info "Optimizer: $opt" - log_info "==========================================" - for model in "${ALL_MODELS[@]}"; do - for bench in "${ALL_BENCHMARKS[@]}"; do - case "$opt" in - gepa) run_gepa "$model" "$bench" || failed=$(( failed + 1 )) ;; - dspy) run_dspy "$model" "$bench" || failed=$(( failed + 1 )) ;; - *) log_error "Unknown optimizer: $opt"; failed=$(( failed + 1 )) ;; - esac - echo "" - done - done - done - - print_summary - - if [[ $failed -gt 0 ]]; then - log_error "$failed run(s) failed. Check log for details: $LOG_FILE" - exit 1 - fi - - log_ok "All Track B runs completed successfully." -} - -main "$@" diff --git a/scripts/experiments/run_track_d_lora_sft.sh b/scripts/experiments/run_track_d_lora_sft.sh deleted file mode 100644 index e90b7415..00000000 --- a/scripts/experiments/run_track_d_lora_sft.sh +++ /dev/null @@ -1,778 +0,0 @@ -#!/usr/bin/env bash -# ============================================================================= -# Track D: LoRA / SFT Intelligence Optimization -# NeurIPS 2026 — Intelligence Optimization Experiments -# -# Runs LoRA and SFT fine-tuning across: -# LoRA models: Qwen-2B, Qwen-9B, Qwen-27B -# SFT models: Qwen-2B, Qwen-9B -# -# After training, runs fast-benchmark eval on every checkpoint. -# -# Usage: -# bash scripts/experiments/run_track_d_lora_sft.sh -# bash scripts/experiments/run_track_d_lora_sft.sh --method lora -# bash scripts/experiments/run_track_d_lora_sft.sh --method sft --model qwen-2b -# bash scripts/experiments/run_track_d_lora_sft.sh --skip-eval -# -# Expected runtime per run: -# Qwen-2B LoRA (~4-8 h, 1x H100, GPU 0) -# Qwen-9B LoRA (~8-16 h, 1x H100, GPU 1) -# Qwen-27B LoRA (~16-24 h, 2x H100, GPUs 2-3) -# Qwen-2B SFT (~8-16 h, 1x H100, GPU 4) -# Qwen-9B SFT (~16-24 h, 2x H100, GPUs 5-6) -# -# Total wall-clock (all in parallel): ~24 h on a 7-8x H100 node -# Total GPU-hours: ~100-200 H100-hours -# -# ============================================================================= -# GPU Allocation — recommended for a single 8x H100 node -# ============================================================================= -# -# Run D1 (Qwen-2B LoRA) on GPU 0: -# CUDA_VISIBLE_DEVICES=0 bash scripts/experiments/run_track_d_lora_sft.sh \ -# --method lora --model qwen-2b & -# -# Run D2 (Qwen-9B LoRA) on GPU 1: -# CUDA_VISIBLE_DEVICES=1 bash scripts/experiments/run_track_d_lora_sft.sh \ -# --method lora --model qwen-9b & -# -# Run D3 (Qwen-27B LoRA) on GPUs 2-3: -# CUDA_VISIBLE_DEVICES=2,3 bash scripts/experiments/run_track_d_lora_sft.sh \ -# --method lora --model qwen-27b & -# -# Run D4 (Qwen-2B SFT) on GPU 4: -# CUDA_VISIBLE_DEVICES=4 bash scripts/experiments/run_track_d_lora_sft.sh \ -# --method sft --model qwen-2b & -# -# Run D5 (Qwen-9B SFT) on GPUs 5-6: -# CUDA_VISIBLE_DEVICES=5,6 bash scripts/experiments/run_track_d_lora_sft.sh \ -# --method sft --model qwen-9b & -# -# ============================================================================= -# Training datasets (downloaded automatically if HF_TOKEN is set) -# ============================================================================= -# -# Primary agentic traces: -# - neulab/agent-data-collection -# - GAIR/AgentInstruct -# -# Supplementary reasoning: -# - GeneralThought-430K-filtered -# - GLM-4.7-flash SFT traces (168K + 57K) -# -# ============================================================================= - -set -euo pipefail - -# --------------------------------------------------------------------------- -# Defaults -# --------------------------------------------------------------------------- -REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" -RESULTS_BASE="${REPO_ROOT}/results/neurips-2026/intelligence-optimization" -LOG_DIR="${REPO_ROOT}/results/neurips-2026/logs" -TIMESTAMP="$(date +%Y%m%d_%H%M%S)" - -# Default: run all methods and models -ALL_LORA_MODELS=(qwen-2b qwen-9b qwen-27b) -ALL_SFT_MODELS=(qwen-2b qwen-9b) -FAST_BENCHMARKS=(toolcall15 pinchbench taubench) - -# CLI overrides -FILTER_METHOD="" -FILTER_MODEL="" -SKIP_EVAL=false - -# Training hyperparameters (override per model below) -LORA_RANK=64 -LORA_ALPHA=128 -LORA_DROPOUT=0.05 -LORA_EPOCHS=3 -LORA_LR=2e-4 -LORA_BATCH_SIZE=8 -LORA_GRAD_ACCUM=4 - -SFT_EPOCHS=3 -SFT_LR=1e-5 -SFT_BATCH_SIZE=4 -SFT_GRAD_ACCUM=8 - -MAX_SEQ_LEN=8192 - -# Fast-eval settings -EVAL_MAX_SAMPLES=20 - -# --------------------------------------------------------------------------- -# HuggingFace model IDs -# --------------------------------------------------------------------------- -model_hf_id() { - case "$1" in - qwen-2b) echo "Qwen/Qwen2.5-1.5B-Instruct" ;; - qwen-9b) echo "Qwen/Qwen2.5-7B-Instruct" ;; - qwen-27b) echo "Qwen/Qwen2.5-32B-Instruct" ;; - *) - echo "ERROR: unknown model $1" >&2 - exit 1 ;; - esac -} - -# --------------------------------------------------------------------------- -# Parse CLI flags -# --------------------------------------------------------------------------- -while [[ $# -gt 0 ]]; do - case "$1" in - --method) - FILTER_METHOD="$2"; shift 2 ;; - --model) - FILTER_MODEL="$2"; shift 2 ;; - --skip-eval) - SKIP_EVAL=true; shift ;; - --lora-rank) - LORA_RANK="$2"; shift 2 ;; - --lora-epochs) - LORA_EPOCHS="$2"; shift 2 ;; - --sft-epochs) - SFT_EPOCHS="$2"; shift 2 ;; - -h|--help) - sed -n '2,35p' "$0" | grep '^#' | sed 's/^# \?//' - exit 0 ;; - *) - echo "Unknown flag: $1"; exit 1 ;; - esac -done - -# Apply model filter -if [[ -n "$FILTER_MODEL" ]]; then - ALL_LORA_MODELS=("$FILTER_MODEL") - ALL_SFT_MODELS=("$FILTER_MODEL") -fi - -# Apply method filter -RUN_LORA=true -RUN_SFT=true -if [[ "$FILTER_METHOD" == "lora" ]]; then - RUN_SFT=false -elif [[ "$FILTER_METHOD" == "sft" ]]; then - RUN_LORA=false -fi - -# --------------------------------------------------------------------------- -# Logging helpers -# --------------------------------------------------------------------------- -mkdir -p "$LOG_DIR" -LOG_FILE="${LOG_DIR}/track_d_${TIMESTAMP}.log" - -log() { - local level="$1"; shift - local msg="[$(date '+%Y-%m-%d %H:%M:%S')] [$level] $*" - echo "$msg" - echo "$msg" >> "$LOG_FILE" -} - -log_info() { log "INFO " "$@"; } -log_ok() { log "OK " "$@"; } -log_warn() { log "WARN " "$@"; } -log_error() { log "ERROR" "$@"; } - -# --------------------------------------------------------------------------- -# Environment setup -# --------------------------------------------------------------------------- -setup_env() { - log_info "=== Track D: LoRA/SFT Training ===" - log_info "Repo: $REPO_ROOT" - log_info "Log: $LOG_FILE" - log_info "Results: $RESULTS_BASE" - echo "" - - # Check we are in the repo root - if [[ ! -f "${REPO_ROOT}/pyproject.toml" ]]; then - log_error "pyproject.toml not found — is REPO_ROOT set correctly? ($REPO_ROOT)" - exit 1 - fi - - # Install/sync dependencies - log_info "Running uv sync..." - cd "$REPO_ROOT" - uv sync --extra dev 2>&1 | tail -5 - log_ok "uv sync complete" - - # HuggingFace token — required to download gated Qwen models - if [[ -z "${HF_TOKEN:-}" ]]; then - log_warn "HF_TOKEN is not set." - log_warn " If Qwen models are gated, set: export HF_TOKEN=hf_..." - log_warn " Or pre-download them with: huggingface-cli download " - else - log_ok "HF_TOKEN is set" - # Log in so huggingface_hub uses the token - uv run python -c " -import huggingface_hub -huggingface_hub.login(token='${HF_TOKEN}', add_to_git_credential=False) -print('Logged in to HuggingFace Hub') -" 2>&1 | tee -a "$LOG_FILE" - fi - - # Check for GPU - if ! command -v nvidia-smi &>/dev/null; then - log_warn "nvidia-smi not found — ensure CUDA is available for training." - else - log_info "GPU status:" - nvidia-smi --query-gpu=index,name,memory.total,memory.free \ - --format=csv,noheader 2>&1 | while IFS= read -r line; do - log_info " $line" - done - fi - - # Check for trl / peft / transformers (training stack) - uv run python -c " -import importlib, sys -missing = [] -for pkg in ['transformers', 'peft', 'trl', 'datasets', 'accelerate', 'bitsandbytes']: - if importlib.util.find_spec(pkg) is None: - missing.append(pkg) -if missing: - print('MISSING packages:', missing) - sys.exit(1) -else: - print('Training stack OK: transformers, peft, trl, datasets, accelerate, bitsandbytes') -" 2>&1 | tee -a "$LOG_FILE" || { - log_error "Some training packages are missing. Install with:" - log_error " uv add transformers peft trl datasets accelerate bitsandbytes" - exit 1 - } - - echo "" -} - -# --------------------------------------------------------------------------- -# Download / verify training dataset -# --------------------------------------------------------------------------- -prepare_dataset() { - local method="$1" - local model="$2" - local data_dir="${REPO_ROOT}/results/neurips-2026/training-data" - mkdir -p "$data_dir" - - log_info "Preparing training dataset for $method/$model..." - - uv run python - <&1 | tee -a "$LOG_FILE" -from datasets import load_dataset -import json -from pathlib import Path - -data_dir = Path("${data_dir}") -output_path = data_dir / "${method}_${model}_train.jsonl" - -if output_path.exists(): - lines = output_path.read_text().count('\n') - print(f"Dataset already exists: {output_path} ({lines} examples)") -else: - print("Downloading neulab/agent-data-collection...") - ds = load_dataset("neulab/agent-data-collection", split="train") - print(f"Raw dataset size: {len(ds)}") - - # Filter to reasonable-length examples for agentic fine-tuning - examples = [] - for ex in ds: - messages = ex.get("messages") or ex.get("conversations") or [] - if not messages: - continue - total_len = sum(len(str(m)) for m in messages) - if 200 <= total_len <= 16000: - examples.append({"messages": messages}) - - print(f"Filtered dataset size: {len(examples)}") - - with output_path.open("w") as f: - for ex in examples: - f.write(json.dumps(ex) + "\n") - print(f"Saved to {output_path}") -PYEOF - - echo "${data_dir}/${method}_${model}_train.jsonl" -} - -# --------------------------------------------------------------------------- -# LoRA training for one model -# --------------------------------------------------------------------------- -run_lora() { - local model="$1" - local hf_id - hf_id="$(model_hf_id "$model")" - local out_dir="${RESULTS_BASE}/lora/${model}" - local checkpoint_dir="${out_dir}/checkpoint" - mkdir -p "$out_dir" "$checkpoint_dir" - - log_info "==========================================" - log_info "LoRA training: $model ($hf_id)" - log_info " Output: $out_dir" - log_info " LoRA rank=$LORA_RANK alpha=$LORA_ALPHA dropout=$LORA_DROPOUT" - log_info " Epochs=$LORA_EPOCHS LR=$LORA_LR batch=$LORA_BATCH_SIZE grad_accum=$LORA_GRAD_ACCUM" - log_info "==========================================" - - local dataset_path - dataset_path="$(prepare_dataset lora "$model")" - - local t0 - t0="$(date +%s)" - - # Number of GPUs visible - local n_gpus - n_gpus="$(python3 -c "import torch; print(torch.cuda.device_count())" 2>/dev/null || echo 1)" - log_info "Training on $n_gpus GPU(s)" - - if [[ "$n_gpus" -gt 1 ]]; then - LAUNCHER="uv run torchrun --nproc_per_node=$n_gpus" - else - LAUNCHER="uv run python" - fi - - $LAUNCHER - <&1 | tee -a "$LOG_FILE" -import json -import math -import os -from pathlib import Path - -import torch -from datasets import load_dataset -from peft import LoraConfig, TaskType, get_peft_model -from transformers import ( - AutoModelForCausalLM, - AutoTokenizer, - DataCollatorForSeq2Seq, - Trainer, - TrainingArguments, -) - -hf_id = "${hf_id}" -out_dir = "${out_dir}" -ckpt = "${checkpoint_dir}" -data_path = "${dataset_path}" - -# ---- Tokenizer ---- -print(f"Loading tokenizer: {hf_id}") -tokenizer = AutoTokenizer.from_pretrained(hf_id, trust_remote_code=True) -if tokenizer.pad_token is None: - tokenizer.pad_token = tokenizer.eos_token - -# ---- Dataset ---- -print(f"Loading dataset: {data_path}") -raw = load_dataset("json", data_files=data_path, split="train") -raw = raw.train_test_split(test_size=0.02, seed=42) - -def tokenize(example): - messages = example["messages"] - text = tokenizer.apply_chat_template( - messages, - tokenize=False, - add_generation_prompt=False, - ) - enc = tokenizer(text, truncation=True, max_length=${MAX_SEQ_LEN}) - enc["labels"] = enc["input_ids"].copy() - return enc - -print("Tokenizing dataset...") -tok_ds = raw.map(tokenize, remove_columns=raw["train"].column_names, num_proc=4) -print(f"Train: {len(tok_ds['train'])} Eval: {len(tok_ds['test'])}") - -# ---- Model ---- -print(f"Loading model: {hf_id}") -model = AutoModelForCausalLM.from_pretrained( - hf_id, - torch_dtype=torch.bfloat16, - trust_remote_code=True, - device_map="auto", -) - -# ---- LoRA ---- -lora_config = LoraConfig( - task_type=TaskType.CAUSAL_LM, - r=${LORA_RANK}, - lora_alpha=${LORA_ALPHA}, - lora_dropout=${LORA_DROPOUT}, - target_modules=["q_proj", "k_proj", "v_proj", "o_proj", - "gate_proj", "up_proj", "down_proj"], - bias="none", -) -model = get_peft_model(model, lora_config) -model.print_trainable_parameters() - -# ---- Training args ---- -steps_per_epoch = math.ceil(len(tok_ds["train"]) / (${LORA_BATCH_SIZE} * ${LORA_GRAD_ACCUM})) -total_steps = steps_per_epoch * ${LORA_EPOCHS} -save_steps = max(50, steps_per_epoch // 2) - -args = TrainingArguments( - output_dir=ckpt, - num_train_epochs=${LORA_EPOCHS}, - per_device_train_batch_size=${LORA_BATCH_SIZE}, - gradient_accumulation_steps=${LORA_GRAD_ACCUM}, - learning_rate=${LORA_LR}, - warmup_ratio=0.05, - lr_scheduler_type="cosine", - logging_steps=10, - save_steps=save_steps, - eval_strategy="steps", - eval_steps=save_steps, - load_best_model_at_end=True, - metric_for_best_model="eval_loss", - bf16=True, - gradient_checkpointing=True, - dataloader_num_workers=4, - report_to="none", - save_total_limit=3, -) - -# ---- Trainer ---- -collator = DataCollatorForSeq2Seq(tokenizer, model=model, padding=True) -trainer = Trainer( - model=model, - args=args, - train_dataset=tok_ds["train"], - eval_dataset=tok_ds["test"], - data_collator=collator, -) - -print(f"Starting LoRA training: {total_steps} total steps") -trainer.train() - -# Save final adapter -final_adapter = os.path.join(out_dir, "lora_adapter_final") -model.save_pretrained(final_adapter) -tokenizer.save_pretrained(final_adapter) -print(f"Final LoRA adapter saved to {final_adapter}") - -# Save training metadata -meta = { - "model": hf_id, - "method": "lora", - "lora_rank": ${LORA_RANK}, - "lora_alpha": ${LORA_ALPHA}, - "epochs": ${LORA_EPOCHS}, - "train_examples": len(tok_ds["train"]), - "final_train_loss": trainer.state.log_history[-1].get("loss"), -} -with open(os.path.join(out_dir, "training_meta.json"), "w") as f: - import json; json.dump(meta, f, indent=2) -print("Training metadata saved.") -PYEOF - - local exit_code=${PIPESTATUS[0]} - local t1 - t1="$(date +%s)" - local elapsed=$(( t1 - t0 )) - - if [[ $exit_code -eq 0 ]]; then - log_ok "LoRA $model done in ${elapsed}s (~$(( elapsed / 3600 ))h $(( (elapsed % 3600) / 60 ))m)" - return 0 - else - log_error "LoRA $model FAILED (exit $exit_code) after ${elapsed}s" - return $exit_code - fi -} - -# --------------------------------------------------------------------------- -# SFT (full fine-tuning) for one model -# --------------------------------------------------------------------------- -run_sft() { - local model="$1" - local hf_id - hf_id="$(model_hf_id "$model")" - local out_dir="${RESULTS_BASE}/sft/${model}" - local checkpoint_dir="${out_dir}/checkpoint" - mkdir -p "$out_dir" "$checkpoint_dir" - - log_info "==========================================" - log_info "SFT training: $model ($hf_id)" - log_info " Output: $out_dir" - log_info " Epochs=$SFT_EPOCHS LR=$SFT_LR batch=$SFT_BATCH_SIZE grad_accum=$SFT_GRAD_ACCUM" - log_info "==========================================" - - local dataset_path - dataset_path="$(prepare_dataset sft "$model")" - - local t0 - t0="$(date +%s)" - - local n_gpus - n_gpus="$(python3 -c "import torch; print(torch.cuda.device_count())" 2>/dev/null || echo 1)" - log_info "Training on $n_gpus GPU(s)" - - if [[ "$n_gpus" -gt 1 ]]; then - LAUNCHER="uv run torchrun --nproc_per_node=$n_gpus" - else - LAUNCHER="uv run python" - fi - - $LAUNCHER - <&1 | tee -a "$LOG_FILE" -import json -import math -import os -from pathlib import Path - -import torch -from datasets import load_dataset -from transformers import ( - AutoModelForCausalLM, - AutoTokenizer, - DataCollatorForSeq2Seq, - Trainer, - TrainingArguments, -) - -hf_id = "${hf_id}" -out_dir = "${out_dir}" -ckpt = "${checkpoint_dir}" -data_path = "${dataset_path}" - -# ---- Tokenizer ---- -print(f"Loading tokenizer: {hf_id}") -tokenizer = AutoTokenizer.from_pretrained(hf_id, trust_remote_code=True) -if tokenizer.pad_token is None: - tokenizer.pad_token = tokenizer.eos_token - -# ---- Dataset ---- -print(f"Loading dataset: {data_path}") -raw = load_dataset("json", data_files=data_path, split="train") -raw = raw.train_test_split(test_size=0.02, seed=42) - -def tokenize(example): - messages = example["messages"] - text = tokenizer.apply_chat_template( - messages, - tokenize=False, - add_generation_prompt=False, - ) - enc = tokenizer(text, truncation=True, max_length=${MAX_SEQ_LEN}) - enc["labels"] = enc["input_ids"].copy() - return enc - -print("Tokenizing dataset...") -tok_ds = raw.map(tokenize, remove_columns=raw["train"].column_names, num_proc=4) -print(f"Train: {len(tok_ds['train'])} Eval: {len(tok_ds['test'])}") - -# ---- Model (4-bit quantized to fit on fewer GPUs) ---- -print(f"Loading model: {hf_id}") -from transformers import BitsAndBytesConfig -bnb_config = BitsAndBytesConfig( - load_in_4bit=True, - bnb_4bit_compute_dtype=torch.bfloat16, - bnb_4bit_quant_type="nf4", - bnb_4bit_use_double_quant=True, -) -model = AutoModelForCausalLM.from_pretrained( - hf_id, - quantization_config=bnb_config, - trust_remote_code=True, - device_map="auto", -) -model.config.use_cache = False - -# ---- Training args ---- -steps_per_epoch = math.ceil(len(tok_ds["train"]) / (${SFT_BATCH_SIZE} * ${SFT_GRAD_ACCUM})) -total_steps = steps_per_epoch * ${SFT_EPOCHS} -save_steps = max(50, steps_per_epoch // 2) - -args = TrainingArguments( - output_dir=ckpt, - num_train_epochs=${SFT_EPOCHS}, - per_device_train_batch_size=${SFT_BATCH_SIZE}, - gradient_accumulation_steps=${SFT_GRAD_ACCUM}, - learning_rate=${SFT_LR}, - warmup_ratio=0.03, - lr_scheduler_type="cosine", - logging_steps=10, - save_steps=save_steps, - eval_strategy="steps", - eval_steps=save_steps, - load_best_model_at_end=True, - metric_for_best_model="eval_loss", - bf16=True, - gradient_checkpointing=True, - dataloader_num_workers=4, - report_to="none", - save_total_limit=3, -) - -# ---- Trainer ---- -collator = DataCollatorForSeq2Seq(tokenizer, model=model, padding=True) -trainer = Trainer( - model=model, - args=args, - train_dataset=tok_ds["train"], - eval_dataset=tok_ds["test"], - data_collator=collator, -) - -print(f"Starting SFT training: {total_steps} total steps") -trainer.train() - -# Save final model -final_model = os.path.join(out_dir, "sft_model_final") -model.save_pretrained(final_model) -tokenizer.save_pretrained(final_model) -print(f"Final SFT model saved to {final_model}") - -# Save training metadata -meta = { - "model": hf_id, - "method": "sft", - "epochs": ${SFT_EPOCHS}, - "train_examples": len(tok_ds["train"]), - "final_train_loss": trainer.state.log_history[-1].get("loss"), -} -with open(os.path.join(out_dir, "training_meta.json"), "w") as f: - import json; json.dump(meta, f, indent=2) -print("Training metadata saved.") -PYEOF - - local exit_code=${PIPESTATUS[0]} - local t1 - t1="$(date +%s)" - local elapsed=$(( t1 - t0 )) - - if [[ $exit_code -eq 0 ]]; then - log_ok "SFT $model done in ${elapsed}s (~$(( elapsed / 3600 ))h $(( (elapsed % 3600) / 60 ))m)" - return 0 - else - log_error "SFT $model FAILED (exit $exit_code) after ${elapsed}s" - return $exit_code - fi -} - -# --------------------------------------------------------------------------- -# Post-training eval: run fast benchmarks on a checkpoint -# --------------------------------------------------------------------------- -run_eval() { - local method="$1" # lora or sft - local model="$2" - local out_dir="${RESULTS_BASE}/${method}/${model}" - - if [[ "$SKIP_EVAL" == "true" ]]; then - log_info "Skipping eval (--skip-eval)" - return 0 - fi - - # Locate the final checkpoint / adapter - local checkpoint="" - if [[ "$method" == "lora" ]]; then - checkpoint="${out_dir}/lora_adapter_final" - else - checkpoint="${out_dir}/sft_model_final" - fi - - if [[ ! -d "$checkpoint" ]]; then - log_warn "Checkpoint not found for $method/$model at $checkpoint — skipping eval" - return 0 - fi - - log_info "--- Post-training eval: $method/$model ---" - log_info " Checkpoint: $checkpoint" - - for bench in "${FAST_BENCHMARKS[@]}"; do - local eval_out="${out_dir}/eval/${bench}" - mkdir -p "$eval_out" - log_info " Eval: $bench -> $eval_out" - - uv run python - <&1 | tee -a "$LOG_FILE" -import subprocess, sys - -cmd = [ - "uv", "run", "python", "-m", "openjarvis.evals", "run", - "--model-path", "${checkpoint}", - "--model-id", "${model}-${method}", - "--benchmark", "${bench}", - "--max-samples", "${EVAL_MAX_SAMPLES}", - "--output", "${eval_out}", -] -print("Running:", " ".join(cmd)) -result = subprocess.run(cmd, capture_output=False) -sys.exit(result.returncode) -PYEOF - - local eval_exit=${PIPESTATUS[0]} - if [[ $eval_exit -eq 0 ]]; then - log_ok " Eval $method/$model/$bench OK" - else - log_warn " Eval $method/$model/$bench returned exit $eval_exit (non-fatal)" - fi - done -} - -# --------------------------------------------------------------------------- -# Summary -# --------------------------------------------------------------------------- -print_summary() { - echo "" - log_info "=== Track D Complete ===" - log_info "Results written to:" - - if [[ "$RUN_LORA" == "true" ]]; then - for model in "${ALL_LORA_MODELS[@]}"; do - local out="${RESULTS_BASE}/lora/${model}" - if [[ -d "$out" ]]; then - log_ok " lora/$model -> $out" - else - log_warn " lora/$model -> MISSING ($out)" - fi - done - fi - - if [[ "$RUN_SFT" == "true" ]]; then - for model in "${ALL_SFT_MODELS[@]}"; do - local out="${RESULTS_BASE}/sft/${model}" - if [[ -d "$out" ]]; then - log_ok " sft/$model -> $out" - else - log_warn " sft/$model -> MISSING ($out)" - fi - done - fi - - log_info "Full log: $LOG_FILE" -} - -# --------------------------------------------------------------------------- -# Main -# --------------------------------------------------------------------------- -main() { - setup_env - - local failed=0 - - # ---- LoRA runs ---- - if [[ "$RUN_LORA" == "true" ]]; then - log_info "==========================================" - log_info "Starting LoRA training runs" - log_info "Models: ${ALL_LORA_MODELS[*]}" - log_info "==========================================" - for model in "${ALL_LORA_MODELS[@]}"; do - run_lora "$model" || { failed=$(( failed + 1 )); log_error "LoRA $model failed, continuing..."; } - run_eval lora "$model" - echo "" - done - fi - - # ---- SFT runs ---- - if [[ "$RUN_SFT" == "true" ]]; then - log_info "==========================================" - log_info "Starting SFT training runs" - log_info "Models: ${ALL_SFT_MODELS[*]}" - log_info "==========================================" - for model in "${ALL_SFT_MODELS[@]}"; do - run_sft "$model" || { failed=$(( failed + 1 )); log_error "SFT $model failed, continuing..."; } - run_eval sft "$model" - echo "" - done - fi - - print_summary - - if [[ $failed -gt 0 ]]; then - log_error "$failed training run(s) failed. Check log: $LOG_FILE" - exit 1 - fi - - log_ok "All Track D runs completed successfully." -} - -main "$@" diff --git a/scripts/oauth_all.py b/scripts/oauth_all.py index 7e969c27..50755171 100644 --- a/scripts/oauth_all.py +++ b/scripts/oauth_all.py @@ -14,7 +14,7 @@ import os import webbrowser from http.server import BaseHTTPRequestHandler, HTTPServer from pathlib import Path -from typing import Any, Dict, List, Optional +from typing import Any, Dict, List from urllib.parse import parse_qs, urlencode, urlparse import httpx @@ -44,16 +44,31 @@ def _load_creds(filename: str) -> Dict[str, str]: _google = _load_creds("google.json") -GOOGLE_CLIENT_ID = os.environ.get("OPENJARVIS_GOOGLE_CLIENT_ID", "") or _google["client_id"] -GOOGLE_CLIENT_SECRET = os.environ.get("OPENJARVIS_GOOGLE_CLIENT_SECRET", "") or _google["client_secret"] +GOOGLE_CLIENT_ID = ( + os.environ.get("OPENJARVIS_GOOGLE_CLIENT_ID", "") or _google["client_id"] +) +GOOGLE_CLIENT_SECRET = ( + os.environ.get("OPENJARVIS_GOOGLE_CLIENT_SECRET", "") + or _google["client_secret"] +) _strava = _load_creds("strava.json") -STRAVA_CLIENT_ID = os.environ.get("OPENJARVIS_STRAVA_CLIENT_ID", "") or _strava["client_id"] -STRAVA_CLIENT_SECRET = os.environ.get("OPENJARVIS_STRAVA_CLIENT_SECRET", "") or _strava["client_secret"] +STRAVA_CLIENT_ID = ( + os.environ.get("OPENJARVIS_STRAVA_CLIENT_ID", "") or _strava["client_id"] +) +STRAVA_CLIENT_SECRET = ( + os.environ.get("OPENJARVIS_STRAVA_CLIENT_SECRET", "") + or _strava["client_secret"] +) _spotify = _load_creds("spotify.json") -SPOTIFY_CLIENT_ID = os.environ.get("OPENJARVIS_SPOTIFY_CLIENT_ID", "") or _spotify["client_id"] -SPOTIFY_CLIENT_SECRET = os.environ.get("OPENJARVIS_SPOTIFY_CLIENT_SECRET", "") or _spotify["client_secret"] +SPOTIFY_CLIENT_ID = ( + os.environ.get("OPENJARVIS_SPOTIFY_CLIENT_ID", "") or _spotify["client_id"] +) +SPOTIFY_CLIENT_SECRET = ( + os.environ.get("OPENJARVIS_SPOTIFY_CLIENT_SECRET", "") + or _spotify["client_secret"] +) # ── Generic OAuth helpers ──────────────────────────────────────────────────── @@ -128,7 +143,7 @@ def do_google() -> None: "prompt": "consent", }) ) - print(f" Opening browser...") + print(" Opening browser...") webbrowser.open(url) code = _wait_for_code() @@ -176,7 +191,7 @@ def do_strava() -> None: "scope": "activity:read_all", }) ) - print(f" Opening browser...") + print(" Opening browser...") webbrowser.open(url) code = _wait_for_code() @@ -208,7 +223,6 @@ def do_strava() -> None: def _make_self_signed_cert() -> tuple[str, str]: """Generate a temporary self-signed cert for localhost HTTPS.""" - import ssl import subprocess import tempfile @@ -288,7 +302,7 @@ def do_spotify() -> None: "scope": "user-read-recently-played", }) ) - print(f" Opening browser...") + print(" Opening browser...") webbrowser.open(url) code = _wait_for_code(port=spotify_port) @@ -323,7 +337,9 @@ def do_spotify() -> None: # ── Main ───────────────────────────────────────────────────────────────────── if __name__ == "__main__": - parser = argparse.ArgumentParser(description="Run OAuth flows for OpenJarvis connectors") + parser = argparse.ArgumentParser( + description="Run OAuth flows for OpenJarvis connectors" + ) parser.add_argument("--google", action="store_true", help="Only Google") parser.add_argument("--strava", action="store_true", help="Only Strava") parser.add_argument("--spotify", action="store_true", help="Only Spotify") diff --git a/scripts/smoke_framework_comparison.sh b/scripts/smoke_framework_comparison.sh index f9ad6faa..38791d5e 100755 --- a/scripts/smoke_framework_comparison.sh +++ b/scripts/smoke_framework_comparison.sh @@ -47,27 +47,45 @@ print(' all pins OK') echo "==> Running one-task smoke per (framework, benchmark)" mkdir -p results/smoke +CONFIG_DIR="results/smoke/configs" +mkdir -p "$CONFIG_DIR" SMOKE_BENCHES=(toolcall15 pinchbench gaia) SMOKE_FRAMEWORKS=(hermes openclaw openjarvis) SMOKE_MODEL="qwen-9b" for fwk in "${SMOKE_FRAMEWORKS[@]}"; do for bench in "${SMOKE_BENCHES[@]}"; do - config="src/openjarvis/evals/configs/framework_comparison/${bench}-${fwk}-${SMOKE_MODEL}.toml" - if [ ! -f "$config" ]; then - echo " ! missing config: $config (run make_configs --all-tier1 (or materialize the configs you need))" - continue - fi + uv run python -m openjarvis.evals.comparison.make_configs \ + --framework "$fwk" \ + --model "$SMOKE_MODEL" \ + --benchmark "$bench" \ + --output-dir "$CONFIG_DIR" >/dev/null + config="${CONFIG_DIR}/${bench}-${fwk}-${SMOKE_MODEL}.toml" + run_dir="results/smoke/${fwk}/${SMOKE_MODEL}/${bench}/" + JARVIS_BACKEND_BASE_URL="$JARVIS_MOCK_LLM_URL" uv run python - "$config" "$run_dir" <<'PY' +from pathlib import Path +import re +import sys + +path = Path(sys.argv[1]) +run_dir = sys.argv[2] +text = path.read_text() +text = re.sub(r'^output_dir = ".*"$', f'output_dir = "{run_dir}"', text, flags=re.M) +text = re.sub(r"^max_samples = \d+$", "max_samples = 1", text, flags=re.M) +path.write_text(text) +PY echo " ▸ $fwk × $bench" - uv run python -m openjarvis.evals run --config "$config" --max-samples 1 \ - --output-dir "results/smoke/${fwk}/${SMOKE_MODEL}/${bench}/" \ + mkdir -p "$run_dir" + JARVIS_BACKEND_BASE_URL="$JARVIS_MOCK_LLM_URL" \ + JARVIS_BACKEND_API_KEY="${JARVIS_BACKEND_API_KEY:-smoke}" \ + uv run python -m openjarvis.evals run --config "$config" \ || echo " FAILED (continuing)" done done echo "==> Generating T1 from smoke results" uv run python -m openjarvis.evals.comparison.table_gen \ - --results-glob "results/smoke/**/summary.json" \ + --results-glob "results/smoke/**/*.summary.json" \ --tables T1 \ --output-dir results/smoke/tables/ diff --git a/src/openjarvis/evals/backends/external/_runners/openclaw_runner.mjs b/src/openjarvis/evals/backends/external/_runners/openclaw_runner.mjs index d199a740..242e68fc 100644 --- a/src/openjarvis/evals/backends/external/_runners/openclaw_runner.mjs +++ b/src/openjarvis/evals/backends/external/_runners/openclaw_runner.mjs @@ -6,13 +6,14 @@ // --task --model --base-url --api-key \ // --output-json [--workspace ] // -// Loads OpenClaw from $OPENCLAW_PATH and runs `openclaw chat --message --json` -// (or equivalent non-interactive entry). Emits a JSON dict matching the +// Loads OpenClaw from $OPENCLAW_PATH and runs `openclaw agent --local +// --message --json`. Emits a JSON dict matching the // _RunnerOutput Pydantic schema in _subprocess_runner.py. -import { writeFileSync, existsSync } from 'node:fs'; +import { mkdtempSync, writeFileSync, existsSync } from 'node:fs'; import { spawn } from 'node:child_process'; import { join } from 'node:path'; +import { tmpdir } from 'node:os'; import { argv, env, exit, chdir } from 'node:process'; function parseArgs(args) { @@ -47,12 +48,30 @@ async function main() { return 3; } + const requestedModel = String(args.model || '').includes('/') + ? String(args.model || '') + : `ollama/${args.model}`; + const runDir = mkdtempSync(join(tmpdir(), 'openjarvis-openclaw-')); + const configPath = join(runDir, 'openclaw.json'); + writeFileSync(configPath, JSON.stringify({ + agents: { + defaults: { + model: { primary: requestedModel }, + models: { [requestedModel]: {} }, + }, + }, + })); + const childEnv = { ...env, OPENCLAW_MODEL: args.model, OPENCLAW_BASE_URL: args.base_url, OPENCLAW_API_KEY: args.api_key, + OPENCLAW_CONFIG_PATH: configPath, }; + if (requestedModel.startsWith('ollama/')) { + childEnv.OLLAMA_API_KEY = args.api_key || 'ollama'; + } // Use the SAME node executable that's running this script — picking up // 'node' from PATH can resolve to a system Node too old for OpenClaw @@ -95,11 +114,18 @@ async function main() { // { response: str, usage: {...}, messages: [{...}], tool_calls: int } try { const parsed = JSON.parse(stdout); - output.content = parsed.response || ''; - output.usage = parsed.usage || {}; - output.trajectory = parsed.messages || []; + const payloads = Array.isArray(parsed.payloads) ? parsed.payloads : []; + const messages = Array.isArray(parsed.messages) ? parsed.messages : []; + const usage = parsed.usage || parsed.meta?.agentMeta?.usage || {}; + output.content = parsed.response || payloads.map((p) => p.text || '').join('\n'); + output.usage = { + prompt_tokens: usage.prompt_tokens ?? usage.input ?? 0, + completion_tokens: usage.completion_tokens ?? usage.output ?? 0, + total_tokens: usage.total_tokens ?? usage.total ?? 0, + }; + output.trajectory = messages; output.tool_calls = parsed.tool_calls || 0; - output.turn_count = (parsed.messages || []).filter( + output.turn_count = messages.filter( (m) => m.role === 'assistant' ).length; } catch (e) { diff --git a/src/openjarvis/evals/comparison/table_gen.py b/src/openjarvis/evals/comparison/table_gen.py index b224590a..575a1660 100644 --- a/src/openjarvis/evals/comparison/table_gen.py +++ b/src/openjarvis/evals/comparison/table_gen.py @@ -38,6 +38,10 @@ class MixedCommitError(TableGenError): """Raised when one (framework, model, benchmark) cell has multiple commits.""" +class NoResultsError(TableGenError): + """Raised when no valid summary files were loaded.""" + + class _MetricStats(BaseModel): mean: float std: float @@ -395,6 +399,11 @@ def main(results_glob: str, tables: str, output_dir: Optional[Path]) -> None: click.echo( f"Loaded {len(frame.df)} metric rows; {frame.unloadable_count} files skipped." ) + if frame.df.is_empty(): + raise click.ClickException( + "No valid summary files matched --results-glob; refusing to emit empty " + "tables." + ) requested = [t.strip() for t in tables.split(",") if t.strip()] for name in requested: diff --git a/src/openjarvis/evals/core/runner.py b/src/openjarvis/evals/core/runner.py index 851a8c8e..9b03d220 100644 --- a/src/openjarvis/evals/core/runner.py +++ b/src/openjarvis/evals/core/runner.py @@ -307,6 +307,38 @@ class EvalRunner: def _process_one(self, record: EvalRecord) -> EvalResult: """Process a single evaluation sample.""" cfg = self._config + + def _backend_error_result(full: dict, message: str) -> EvalResult: + usage = full.get("usage", {}) or {} + energy_j = full.get("energy_joules", 0.0) or 0.0 + power_w = full.get("power_watts") or full.get("peak_power_w") or 0.0 + return EvalResult( + record_id=record.record_id, + model_answer=full.get("content", "") or "", + error=message, + latency_seconds=full.get("latency_seconds", 0.0) or 0.0, + prompt_tokens=usage.get("prompt_tokens", 0), + completion_tokens=usage.get("completion_tokens", 0), + cost_usd=full.get("cost_usd", 0.0) or 0.0, + ttft=full.get("ttft", 0.0) or 0.0, + energy_joules=energy_j, + power_watts=power_w, + gpu_utilization_pct=full.get("gpu_utilization_pct", 0.0) or 0.0, + throughput_tok_per_sec=full.get("throughput_tok_per_sec", 0.0) + or 0.0, + trace_data=full.get("trace_data"), + framework=full.get( + "framework", + getattr(self._backend, "framework_name", "openjarvis"), + ), + framework_commit=full.get( + "framework_commit", + getattr(self._backend, "framework_commit_value", "") or "", + ), + tool_calls=int(full.get("tool_calls", 0) or 0), + turn_count=int(full.get("turn_count", 0) or 0), + ) + try: gen_kwargs: dict = dict( model=cfg.model, @@ -327,6 +359,8 @@ class EvalRunner: **gen_kwargs, ) full = full or {} + if full.get("error"): + return _backend_error_result(full, str(full["error"])) all_tool_results = list( full.get( "tool_results", @@ -346,6 +380,8 @@ class EvalRunner: **gen_kwargs, ) sfull = sfull or {} + if sfull.get("error"): + return _backend_error_result(sfull, str(sfull["error"])) all_tool_results.extend( sfull.get("tool_results", []), ) @@ -363,6 +399,8 @@ class EvalRunner: **gen_kwargs, ) full = full or {} + if full.get("error"): + return _backend_error_result(full, str(full["error"])) content = full.get("content", "") is_correct, scoring_meta = self._scorer.score( record, @@ -378,7 +416,7 @@ class EvalRunner: # 'or 0.0' handles the case where the key is present but the # value is None -- .get(default) only kicks in when missing. energy_j = full.get("energy_joules", 0.0) or 0.0 - power_w = full.get("power_watts", 0.0) or 0.0 + power_w = full.get("power_watts") or full.get("peak_power_w") or 0.0 throughput = full.get("throughput_tok_per_sec", 0.0) or 0.0 accuracy_score = 1.0 if is_correct else 0.0 diff --git a/tests/evals/comparison/test_export_to_table_gen_roundtrip.py b/tests/evals/comparison/test_export_to_table_gen_roundtrip.py index fbe0c378..9ce5679b 100644 --- a/tests/evals/comparison/test_export_to_table_gen_roundtrip.py +++ b/tests/evals/comparison/test_export_to_table_gen_roundtrip.py @@ -122,7 +122,7 @@ class TestSummaryToDictEmitsRequiredFields: completion_tokens=50, cost_usd=0.001, energy_joules=10.0, - power_watts=20.0, + power_watts=30.0, framework="hermes", framework_commit="5d3be898a", tool_calls=2, @@ -156,6 +156,7 @@ class TestSummaryToDictEmitsRequiredFields: assert d["n_tasks"] == 5 assert "accuracy" in d["metrics"] assert "latency_seconds" in d["metrics"] + assert d["metrics"]["peak_power_w"]["mean"] == 30.0 assert d["metrics"]["accuracy"]["n"] == 5 assert d["metrics"]["latency_seconds"]["n"] == 5 # Existing rich schema also present (unchanged) diff --git a/tests/evals/comparison/test_openclaw_runner_contract.py b/tests/evals/comparison/test_openclaw_runner_contract.py new file mode 100644 index 00000000..0db4f5c1 --- /dev/null +++ b/tests/evals/comparison/test_openclaw_runner_contract.py @@ -0,0 +1,72 @@ +"""Contract tests for the OpenClaw subprocess runner.""" + +from __future__ import annotations + +import json +import shutil +import subprocess +from pathlib import Path + +import pytest + + +def test_openclaw_runner_parses_real_agent_json_shape(tmp_path: Path) -> None: + node = shutil.which("node") + if node is None: + pytest.skip("node not available") + + fake_openclaw = tmp_path / "openclaw.mjs" + fake_openclaw.write_text( + "\n".join( + [ + "console.log(JSON.stringify({", + " payloads: [{ text: 'hello from openclaw', mediaUrl: null }],", + " meta: {", + " agentMeta: { usage: { input: 11, output: 7, total: 18 } },", + " },", + "}));", + ] + ), + encoding="utf-8", + ) + + runner = ( + Path(__file__).resolve().parents[3] + / "src/openjarvis/evals/backends/external/_runners/openclaw_runner.mjs" + ) + out_json = tmp_path / "out.json" + env = { + "OPENCLAW_PATH": str(tmp_path), + "HOME": str(tmp_path / "home"), + } + result = subprocess.run( + [ + node, + str(runner), + "--task", + "Say hello.", + "--model", + "qwen3:0.6b", + "--base-url", + "http://127.0.0.1:11434/v1", + "--api-key", + "ollama", + "--output-json", + str(out_json), + ], + env=env, + capture_output=True, + text=True, + timeout=30, + check=False, + ) + + assert result.returncode == 0, result.stderr + data = json.loads(out_json.read_text(encoding="utf-8")) + assert data["content"] == "hello from openclaw" + assert data["usage"] == { + "prompt_tokens": 11, + "completion_tokens": 7, + "total_tokens": 18, + } + assert data["error"] is None diff --git a/tests/evals/comparison/test_runner_error_path_framework.py b/tests/evals/comparison/test_runner_error_path_framework.py index c6272e39..8280dd7a 100644 --- a/tests/evals/comparison/test_runner_error_path_framework.py +++ b/tests/evals/comparison/test_runner_error_path_framework.py @@ -9,6 +9,7 @@ happy-path EvalResult construction. from __future__ import annotations +import json from typing import Any, Dict, Iterable, List, Optional, Tuple from openjarvis.evals.core.dataset import DatasetProvider @@ -33,6 +34,57 @@ class _FailingBackend: pass +class _PeakOnlyBackend: + """Mock external-style backend that reports peak_power_w but no power_watts.""" + + backend_id = "hermes" + framework_name = "hermes" + framework_commit_value = "abc12345" + + def generate(self, *args: Any, **kwargs: Any) -> str: + return "hi" + + def generate_full(self, *args: Any, **kwargs: Any) -> Dict[str, Any]: + return { + "content": "hi", + "usage": {}, + "latency_seconds": 1.0, + "energy_joules": 10.0, + "peak_power_w": 42.0, + "framework": "hermes", + "framework_commit": "abc12345", + } + + def close(self) -> None: + pass + + +class _BackendErrorPayload: + """Mock external backend that returns an error payload instead of raising.""" + + backend_id = "hermes" + framework_name = "hermes" + framework_commit_value = "abc12345" + + def generate(self, *args: Any, **kwargs: Any) -> str: + return "" + + def generate_full(self, *args: Any, **kwargs: Any) -> Dict[str, Any]: + return { + "content": "", + "usage": {"prompt_tokens": 3, "completion_tokens": 0}, + "latency_seconds": 1.5, + "energy_joules": None, + "peak_power_w": 17.0, + "framework": "hermes", + "framework_commit": "abc12345", + "error": "backend reported failure", + } + + def close(self) -> None: + pass + + class _SingleRecordDataset(DatasetProvider): dataset_id = "mock" dataset_name = "mock" @@ -141,6 +193,70 @@ class TestErrorPathFrameworkPropagation: assert results[0].error is not None +class TestExternalTelemetryPropagation: + def test_peak_power_falls_back_to_power_watts(self, tmp_path: Any) -> None: + """External backends report peak_power_w; runner should preserve it + in EvalResult.power_watts so summary/table metrics are populated.""" + cfg = RunConfig( + benchmark="mock", + backend="hermes", + model="mock-model", + max_samples=1, + max_workers=1, + output_path=str(tmp_path / "out.jsonl"), + ) + runner = EvalRunner( + config=cfg, + dataset=_SingleRecordDataset(), + backend=_PeakOnlyBackend(), # type: ignore[arg-type] + scorer=_NoopScorer(), + ) + runner.run() + + assert runner.results[0].power_watts == 42.0 + summary = json.loads((tmp_path / "out.summary.json").read_text()) + assert summary["metrics"]["peak_power_w"]["mean"] == 42.0 + + +class TestBackendErrorPayloadPropagation: + def test_backend_error_payload_marks_result_error(self, tmp_path: Any) -> None: + """Backends can return structured error payloads without raising; + those must count as errors rather than scored empty answers.""" + cfg = RunConfig( + benchmark="mock", + backend="hermes", + model="mock-model", + max_samples=1, + max_workers=1, + output_path=str(tmp_path / "out.jsonl"), + ) + runner = EvalRunner( + config=cfg, + dataset=_SingleRecordDataset(), + backend=_BackendErrorPayload(), # type: ignore[arg-type] + scorer=_NoopScorer(), + ) + summary = runner.run() + + result = runner.results[0] + assert result.error == "backend reported failure" + assert result.is_correct is None + assert result.framework == "hermes" + assert result.framework_commit == "abc12345" + assert result.power_watts == 17.0 + assert result.prompt_tokens == 3 + assert summary.errors == 1 + assert summary.scored_samples == 0 + + summary_data = json.loads((tmp_path / "out.summary.json").read_text()) + assert summary_data["errors"] == 1 + assert summary_data["metrics"]["accuracy"] == { + "mean": 0.0, + "std": 0.0, + "n": 0, + } + + class _MockBackendWithCommit: """Mock backend that exposes framework_name AND framework_commit_value.""" diff --git a/tests/evals/comparison/test_table_gen.py b/tests/evals/comparison/test_table_gen.py index 18c2fcf8..bf82daa5 100644 --- a/tests/evals/comparison/test_table_gen.py +++ b/tests/evals/comparison/test_table_gen.py @@ -13,6 +13,7 @@ from openjarvis.evals.comparison.table_gen import ( # noqa: E402 MixedCommitError, ResultsFrame, load_results, + main, ) @@ -70,6 +71,23 @@ class TestLoadResults: with pytest.raises(MixedCommitError, match="abc123.*zzz999"): load_results(str(tmp_path / "**" / "summary.json")) + def test_cli_refuses_empty_result_set(self, tmp_path: Path) -> None: + from click.testing import CliRunner + + result = CliRunner().invoke( + main, + [ + "--results-glob", + str(tmp_path / "**" / "summary.json"), + "--tables", + "T1", + "--output-dir", + str(tmp_path / "tables"), + ], + ) + assert result.exit_code != 0 + assert "No valid summary files matched" in result.output + class TestRenderBooktabs: def test_emits_valid_tabular(self) -> None: