mirror of
https://github.com/open-jarvis/OpenJarvis.git
synced 2026-07-27 21:05:34 +00:00
fix(evals): clean up AI stack runtime harness (#320)
Co-authored-by: krypticmouse <herumbshandilya123@gmail.com>
This commit is contained in:
Avanika Narayan
co-authored by
krypticmouse
krypticmouse
parent
f41cf420be
commit
a689be0c69
@@ -1,246 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""A1: Seed feedback on all traces in the local M1 traces.db.
|
||||
|
||||
- Judges each unscored trace with Sonnet 4.6 using the calibration-validated prompt
|
||||
- Parallelized via ThreadPoolExecutor (8 workers) for I/O-bound API calls
|
||||
- Writes feedback to local traces.db via TraceStore.update_feedback
|
||||
- Logs every call to a JSONL audit file
|
||||
- Idempotent: skips traces that already have feedback (safe to re-run)
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import sqlite3
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
|
||||
from openjarvis.core.types import Message, Role
|
||||
from openjarvis.engine.cloud import CloudEngine
|
||||
from openjarvis.traces.store import TraceStore
|
||||
|
||||
HOME = Path(os.environ.get("OPENJARVIS_HOME", "/scratch/user/jonsaadfalcon/openjarvis-m1"))
|
||||
DB = HOME / "traces.db"
|
||||
LOG = HOME / "a1_feedback_log.jsonl"
|
||||
MODEL = "claude-sonnet-4-6"
|
||||
MAX_WORKERS = 8
|
||||
|
||||
JUDGE_PROMPT = """\
|
||||
You are evaluating whether an AI agent successfully completed its assigned task.
|
||||
|
||||
Assign a SCORE from the set {{0.2, 0.4, 0.6, 0.8}} using this rubric:
|
||||
|
||||
- 0.8 = Clean success. Task completed correctly. Minor stylistic issues don't affect correctness.
|
||||
- 0.6 = Partial. Real progress made but the answer has real gaps — missed requirement, incomplete output, recovered from errors but final result imperfect.
|
||||
- 0.4 = Poor. Some progress but the result is clearly incomplete, wrong, or the agent got mostly stuck.
|
||||
- 0.2 = Failure. Agent crashed, got stuck in a loop, gave up, hit a budget/poll/token limit before finishing, or produced no usable result.
|
||||
|
||||
IMPORTANT: Do not trust the agent's own self-assessment. Agents often narrate "I am stuck" or "I hit an error" — those are failure signals. Agents sometimes claim success when the actual output is incomplete — look at the concrete result, not the rhetoric.
|
||||
|
||||
TASK QUERY (first 1200 chars):
|
||||
<<<
|
||||
{query}
|
||||
>>>
|
||||
|
||||
AGENT FINAL RESULT (first 2500 chars):
|
||||
<<<
|
||||
{result_head}
|
||||
>>>
|
||||
{tail_section}
|
||||
Respond in EXACTLY this format, nothing else:
|
||||
SCORE=<one of 0.2, 0.4, 0.6, 0.8>
|
||||
REASON=<one brief sentence>
|
||||
"""
|
||||
|
||||
|
||||
SCORE_RE = re.compile(r"SCORE\s*=\s*(0?\.[2468])", re.IGNORECASE)
|
||||
REASON_RE = re.compile(r"REASON\s*=\s*(.+?)\s*$", re.IGNORECASE | re.DOTALL)
|
||||
|
||||
|
||||
def build_prompt(query: str, result: str) -> str:
|
||||
q = (query or "")[:1200]
|
||||
head = (result or "")[:2500]
|
||||
if result and len(result) > 3000:
|
||||
tail = f"\nAGENT FINAL RESULT (last 500 chars):\n<<<\n{result[-500:]}\n>>>\n"
|
||||
else:
|
||||
tail = ""
|
||||
return JUDGE_PROMPT.format(query=q, result_head=head, tail_section=tail)
|
||||
|
||||
|
||||
def judge_one(ce: CloudEngine, trace_id: str, query: str, result: str) -> dict:
|
||||
prompt = build_prompt(query, result)
|
||||
t0 = time.time()
|
||||
try:
|
||||
resp = ce.generate(
|
||||
messages=[Message(role=Role.USER, content=prompt)],
|
||||
model=MODEL,
|
||||
max_tokens=150,
|
||||
temperature=0.0,
|
||||
)
|
||||
content = resp.get("content", "") or ""
|
||||
cost = resp.get("cost_usd", 0.0) or 0.0
|
||||
usage = resp.get("usage", {}) or {}
|
||||
in_tok = usage.get("prompt_tokens", 0) or usage.get("input_tokens", 0)
|
||||
out_tok = usage.get("completion_tokens", 0) or usage.get("output_tokens", 0)
|
||||
|
||||
m = SCORE_RE.search(content)
|
||||
score = float(m.group(1)) if m else None
|
||||
mr = REASON_RE.search(content)
|
||||
reason = mr.group(1).strip() if mr else "(parse failed)"
|
||||
|
||||
return {
|
||||
"trace_id": trace_id,
|
||||
"score": score,
|
||||
"reason": reason,
|
||||
"raw": content,
|
||||
"cost": cost,
|
||||
"input_tokens": in_tok,
|
||||
"output_tokens": out_tok,
|
||||
"elapsed": time.time() - t0,
|
||||
"judged_at": datetime.utcnow().isoformat(timespec="seconds") + "Z",
|
||||
"error": None,
|
||||
}
|
||||
except Exception as e:
|
||||
return {
|
||||
"trace_id": trace_id,
|
||||
"score": None,
|
||||
"reason": None,
|
||||
"raw": None,
|
||||
"cost": 0.0,
|
||||
"input_tokens": 0,
|
||||
"output_tokens": 0,
|
||||
"elapsed": time.time() - t0,
|
||||
"judged_at": datetime.utcnow().isoformat(timespec="seconds") + "Z",
|
||||
"error": f"{type(e).__name__}: {e}",
|
||||
}
|
||||
|
||||
|
||||
def main() -> int:
|
||||
if not os.environ.get("ANTHROPIC_API_KEY"):
|
||||
print("ERROR: ANTHROPIC_API_KEY not set", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
store = TraceStore(DB)
|
||||
# Use the TraceStore's connection for the initial read
|
||||
conn = store._conn
|
||||
conn.row_factory = sqlite3.Row
|
||||
|
||||
# Pull all traces that need scoring
|
||||
rows = list(conn.execute(
|
||||
"SELECT trace_id, query, result, agent, model FROM traces "
|
||||
"WHERE feedback IS NULL"
|
||||
))
|
||||
already_scored = conn.execute(
|
||||
"SELECT COUNT(*) FROM traces WHERE feedback IS NOT NULL"
|
||||
).fetchone()[0]
|
||||
|
||||
print(f"traces.db: {conn.execute('SELECT COUNT(*) FROM traces').fetchone()[0]} total")
|
||||
print(f" already scored: {already_scored}")
|
||||
print(f" to judge: {len(rows)}")
|
||||
print(f"parallelism: {MAX_WORKERS} workers")
|
||||
print(f"log: {LOG}")
|
||||
|
||||
if not rows:
|
||||
print("Nothing to do.")
|
||||
return 0
|
||||
|
||||
ce = CloudEngine()
|
||||
write_lock = threading.Lock()
|
||||
log_lock = threading.Lock()
|
||||
log_fp = open(LOG, "a", encoding="utf-8")
|
||||
|
||||
# Write a header line marking this run
|
||||
log_fp.write(json.dumps({
|
||||
"_run_started": datetime.utcnow().isoformat(timespec="seconds") + "Z",
|
||||
"model": MODEL,
|
||||
"workers": MAX_WORKERS,
|
||||
"to_judge": len(rows),
|
||||
"already_scored": already_scored,
|
||||
}) + "\n")
|
||||
log_fp.flush()
|
||||
|
||||
total_cost = 0.0
|
||||
done = 0
|
||||
errors = 0
|
||||
score_counts: dict = {}
|
||||
t_start = time.time()
|
||||
|
||||
def on_result(res: dict) -> None:
|
||||
nonlocal total_cost, done, errors
|
||||
trace_id = res["trace_id"]
|
||||
|
||||
# Write feedback to DB (if we got a valid score)
|
||||
if res["score"] is not None and res["error"] is None:
|
||||
with write_lock:
|
||||
store.update_feedback(trace_id, res["score"])
|
||||
conn.commit()
|
||||
else:
|
||||
errors += 1
|
||||
|
||||
with log_lock:
|
||||
log_fp.write(json.dumps(res) + "\n")
|
||||
log_fp.flush()
|
||||
|
||||
total_cost += res["cost"] or 0.0
|
||||
done += 1
|
||||
score_counts[res["score"]] = score_counts.get(res["score"], 0) + 1
|
||||
|
||||
# Progress every 50 or on error
|
||||
if done % 50 == 0 or res["error"]:
|
||||
elapsed = time.time() - t_start
|
||||
rate = done / max(0.001, elapsed)
|
||||
eta = (len(rows) - done) / max(0.001, rate)
|
||||
tag = "ERR " if res["error"] else " "
|
||||
print(
|
||||
f"{tag}[{done:4}/{len(rows)}] score={res['score']} "
|
||||
f"cost=${total_cost:.3f} "
|
||||
f"rate={rate:.1f}/s eta={eta:.0f}s "
|
||||
f"errors={errors}"
|
||||
)
|
||||
if res["error"]:
|
||||
print(f" ERROR on {trace_id[:12]}: {res['error']}")
|
||||
|
||||
with ThreadPoolExecutor(max_workers=MAX_WORKERS) as pool:
|
||||
futures = [
|
||||
pool.submit(judge_one, ce, r["trace_id"], r["query"] or "", r["result"] or "")
|
||||
for r in rows
|
||||
]
|
||||
for fut in as_completed(futures):
|
||||
res = fut.result()
|
||||
on_result(res)
|
||||
|
||||
log_fp.close()
|
||||
|
||||
elapsed = time.time() - t_start
|
||||
print(f"\n{'='*60}")
|
||||
print(f"A1 COMPLETE in {elapsed:.1f}s ({elapsed/60:.1f}min)")
|
||||
print(f"Total cost: ${total_cost:.4f}")
|
||||
print(f"Errors: {errors}/{len(rows)}")
|
||||
print(f"Score distribution:")
|
||||
for k in sorted(score_counts.keys(), key=lambda x: (x is None, x)):
|
||||
v = score_counts[k]
|
||||
pct = 100 * v / len(rows)
|
||||
label = {0.2: "failure", 0.4: "poor", 0.6: "partial", 0.8: "clean", None: "ERROR"}.get(k, "?")
|
||||
print(f" {k} ({label}): {v} ({pct:.1f}%)")
|
||||
|
||||
# Verify by re-counting from DB
|
||||
with_fb = conn.execute(
|
||||
"SELECT COUNT(*) FROM traces WHERE feedback IS NOT NULL"
|
||||
).fetchone()[0]
|
||||
above_gate = conn.execute(
|
||||
"SELECT COUNT(*) FROM traces WHERE feedback >= 0.7"
|
||||
).fetchone()[0]
|
||||
print(f"\nPost-A1 DB state:")
|
||||
print(f" traces with feedback: {with_fb}")
|
||||
print(f" traces passing 0.7 gate (eligible for personal benchmark): {above_gate}")
|
||||
return 0 if errors == 0 else 2
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -1,304 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Generate all distillation experiment TOML configs.
|
||||
|
||||
Produces configs for 7 experiment axes × multiple settings.
|
||||
Run: python scripts/experiments/generate_distillation_configs.py
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import itertools
|
||||
from pathlib import Path
|
||||
|
||||
CONFIGS_DIR = Path("src/openjarvis/evals/configs/distillation")
|
||||
|
||||
# ── Teacher models ──────────────────────────────────────────────────────────
|
||||
TEACHERS = {
|
||||
"opus": {
|
||||
"model": "claude-opus-4-6",
|
||||
"engine": "cloud",
|
||||
"provider": "anthropic",
|
||||
},
|
||||
"gpt54": {
|
||||
"model": "gpt-5.4",
|
||||
"engine": "cloud",
|
||||
"provider": "openai",
|
||||
},
|
||||
"gemini": {
|
||||
"model": "gemini-3.1-pro-preview",
|
||||
"engine": "cloud",
|
||||
"provider": "google",
|
||||
},
|
||||
"qwen397b": {
|
||||
"model": "Qwen/Qwen3.5-397B-A17B-FP8",
|
||||
"engine": "vllm",
|
||||
"provider": "local",
|
||||
"note": "# Requires 8×H100, vLLM serve on port 8010",
|
||||
},
|
||||
}
|
||||
|
||||
# ── Student models ──────────────────────────────────────────────────────────
|
||||
# Served via vLLM on this H100 node. 27B uses the FP8 weights that fit on a
|
||||
# single H100; 2B and 9B use standard FP16.
|
||||
STUDENTS = {
|
||||
"2b": {"model": "Qwen/Qwen3.5-2B", "engine": "vllm", "port": 8000},
|
||||
"9b": {"model": "Qwen/Qwen3.5-9B", "engine": "vllm", "port": 8001},
|
||||
"27b": {"model": "Qwen/Qwen3.5-27B-FP8", "engine": "vllm", "port": 8002},
|
||||
}
|
||||
|
||||
# ── Benchmarks ──────────────────────────────────────────────────────────────
|
||||
BENCHMARKS = {
|
||||
"pb": "pinchbench",
|
||||
"tc15": "toolcall15",
|
||||
"tb": "taubench",
|
||||
}
|
||||
|
||||
# ── Data configs ────────────────────────────────────────────────────────────
|
||||
DATA_CONFIGS = {
|
||||
"C1": {
|
||||
"desc": "Zero test data — external traces only (GeneralThought + ADP)",
|
||||
"trace_source": "external",
|
||||
"benchmark_queries_visible": False,
|
||||
},
|
||||
"C2": {
|
||||
"desc": "Test queries only — benchmark traces visible, answers hidden",
|
||||
"trace_source": "benchmark",
|
||||
"benchmark_queries_visible": True,
|
||||
},
|
||||
"C3": {
|
||||
"desc": "Test queries + external — both benchmark and external traces",
|
||||
"trace_source": "both",
|
||||
"benchmark_queries_visible": True,
|
||||
},
|
||||
}
|
||||
|
||||
# ── Budget presets ──────────────────────────────────────────────────────────
|
||||
BUDGETS = {
|
||||
"minimal": {"max_tool_calls": 5, "max_cost": 0.50},
|
||||
"standard": {"max_tool_calls": 15, "max_cost": 2.00},
|
||||
"thorough": {"max_tool_calls": 30, "max_cost": 5.00},
|
||||
"exhaustive": {"max_tool_calls": 50, "max_cost": 10.00},
|
||||
}
|
||||
|
||||
# ── Gate presets ────────────────────────────────────────────────────────────
|
||||
GATES = {
|
||||
"permissive": {"min_improvement": 0.0, "max_regression": 0.10},
|
||||
"standard": {"min_improvement": 0.0, "max_regression": 0.05},
|
||||
"strict": {"min_improvement": 0.02, "max_regression": 0.02},
|
||||
"none": {"min_improvement": -1.0, "max_regression": 1.0},
|
||||
}
|
||||
|
||||
# ── Autonomy presets ────────────────────────────────────────────────────────
|
||||
AUTONOMY_MODES = ["auto", "tiered", "manual"]
|
||||
|
||||
|
||||
def render_config(
|
||||
*,
|
||||
experiment: str,
|
||||
teacher_key: str,
|
||||
student_key: str,
|
||||
benchmark_key: str,
|
||||
data_config_key: str = "C2",
|
||||
budget_key: str = "standard",
|
||||
gate_key: str = "standard",
|
||||
autonomy: str = "auto",
|
||||
iterative_sessions: int = 1,
|
||||
) -> str:
|
||||
"""Render a TOML config string."""
|
||||
teacher = TEACHERS[teacher_key]
|
||||
student = STUDENTS[student_key]
|
||||
benchmark = BENCHMARKS[benchmark_key]
|
||||
data_cfg = DATA_CONFIGS[data_config_key]
|
||||
budget = BUDGETS[budget_key]
|
||||
gate = GATES[gate_key]
|
||||
|
||||
note = teacher.get("note", "")
|
||||
note_line = f"\n{note}" if note else ""
|
||||
|
||||
return f"""\
|
||||
# Distillation Experiment Config
|
||||
# Experiment: {experiment}
|
||||
# Teacher: {teacher["model"]} ({teacher["provider"]})
|
||||
# Student: {student["model"]} ({student["engine"]})
|
||||
# Benchmark: {benchmark}
|
||||
# Data config: {data_config_key} — {data_cfg["desc"]}
|
||||
# Budget: {budget_key} ({budget["max_tool_calls"]} tool calls, ${budget["max_cost"]:.2f})
|
||||
# Gate: {gate_key} (min_improvement={gate["min_improvement"]}, max_regression={gate["max_regression"]})
|
||||
# Autonomy: {autonomy}
|
||||
# Iterative sessions: {iterative_sessions}
|
||||
{note_line}
|
||||
|
||||
[intelligence]
|
||||
default_model = "{student["model"]}"
|
||||
|
||||
[engine]
|
||||
default = "{student["engine"]}"
|
||||
|
||||
[engine.vllm]
|
||||
host = "http://localhost:{student.get("port", 8000)}"
|
||||
|
||||
[learning.distillation]
|
||||
enabled = true
|
||||
autonomy_mode = "{autonomy}"
|
||||
teacher_model = "{teacher["model"]}"
|
||||
max_cost_per_session_usd = {budget["max_cost"]}
|
||||
max_tool_calls_per_diagnosis = {budget["max_tool_calls"]}
|
||||
|
||||
[learning.distillation.gate]
|
||||
min_improvement = {gate["min_improvement"]}
|
||||
max_regression = {gate["max_regression"]}
|
||||
benchmark_subsample_size = 50
|
||||
|
||||
[learning.distillation.benchmark]
|
||||
synthesis_feedback_threshold = 0.7
|
||||
max_benchmark_size = 200
|
||||
|
||||
[learning.distillation.experiment]
|
||||
# Metadata for the experiment runner (not read by distillation itself)
|
||||
experiment_id = "{experiment}"
|
||||
teacher_key = "{teacher_key}"
|
||||
student_key = "{student_key}"
|
||||
benchmark = "{benchmark}"
|
||||
data_config = "{data_config_key}"
|
||||
trace_source = "{data_cfg["trace_source"]}"
|
||||
benchmark_queries_visible = {str(data_cfg["benchmark_queries_visible"]).lower()}
|
||||
budget_key = "{budget_key}"
|
||||
gate_key = "{gate_key}"
|
||||
iterative_sessions = {iterative_sessions}
|
||||
"""
|
||||
|
||||
|
||||
def write_config(subdir: str, filename: str, content: str) -> Path:
|
||||
path = CONFIGS_DIR / subdir / filename
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text(content, encoding="utf-8")
|
||||
return path
|
||||
|
||||
|
||||
def generate_all() -> int:
|
||||
count = 0
|
||||
|
||||
# ── Exp 1a: Teacher Model Ablation ────────────────────────────────────
|
||||
# Fix: S-9b, B-standard, A-auto, G-standard, I-single
|
||||
# Vary: teacher × benchmark × data_config
|
||||
for teacher_key, bench_key, dc_key in itertools.product(
|
||||
TEACHERS, BENCHMARKS, DATA_CONFIGS
|
||||
):
|
||||
filename = f"{teacher_key}-9b-{bench_key}-{dc_key}.toml"
|
||||
content = render_config(
|
||||
experiment=f"exp1a-teacher/{teacher_key}-{bench_key}-{dc_key}",
|
||||
teacher_key=teacher_key,
|
||||
student_key="9b",
|
||||
benchmark_key=bench_key,
|
||||
data_config_key=dc_key,
|
||||
)
|
||||
write_config("exp1a-teacher", filename, content)
|
||||
count += 1
|
||||
|
||||
# ── Exp 1b: Budget Ablation ───────────────────────────────────────────
|
||||
# Fix: S-9b, T-sonnet(opus for quality), A-auto, G-standard, I-single
|
||||
# Vary: budget × benchmark × data_config
|
||||
for budget_key, bench_key, dc_key in itertools.product(
|
||||
BUDGETS, BENCHMARKS, DATA_CONFIGS
|
||||
):
|
||||
filename = f"{budget_key}-9b-{bench_key}-{dc_key}.toml"
|
||||
content = render_config(
|
||||
experiment=f"exp1b-budget/{budget_key}-{bench_key}-{dc_key}",
|
||||
teacher_key="opus",
|
||||
student_key="9b",
|
||||
benchmark_key=bench_key,
|
||||
data_config_key=dc_key,
|
||||
budget_key=budget_key,
|
||||
)
|
||||
write_config("exp1b-budget", filename, content)
|
||||
count += 1
|
||||
|
||||
# ── Exp 1c: Student Model Scaling ─────────────────────────────────────
|
||||
# Fix: T-opus, B-standard, A-auto, G-standard, I-single
|
||||
# Vary: student × benchmark × data_config
|
||||
for student_key, bench_key, dc_key in itertools.product(
|
||||
STUDENTS, BENCHMARKS, DATA_CONFIGS
|
||||
):
|
||||
filename = f"opus-{student_key}-{bench_key}-{dc_key}.toml"
|
||||
content = render_config(
|
||||
experiment=f"exp1c-student/opus-{student_key}-{bench_key}-{dc_key}",
|
||||
teacher_key="opus",
|
||||
student_key=student_key,
|
||||
benchmark_key=bench_key,
|
||||
data_config_key=dc_key,
|
||||
)
|
||||
write_config("exp1c-student", filename, content)
|
||||
count += 1
|
||||
|
||||
# ── Exp 2a: Gate Strictness ───────────────────────────────────────────
|
||||
# Fix: S-9b, T-opus, B-standard, A-auto, I-single
|
||||
# Vary: gate × benchmark
|
||||
for gate_key, bench_key in itertools.product(GATES, BENCHMARKS):
|
||||
filename = f"{gate_key}-9b-{bench_key}.toml"
|
||||
content = render_config(
|
||||
experiment=f"exp2a-gate/{gate_key}-{bench_key}",
|
||||
teacher_key="opus",
|
||||
student_key="9b",
|
||||
benchmark_key=bench_key,
|
||||
gate_key=gate_key,
|
||||
)
|
||||
write_config("exp2a-gate", filename, content)
|
||||
count += 1
|
||||
|
||||
# ── Exp 2b: Autonomy Mode ────────────────────────────────────────────
|
||||
# Fix: S-9b, T-opus, B-standard, G-standard, I-single
|
||||
# Vary: autonomy × benchmark
|
||||
for autonomy, bench_key in itertools.product(AUTONOMY_MODES, BENCHMARKS):
|
||||
filename = f"{autonomy}-9b-{bench_key}.toml"
|
||||
content = render_config(
|
||||
experiment=f"exp2b-autonomy/{autonomy}-{bench_key}",
|
||||
teacher_key="opus",
|
||||
student_key="9b",
|
||||
benchmark_key=bench_key,
|
||||
autonomy=autonomy,
|
||||
)
|
||||
write_config("exp2b-autonomy", filename, content)
|
||||
count += 1
|
||||
|
||||
# ── Exp 3a: Iterative Sessions ───────────────────────────────────────
|
||||
# Fix: S-9b, T-opus, B-standard, A-auto, G-standard
|
||||
# Vary: number of chained sessions × benchmark
|
||||
for n_sessions, bench_key in itertools.product([1, 3, 5], BENCHMARKS):
|
||||
filename = f"iter{n_sessions}-9b-{bench_key}.toml"
|
||||
content = render_config(
|
||||
experiment=f"exp3a-iterative/iter{n_sessions}-{bench_key}",
|
||||
teacher_key="opus",
|
||||
student_key="9b",
|
||||
benchmark_key=bench_key,
|
||||
iterative_sessions=n_sessions,
|
||||
)
|
||||
write_config("exp3a-iterative", filename, content)
|
||||
count += 1
|
||||
|
||||
# ── Exp 3b: Cross-Benchmark Transfer ─────────────────────────────────
|
||||
# Optimize using traces from benchmark X, eval on benchmark Y
|
||||
for opt_bench, eval_bench in itertools.permutations(BENCHMARKS, 2):
|
||||
filename = f"opt-{opt_bench}-eval-{eval_bench}-9b.toml"
|
||||
content = render_config(
|
||||
experiment=f"exp3b-transfer/opt-{opt_bench}-eval-{eval_bench}",
|
||||
teacher_key="opus",
|
||||
student_key="9b",
|
||||
benchmark_key=opt_bench, # Traces from this benchmark
|
||||
)
|
||||
# Add eval benchmark as metadata
|
||||
content += f'\neval_benchmark = "{BENCHMARKS[eval_bench]}"\n'
|
||||
write_config("exp3b-transfer", filename, content)
|
||||
count += 1
|
||||
|
||||
return count
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
n = generate_all()
|
||||
print(f"Generated {n} config files in {CONFIGS_DIR}/")
|
||||
|
||||
# Print summary
|
||||
for subdir in sorted(CONFIGS_DIR.iterdir()):
|
||||
if subdir.is_dir():
|
||||
files = list(subdir.glob("*.toml"))
|
||||
print(f" {subdir.name}/: {len(files)} configs")
|
||||
@@ -1,133 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""M2: Collect distilled eval results and produce comparison table.
|
||||
|
||||
Reads .summary.json files from results/neurips-2026/{distilled,baselines}/
|
||||
and produces a before/after comparison against the Step 1 baseline numbers.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
# Jon's Step 1 baselines — used when a local baseline result doesn't exist
|
||||
STEP1_BASELINES = {
|
||||
"2b": {"toolcall15": 33.3, "pinchbench": 69.6, "livecodebench": 5.6, "taubench": 70.0, "taubench-telecom": 0.0, "gaia": 0.0, "liveresearch": 0.0, "liveresearchbench": None},
|
||||
"9b": {"toolcall15": 46.7, "pinchbench": 95.7, "livecodebench": 17.6, "taubench": 85.0, "taubench-telecom": 80.0, "gaia": 38.0, "liveresearch": 75.0, "liveresearchbench": None},
|
||||
"27b": {"toolcall15": 40.0, "pinchbench": 65.2, "livecodebench": 20.0, "taubench": 75.0, "taubench-telecom": 75.0, "gaia": 48.0, "liveresearch": 66.7, "liveresearchbench": None},
|
||||
}
|
||||
|
||||
# Which benchmarks go through the agent layer (where distillation edits actually apply)
|
||||
AGENT_BENCHMARKS = {"pinchbench", "gaia", "liveresearch"}
|
||||
|
||||
DISTILLED_ROOT = Path("results/neurips-2026/distilled")
|
||||
BASELINE_ROOT = Path("results/neurips-2026/baselines")
|
||||
|
||||
|
||||
def find_summary(root: Path, size: str, bench: str) -> Path | None:
|
||||
"""Find the summary JSON for a model × benchmark run."""
|
||||
# Expected path: root/qwen-{size}/{bench}/{bench}_Qwen-Qwen3.5-{size}.summary.json
|
||||
candidates = list(root.glob(f"qwen-{size}/{bench}/*.summary.json"))
|
||||
return candidates[0] if candidates else None
|
||||
|
||||
|
||||
def load_accuracy(summary_path: Path) -> float | None:
|
||||
"""Extract overall accuracy from a summary.json file."""
|
||||
try:
|
||||
d = json.loads(summary_path.read_text())
|
||||
# The summary has various shapes; try a few
|
||||
for key in ["overall_accuracy", "accuracy", "overall_score"]:
|
||||
if key in d:
|
||||
return float(d[key]) * 100 if d[key] <= 1.0 else float(d[key])
|
||||
# Try nested
|
||||
if "results" in d:
|
||||
for r in d["results"]:
|
||||
if "accuracy" in r:
|
||||
return float(r["accuracy"]) * 100 if r["accuracy"] <= 1.0 else float(r["accuracy"])
|
||||
except Exception as e:
|
||||
print(f" error reading {summary_path}: {e}", file=sys.stderr)
|
||||
return None
|
||||
|
||||
|
||||
def main() -> int:
|
||||
print("=" * 100)
|
||||
print("M2 Distilled vs Baseline Comparison")
|
||||
print("=" * 100)
|
||||
print()
|
||||
print(f"{'Model':8} {'Benchmark':20} {'Baseline':>10} {'Distilled':>10} {'Delta':>10} {'Agent?':>10}")
|
||||
print("-" * 100)
|
||||
|
||||
benchmarks = [
|
||||
"toolcall15", "pinchbench", "livecodebench",
|
||||
"taubench", "taubench-telecom",
|
||||
"gaia", "liveresearch", "liveresearchbench",
|
||||
]
|
||||
|
||||
summary_rows = []
|
||||
for size in ["2b", "9b", "27b"]:
|
||||
for bench in benchmarks:
|
||||
# Baseline: prefer local file (if run this session), fall back to Step 1 numbers
|
||||
baseline_path = find_summary(BASELINE_ROOT, size, bench)
|
||||
if baseline_path:
|
||||
baseline = load_accuracy(baseline_path)
|
||||
base_source = "local"
|
||||
else:
|
||||
baseline = STEP1_BASELINES[size].get(bench)
|
||||
base_source = "step1"
|
||||
|
||||
# Distilled: must be local from this session
|
||||
distilled_path = find_summary(DISTILLED_ROOT, size, bench)
|
||||
distilled = load_accuracy(distilled_path) if distilled_path else None
|
||||
|
||||
# Format
|
||||
b_str = f"{baseline:.1f}%" if baseline is not None else "—"
|
||||
d_str = f"{distilled:.1f}%" if distilled is not None else "pending"
|
||||
if baseline is not None and distilled is not None:
|
||||
delta = distilled - baseline
|
||||
d_sign = "+" if delta >= 0 else ""
|
||||
delta_str = f"{d_sign}{delta:.1f}%"
|
||||
else:
|
||||
delta_str = "—"
|
||||
agent = "AGENT" if bench in AGENT_BENCHMARKS else "direct"
|
||||
|
||||
print(f"qwen-{size:4} {bench:20} {b_str:>10} {d_str:>10} {delta_str:>10} {agent:>10}")
|
||||
summary_rows.append({
|
||||
"model": f"qwen-{size}",
|
||||
"benchmark": bench,
|
||||
"baseline": baseline,
|
||||
"distilled": distilled,
|
||||
"delta": distilled - baseline if (baseline is not None and distilled is not None) else None,
|
||||
"agent_benchmark": bench in AGENT_BENCHMARKS,
|
||||
})
|
||||
print()
|
||||
|
||||
# Aggregate: agent vs direct benchmark deltas
|
||||
print("=" * 100)
|
||||
print("Aggregate deltas by benchmark type (paper finding)")
|
||||
print("=" * 100)
|
||||
agent_deltas = [r["delta"] for r in summary_rows if r["delta"] is not None and r["agent_benchmark"]]
|
||||
direct_deltas = [r["delta"] for r in summary_rows if r["delta"] is not None and not r["agent_benchmark"]]
|
||||
if agent_deltas:
|
||||
mean_agent = sum(agent_deltas) / len(agent_deltas)
|
||||
print(f"Agent benchmarks (PB, GAIA, DeepResearchBench): mean delta = {mean_agent:+.2f}% over {len(agent_deltas)} runs")
|
||||
if direct_deltas:
|
||||
mean_direct = sum(direct_deltas) / len(direct_deltas)
|
||||
print(f"Direct benchmarks (TC15, TauB, TBTel, LRB, LCB): mean delta = {mean_direct:+.2f}% over {len(direct_deltas)} runs")
|
||||
print()
|
||||
|
||||
# Completion progress
|
||||
expected = 24
|
||||
distilled_count = sum(1 for r in summary_rows if r["distilled"] is not None)
|
||||
print(f"Distilled runs complete: {distilled_count}/{expected}")
|
||||
|
||||
# Save JSON
|
||||
out = Path("results/neurips-2026/distillation-m2/m2_comparison.json")
|
||||
out.parent.mkdir(parents=True, exist_ok=True)
|
||||
out.write_text(json.dumps(summary_rows, indent=2, default=str))
|
||||
print(f"Full data: {out}")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -1,307 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""M2: Create distilled eval configs from M1 consensus edits.
|
||||
|
||||
Generates 24 distilled configs (3 models × 8 benchmarks) by cloning
|
||||
baseline configs and applying the consensus edits from M1. Also creates
|
||||
4 missing baseline configs (livecodebench-qwen-9b, liveresearchbench-*).
|
||||
|
||||
Usage: python scripts/experiments/m2_create_distilled_configs.py
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
CONFIGS_DIR = Path("src/openjarvis/evals/configs")
|
||||
M2_DIR = CONFIGS_DIR / "distillation" / "m2"
|
||||
|
||||
# ── Consensus values from M1 (1,131 edits) ──────────────────────────────
|
||||
DISTILLED_TEMP = 0.2 # 84/134 votes (agent benchmarks only)
|
||||
DISTILLED_MAX_TURNS = 15 # 56/125 votes (close: 25 had 49)
|
||||
REMOVE_TOOLS = {"shell_exec", "http_request"} # 13 + 6 votes
|
||||
|
||||
# ── Model specs ──────────────────────────────────────────────────────────
|
||||
MODELS = {
|
||||
"2b": {"name": "Qwen/Qwen3.5-2B", "num_gpus": 1, "port": 8000},
|
||||
"9b": {"name": "Qwen/Qwen3.5-9B", "num_gpus": 1, "port": 8001},
|
||||
"27b": {"name": "Qwen/Qwen3.5-27B-FP8", "num_gpus": 1, "port": 8002},
|
||||
}
|
||||
|
||||
# ── Benchmark specs ──────────────────────────────────────────────────────
|
||||
# Each benchmark defines its baseline config and what changes in distilled.
|
||||
BENCHMARKS = {
|
||||
"toolcall15": {
|
||||
"backend": "jarvis-direct",
|
||||
"baseline_temp": 0.0,
|
||||
"distilled_temp": 0.0, # CONTROL: no change for coding
|
||||
"max_tokens": 4096,
|
||||
"max_samples": None,
|
||||
"judge_model": "gpt-5-mini-2025-08-07",
|
||||
"judge_engine": "cloud",
|
||||
"extra_benchmark_fields": {},
|
||||
},
|
||||
"pinchbench": {
|
||||
"backend": "jarvis-agent",
|
||||
"agent": "native_openhands",
|
||||
"baseline_temp": 0.6,
|
||||
"distilled_temp": DISTILLED_TEMP,
|
||||
"max_tokens": 8192,
|
||||
"max_samples": None,
|
||||
"judge_model": "claude-opus-4-5",
|
||||
"judge_engine": "cloud",
|
||||
"baseline_tools": [
|
||||
"think", "file_read", "file_write", "web_search", "shell_exec",
|
||||
"code_interpreter", "browser_navigate", "image_generate",
|
||||
"calculator", "http_request", "pdf_extract",
|
||||
],
|
||||
"extra_benchmark_fields": {},
|
||||
},
|
||||
"taubench": {
|
||||
"backend": "jarvis-direct",
|
||||
"baseline_temp": 0.7,
|
||||
"distilled_temp": DISTILLED_TEMP,
|
||||
"max_tokens": 4096,
|
||||
"max_samples": 20,
|
||||
"judge_model": "gpt-5-mini-2025-08-07",
|
||||
"judge_engine": "cloud",
|
||||
"extra_benchmark_fields": {"split": "airline,retail"},
|
||||
},
|
||||
"taubench-telecom": {
|
||||
"benchmark_name": "taubench", # same benchmark, different split
|
||||
"backend": "jarvis-direct",
|
||||
"baseline_temp": 0.7,
|
||||
"distilled_temp": DISTILLED_TEMP,
|
||||
"max_tokens": 4096,
|
||||
"max_samples": 20,
|
||||
"judge_model": "gpt-5-mini-2025-08-07",
|
||||
"judge_engine": "cloud",
|
||||
"extra_benchmark_fields": {"split": "telecom"},
|
||||
},
|
||||
"gaia": {
|
||||
"backend": "jarvis-agent",
|
||||
"agent": "monitor_operative",
|
||||
"baseline_temp": 0.6,
|
||||
"distilled_temp": DISTILLED_TEMP,
|
||||
"max_tokens": 8192,
|
||||
"max_samples": 50,
|
||||
"judge_model": "gpt-5-mini-2025-08-07",
|
||||
"judge_engine": "cloud",
|
||||
"baseline_tools": [
|
||||
"think", "calculator", "code_interpreter", "web_search", "file_read",
|
||||
],
|
||||
"extra_benchmark_fields": {},
|
||||
},
|
||||
"liveresearch": {
|
||||
"backend": "jarvis-agent",
|
||||
"agent": "monitor_operative",
|
||||
"baseline_temp": 0.6,
|
||||
"distilled_temp": DISTILLED_TEMP,
|
||||
"max_tokens": 16384,
|
||||
"max_samples": 50,
|
||||
"judge_model": "gpt-5-mini-2025-08-07",
|
||||
"judge_engine": "cloud",
|
||||
"baseline_tools": [
|
||||
"web_search", "file_read", "file_write", "code_interpreter", "think",
|
||||
],
|
||||
"extra_benchmark_fields": {},
|
||||
},
|
||||
"liveresearchbench": {
|
||||
"backend": "jarvis-direct",
|
||||
"baseline_temp": 0.0,
|
||||
"distilled_temp": 0.0, # CONTROL: reasoning benchmark
|
||||
"max_tokens": 8192,
|
||||
"max_samples": 50,
|
||||
"judge_model": "gpt-5-mini-2025-08-07",
|
||||
"judge_engine": "cloud",
|
||||
"extra_benchmark_fields": {},
|
||||
},
|
||||
"livecodebench": {
|
||||
"backend": "jarvis-direct",
|
||||
"baseline_temp": 0.0,
|
||||
"distilled_temp": 0.0, # CONTROL: coding benchmark
|
||||
"max_tokens": 4096,
|
||||
"max_samples": 20,
|
||||
"judge_model": "gpt-5-mini-2025-08-07",
|
||||
"judge_engine": "cloud",
|
||||
"extra_benchmark_fields": {},
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def render_config(
|
||||
*,
|
||||
comment: str,
|
||||
meta_name: str,
|
||||
description: str,
|
||||
temperature: float,
|
||||
max_tokens: int,
|
||||
judge_model: str,
|
||||
judge_engine: str,
|
||||
output_dir: str,
|
||||
model_name: str,
|
||||
model_engine: str,
|
||||
num_gpus: int,
|
||||
benchmark_name: str,
|
||||
backend: str,
|
||||
agent: str | None = None,
|
||||
tools: list[str] | None = None,
|
||||
max_samples: int | None = None,
|
||||
extra_benchmark: dict | None = None,
|
||||
seed: int = 42,
|
||||
) -> str:
|
||||
lines = [f"# {comment}"]
|
||||
lines.append(f'[meta]\nname = "{meta_name}"\ndescription = "{description}"\n')
|
||||
lines.append(f"[defaults]\ntemperature = {temperature}\nmax_tokens = {max_tokens}\n")
|
||||
lines.append(f'[judge]\nmodel = "{judge_model}"\ntemperature = 0.0')
|
||||
if judge_engine:
|
||||
lines.append(f'engine = "{judge_engine}"')
|
||||
lines.append(f"max_tokens = 4096\n")
|
||||
lines.append(f'[run]\nmax_workers = 1\noutput_dir = "{output_dir}"\nseed = {seed}\n')
|
||||
lines.append(f'[[models]]\nname = "{model_name}"\nengine = "{model_engine}"\nnum_gpus = {num_gpus}\n')
|
||||
lines.append(f'[[benchmarks]]\nname = "{benchmark_name}"\nbackend = "{backend}"')
|
||||
if agent:
|
||||
lines.append(f'agent = "{agent}"')
|
||||
if max_samples:
|
||||
lines.append(f"max_samples = {max_samples}")
|
||||
if tools:
|
||||
tools_str = ", ".join(f'"{t}"' for t in tools)
|
||||
lines.append(f"tools = [{tools_str}]")
|
||||
if extra_benchmark:
|
||||
for k, v in extra_benchmark.items():
|
||||
if isinstance(v, str):
|
||||
lines.append(f'{k} = "{v}"')
|
||||
else:
|
||||
lines.append(f"{k} = {v}")
|
||||
lines.append("")
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def make_size_label(size: str) -> str:
|
||||
return {"2b": "qwen-2b", "9b": "qwen-9b", "27b": "qwen-27b"}[size]
|
||||
|
||||
|
||||
def generate_missing_baselines() -> int:
|
||||
"""Create baseline configs that don't exist yet."""
|
||||
count = 0
|
||||
|
||||
# livecodebench-qwen-9b (missing)
|
||||
p = CONFIGS_DIR / "livecodebench-qwen-9b.toml"
|
||||
if not p.exists():
|
||||
b = BENCHMARKS["livecodebench"]
|
||||
m = MODELS["9b"]
|
||||
p.write_text(render_config(
|
||||
comment="LiveCodeBench eval: Qwen3.5-9B (vLLM, 1 GPU)",
|
||||
meta_name="livecodebench-qwen-9b",
|
||||
description="LiveCodeBench on Qwen/Qwen3.5-9B (vLLM, 1 GPU)",
|
||||
temperature=b["baseline_temp"],
|
||||
max_tokens=b["max_tokens"],
|
||||
judge_model=b["judge_model"],
|
||||
judge_engine=b["judge_engine"],
|
||||
output_dir="results/neurips-2026/baselines/qwen-9b/livecodebench/",
|
||||
model_name=m["name"], model_engine="vllm", num_gpus=m["num_gpus"],
|
||||
benchmark_name="livecodebench", backend=b["backend"],
|
||||
max_samples=b["max_samples"],
|
||||
))
|
||||
count += 1
|
||||
print(f" created {p}")
|
||||
|
||||
# liveresearchbench-qwen-{2b,9b,27b}
|
||||
for size, m in MODELS.items():
|
||||
sl = make_size_label(size)
|
||||
p = CONFIGS_DIR / f"liveresearchbench-{sl}.toml"
|
||||
if not p.exists():
|
||||
b = BENCHMARKS["liveresearchbench"]
|
||||
p.write_text(render_config(
|
||||
comment=f"LiveResearchBench (Salesforce): Qwen3.5-{size.upper()} (vLLM)",
|
||||
meta_name=f"liveresearchbench-{sl}",
|
||||
description=f"LiveResearchBench on {m['name']} (vLLM)",
|
||||
temperature=b["baseline_temp"],
|
||||
max_tokens=b["max_tokens"],
|
||||
judge_model=b["judge_model"],
|
||||
judge_engine=b["judge_engine"],
|
||||
output_dir=f"results/neurips-2026/baselines/{sl}/liveresearchbench/",
|
||||
model_name=m["name"], model_engine="vllm", num_gpus=m["num_gpus"],
|
||||
benchmark_name="liveresearchbench", backend=b["backend"],
|
||||
max_samples=b["max_samples"],
|
||||
))
|
||||
count += 1
|
||||
print(f" created {p}")
|
||||
|
||||
return count
|
||||
|
||||
|
||||
def generate_distilled_configs() -> int:
|
||||
"""Create distilled configs for all 24 model × benchmark combos."""
|
||||
M2_DIR.mkdir(parents=True, exist_ok=True)
|
||||
count = 0
|
||||
|
||||
for size, model in MODELS.items():
|
||||
sl = make_size_label(size)
|
||||
for bench_key, bench in BENCHMARKS.items():
|
||||
bench_name = bench.get("benchmark_name", bench_key)
|
||||
is_agent = bench["backend"] == "jarvis-agent"
|
||||
temp = bench["distilled_temp"]
|
||||
|
||||
# Tool list: remove broken tools for agent benchmarks
|
||||
tools = None
|
||||
if is_agent and "baseline_tools" in bench:
|
||||
tools = [t for t in bench["baseline_tools"]
|
||||
if t not in REMOVE_TOOLS]
|
||||
|
||||
fname = f"{bench_key}-{sl}-distilled.toml"
|
||||
out_path = M2_DIR / fname
|
||||
|
||||
# Determine what changed for the comment
|
||||
changes = []
|
||||
if temp != bench["baseline_temp"]:
|
||||
changes.append(f"temp {bench['baseline_temp']}→{temp}")
|
||||
if tools and set(tools) != set(bench.get("baseline_tools", [])):
|
||||
removed = set(bench.get("baseline_tools", [])) - set(tools)
|
||||
changes.append(f"removed {removed}")
|
||||
if is_agent:
|
||||
changes.append(f"max_turns 10→{DISTILLED_MAX_TURNS} (via OPENJARVIS_CONFIG)")
|
||||
change_str = "; ".join(changes) if changes else "CONTROL (no change)"
|
||||
|
||||
out_path.write_text(render_config(
|
||||
comment=f"M2 DISTILLED: {bench_key} × {model['name']} — {change_str}",
|
||||
meta_name=f"{bench_key}-{sl}-distilled",
|
||||
description=f"Distilled {bench_key} on {model['name']}",
|
||||
temperature=temp,
|
||||
max_tokens=bench["max_tokens"],
|
||||
judge_model=bench["judge_model"],
|
||||
judge_engine=bench["judge_engine"],
|
||||
output_dir=f"results/neurips-2026/distilled/{sl}/{bench_key}/",
|
||||
model_name=model["name"], model_engine="vllm", num_gpus=model["num_gpus"],
|
||||
benchmark_name=bench_name, backend=bench["backend"],
|
||||
agent=bench.get("agent"),
|
||||
tools=tools,
|
||||
max_samples=bench.get("max_samples"),
|
||||
extra_benchmark=bench.get("extra_benchmark_fields"),
|
||||
))
|
||||
count += 1
|
||||
|
||||
return count
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
print("=== Creating missing baseline configs ===")
|
||||
n_base = generate_missing_baselines()
|
||||
print(f"Created {n_base} missing baseline configs\n")
|
||||
|
||||
print("=== Creating distilled M2 configs ===")
|
||||
n_dist = generate_distilled_configs()
|
||||
print(f"Created {n_dist} distilled configs in {M2_DIR}/\n")
|
||||
|
||||
# Summary
|
||||
print("=== Change matrix ===")
|
||||
print(f"{'Benchmark':20} {'Backend':14} {'Temp Δ':12} {'Tool Δ':20} {'max_turns Δ':12}")
|
||||
print("-" * 80)
|
||||
for bk, b in BENCHMARKS.items():
|
||||
is_agent = b["backend"] == "jarvis-agent"
|
||||
temp_change = f"{b['baseline_temp']}→{b['distilled_temp']}" if b["baseline_temp"] != b["distilled_temp"] else "—"
|
||||
tool_change = "—"
|
||||
if is_agent and "baseline_tools" in b:
|
||||
removed = REMOVE_TOOLS & set(b.get("baseline_tools", []))
|
||||
tool_change = f"-{removed}" if removed else "—"
|
||||
mt_change = f"10→{DISTILLED_MAX_TURNS}" if is_agent else "—"
|
||||
print(f"{bk:20} {b['backend']:14} {temp_change:12} {str(tool_change):20} {mt_change:12}")
|
||||
@@ -1,183 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
# ──────────────────────────────────────────────────────────────────────────────
|
||||
# M2: Run distilled eval configs — resumable, agent-benchmarks-first ordering
|
||||
#
|
||||
# Assumes vLLM already running: 2B:8000, 9B:8001, 27B-FP8:8002
|
||||
#
|
||||
# Usage:
|
||||
# bash m2_run_distilled_evals.sh # all
|
||||
# bash m2_run_distilled_evals.sh 9b # 9b only
|
||||
# bash m2_run_distilled_evals.sh 9b gaia # 9b + gaia only
|
||||
# ──────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
set -uo pipefail
|
||||
|
||||
VENV=".venv/bin/python"
|
||||
M2_CONFIGS="src/openjarvis/evals/configs/distillation/m2"
|
||||
BASELINE_CONFIGS="src/openjarvis/evals/configs"
|
||||
M2_HOME="/scratch/user/jonsaadfalcon/openjarvis-m2"
|
||||
MODEL_FILTER=${1:-all}
|
||||
BENCH_FILTER=${2:-all}
|
||||
FORCE=${FORCE:-0} # set FORCE=1 to re-run completed configs
|
||||
|
||||
RED='\033[0;31m'; GREEN='\033[0;32m'; YELLOW='\033[1;33m'; BLUE='\033[0;34m'; NC='\033[0m'
|
||||
log() { echo -e "${BLUE}[m2]${NC} $*"; }
|
||||
ok() { echo -e "${GREEN}[ OK ]${NC} $*"; }
|
||||
warn() { echo -e "${YELLOW}[WARN]${NC} $*"; }
|
||||
fail() { echo -e "${RED}[FAIL]${NC} $*"; }
|
||||
skip() { echo -e "${YELLOW}[SKIP]${NC} $*"; }
|
||||
|
||||
declare -A MODEL_PORT=( [2b]=8000 [9b]=8001 [27b]=8002 )
|
||||
|
||||
# Agent benchmarks FIRST (where distillation impact is expected)
|
||||
AGENT_BENCHMARKS="pinchbench gaia liveresearch"
|
||||
DIRECT_BENCHMARKS="toolcall15 taubench taubench-telecom livecodebench liveresearchbench"
|
||||
ALL_BENCHMARKS="${AGENT_BENCHMARKS} ${DIRECT_BENCHMARKS}"
|
||||
|
||||
check_vllm() {
|
||||
for size in 2b 9b 27b; do
|
||||
[ "$MODEL_FILTER" != "all" ] && [ "$MODEL_FILTER" != "$size" ] && continue
|
||||
local port=${MODEL_PORT[$size]}
|
||||
if ! curl -sf "http://localhost:${port}/v1/models" >/dev/null 2>&1; then
|
||||
fail "vLLM ${size} not responding on port ${port}"
|
||||
return 1
|
||||
fi
|
||||
ok "vLLM ${size} healthy on port ${port}"
|
||||
done
|
||||
}
|
||||
|
||||
# Check if a run already completed (summary.json exists AND has a real accuracy).
|
||||
# A summary with errors=total_samples (like my earlier broken-routing tests)
|
||||
# is treated as incomplete and re-run.
|
||||
is_complete() {
|
||||
local summary_path=$1
|
||||
[ -f "$summary_path" ] || return 1
|
||||
python3 -c "
|
||||
import json, sys
|
||||
try:
|
||||
d = json.load(open('$summary_path'))
|
||||
total = d.get('total_samples', 0)
|
||||
errors = d.get('errors', 0)
|
||||
scored = d.get('scored_samples', 0)
|
||||
# Complete if at least some samples were scored successfully
|
||||
sys.exit(0 if scored > 0 else 1)
|
||||
except Exception:
|
||||
sys.exit(1)
|
||||
" 2>/dev/null
|
||||
}
|
||||
|
||||
run_eval() {
|
||||
local config_path=$1 label=$2 size=$3 summary_path=$4 use_distilled=${5:-false}
|
||||
|
||||
if [ "$FORCE" != "1" ] && is_complete "$summary_path"; then
|
||||
skip "${label} [already complete]"
|
||||
return 0
|
||||
fi
|
||||
|
||||
local oj_config="${M2_HOME}/config-baseline-${size}.toml"
|
||||
[ "$use_distilled" = "true" ] && oj_config="${M2_HOME}/config-${size}.toml"
|
||||
|
||||
log "Running: ${label} [$(basename ${oj_config})]"
|
||||
env OPENJARVIS_CONFIG="${oj_config}" ${VENV} -m openjarvis.evals run -c "${config_path}" 2>&1
|
||||
local rc=$?
|
||||
if [ $rc -eq 0 ] && is_complete "$summary_path"; then
|
||||
ok "Done: ${label}"
|
||||
else
|
||||
warn "Failed: ${label} (rc=$rc)"
|
||||
fi
|
||||
}
|
||||
|
||||
# Derive the expected summary.json path for a given distilled config
|
||||
summary_for_distilled() {
|
||||
local bench=$1 size=$2
|
||||
# Output dir from the config template: results/neurips-2026/distilled/qwen-{size}/{bench}/
|
||||
# Summary file pattern: {bench}_Qwen-Qwen3.5-{size}.summary.json
|
||||
local model_slug
|
||||
if [ "$size" = "27b" ]; then model_slug="Qwen-Qwen3.5-27B-FP8"
|
||||
else model_slug="Qwen-Qwen3.5-${size}"; fi
|
||||
# Uppercase B for the model slug
|
||||
model_slug=$(echo "$model_slug" | sed 's/-\([0-9][0-9]*\)b/-\1B/g')
|
||||
# For taubench-telecom, the benchmark name in output is "taubench" not "taubench-telecom"
|
||||
local bench_fname=$bench
|
||||
[ "$bench" = "taubench-telecom" ] && bench_fname="taubench"
|
||||
echo "results/neurips-2026/distilled/qwen-${size}/${bench}/${bench_fname}_${model_slug}.summary.json"
|
||||
}
|
||||
|
||||
summary_for_baseline() {
|
||||
local bench=$1 size=$2
|
||||
local model_slug
|
||||
if [ "$size" = "27b" ]; then model_slug="Qwen-Qwen3.5-27B-FP8"
|
||||
else model_slug="Qwen-Qwen3.5-${size}"; fi
|
||||
model_slug=$(echo "$model_slug" | sed 's/-\([0-9][0-9]*\)b/-\1B/g')
|
||||
local bench_fname=$bench
|
||||
[ "$bench" = "taubench-telecom" ] && bench_fname="taubench"
|
||||
echo "results/neurips-2026/baselines/qwen-${size}/${bench}/${bench_fname}_${model_slug}.summary.json"
|
||||
}
|
||||
|
||||
log "M2 Distilled Eval Runner (resumable, agent-first)"
|
||||
log "Model filter: ${MODEL_FILTER} Benchmark filter: ${BENCH_FILTER} FORCE=${FORCE}"
|
||||
|
||||
check_vllm || exit 1
|
||||
|
||||
start_time=$(date +%s)
|
||||
|
||||
# Phase B1: DISTILLED agent benchmarks (highest-priority: PinchBench, GAIA, DeepResearchBench)
|
||||
log "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
log "Phase B1: DISTILLED agent benchmarks (9 runs — the critical data)"
|
||||
log "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
for size in 2b 9b 27b; do
|
||||
[ "$MODEL_FILTER" != "all" ] && [ "$MODEL_FILTER" != "$size" ] && continue
|
||||
for bench in ${AGENT_BENCHMARKS}; do
|
||||
[ "$BENCH_FILTER" != "all" ] && [ "$BENCH_FILTER" != "$bench" ] && continue
|
||||
cfg="${M2_CONFIGS}/${bench}-qwen-${size}-distilled.toml"
|
||||
sum=$(summary_for_distilled "$bench" "$size")
|
||||
[ -f "$cfg" ] && run_eval "$cfg" "DISTILLED ${bench}-qwen-${size}" "${size}" "$sum" true
|
||||
done
|
||||
done
|
||||
|
||||
# Phase B2: DISTILLED direct benchmarks (controls — should show minimal delta)
|
||||
log "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
log "Phase B2: DISTILLED direct benchmarks (15 runs — controls)"
|
||||
log "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
for size in 2b 9b 27b; do
|
||||
[ "$MODEL_FILTER" != "all" ] && [ "$MODEL_FILTER" != "$size" ] && continue
|
||||
for bench in ${DIRECT_BENCHMARKS}; do
|
||||
[ "$BENCH_FILTER" != "all" ] && [ "$BENCH_FILTER" != "$bench" ] && continue
|
||||
cfg="${M2_CONFIGS}/${bench}-qwen-${size}-distilled.toml"
|
||||
sum=$(summary_for_distilled "$bench" "$size")
|
||||
[ -f "$cfg" ] && run_eval "$cfg" "DISTILLED ${bench}-qwen-${size}" "${size}" "$sum" true
|
||||
done
|
||||
done
|
||||
|
||||
# Phase A: LiveResearchBench baselines (last, since Step 1 baselines exist for other benchmarks)
|
||||
log "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
log "Phase A: LiveResearchBench baselines (3 runs — new benchmark only)"
|
||||
log "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
for size in 2b 9b 27b; do
|
||||
[ "$MODEL_FILTER" != "all" ] && [ "$MODEL_FILTER" != "$size" ] && continue
|
||||
[ "$BENCH_FILTER" != "all" ] && [ "$BENCH_FILTER" != "liveresearchbench" ] && continue
|
||||
cfg="${BASELINE_CONFIGS}/liveresearchbench-qwen-${size}.toml"
|
||||
sum=$(summary_for_baseline "liveresearchbench" "$size")
|
||||
[ -f "$cfg" ] && run_eval "$cfg" "BASELINE liveresearchbench-qwen-${size}" "${size}" "$sum" false
|
||||
done
|
||||
|
||||
# Phase C: Spot-check 2 baselines against Jon's Step 1 numbers
|
||||
log "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
log "Phase C: Spot-check baselines (TC15-9B, GAIA-9B)"
|
||||
log "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
if [ "$MODEL_FILTER" = "all" ] || [ "$MODEL_FILTER" = "9b" ]; then
|
||||
for bench in toolcall15 gaia; do
|
||||
[ "$BENCH_FILTER" != "all" ] && [ "$BENCH_FILTER" != "$bench" ] && continue
|
||||
cfg="${BASELINE_CONFIGS}/${bench}-qwen-9b.toml"
|
||||
sum=$(summary_for_baseline "$bench" "9b")
|
||||
[ -f "$cfg" ] && run_eval "$cfg" "SPOTCHECK ${bench}-qwen-9b" "9b" "$sum" false
|
||||
done
|
||||
fi
|
||||
|
||||
end_time=$(date +%s)
|
||||
elapsed=$((end_time - start_time))
|
||||
|
||||
log "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
ok "M2 complete in ${elapsed}s ($(( elapsed / 3600 ))h $(( (elapsed % 3600) / 60 ))m)"
|
||||
log "Distilled results: results/neurips-2026/distilled/"
|
||||
log "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
@@ -1,581 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""M3: Empirical hill-climbing with an LLM proposer.
|
||||
|
||||
Replaces M1's open-loop "aggregate consensus across sessions" with a closed
|
||||
loop:
|
||||
|
||||
For each target (student, benchmark, agent):
|
||||
for round in 1..N:
|
||||
edit = teacher.propose_one(history_with_measured_deltas)
|
||||
score_new = eval_subsample(apply(edit))
|
||||
if score_new > current_score: accept
|
||||
|
||||
Every proposal is empirically verified before the next is proposed, and the
|
||||
teacher sees measured deltas (not just traces) in its context.
|
||||
|
||||
Usage:
|
||||
python scripts/experiments/m3_hill_climb.py \\
|
||||
--student 9b --benchmark liveresearch \\
|
||||
--rounds 4 --k-subsample 8 --k-final 30
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import time
|
||||
from dataclasses import asdict, dataclass, field
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from openjarvis.core.types import Message, Role
|
||||
from openjarvis.engine.cloud import CloudEngine
|
||||
|
||||
|
||||
# ═══════════════════════════════════════════════════════════════════════════
|
||||
# Config & constants
|
||||
# ═══════════════════════════════════════════════════════════════════════════
|
||||
|
||||
STUDENT = {
|
||||
"2b": {"name": "Qwen/Qwen3.5-2B", "port": 8000, "gpu": 4},
|
||||
"9b": {"name": "Qwen/Qwen3.5-9B", "port": 8001, "gpu": 5},
|
||||
"27b": {"name": "Qwen/Qwen3.5-27B-FP8", "port": 8002, "gpu": 6},
|
||||
}
|
||||
|
||||
# Per-benchmark defaults (backend, baseline config)
|
||||
BENCHMARK = {
|
||||
"liveresearch": {
|
||||
"backend": "jarvis-agent",
|
||||
"agent": "monitor_operative",
|
||||
"baseline_temp": 0.6,
|
||||
"baseline_max_tokens": 16384,
|
||||
"baseline_max_turns": 10,
|
||||
"baseline_tools": ["web_search", "file_read", "file_write",
|
||||
"code_interpreter", "think"],
|
||||
"max_samples_final": 50, # final eval
|
||||
"judge": "gpt-5-mini-2025-08-07",
|
||||
},
|
||||
"gaia": {
|
||||
"backend": "jarvis-agent",
|
||||
"agent": "monitor_operative",
|
||||
"baseline_temp": 0.6,
|
||||
"baseline_max_tokens": 8192,
|
||||
"baseline_max_turns": 10,
|
||||
"baseline_tools": ["think", "calculator", "code_interpreter",
|
||||
"web_search", "file_read"],
|
||||
"max_samples_final": 50,
|
||||
"judge": "gpt-5-mini-2025-08-07",
|
||||
},
|
||||
"pinchbench": {
|
||||
"backend": "jarvis-agent",
|
||||
"agent": "native_openhands",
|
||||
"baseline_temp": 0.6,
|
||||
"baseline_max_tokens": 8192,
|
||||
"baseline_max_turns": 10,
|
||||
"baseline_tools": ["think", "file_read", "file_write", "web_search",
|
||||
"shell_exec", "code_interpreter", "browser_navigate",
|
||||
"image_generate", "calculator", "http_request",
|
||||
"pdf_extract"],
|
||||
"max_samples_final": 23,
|
||||
"judge": "claude-opus-4-5",
|
||||
},
|
||||
}
|
||||
|
||||
AVAILABLE_TOOLS = [
|
||||
"think", "file_read", "file_write", "web_search", "shell_exec",
|
||||
"code_interpreter", "browser_navigate", "image_generate", "calculator",
|
||||
"http_request", "pdf_extract", "pdf_reader", "list_directory",
|
||||
]
|
||||
|
||||
|
||||
# ═══════════════════════════════════════════════════════════════════════════
|
||||
# Proposer (LLM)
|
||||
# ═══════════════════════════════════════════════════════════════════════════
|
||||
|
||||
|
||||
PROPOSER_SYSTEM = """\
|
||||
You are optimizing an OpenJarvis agent configuration for maximum accuracy on \
|
||||
a benchmark. You propose ONE config edit per round. After each proposal, the \
|
||||
edit is applied and the benchmark is run on a subsample; you then see the \
|
||||
measured score delta and decide the next edit.
|
||||
|
||||
Your job is to find the config that maximizes measured accuracy.
|
||||
|
||||
EDIT GRAMMAR — return JSON with "op" and the parameter fields at top level:
|
||||
|
||||
{"op": "<op_name>", <param_fields>, "rationale": "<one sentence>"}
|
||||
|
||||
Valid ops and their parameter fields:
|
||||
|
||||
1. set_temperature: "value" (float, 0.0..1.0)
|
||||
2. set_max_turns: "value" (int, 1..100)
|
||||
3. set_max_tokens: "value" (int, 512..32768)
|
||||
4. add_tool: "tool_name" (string; must be in AVAILABLE_TOOLS, not already active)
|
||||
5. remove_tool: "tool_name" (string; must be in current tools)
|
||||
6. noop: (no params; propose only if you believe no further edit will help)
|
||||
|
||||
CONCRETE EXAMPLES:
|
||||
{"op": "set_temperature", "value": 0.3, "rationale": "Reduce loop risk."}
|
||||
{"op": "set_max_turns", "value": 20, "rationale": "More turns for research tasks."}
|
||||
{"op": "add_tool", "tool_name": "pdf_extract", "rationale": "Tasks require PDF reading."}
|
||||
{"op": "remove_tool", "tool_name": "shell_exec", "rationale": "Tool is broken in this env."}
|
||||
{"op": "noop", "rationale": "Current config seems optimal."}
|
||||
|
||||
EXPLORATION BIAS:
|
||||
The config space has 5 distinct axes: temperature, max_turns, max_tokens,
|
||||
tool additions (add_tool), tool removals (remove_tool). Before proposing a
|
||||
second edit on an axis you've already tried, consider whether an untried
|
||||
axis might reveal a larger gain. Tool-list edits (add/remove) often matter
|
||||
more than numeric hyperparameters on benchmarks where the agent uses tools.
|
||||
|
||||
Return ONLY the JSON object, no preamble, no code fences."""
|
||||
|
||||
|
||||
def build_user_prompt(
|
||||
*, benchmark: str, student: str, agent: str,
|
||||
baseline_score: float, current_score: float, current_config: dict,
|
||||
edit_history: list[dict], available_tools: list[str],
|
||||
sample_queries: list[str],
|
||||
) -> str:
|
||||
hist_lines = []
|
||||
for i, e in enumerate(edit_history, 1):
|
||||
delta = e["score_after"] - e["score_before"]
|
||||
status = "ACCEPTED" if e["accepted"] else "REJECTED"
|
||||
hist_lines.append(
|
||||
f" Round {i}: {json.dumps(e['edit'])} "
|
||||
f"→ score {e['score_before']:.1f}% → {e['score_after']:.1f}% "
|
||||
f"(Δ {delta:+.1f}, {status})"
|
||||
)
|
||||
hist = "\n".join(hist_lines) if hist_lines else " (no edits yet — baseline is the starting point)"
|
||||
|
||||
samples = "\n".join(f" - {q[:150]}..." for q in sample_queries[:3])
|
||||
|
||||
unused_tools = [t for t in available_tools if t not in current_config["tools"]]
|
||||
|
||||
return f"""\
|
||||
TARGET:
|
||||
student: {student} (vLLM-served Qwen3.5)
|
||||
benchmark: {benchmark}
|
||||
agent: {agent} (uses OpenAI-format structured tool calls)
|
||||
|
||||
CURRENT CONFIG:
|
||||
temperature = {current_config['temperature']}
|
||||
max_turns = {current_config['max_turns']}
|
||||
max_tokens = {current_config['max_tokens']}
|
||||
tools = {current_config['tools']}
|
||||
|
||||
TOOLS NOT CURRENTLY ACTIVE (available to add):
|
||||
{unused_tools}
|
||||
|
||||
BASELINE (unedited) SCORE: {baseline_score:.1f}%
|
||||
CURRENT BEST SCORE: {current_score:.1f}%
|
||||
|
||||
EDIT HISTORY (with measured deltas):
|
||||
{hist}
|
||||
|
||||
SAMPLE TASKS FROM THIS BENCHMARK:
|
||||
{samples}
|
||||
|
||||
Propose ONE edit that you predict will improve the measured accuracy. Consider \
|
||||
the edit history — do not repeat proposals that were rejected. If you believe \
|
||||
further edits will not help, propose noop.
|
||||
|
||||
Return JSON only."""
|
||||
|
||||
|
||||
def call_proposer(
|
||||
engine: CloudEngine, system: str, user: str, model: str = "claude-sonnet-4-6"
|
||||
) -> dict:
|
||||
resp = engine.generate(
|
||||
messages=[
|
||||
Message(role=Role.SYSTEM, content=system),
|
||||
Message(role=Role.USER, content=user),
|
||||
],
|
||||
model=model, max_tokens=600, temperature=0.3,
|
||||
)
|
||||
content = (resp.get("content") or "").strip()
|
||||
# Be forgiving about code fences
|
||||
if content.startswith("```"):
|
||||
content = re.sub(r"^```(?:json)?\s*", "", content)
|
||||
content = re.sub(r"\s*```\s*$", "", content)
|
||||
# Find the JSON object
|
||||
m = re.search(r"\{[\s\S]*\}", content)
|
||||
if not m:
|
||||
raise ValueError(f"No JSON object found in proposer output: {content[:200]}")
|
||||
return json.loads(m.group(0))
|
||||
|
||||
|
||||
# ═══════════════════════════════════════════════════════════════════════════
|
||||
# Edit application & evaluation
|
||||
# ═══════════════════════════════════════════════════════════════════════════
|
||||
|
||||
|
||||
@dataclass
|
||||
class Config:
|
||||
temperature: float
|
||||
max_turns: int
|
||||
max_tokens: int
|
||||
tools: list[str]
|
||||
|
||||
|
||||
def apply_edit(cfg: Config, edit: dict) -> Config:
|
||||
"""Apply an edit. Tolerant of both flat and nested (params) forms."""
|
||||
op = edit["op"]
|
||||
# Merge top-level edit fields with params for flat-or-nested tolerance
|
||||
p = {**edit.get("params", {}), **{k: v for k, v in edit.items()
|
||||
if k not in ("op", "params", "rationale")}}
|
||||
new = Config(
|
||||
temperature=cfg.temperature, max_turns=cfg.max_turns,
|
||||
max_tokens=cfg.max_tokens, tools=list(cfg.tools),
|
||||
)
|
||||
if op == "noop":
|
||||
return new
|
||||
if op == "set_temperature":
|
||||
new.temperature = float(p["value"])
|
||||
elif op == "set_max_turns":
|
||||
new.max_turns = int(p["value"])
|
||||
elif op == "set_max_tokens":
|
||||
new.max_tokens = int(p["value"])
|
||||
elif op == "add_tool":
|
||||
tool = p["tool_name"]
|
||||
if tool not in new.tools:
|
||||
new.tools.append(tool)
|
||||
elif op == "remove_tool":
|
||||
tool = p["tool_name"]
|
||||
new.tools = [t for t in new.tools if t != tool]
|
||||
else:
|
||||
raise ValueError(f"unknown edit op: {op}")
|
||||
return new
|
||||
|
||||
|
||||
def write_eval_toml(
|
||||
*, bench: str, bench_spec: dict, student: dict, cfg: Config,
|
||||
k_samples: int, output_dir: Path,
|
||||
) -> Path:
|
||||
agent_line = f'agent = "{bench_spec["agent"]}"' if bench_spec.get("agent") else ""
|
||||
tools_str = "[" + ", ".join(f'"{t}"' for t in cfg.tools) + "]"
|
||||
toml = f"""\
|
||||
[meta]
|
||||
name = "m3-{bench}-{student['name'].replace('/', '-')}"
|
||||
description = "M3 hill-climb round"
|
||||
|
||||
[defaults]
|
||||
temperature = {cfg.temperature}
|
||||
max_tokens = {cfg.max_tokens}
|
||||
|
||||
[judge]
|
||||
model = "{bench_spec['judge']}"
|
||||
temperature = 0.0
|
||||
max_tokens = 4096
|
||||
engine = "cloud"
|
||||
|
||||
[run]
|
||||
max_workers = 1
|
||||
output_dir = "{output_dir}"
|
||||
seed = 42
|
||||
|
||||
[[models]]
|
||||
name = "{student['name']}"
|
||||
engine = "vllm"
|
||||
num_gpus = 1
|
||||
|
||||
[[benchmarks]]
|
||||
name = "{bench}"
|
||||
backend = "{bench_spec['backend']}"
|
||||
{agent_line}
|
||||
max_samples = {k_samples}
|
||||
tools = {tools_str}
|
||||
"""
|
||||
path = output_dir / "eval.toml"
|
||||
output_dir.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text(toml)
|
||||
return path
|
||||
|
||||
|
||||
def write_openjarvis_config(home: Path, port: int, max_turns: int) -> Path:
|
||||
p = home / "global-config.toml"
|
||||
p.parent.mkdir(parents=True, exist_ok=True)
|
||||
p.write_text(f"""\
|
||||
[agent]
|
||||
max_turns = {max_turns}
|
||||
|
||||
[engine]
|
||||
default = "vllm"
|
||||
|
||||
[engine.vllm]
|
||||
host = "http://localhost:{port}"
|
||||
""")
|
||||
return p
|
||||
|
||||
|
||||
def run_eval(eval_toml: Path, oj_config: Path) -> tuple[float, int, int]:
|
||||
"""Run one eval. Returns (accuracy_pct, scored, total)."""
|
||||
env = {**os.environ, "OPENJARVIS_CONFIG": str(oj_config)}
|
||||
result = subprocess.run(
|
||||
[".venv/bin/python", "-m", "openjarvis.evals", "run", "-c", str(eval_toml)],
|
||||
capture_output=True, text=True, env=env, timeout=7200,
|
||||
)
|
||||
# Find the summary.json
|
||||
out_dir = eval_toml.parent
|
||||
sums = list(out_dir.glob("**/*.summary.json"))
|
||||
if not sums:
|
||||
print(f"[m3] WARNING: no summary.json in {out_dir}")
|
||||
print(f"[m3] stderr tail: {result.stderr[-500:]}")
|
||||
return 0.0, 0, 0
|
||||
d = json.loads(sums[0].read_text())
|
||||
acc = d.get("accuracy", 0.0)
|
||||
acc_pct = acc * 100 if acc <= 1.0 else acc
|
||||
return acc_pct, d.get("scored_samples", 0), d.get("total_samples", 0)
|
||||
|
||||
|
||||
# ═══════════════════════════════════════════════════════════════════════════
|
||||
# Benchmark sample loader (for proposer context)
|
||||
# ═══════════════════════════════════════════════════════════════════════════
|
||||
|
||||
|
||||
def load_sample_queries(bench: str, n: int = 3) -> list[str]:
|
||||
try:
|
||||
if bench == "liveresearch":
|
||||
from openjarvis.evals.datasets.liveresearch import LiveResearchBenchDataset
|
||||
ds = LiveResearchBenchDataset()
|
||||
ds.load(max_samples=n)
|
||||
elif bench == "pinchbench":
|
||||
from openjarvis.evals.datasets.pinchbench import PinchBenchDataset
|
||||
ds = PinchBenchDataset()
|
||||
elif bench == "gaia":
|
||||
from openjarvis.evals.datasets.gaia import GAIADataset
|
||||
ds = GAIADataset()
|
||||
if hasattr(ds, "load"):
|
||||
ds.load(max_samples=n)
|
||||
else:
|
||||
return []
|
||||
return [r.problem for r in list(ds.iter_records())[:n]]
|
||||
except Exception as e:
|
||||
print(f"[m3] WARNING: could not load samples for {bench}: {e}")
|
||||
return []
|
||||
|
||||
|
||||
# ═══════════════════════════════════════════════════════════════════════════
|
||||
# Main hill-climb loop
|
||||
# ═══════════════════════════════════════════════════════════════════════════
|
||||
|
||||
|
||||
def hill_climb(args) -> dict:
|
||||
bench_spec = BENCHMARK[args.benchmark]
|
||||
student = STUDENT[args.student]
|
||||
|
||||
# State dir (resumable)
|
||||
base_dir = Path(args.out_dir) / f"{args.student}-{args.benchmark}"
|
||||
base_dir.mkdir(parents=True, exist_ok=True)
|
||||
state_path = base_dir / "state.json"
|
||||
|
||||
# Load or initialize state
|
||||
if state_path.exists() and not args.fresh:
|
||||
state = json.loads(state_path.read_text())
|
||||
print(f"[m3] Resumed state from round {len(state['history'])}/{args.rounds}")
|
||||
else:
|
||||
baseline_cfg_dict = {
|
||||
"temperature": bench_spec["baseline_temp"],
|
||||
"max_turns": bench_spec["baseline_max_turns"],
|
||||
"max_tokens": bench_spec["baseline_max_tokens"],
|
||||
"tools": list(bench_spec["baseline_tools"]),
|
||||
}
|
||||
# ALWAYS measure the baseline today first (unless user provides --trust-baseline).
|
||||
# This prevents anchoring to an unreproducible Step 1 number.
|
||||
# We measure at k=k_final so the final delta is like-for-like.
|
||||
if args.trust_baseline and args.baseline_score is not None:
|
||||
measured_baseline = args.baseline_score
|
||||
measured_baseline_k = None
|
||||
print(f"[m3] Trusting provided baseline score: {measured_baseline:.1f}% "
|
||||
f"(--trust-baseline set)")
|
||||
else:
|
||||
print(f"[m3] Measuring today's baseline with k={args.k_final} "
|
||||
f"(matches k_final for clean like-for-like delta)...")
|
||||
bl_dir = base_dir / "baseline_measure"
|
||||
bl_toml = write_eval_toml(
|
||||
bench=args.benchmark, bench_spec=bench_spec, student=student,
|
||||
cfg=Config(**baseline_cfg_dict),
|
||||
k_samples=args.k_final, output_dir=bl_dir,
|
||||
)
|
||||
bl_oj = write_openjarvis_config(bl_dir, student["port"], baseline_cfg_dict["max_turns"])
|
||||
t0 = time.monotonic()
|
||||
measured_baseline, bl_scored, bl_total = run_eval(bl_toml, bl_oj)
|
||||
measured_baseline_k = bl_scored
|
||||
print(f"[m3] Measured baseline: {measured_baseline:.1f}% "
|
||||
f"({bl_scored}/{bl_total}) in {(time.monotonic() - t0)/60:.1f} min")
|
||||
if args.baseline_score is not None:
|
||||
print(f"[m3] (reference: --baseline-score was {args.baseline_score:.1f}%, "
|
||||
f"drift = {measured_baseline - args.baseline_score:+.1f})")
|
||||
|
||||
state = {
|
||||
"args": vars(args),
|
||||
"benchmark": args.benchmark,
|
||||
"student": student["name"],
|
||||
"agent": bench_spec["agent"],
|
||||
"baseline_config": baseline_cfg_dict,
|
||||
"baseline_score": measured_baseline,
|
||||
"baseline_score_reference": args.baseline_score,
|
||||
"current_config": dict(baseline_cfg_dict),
|
||||
"current_score": measured_baseline,
|
||||
"history": [],
|
||||
}
|
||||
|
||||
# Save state helper
|
||||
def save():
|
||||
state_path.write_text(json.dumps(state, indent=2, default=str))
|
||||
|
||||
save()
|
||||
|
||||
# Sample queries for proposer context
|
||||
sample_queries = load_sample_queries(args.benchmark)
|
||||
|
||||
engine = CloudEngine()
|
||||
|
||||
# Hill-climb rounds
|
||||
for round_num in range(len(state["history"]) + 1, args.rounds + 1):
|
||||
print(f"\n{'═' * 70}")
|
||||
print(f"[m3] Round {round_num}/{args.rounds}")
|
||||
print(f"[m3] current_score = {state['current_score']:.1f}%")
|
||||
print(f"[m3] current_config = {state['current_config']}")
|
||||
|
||||
# Propose
|
||||
user_prompt = build_user_prompt(
|
||||
benchmark=args.benchmark, student=student["name"],
|
||||
agent=bench_spec["agent"],
|
||||
baseline_score=state["baseline_score"],
|
||||
current_score=state["current_score"],
|
||||
current_config=state["current_config"],
|
||||
edit_history=state["history"],
|
||||
available_tools=AVAILABLE_TOOLS,
|
||||
sample_queries=sample_queries,
|
||||
)
|
||||
try:
|
||||
edit = call_proposer(engine, PROPOSER_SYSTEM, user_prompt,
|
||||
model=args.proposer_model)
|
||||
except Exception as e:
|
||||
print(f"[m3] Proposer failed: {e}. Ending hill-climb.")
|
||||
break
|
||||
|
||||
print(f"[m3] Proposed: {json.dumps(edit)}")
|
||||
|
||||
if edit.get("op") == "noop":
|
||||
print(f"[m3] Teacher proposed noop; stopping.")
|
||||
break
|
||||
|
||||
# Apply and evaluate subsample
|
||||
try:
|
||||
candidate = apply_edit(Config(**state["current_config"]), edit)
|
||||
except Exception as e:
|
||||
print(f"[m3] apply_edit failed: {e}. Recording as malformed edit.")
|
||||
# Record as a rejected malformed edit so teacher won't repeat
|
||||
state["history"].append({
|
||||
"round": round_num, "edit": edit,
|
||||
"config_after": None,
|
||||
"score_before": state["current_score"],
|
||||
"score_after": state["current_score"], # no change
|
||||
"scored": 0, "total": 0, "elapsed_seconds": 0,
|
||||
"accepted": False,
|
||||
"error": f"malformed_edit: {e}",
|
||||
})
|
||||
save()
|
||||
continue
|
||||
|
||||
round_dir = base_dir / f"round_{round_num}"
|
||||
eval_toml = write_eval_toml(
|
||||
bench=args.benchmark, bench_spec=bench_spec, student=student,
|
||||
cfg=candidate, k_samples=args.k_subsample,
|
||||
output_dir=round_dir,
|
||||
)
|
||||
oj_cfg = write_openjarvis_config(round_dir, student["port"], candidate.max_turns)
|
||||
|
||||
print(f"[m3] Running k={args.k_subsample} subsample eval...")
|
||||
t0 = time.monotonic()
|
||||
acc, scored, total = run_eval(eval_toml, oj_cfg)
|
||||
elapsed = time.monotonic() - t0
|
||||
print(f"[m3] Subsample score: {acc:.1f}% ({scored}/{total}) in {elapsed/60:.1f} min")
|
||||
|
||||
score_before = state["current_score"]
|
||||
delta = acc - score_before
|
||||
accepted = delta > args.accept_threshold
|
||||
|
||||
# Record history
|
||||
state["history"].append({
|
||||
"round": round_num, "edit": edit,
|
||||
"config_after": asdict(candidate),
|
||||
"score_before": score_before, "score_after": acc,
|
||||
"scored": scored, "total": total, "elapsed_seconds": elapsed,
|
||||
"accepted": accepted,
|
||||
})
|
||||
|
||||
if accepted:
|
||||
state["current_config"] = asdict(candidate)
|
||||
state["current_score"] = acc
|
||||
print(f"[m3] ACCEPTED (Δ={delta:+.1f})")
|
||||
else:
|
||||
print(f"[m3] REJECTED (Δ={delta:+.1f} ≤ {args.accept_threshold})")
|
||||
|
||||
save()
|
||||
|
||||
# Final eval with current config
|
||||
print(f"\n{'═' * 70}")
|
||||
print(f"[m3] Final eval with best config: {state['current_config']}")
|
||||
final_dir = base_dir / "final"
|
||||
final_cfg = Config(**state["current_config"])
|
||||
eval_toml = write_eval_toml(
|
||||
bench=args.benchmark, bench_spec=bench_spec, student=student,
|
||||
cfg=final_cfg, k_samples=args.k_final,
|
||||
output_dir=final_dir,
|
||||
)
|
||||
oj_cfg = write_openjarvis_config(final_dir, student["port"], final_cfg.max_turns)
|
||||
|
||||
t0 = time.monotonic()
|
||||
final_acc, final_scored, final_total = run_eval(eval_toml, oj_cfg)
|
||||
elapsed = time.monotonic() - t0
|
||||
|
||||
state["final_score"] = final_acc
|
||||
state["final_scored"] = final_scored
|
||||
state["final_total"] = final_total
|
||||
state["final_elapsed_seconds"] = elapsed
|
||||
save()
|
||||
|
||||
print(f"\n{'═' * 70}")
|
||||
print(f"[m3] DONE")
|
||||
print(f"[m3] baseline = {state['baseline_score']:.1f}%")
|
||||
print(f"[m3] final = {final_acc:.1f}% ({final_scored}/{final_total}) in {elapsed/60:.1f} min")
|
||||
print(f"[m3] Δ vs baseline = {final_acc - state['baseline_score']:+.1f}")
|
||||
print(f"[m3] state: {state_path}")
|
||||
|
||||
return state
|
||||
|
||||
|
||||
def main() -> int:
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--student", required=True, choices=list(STUDENT))
|
||||
ap.add_argument("--benchmark", required=True, choices=list(BENCHMARK))
|
||||
ap.add_argument("--rounds", type=int, default=4)
|
||||
ap.add_argument("--k-subsample", type=int, default=8)
|
||||
ap.add_argument("--k-final", type=int, default=30)
|
||||
ap.add_argument("--baseline-score", type=float, default=None,
|
||||
help="Optional reference baseline (shown alongside measured). "
|
||||
"Hill-climb always measures today's baseline unless --trust-baseline.")
|
||||
ap.add_argument("--trust-baseline", action="store_true",
|
||||
help="Skip baseline re-measurement, trust --baseline-score.")
|
||||
ap.add_argument("--accept-threshold", type=float, default=0.0,
|
||||
help="Accept edit if score Δ > this (default: 0)")
|
||||
ap.add_argument("--proposer-model", default="claude-sonnet-4-6")
|
||||
ap.add_argument("--out-dir", default="results/neurips-2026/distillation-m3")
|
||||
ap.add_argument("--fresh", action="store_true",
|
||||
help="Overwrite existing state and start fresh")
|
||||
args = ap.parse_args()
|
||||
|
||||
result = hill_climb(args)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -1,307 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
# ──────────────────────────────────────────────────────────────────────────────
|
||||
# Run distillation ablation experiments
|
||||
#
|
||||
# Prerequisites:
|
||||
# - Ollama running with qwen3.5:{2b,9b,27b}
|
||||
# - ANTHROPIC_API_KEY set (for Opus teacher)
|
||||
# - OPENAI_API_KEY set (for GPT-5.4 teacher)
|
||||
# - GOOGLE_API_KEY set (for Gemini teacher)
|
||||
# - For Qwen-397B teacher: vLLM serving on port 8010 with 8×H100
|
||||
# - Traces seeded with feedback (run A1 blocker first)
|
||||
# - jarvis learning init already run
|
||||
#
|
||||
# Usage:
|
||||
# bash scripts/experiments/run_distillation_experiments.sh # Run all
|
||||
# bash scripts/experiments/run_distillation_experiments.sh exp1a # Run Phase 1a only
|
||||
# bash scripts/experiments/run_distillation_experiments.sh exp1a opus # Single config
|
||||
# ──────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
CONFIGS_DIR="src/openjarvis/evals/configs/distillation"
|
||||
RESULTS_DIR="results/neurips-2026/agent-optimization/distillation"
|
||||
EXPERIMENT=${1:-all}
|
||||
FILTER=${2:-}
|
||||
|
||||
# ── Colors ───────────────────────────────────────────────────────────────────
|
||||
RED='\033[0;31m'
|
||||
GREEN='\033[0;32m'
|
||||
YELLOW='\033[1;33m'
|
||||
BLUE='\033[0;34m'
|
||||
NC='\033[0m'
|
||||
|
||||
log() { echo -e "${BLUE}[distill]${NC} $*"; }
|
||||
ok() { echo -e "${GREEN}[ OK ]${NC} $*"; }
|
||||
warn() { echo -e "${YELLOW}[ WARN ]${NC} $*"; }
|
||||
fail() { echo -e "${RED}[ FAIL ]${NC} $*"; }
|
||||
|
||||
# ── Preflight checks ────────────────────────────────────────────────────────
|
||||
check_prereqs() {
|
||||
log "Preflight checks..."
|
||||
|
||||
# Check API keys
|
||||
if [ -z "${ANTHROPIC_API_KEY:-}" ]; then
|
||||
warn "ANTHROPIC_API_KEY not set — Opus teacher experiments will fail"
|
||||
fi
|
||||
if [ -z "${OPENAI_API_KEY:-}" ]; then
|
||||
warn "OPENAI_API_KEY not set — GPT-5.4 teacher experiments will fail"
|
||||
fi
|
||||
if [ -z "${GOOGLE_API_KEY:-}" ]; then
|
||||
warn "GOOGLE_API_KEY not set — Gemini teacher experiments will fail"
|
||||
fi
|
||||
|
||||
# Check Ollama
|
||||
if ! ollama list &>/dev/null; then
|
||||
fail "Ollama not running. Start it first."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Check student models
|
||||
for model in qwen3.5:2b qwen3.5:9b qwen3.5:27b; do
|
||||
if ! ollama list 2>/dev/null | grep -q "$model"; then
|
||||
warn "Model $model not found in Ollama. Pull with: ollama pull $model"
|
||||
fi
|
||||
done
|
||||
|
||||
# Check distillation init
|
||||
if [ ! -d "$HOME/.openjarvis/learning" ]; then
|
||||
log "Running jarvis learning init..."
|
||||
uv run jarvis learning init
|
||||
fi
|
||||
|
||||
ok "Preflight complete"
|
||||
}
|
||||
|
||||
# ── Run a single distillation session ────────────────────────────────────────
|
||||
run_session() {
|
||||
local config_file=$1
|
||||
local experiment_name
|
||||
experiment_name=$(basename "$(dirname "$config_file")")
|
||||
local config_name
|
||||
config_name=$(basename "${config_file%.toml}")
|
||||
local output_dir="${RESULTS_DIR}/${experiment_name}/${config_name}"
|
||||
|
||||
# Skip if already completed
|
||||
if [ -f "${output_dir}/session/session.json" ]; then
|
||||
ok "SKIP ${experiment_name}/${config_name} (already done)"
|
||||
return 0
|
||||
fi
|
||||
|
||||
log "──────────────────────────────────────────────────────"
|
||||
log "Experiment: ${experiment_name}/${config_name}"
|
||||
log "Config: ${config_file}"
|
||||
log "Output: ${output_dir}"
|
||||
log "──────────────────────────────────────────────────────"
|
||||
|
||||
mkdir -p "${output_dir}"
|
||||
|
||||
# Extract metadata from config
|
||||
local teacher_model
|
||||
teacher_model=$(grep 'teacher_model' "$config_file" | head -1 | sed 's/.*= *"\(.*\)"/\1/')
|
||||
local student_model
|
||||
student_model=$(grep 'default_model' "$config_file" | head -1 | sed 's/.*= *"\(.*\)"/\1/')
|
||||
local benchmark
|
||||
benchmark=$(grep '^benchmark ' "$config_file" | head -1 | sed 's/.*= *"\(.*\)"/\1/')
|
||||
local data_config
|
||||
data_config=$(grep 'data_config' "$config_file" | head -1 | sed 's/.*= *"\(.*\)"/\1/')
|
||||
local iterative
|
||||
iterative=$(grep 'iterative_sessions' "$config_file" | head -1 | sed 's/.*= *//')
|
||||
|
||||
log "Teacher: ${teacher_model}"
|
||||
log "Student: ${student_model}"
|
||||
log "Data: ${data_config:-C2}"
|
||||
log "Iter: ${iterative:-1}"
|
||||
|
||||
# ── Step 1: Seed traces based on data config ─────────────────────────
|
||||
# (In a full implementation, this would filter/prepare the TraceStore
|
||||
# based on C1/C2/C3. For now we use whatever traces exist.)
|
||||
|
||||
# ── Step 2: Run distillation session ─────────────────────────────────
|
||||
local n_sessions=${iterative:-1}
|
||||
local session_num=1
|
||||
local prev_session_id=""
|
||||
|
||||
while [ "$session_num" -le "$n_sessions" ]; do
|
||||
log "Session ${session_num}/${n_sessions}..."
|
||||
|
||||
local session_output="${output_dir}/session_${session_num}"
|
||||
mkdir -p "${session_output}"
|
||||
|
||||
# Run the distillation session via Python
|
||||
# (jarvis learning run doesn't support all config params yet,
|
||||
# so we call the orchestrator directly)
|
||||
uv run python << PYEOF > "${session_output}/run.log" 2>&1 || true
|
||||
import json, os, shutil, sys
|
||||
from pathlib import Path
|
||||
|
||||
from openjarvis.engine.cloud import CloudEngine
|
||||
from openjarvis.evals.backends.jarvis_direct import JarvisDirectBackend
|
||||
from openjarvis.traces.store import TraceStore
|
||||
from openjarvis.learning.distillation.checkpoint.store import CheckpointStore
|
||||
from openjarvis.learning.distillation.models import AutonomyMode
|
||||
from openjarvis.learning.distillation.orchestrator import DistillationOrchestrator
|
||||
from openjarvis.learning.distillation.storage.session_store import SessionStore
|
||||
from openjarvis.learning.distillation.student_runner import (
|
||||
VLLMStudentRunner,
|
||||
build_benchmark_samples_from_traces,
|
||||
)
|
||||
from openjarvis.learning.distillation.triggers import OnDemandTrigger
|
||||
from openjarvis.learning.optimize.feedback.judge import TraceJudge
|
||||
|
||||
home = Path(os.environ.get("OPENJARVIS_HOME", str(Path.home() / ".openjarvis")))
|
||||
|
||||
# Read config params
|
||||
teacher_model = "${teacher_model}"
|
||||
student_model = "${student_model}"
|
||||
autonomy = "auto"
|
||||
max_cost = float("$(grep 'max_cost_per_session_usd' "$config_file" | head -1 | sed 's/.*= *//')")
|
||||
max_tools = int("$(grep 'max_tool_calls_per_diagnosis' "$config_file" | head -1 | sed 's/.*= *//')")
|
||||
|
||||
# Real student runner via vLLM
|
||||
vllm_host = os.environ.get("VLLM_HOST", "http://localhost:8001")
|
||||
student_runner = VLLMStudentRunner(
|
||||
host=vllm_host,
|
||||
model=student_model,
|
||||
)
|
||||
|
||||
# Real judge via cloud LLM
|
||||
cloud_engine = CloudEngine()
|
||||
judge_backend = JarvisDirectBackend(engine_key="cloud")
|
||||
judge = TraceJudge(backend=judge_backend, model="gpt-5-mini-2025-08-07")
|
||||
|
||||
# Build benchmark samples from existing traces
|
||||
trace_store = TraceStore(home / "traces.db")
|
||||
benchmark_samples = build_benchmark_samples_from_traces(trace_store, limit=50)
|
||||
|
||||
orch = DistillationOrchestrator(
|
||||
teacher_engine=cloud_engine,
|
||||
teacher_model=teacher_model,
|
||||
trace_store=trace_store,
|
||||
benchmark_samples=benchmark_samples,
|
||||
student_runner=student_runner,
|
||||
judge=judge,
|
||||
session_store=SessionStore(home / "learning" / "learning.db"),
|
||||
checkpoint_store=CheckpointStore(home),
|
||||
openjarvis_home=home,
|
||||
autonomy_mode=AutonomyMode.AUTO,
|
||||
scorer=None,
|
||||
min_traces=10,
|
||||
max_cost_usd=max_cost,
|
||||
max_tool_calls=max_tools,
|
||||
)
|
||||
session = orch.run(OnDemandTrigger())
|
||||
|
||||
# Save results
|
||||
result = {
|
||||
"session_id": session.id,
|
||||
"status": session.status.value,
|
||||
"cost_usd": session.teacher_cost_usd,
|
||||
"edits_total": len(session.edit_outcomes),
|
||||
"edits_applied": len([o for o in session.edit_outcomes if o.status == "applied"]),
|
||||
"edits_rejected": len([o for o in session.edit_outcomes if o.status == "rejected_by_gate"]),
|
||||
"error": session.error,
|
||||
}
|
||||
Path("${session_output}/result.json").write_text(json.dumps(result, indent=2))
|
||||
|
||||
# Copy session artifacts
|
||||
sd = home / "learning" / "sessions" / session.id
|
||||
if sd.exists():
|
||||
shutil.copytree(sd, Path("${session_output}/artifacts"), dirs_exist_ok=True)
|
||||
|
||||
print(json.dumps(result, indent=2))
|
||||
PYEOF
|
||||
|
||||
# Check result
|
||||
if [ -f "${session_output}/result.json" ]; then
|
||||
local status
|
||||
status=$(python3 -c "import json; print(json.load(open('${session_output}/result.json'))['status'])")
|
||||
local cost
|
||||
cost=$(python3 -c "import json; print(f\"\${json.load(open('${session_output}/result.json'))['cost_usd']:.4f}\")")
|
||||
local applied
|
||||
applied=$(python3 -c "import json; print(json.load(open('${session_output}/result.json'))['edits_applied'])")
|
||||
|
||||
if [ "$status" = "completed" ]; then
|
||||
ok "Session ${session_num}: status=${status}, cost=\$${cost}, applied=${applied}"
|
||||
else
|
||||
warn "Session ${session_num}: status=${status}, cost=\$${cost}"
|
||||
fi
|
||||
else
|
||||
fail "Session ${session_num}: no result.json (check ${session_output}/run.log)"
|
||||
fi
|
||||
|
||||
session_num=$((session_num + 1))
|
||||
done
|
||||
|
||||
ok "Done: ${experiment_name}/${config_name}"
|
||||
}
|
||||
|
||||
# ── Run experiment group ─────────────────────────────────────────────────────
|
||||
run_experiment() {
|
||||
local exp_dir=$1
|
||||
local exp_name
|
||||
exp_name=$(basename "$exp_dir")
|
||||
|
||||
log "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
log "EXPERIMENT GROUP: ${exp_name}"
|
||||
log "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
|
||||
local count=0
|
||||
local total
|
||||
total=$(ls "${exp_dir}"/*.toml 2>/dev/null | wc -l)
|
||||
|
||||
for config in "${exp_dir}"/*.toml; do
|
||||
[ -f "$config" ] || continue
|
||||
|
||||
# Apply filter if specified
|
||||
if [ -n "${FILTER}" ] && ! echo "$config" | grep -q "${FILTER}"; then
|
||||
continue
|
||||
fi
|
||||
|
||||
count=$((count + 1))
|
||||
log "[${count}/${total}] $(basename "$config")"
|
||||
run_session "$config"
|
||||
done
|
||||
|
||||
ok "Experiment group ${exp_name}: ${count} configs processed"
|
||||
}
|
||||
|
||||
# ── Main ─────────────────────────────────────────────────────────────────────
|
||||
main() {
|
||||
check_prereqs
|
||||
|
||||
log "Starting distillation experiments"
|
||||
log "Experiment filter: ${EXPERIMENT}"
|
||||
log "Config filter: ${FILTER:-none}"
|
||||
|
||||
local start_time
|
||||
start_time=$(date +%s)
|
||||
|
||||
if [ "$EXPERIMENT" = "all" ]; then
|
||||
# Run in priority order
|
||||
for exp in exp1a-teacher exp1b-budget exp1c-student \
|
||||
exp2a-gate exp2b-autonomy \
|
||||
exp3a-iterative exp3b-transfer; do
|
||||
if [ -d "${CONFIGS_DIR}/${exp}" ]; then
|
||||
run_experiment "${CONFIGS_DIR}/${exp}"
|
||||
fi
|
||||
done
|
||||
elif [ -d "${CONFIGS_DIR}/${EXPERIMENT}" ]; then
|
||||
run_experiment "${CONFIGS_DIR}/${EXPERIMENT}"
|
||||
else
|
||||
fail "Unknown experiment: ${EXPERIMENT}"
|
||||
echo "Available: exp1a-teacher exp1b-budget exp1c-student exp2a-gate exp2b-autonomy exp3a-iterative exp3b-transfer"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
local end_time
|
||||
end_time=$(date +%s)
|
||||
local elapsed=$((end_time - start_time))
|
||||
|
||||
log "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
ok "All experiments complete in ${elapsed}s"
|
||||
log "Results in: ${RESULTS_DIR}/"
|
||||
log "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
}
|
||||
|
||||
main "$@"
|
||||
@@ -1,375 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
# =============================================================================
|
||||
# Track B: GEPA/DSPy Agent Optimization
|
||||
# NeurIPS 2026 — Agent Optimization Experiments
|
||||
#
|
||||
# Runs GEPA and DSPy BootstrapFewShot across:
|
||||
# Models: qwen-9b, qwen-27b, qwen-35b
|
||||
# Benchmarks: toolcall15, pinchbench, taubench
|
||||
#
|
||||
# Usage:
|
||||
# bash scripts/experiments/run_track_b_gepa_dspy.sh
|
||||
# bash scripts/experiments/run_track_b_gepa_dspy.sh --model qwen-9b --benchmark pinchbench
|
||||
# bash scripts/experiments/run_track_b_gepa_dspy.sh --optimizer gepa
|
||||
# bash scripts/experiments/run_track_b_gepa_dspy.sh --optimizer dspy
|
||||
#
|
||||
# Expected runtime: ~2-4 hours per GEPA run, ~1-2 hours per DSPy run
|
||||
# Total wall-clock: ~12-18 hours (parallelized across GPUs)
|
||||
# Estimated API cost: ~$90 GEPA + ~$90 DSPy = ~$180 total
|
||||
#
|
||||
# =============================================================================
|
||||
# vLLM Serving Commands (run these BEFORE this script on the GPU node)
|
||||
# =============================================================================
|
||||
#
|
||||
# GPU 0 — Qwen-9B (1x GPU):
|
||||
# CUDA_VISIBLE_DEVICES=0 vllm serve Qwen/Qwen2.5-7B-Instruct \
|
||||
# --model Qwen/Qwen2.5-7B-Instruct \
|
||||
# --served-model-name qwen-9b \
|
||||
# --port 8001 --host 0.0.0.0 \
|
||||
# --max-model-len 32768 --gpu-memory-utilization 0.9 &
|
||||
#
|
||||
# GPU 1 — Qwen-27B (1-2x GPU):
|
||||
# CUDA_VISIBLE_DEVICES=1,2 vllm serve Qwen/Qwen2.5-32B-Instruct \
|
||||
# --model Qwen/Qwen2.5-32B-Instruct \
|
||||
# --served-model-name qwen-27b \
|
||||
# --port 8002 --host 0.0.0.0 \
|
||||
# --tensor-parallel-size 2 \
|
||||
# --max-model-len 32768 --gpu-memory-utilization 0.9 &
|
||||
#
|
||||
# GPU 3 — Qwen-35B (1-2x GPU):
|
||||
# CUDA_VISIBLE_DEVICES=3,4 vllm serve Qwen/Qwen2.5-72B-Instruct \
|
||||
# --model Qwen/Qwen2.5-72B-Instruct \
|
||||
# --served-model-name qwen-35b \
|
||||
# --port 8003 --host 0.0.0.0 \
|
||||
# --tensor-parallel-size 2 \
|
||||
# --max-model-len 32768 --gpu-memory-utilization 0.9 &
|
||||
#
|
||||
# Wait for all servers to be healthy:
|
||||
# sleep 60 && curl -s http://localhost:8001/health && \
|
||||
# curl -s http://localhost:8002/health && \
|
||||
# curl -s http://localhost:8003/health
|
||||
#
|
||||
# =============================================================================
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Defaults
|
||||
# ---------------------------------------------------------------------------
|
||||
REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
|
||||
RESULTS_BASE="${REPO_ROOT}/results/neurips-2026/agent-optimization"
|
||||
LOG_DIR="${REPO_ROOT}/results/neurips-2026/logs"
|
||||
TIMESTAMP="$(date +%Y%m%d_%H%M%S)"
|
||||
|
||||
ALL_MODELS=(qwen-9b qwen-27b qwen-35b)
|
||||
ALL_BENCHMARKS=(toolcall15 pinchbench taubench)
|
||||
ALL_OPTIMIZERS=(gepa dspy)
|
||||
|
||||
# Override defaults with CLI flags
|
||||
FILTER_MODEL=""
|
||||
FILTER_BENCHMARK=""
|
||||
FILTER_OPTIMIZER=""
|
||||
|
||||
# GEPA settings
|
||||
GEPA_TRIALS=20
|
||||
GEPA_MAX_SAMPLES=50
|
||||
GEPA_OPTIMIZER_MODEL="claude-sonnet-4-6"
|
||||
|
||||
# DSPy settings
|
||||
DSPY_OPTIMIZER="BootstrapFewShotWithRandomSearch"
|
||||
DSPY_TEACHER_LM="claude-sonnet-4-6"
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Parse CLI flags
|
||||
# ---------------------------------------------------------------------------
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case "$1" in
|
||||
--model)
|
||||
FILTER_MODEL="$2"; shift 2 ;;
|
||||
--benchmark)
|
||||
FILTER_BENCHMARK="$2"; shift 2 ;;
|
||||
--optimizer)
|
||||
FILTER_OPTIMIZER="$2"; shift 2 ;;
|
||||
--gepa-trials)
|
||||
GEPA_TRIALS="$2"; shift 2 ;;
|
||||
--gepa-max-samples)
|
||||
GEPA_MAX_SAMPLES="$2"; shift 2 ;;
|
||||
--dspy-optimizer)
|
||||
DSPY_OPTIMIZER="$2"; shift 2 ;;
|
||||
-h|--help)
|
||||
sed -n '2,30p' "$0" | grep '^#' | sed 's/^# \?//'
|
||||
exit 0 ;;
|
||||
*)
|
||||
echo "Unknown flag: $1"; exit 1 ;;
|
||||
esac
|
||||
done
|
||||
|
||||
# Apply filters
|
||||
if [[ -n "$FILTER_MODEL" ]]; then
|
||||
ALL_MODELS=("$FILTER_MODEL")
|
||||
fi
|
||||
if [[ -n "$FILTER_BENCHMARK" ]]; then
|
||||
ALL_BENCHMARKS=("$FILTER_BENCHMARK")
|
||||
fi
|
||||
if [[ -n "$FILTER_OPTIMIZER" ]]; then
|
||||
ALL_OPTIMIZERS=("$FILTER_OPTIMIZER")
|
||||
fi
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Logging helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
mkdir -p "$LOG_DIR"
|
||||
LOG_FILE="${LOG_DIR}/track_b_${TIMESTAMP}.log"
|
||||
|
||||
log() {
|
||||
local level="$1"; shift
|
||||
local msg="[$(date '+%Y-%m-%d %H:%M:%S')] [$level] $*"
|
||||
echo "$msg"
|
||||
echo "$msg" >> "$LOG_FILE"
|
||||
}
|
||||
|
||||
log_info() { log "INFO " "$@"; }
|
||||
log_ok() { log "OK " "$@"; }
|
||||
log_warn() { log "WARN " "$@"; }
|
||||
log_error() { log "ERROR" "$@"; }
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Environment setup
|
||||
# ---------------------------------------------------------------------------
|
||||
setup_env() {
|
||||
log_info "=== Track B: GEPA/DSPy Optimization ==="
|
||||
log_info "Repo: $REPO_ROOT"
|
||||
log_info "Log: $LOG_FILE"
|
||||
log_info "Models: ${ALL_MODELS[*]}"
|
||||
log_info "Benchmarks: ${ALL_BENCHMARKS[*]}"
|
||||
log_info "Optimizers: ${ALL_OPTIMIZERS[*]}"
|
||||
echo ""
|
||||
|
||||
# Check we are in the repo root
|
||||
if [[ ! -f "${REPO_ROOT}/pyproject.toml" ]]; then
|
||||
log_error "pyproject.toml not found — is REPO_ROOT set correctly? ($REPO_ROOT)"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Install/sync dependencies
|
||||
log_info "Running uv sync..."
|
||||
cd "$REPO_ROOT"
|
||||
uv sync --extra dev 2>&1 | tail -5
|
||||
log_ok "uv sync complete"
|
||||
|
||||
# Check required API keys
|
||||
if [[ -z "${ANTHROPIC_API_KEY:-}" ]]; then
|
||||
log_error "ANTHROPIC_API_KEY is not set. Required for the optimizer teacher model."
|
||||
log_error " export ANTHROPIC_API_KEY=sk-ant-..."
|
||||
exit 1
|
||||
fi
|
||||
log_ok "ANTHROPIC_API_KEY is set"
|
||||
|
||||
# Optional: OpenAI key (used if teacher_lm is an OpenAI model)
|
||||
if [[ -z "${OPENAI_API_KEY:-}" ]]; then
|
||||
log_warn "OPENAI_API_KEY not set (only needed if using OpenAI teacher models)"
|
||||
fi
|
||||
|
||||
echo ""
|
||||
log_info "Model-to-port mapping (vLLM must be pre-started on these ports):"
|
||||
log_info " qwen-9b -> http://localhost:8001"
|
||||
log_info " qwen-27b -> http://localhost:8002"
|
||||
log_info " qwen-35b -> http://localhost:8003"
|
||||
echo ""
|
||||
}
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Model port lookup
|
||||
# ---------------------------------------------------------------------------
|
||||
model_port() {
|
||||
case "$1" in
|
||||
qwen-9b) echo 8001 ;;
|
||||
qwen-27b) echo 8002 ;;
|
||||
qwen-35b) echo 8003 ;;
|
||||
*)
|
||||
log_error "Unknown model: $1"
|
||||
exit 1 ;;
|
||||
esac
|
||||
}
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Health check: verify the vLLM server for a model is reachable
|
||||
# ---------------------------------------------------------------------------
|
||||
check_server_health() {
|
||||
local model="$1"
|
||||
local port
|
||||
port="$(model_port "$model")"
|
||||
local url="http://localhost:${port}/health"
|
||||
if curl -sf "$url" > /dev/null 2>&1; then
|
||||
log_ok "vLLM server for $model is healthy at port $port"
|
||||
return 0
|
||||
else
|
||||
log_error "vLLM server for $model NOT reachable at $url"
|
||||
log_error "Start it with the vLLM commands in the script header."
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# GEPA optimization for one (model, benchmark) pair
|
||||
# ---------------------------------------------------------------------------
|
||||
run_gepa() {
|
||||
local model="$1"
|
||||
local bench="$2"
|
||||
local port
|
||||
port="$(model_port "$model")"
|
||||
local out_dir="${RESULTS_BASE}/gepa/${model}/${bench}"
|
||||
|
||||
log_info "--- GEPA: $model × $bench ---"
|
||||
log_info " Output dir: $out_dir"
|
||||
log_info " Trials: $GEPA_TRIALS Max-samples: $GEPA_MAX_SAMPLES"
|
||||
mkdir -p "$out_dir"
|
||||
|
||||
# Record start time
|
||||
local t0
|
||||
t0="$(date +%s)"
|
||||
|
||||
OPENAI_API_BASE="http://localhost:${port}/v1" \
|
||||
uv run jarvis optimize run \
|
||||
--benchmark "$bench" \
|
||||
--model "$model" \
|
||||
--optimizer-model "$GEPA_OPTIMIZER_MODEL" \
|
||||
--trials "$GEPA_TRIALS" \
|
||||
--max-samples "$GEPA_MAX_SAMPLES" \
|
||||
--output-dir "$out_dir" \
|
||||
2>&1 | tee -a "$LOG_FILE"
|
||||
|
||||
local exit_code=${PIPESTATUS[0]}
|
||||
local t1
|
||||
t1="$(date +%s)"
|
||||
local elapsed=$(( t1 - t0 ))
|
||||
|
||||
if [[ $exit_code -eq 0 ]]; then
|
||||
log_ok "GEPA $model/$bench done in ${elapsed}s"
|
||||
else
|
||||
log_error "GEPA $model/$bench FAILED (exit $exit_code) after ${elapsed}s"
|
||||
return $exit_code
|
||||
fi
|
||||
}
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# DSPy optimization for one (model, benchmark) pair
|
||||
# ---------------------------------------------------------------------------
|
||||
run_dspy() {
|
||||
local model="$1"
|
||||
local bench="$2"
|
||||
local port
|
||||
port="$(model_port "$model")"
|
||||
local out_dir="${RESULTS_BASE}/dspy/${model}/${bench}"
|
||||
|
||||
log_info "--- DSPy: $model × $bench ---"
|
||||
log_info " Output dir: $out_dir"
|
||||
log_info " Teleprompter: $DSPY_OPTIMIZER Teacher: $DSPY_TEACHER_LM"
|
||||
mkdir -p "$out_dir"
|
||||
|
||||
local t0
|
||||
t0="$(date +%s)"
|
||||
|
||||
OPENAI_API_BASE="http://localhost:${port}/v1" \
|
||||
uv run python - <<PYEOF 2>&1 | tee -a "$LOG_FILE"
|
||||
import sys
|
||||
from openjarvis.learning.agents.dspy_optimizer import DSPyAgentOptimizer
|
||||
from openjarvis.core.config import DSPyOptimizerConfig
|
||||
from openjarvis.traces.store import TraceStore
|
||||
|
||||
store = TraceStore()
|
||||
config = DSPyOptimizerConfig(
|
||||
optimizer="${DSPY_OPTIMIZER}",
|
||||
teacher_lm="${DSPY_TEACHER_LM}",
|
||||
config_dir="${out_dir}",
|
||||
benchmark="${bench}",
|
||||
agent_filter="${model}",
|
||||
)
|
||||
result = DSPyAgentOptimizer(config).optimize(store)
|
||||
print(f"DSPy result for ${model}/${bench}: {result}")
|
||||
if result.get("status") not in ("ok", "success", "done"):
|
||||
sys.exit(1)
|
||||
PYEOF
|
||||
|
||||
local exit_code=${PIPESTATUS[0]}
|
||||
local t1
|
||||
t1="$(date +%s)"
|
||||
local elapsed=$(( t1 - t0 ))
|
||||
|
||||
if [[ $exit_code -eq 0 ]]; then
|
||||
log_ok "DSPy $model/$bench done in ${elapsed}s"
|
||||
else
|
||||
log_error "DSPy $model/$bench FAILED (exit $exit_code) after ${elapsed}s"
|
||||
return $exit_code
|
||||
fi
|
||||
}
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Summary: print result file locations
|
||||
# ---------------------------------------------------------------------------
|
||||
print_summary() {
|
||||
echo ""
|
||||
log_info "=== Track B Complete ==="
|
||||
log_info "Results written to:"
|
||||
for opt in "${ALL_OPTIMIZERS[@]}"; do
|
||||
for model in "${ALL_MODELS[@]}"; do
|
||||
for bench in "${ALL_BENCHMARKS[@]}"; do
|
||||
local out_dir="${RESULTS_BASE}/${opt}/${model}/${bench}"
|
||||
if [[ -d "$out_dir" ]]; then
|
||||
log_ok " $opt/$model/$bench -> $out_dir"
|
||||
else
|
||||
log_warn " $opt/$model/$bench -> MISSING ($out_dir)"
|
||||
fi
|
||||
done
|
||||
done
|
||||
done
|
||||
log_info "Full log: $LOG_FILE"
|
||||
}
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Main
|
||||
# ---------------------------------------------------------------------------
|
||||
main() {
|
||||
setup_env
|
||||
|
||||
local failed=0
|
||||
|
||||
# Pre-flight: check vLLM servers for all target models
|
||||
log_info "Checking vLLM server health..."
|
||||
for model in "${ALL_MODELS[@]}"; do
|
||||
check_server_health "$model" || failed=$(( failed + 1 ))
|
||||
done
|
||||
if [[ $failed -gt 0 ]]; then
|
||||
log_error "$failed vLLM server(s) not reachable. Start them first (see header)."
|
||||
exit 1
|
||||
fi
|
||||
echo ""
|
||||
|
||||
# Run all requested (optimizer, model, benchmark) combinations
|
||||
for opt in "${ALL_OPTIMIZERS[@]}"; do
|
||||
log_info "=========================================="
|
||||
log_info "Optimizer: $opt"
|
||||
log_info "=========================================="
|
||||
for model in "${ALL_MODELS[@]}"; do
|
||||
for bench in "${ALL_BENCHMARKS[@]}"; do
|
||||
case "$opt" in
|
||||
gepa) run_gepa "$model" "$bench" || failed=$(( failed + 1 )) ;;
|
||||
dspy) run_dspy "$model" "$bench" || failed=$(( failed + 1 )) ;;
|
||||
*) log_error "Unknown optimizer: $opt"; failed=$(( failed + 1 )) ;;
|
||||
esac
|
||||
echo ""
|
||||
done
|
||||
done
|
||||
done
|
||||
|
||||
print_summary
|
||||
|
||||
if [[ $failed -gt 0 ]]; then
|
||||
log_error "$failed run(s) failed. Check log for details: $LOG_FILE"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
log_ok "All Track B runs completed successfully."
|
||||
}
|
||||
|
||||
main "$@"
|
||||
@@ -1,778 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
# =============================================================================
|
||||
# Track D: LoRA / SFT Intelligence Optimization
|
||||
# NeurIPS 2026 — Intelligence Optimization Experiments
|
||||
#
|
||||
# Runs LoRA and SFT fine-tuning across:
|
||||
# LoRA models: Qwen-2B, Qwen-9B, Qwen-27B
|
||||
# SFT models: Qwen-2B, Qwen-9B
|
||||
#
|
||||
# After training, runs fast-benchmark eval on every checkpoint.
|
||||
#
|
||||
# Usage:
|
||||
# bash scripts/experiments/run_track_d_lora_sft.sh
|
||||
# bash scripts/experiments/run_track_d_lora_sft.sh --method lora
|
||||
# bash scripts/experiments/run_track_d_lora_sft.sh --method sft --model qwen-2b
|
||||
# bash scripts/experiments/run_track_d_lora_sft.sh --skip-eval
|
||||
#
|
||||
# Expected runtime per run:
|
||||
# Qwen-2B LoRA (~4-8 h, 1x H100, GPU 0)
|
||||
# Qwen-9B LoRA (~8-16 h, 1x H100, GPU 1)
|
||||
# Qwen-27B LoRA (~16-24 h, 2x H100, GPUs 2-3)
|
||||
# Qwen-2B SFT (~8-16 h, 1x H100, GPU 4)
|
||||
# Qwen-9B SFT (~16-24 h, 2x H100, GPUs 5-6)
|
||||
#
|
||||
# Total wall-clock (all in parallel): ~24 h on a 7-8x H100 node
|
||||
# Total GPU-hours: ~100-200 H100-hours
|
||||
#
|
||||
# =============================================================================
|
||||
# GPU Allocation — recommended for a single 8x H100 node
|
||||
# =============================================================================
|
||||
#
|
||||
# Run D1 (Qwen-2B LoRA) on GPU 0:
|
||||
# CUDA_VISIBLE_DEVICES=0 bash scripts/experiments/run_track_d_lora_sft.sh \
|
||||
# --method lora --model qwen-2b &
|
||||
#
|
||||
# Run D2 (Qwen-9B LoRA) on GPU 1:
|
||||
# CUDA_VISIBLE_DEVICES=1 bash scripts/experiments/run_track_d_lora_sft.sh \
|
||||
# --method lora --model qwen-9b &
|
||||
#
|
||||
# Run D3 (Qwen-27B LoRA) on GPUs 2-3:
|
||||
# CUDA_VISIBLE_DEVICES=2,3 bash scripts/experiments/run_track_d_lora_sft.sh \
|
||||
# --method lora --model qwen-27b &
|
||||
#
|
||||
# Run D4 (Qwen-2B SFT) on GPU 4:
|
||||
# CUDA_VISIBLE_DEVICES=4 bash scripts/experiments/run_track_d_lora_sft.sh \
|
||||
# --method sft --model qwen-2b &
|
||||
#
|
||||
# Run D5 (Qwen-9B SFT) on GPUs 5-6:
|
||||
# CUDA_VISIBLE_DEVICES=5,6 bash scripts/experiments/run_track_d_lora_sft.sh \
|
||||
# --method sft --model qwen-9b &
|
||||
#
|
||||
# =============================================================================
|
||||
# Training datasets (downloaded automatically if HF_TOKEN is set)
|
||||
# =============================================================================
|
||||
#
|
||||
# Primary agentic traces:
|
||||
# - neulab/agent-data-collection
|
||||
# - GAIR/AgentInstruct
|
||||
#
|
||||
# Supplementary reasoning:
|
||||
# - GeneralThought-430K-filtered
|
||||
# - GLM-4.7-flash SFT traces (168K + 57K)
|
||||
#
|
||||
# =============================================================================
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Defaults
|
||||
# ---------------------------------------------------------------------------
|
||||
REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
|
||||
RESULTS_BASE="${REPO_ROOT}/results/neurips-2026/intelligence-optimization"
|
||||
LOG_DIR="${REPO_ROOT}/results/neurips-2026/logs"
|
||||
TIMESTAMP="$(date +%Y%m%d_%H%M%S)"
|
||||
|
||||
# Default: run all methods and models
|
||||
ALL_LORA_MODELS=(qwen-2b qwen-9b qwen-27b)
|
||||
ALL_SFT_MODELS=(qwen-2b qwen-9b)
|
||||
FAST_BENCHMARKS=(toolcall15 pinchbench taubench)
|
||||
|
||||
# CLI overrides
|
||||
FILTER_METHOD=""
|
||||
FILTER_MODEL=""
|
||||
SKIP_EVAL=false
|
||||
|
||||
# Training hyperparameters (override per model below)
|
||||
LORA_RANK=64
|
||||
LORA_ALPHA=128
|
||||
LORA_DROPOUT=0.05
|
||||
LORA_EPOCHS=3
|
||||
LORA_LR=2e-4
|
||||
LORA_BATCH_SIZE=8
|
||||
LORA_GRAD_ACCUM=4
|
||||
|
||||
SFT_EPOCHS=3
|
||||
SFT_LR=1e-5
|
||||
SFT_BATCH_SIZE=4
|
||||
SFT_GRAD_ACCUM=8
|
||||
|
||||
MAX_SEQ_LEN=8192
|
||||
|
||||
# Fast-eval settings
|
||||
EVAL_MAX_SAMPLES=20
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# HuggingFace model IDs
|
||||
# ---------------------------------------------------------------------------
|
||||
model_hf_id() {
|
||||
case "$1" in
|
||||
qwen-2b) echo "Qwen/Qwen2.5-1.5B-Instruct" ;;
|
||||
qwen-9b) echo "Qwen/Qwen2.5-7B-Instruct" ;;
|
||||
qwen-27b) echo "Qwen/Qwen2.5-32B-Instruct" ;;
|
||||
*)
|
||||
echo "ERROR: unknown model $1" >&2
|
||||
exit 1 ;;
|
||||
esac
|
||||
}
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Parse CLI flags
|
||||
# ---------------------------------------------------------------------------
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case "$1" in
|
||||
--method)
|
||||
FILTER_METHOD="$2"; shift 2 ;;
|
||||
--model)
|
||||
FILTER_MODEL="$2"; shift 2 ;;
|
||||
--skip-eval)
|
||||
SKIP_EVAL=true; shift ;;
|
||||
--lora-rank)
|
||||
LORA_RANK="$2"; shift 2 ;;
|
||||
--lora-epochs)
|
||||
LORA_EPOCHS="$2"; shift 2 ;;
|
||||
--sft-epochs)
|
||||
SFT_EPOCHS="$2"; shift 2 ;;
|
||||
-h|--help)
|
||||
sed -n '2,35p' "$0" | grep '^#' | sed 's/^# \?//'
|
||||
exit 0 ;;
|
||||
*)
|
||||
echo "Unknown flag: $1"; exit 1 ;;
|
||||
esac
|
||||
done
|
||||
|
||||
# Apply model filter
|
||||
if [[ -n "$FILTER_MODEL" ]]; then
|
||||
ALL_LORA_MODELS=("$FILTER_MODEL")
|
||||
ALL_SFT_MODELS=("$FILTER_MODEL")
|
||||
fi
|
||||
|
||||
# Apply method filter
|
||||
RUN_LORA=true
|
||||
RUN_SFT=true
|
||||
if [[ "$FILTER_METHOD" == "lora" ]]; then
|
||||
RUN_SFT=false
|
||||
elif [[ "$FILTER_METHOD" == "sft" ]]; then
|
||||
RUN_LORA=false
|
||||
fi
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Logging helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
mkdir -p "$LOG_DIR"
|
||||
LOG_FILE="${LOG_DIR}/track_d_${TIMESTAMP}.log"
|
||||
|
||||
log() {
|
||||
local level="$1"; shift
|
||||
local msg="[$(date '+%Y-%m-%d %H:%M:%S')] [$level] $*"
|
||||
echo "$msg"
|
||||
echo "$msg" >> "$LOG_FILE"
|
||||
}
|
||||
|
||||
log_info() { log "INFO " "$@"; }
|
||||
log_ok() { log "OK " "$@"; }
|
||||
log_warn() { log "WARN " "$@"; }
|
||||
log_error() { log "ERROR" "$@"; }
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Environment setup
|
||||
# ---------------------------------------------------------------------------
|
||||
setup_env() {
|
||||
log_info "=== Track D: LoRA/SFT Training ==="
|
||||
log_info "Repo: $REPO_ROOT"
|
||||
log_info "Log: $LOG_FILE"
|
||||
log_info "Results: $RESULTS_BASE"
|
||||
echo ""
|
||||
|
||||
# Check we are in the repo root
|
||||
if [[ ! -f "${REPO_ROOT}/pyproject.toml" ]]; then
|
||||
log_error "pyproject.toml not found — is REPO_ROOT set correctly? ($REPO_ROOT)"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Install/sync dependencies
|
||||
log_info "Running uv sync..."
|
||||
cd "$REPO_ROOT"
|
||||
uv sync --extra dev 2>&1 | tail -5
|
||||
log_ok "uv sync complete"
|
||||
|
||||
# HuggingFace token — required to download gated Qwen models
|
||||
if [[ -z "${HF_TOKEN:-}" ]]; then
|
||||
log_warn "HF_TOKEN is not set."
|
||||
log_warn " If Qwen models are gated, set: export HF_TOKEN=hf_..."
|
||||
log_warn " Or pre-download them with: huggingface-cli download <model>"
|
||||
else
|
||||
log_ok "HF_TOKEN is set"
|
||||
# Log in so huggingface_hub uses the token
|
||||
uv run python -c "
|
||||
import huggingface_hub
|
||||
huggingface_hub.login(token='${HF_TOKEN}', add_to_git_credential=False)
|
||||
print('Logged in to HuggingFace Hub')
|
||||
" 2>&1 | tee -a "$LOG_FILE"
|
||||
fi
|
||||
|
||||
# Check for GPU
|
||||
if ! command -v nvidia-smi &>/dev/null; then
|
||||
log_warn "nvidia-smi not found — ensure CUDA is available for training."
|
||||
else
|
||||
log_info "GPU status:"
|
||||
nvidia-smi --query-gpu=index,name,memory.total,memory.free \
|
||||
--format=csv,noheader 2>&1 | while IFS= read -r line; do
|
||||
log_info " $line"
|
||||
done
|
||||
fi
|
||||
|
||||
# Check for trl / peft / transformers (training stack)
|
||||
uv run python -c "
|
||||
import importlib, sys
|
||||
missing = []
|
||||
for pkg in ['transformers', 'peft', 'trl', 'datasets', 'accelerate', 'bitsandbytes']:
|
||||
if importlib.util.find_spec(pkg) is None:
|
||||
missing.append(pkg)
|
||||
if missing:
|
||||
print('MISSING packages:', missing)
|
||||
sys.exit(1)
|
||||
else:
|
||||
print('Training stack OK: transformers, peft, trl, datasets, accelerate, bitsandbytes')
|
||||
" 2>&1 | tee -a "$LOG_FILE" || {
|
||||
log_error "Some training packages are missing. Install with:"
|
||||
log_error " uv add transformers peft trl datasets accelerate bitsandbytes"
|
||||
exit 1
|
||||
}
|
||||
|
||||
echo ""
|
||||
}
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Download / verify training dataset
|
||||
# ---------------------------------------------------------------------------
|
||||
prepare_dataset() {
|
||||
local method="$1"
|
||||
local model="$2"
|
||||
local data_dir="${REPO_ROOT}/results/neurips-2026/training-data"
|
||||
mkdir -p "$data_dir"
|
||||
|
||||
log_info "Preparing training dataset for $method/$model..."
|
||||
|
||||
uv run python - <<PYEOF 2>&1 | tee -a "$LOG_FILE"
|
||||
from datasets import load_dataset
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
data_dir = Path("${data_dir}")
|
||||
output_path = data_dir / "${method}_${model}_train.jsonl"
|
||||
|
||||
if output_path.exists():
|
||||
lines = output_path.read_text().count('\n')
|
||||
print(f"Dataset already exists: {output_path} ({lines} examples)")
|
||||
else:
|
||||
print("Downloading neulab/agent-data-collection...")
|
||||
ds = load_dataset("neulab/agent-data-collection", split="train")
|
||||
print(f"Raw dataset size: {len(ds)}")
|
||||
|
||||
# Filter to reasonable-length examples for agentic fine-tuning
|
||||
examples = []
|
||||
for ex in ds:
|
||||
messages = ex.get("messages") or ex.get("conversations") or []
|
||||
if not messages:
|
||||
continue
|
||||
total_len = sum(len(str(m)) for m in messages)
|
||||
if 200 <= total_len <= 16000:
|
||||
examples.append({"messages": messages})
|
||||
|
||||
print(f"Filtered dataset size: {len(examples)}")
|
||||
|
||||
with output_path.open("w") as f:
|
||||
for ex in examples:
|
||||
f.write(json.dumps(ex) + "\n")
|
||||
print(f"Saved to {output_path}")
|
||||
PYEOF
|
||||
|
||||
echo "${data_dir}/${method}_${model}_train.jsonl"
|
||||
}
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# LoRA training for one model
|
||||
# ---------------------------------------------------------------------------
|
||||
run_lora() {
|
||||
local model="$1"
|
||||
local hf_id
|
||||
hf_id="$(model_hf_id "$model")"
|
||||
local out_dir="${RESULTS_BASE}/lora/${model}"
|
||||
local checkpoint_dir="${out_dir}/checkpoint"
|
||||
mkdir -p "$out_dir" "$checkpoint_dir"
|
||||
|
||||
log_info "=========================================="
|
||||
log_info "LoRA training: $model ($hf_id)"
|
||||
log_info " Output: $out_dir"
|
||||
log_info " LoRA rank=$LORA_RANK alpha=$LORA_ALPHA dropout=$LORA_DROPOUT"
|
||||
log_info " Epochs=$LORA_EPOCHS LR=$LORA_LR batch=$LORA_BATCH_SIZE grad_accum=$LORA_GRAD_ACCUM"
|
||||
log_info "=========================================="
|
||||
|
||||
local dataset_path
|
||||
dataset_path="$(prepare_dataset lora "$model")"
|
||||
|
||||
local t0
|
||||
t0="$(date +%s)"
|
||||
|
||||
# Number of GPUs visible
|
||||
local n_gpus
|
||||
n_gpus="$(python3 -c "import torch; print(torch.cuda.device_count())" 2>/dev/null || echo 1)"
|
||||
log_info "Training on $n_gpus GPU(s)"
|
||||
|
||||
if [[ "$n_gpus" -gt 1 ]]; then
|
||||
LAUNCHER="uv run torchrun --nproc_per_node=$n_gpus"
|
||||
else
|
||||
LAUNCHER="uv run python"
|
||||
fi
|
||||
|
||||
$LAUNCHER - <<PYEOF 2>&1 | tee -a "$LOG_FILE"
|
||||
import json
|
||||
import math
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
import torch
|
||||
from datasets import load_dataset
|
||||
from peft import LoraConfig, TaskType, get_peft_model
|
||||
from transformers import (
|
||||
AutoModelForCausalLM,
|
||||
AutoTokenizer,
|
||||
DataCollatorForSeq2Seq,
|
||||
Trainer,
|
||||
TrainingArguments,
|
||||
)
|
||||
|
||||
hf_id = "${hf_id}"
|
||||
out_dir = "${out_dir}"
|
||||
ckpt = "${checkpoint_dir}"
|
||||
data_path = "${dataset_path}"
|
||||
|
||||
# ---- Tokenizer ----
|
||||
print(f"Loading tokenizer: {hf_id}")
|
||||
tokenizer = AutoTokenizer.from_pretrained(hf_id, trust_remote_code=True)
|
||||
if tokenizer.pad_token is None:
|
||||
tokenizer.pad_token = tokenizer.eos_token
|
||||
|
||||
# ---- Dataset ----
|
||||
print(f"Loading dataset: {data_path}")
|
||||
raw = load_dataset("json", data_files=data_path, split="train")
|
||||
raw = raw.train_test_split(test_size=0.02, seed=42)
|
||||
|
||||
def tokenize(example):
|
||||
messages = example["messages"]
|
||||
text = tokenizer.apply_chat_template(
|
||||
messages,
|
||||
tokenize=False,
|
||||
add_generation_prompt=False,
|
||||
)
|
||||
enc = tokenizer(text, truncation=True, max_length=${MAX_SEQ_LEN})
|
||||
enc["labels"] = enc["input_ids"].copy()
|
||||
return enc
|
||||
|
||||
print("Tokenizing dataset...")
|
||||
tok_ds = raw.map(tokenize, remove_columns=raw["train"].column_names, num_proc=4)
|
||||
print(f"Train: {len(tok_ds['train'])} Eval: {len(tok_ds['test'])}")
|
||||
|
||||
# ---- Model ----
|
||||
print(f"Loading model: {hf_id}")
|
||||
model = AutoModelForCausalLM.from_pretrained(
|
||||
hf_id,
|
||||
torch_dtype=torch.bfloat16,
|
||||
trust_remote_code=True,
|
||||
device_map="auto",
|
||||
)
|
||||
|
||||
# ---- LoRA ----
|
||||
lora_config = LoraConfig(
|
||||
task_type=TaskType.CAUSAL_LM,
|
||||
r=${LORA_RANK},
|
||||
lora_alpha=${LORA_ALPHA},
|
||||
lora_dropout=${LORA_DROPOUT},
|
||||
target_modules=["q_proj", "k_proj", "v_proj", "o_proj",
|
||||
"gate_proj", "up_proj", "down_proj"],
|
||||
bias="none",
|
||||
)
|
||||
model = get_peft_model(model, lora_config)
|
||||
model.print_trainable_parameters()
|
||||
|
||||
# ---- Training args ----
|
||||
steps_per_epoch = math.ceil(len(tok_ds["train"]) / (${LORA_BATCH_SIZE} * ${LORA_GRAD_ACCUM}))
|
||||
total_steps = steps_per_epoch * ${LORA_EPOCHS}
|
||||
save_steps = max(50, steps_per_epoch // 2)
|
||||
|
||||
args = TrainingArguments(
|
||||
output_dir=ckpt,
|
||||
num_train_epochs=${LORA_EPOCHS},
|
||||
per_device_train_batch_size=${LORA_BATCH_SIZE},
|
||||
gradient_accumulation_steps=${LORA_GRAD_ACCUM},
|
||||
learning_rate=${LORA_LR},
|
||||
warmup_ratio=0.05,
|
||||
lr_scheduler_type="cosine",
|
||||
logging_steps=10,
|
||||
save_steps=save_steps,
|
||||
eval_strategy="steps",
|
||||
eval_steps=save_steps,
|
||||
load_best_model_at_end=True,
|
||||
metric_for_best_model="eval_loss",
|
||||
bf16=True,
|
||||
gradient_checkpointing=True,
|
||||
dataloader_num_workers=4,
|
||||
report_to="none",
|
||||
save_total_limit=3,
|
||||
)
|
||||
|
||||
# ---- Trainer ----
|
||||
collator = DataCollatorForSeq2Seq(tokenizer, model=model, padding=True)
|
||||
trainer = Trainer(
|
||||
model=model,
|
||||
args=args,
|
||||
train_dataset=tok_ds["train"],
|
||||
eval_dataset=tok_ds["test"],
|
||||
data_collator=collator,
|
||||
)
|
||||
|
||||
print(f"Starting LoRA training: {total_steps} total steps")
|
||||
trainer.train()
|
||||
|
||||
# Save final adapter
|
||||
final_adapter = os.path.join(out_dir, "lora_adapter_final")
|
||||
model.save_pretrained(final_adapter)
|
||||
tokenizer.save_pretrained(final_adapter)
|
||||
print(f"Final LoRA adapter saved to {final_adapter}")
|
||||
|
||||
# Save training metadata
|
||||
meta = {
|
||||
"model": hf_id,
|
||||
"method": "lora",
|
||||
"lora_rank": ${LORA_RANK},
|
||||
"lora_alpha": ${LORA_ALPHA},
|
||||
"epochs": ${LORA_EPOCHS},
|
||||
"train_examples": len(tok_ds["train"]),
|
||||
"final_train_loss": trainer.state.log_history[-1].get("loss"),
|
||||
}
|
||||
with open(os.path.join(out_dir, "training_meta.json"), "w") as f:
|
||||
import json; json.dump(meta, f, indent=2)
|
||||
print("Training metadata saved.")
|
||||
PYEOF
|
||||
|
||||
local exit_code=${PIPESTATUS[0]}
|
||||
local t1
|
||||
t1="$(date +%s)"
|
||||
local elapsed=$(( t1 - t0 ))
|
||||
|
||||
if [[ $exit_code -eq 0 ]]; then
|
||||
log_ok "LoRA $model done in ${elapsed}s (~$(( elapsed / 3600 ))h $(( (elapsed % 3600) / 60 ))m)"
|
||||
return 0
|
||||
else
|
||||
log_error "LoRA $model FAILED (exit $exit_code) after ${elapsed}s"
|
||||
return $exit_code
|
||||
fi
|
||||
}
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# SFT (full fine-tuning) for one model
|
||||
# ---------------------------------------------------------------------------
|
||||
run_sft() {
|
||||
local model="$1"
|
||||
local hf_id
|
||||
hf_id="$(model_hf_id "$model")"
|
||||
local out_dir="${RESULTS_BASE}/sft/${model}"
|
||||
local checkpoint_dir="${out_dir}/checkpoint"
|
||||
mkdir -p "$out_dir" "$checkpoint_dir"
|
||||
|
||||
log_info "=========================================="
|
||||
log_info "SFT training: $model ($hf_id)"
|
||||
log_info " Output: $out_dir"
|
||||
log_info " Epochs=$SFT_EPOCHS LR=$SFT_LR batch=$SFT_BATCH_SIZE grad_accum=$SFT_GRAD_ACCUM"
|
||||
log_info "=========================================="
|
||||
|
||||
local dataset_path
|
||||
dataset_path="$(prepare_dataset sft "$model")"
|
||||
|
||||
local t0
|
||||
t0="$(date +%s)"
|
||||
|
||||
local n_gpus
|
||||
n_gpus="$(python3 -c "import torch; print(torch.cuda.device_count())" 2>/dev/null || echo 1)"
|
||||
log_info "Training on $n_gpus GPU(s)"
|
||||
|
||||
if [[ "$n_gpus" -gt 1 ]]; then
|
||||
LAUNCHER="uv run torchrun --nproc_per_node=$n_gpus"
|
||||
else
|
||||
LAUNCHER="uv run python"
|
||||
fi
|
||||
|
||||
$LAUNCHER - <<PYEOF 2>&1 | tee -a "$LOG_FILE"
|
||||
import json
|
||||
import math
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
import torch
|
||||
from datasets import load_dataset
|
||||
from transformers import (
|
||||
AutoModelForCausalLM,
|
||||
AutoTokenizer,
|
||||
DataCollatorForSeq2Seq,
|
||||
Trainer,
|
||||
TrainingArguments,
|
||||
)
|
||||
|
||||
hf_id = "${hf_id}"
|
||||
out_dir = "${out_dir}"
|
||||
ckpt = "${checkpoint_dir}"
|
||||
data_path = "${dataset_path}"
|
||||
|
||||
# ---- Tokenizer ----
|
||||
print(f"Loading tokenizer: {hf_id}")
|
||||
tokenizer = AutoTokenizer.from_pretrained(hf_id, trust_remote_code=True)
|
||||
if tokenizer.pad_token is None:
|
||||
tokenizer.pad_token = tokenizer.eos_token
|
||||
|
||||
# ---- Dataset ----
|
||||
print(f"Loading dataset: {data_path}")
|
||||
raw = load_dataset("json", data_files=data_path, split="train")
|
||||
raw = raw.train_test_split(test_size=0.02, seed=42)
|
||||
|
||||
def tokenize(example):
|
||||
messages = example["messages"]
|
||||
text = tokenizer.apply_chat_template(
|
||||
messages,
|
||||
tokenize=False,
|
||||
add_generation_prompt=False,
|
||||
)
|
||||
enc = tokenizer(text, truncation=True, max_length=${MAX_SEQ_LEN})
|
||||
enc["labels"] = enc["input_ids"].copy()
|
||||
return enc
|
||||
|
||||
print("Tokenizing dataset...")
|
||||
tok_ds = raw.map(tokenize, remove_columns=raw["train"].column_names, num_proc=4)
|
||||
print(f"Train: {len(tok_ds['train'])} Eval: {len(tok_ds['test'])}")
|
||||
|
||||
# ---- Model (4-bit quantized to fit on fewer GPUs) ----
|
||||
print(f"Loading model: {hf_id}")
|
||||
from transformers import BitsAndBytesConfig
|
||||
bnb_config = BitsAndBytesConfig(
|
||||
load_in_4bit=True,
|
||||
bnb_4bit_compute_dtype=torch.bfloat16,
|
||||
bnb_4bit_quant_type="nf4",
|
||||
bnb_4bit_use_double_quant=True,
|
||||
)
|
||||
model = AutoModelForCausalLM.from_pretrained(
|
||||
hf_id,
|
||||
quantization_config=bnb_config,
|
||||
trust_remote_code=True,
|
||||
device_map="auto",
|
||||
)
|
||||
model.config.use_cache = False
|
||||
|
||||
# ---- Training args ----
|
||||
steps_per_epoch = math.ceil(len(tok_ds["train"]) / (${SFT_BATCH_SIZE} * ${SFT_GRAD_ACCUM}))
|
||||
total_steps = steps_per_epoch * ${SFT_EPOCHS}
|
||||
save_steps = max(50, steps_per_epoch // 2)
|
||||
|
||||
args = TrainingArguments(
|
||||
output_dir=ckpt,
|
||||
num_train_epochs=${SFT_EPOCHS},
|
||||
per_device_train_batch_size=${SFT_BATCH_SIZE},
|
||||
gradient_accumulation_steps=${SFT_GRAD_ACCUM},
|
||||
learning_rate=${SFT_LR},
|
||||
warmup_ratio=0.03,
|
||||
lr_scheduler_type="cosine",
|
||||
logging_steps=10,
|
||||
save_steps=save_steps,
|
||||
eval_strategy="steps",
|
||||
eval_steps=save_steps,
|
||||
load_best_model_at_end=True,
|
||||
metric_for_best_model="eval_loss",
|
||||
bf16=True,
|
||||
gradient_checkpointing=True,
|
||||
dataloader_num_workers=4,
|
||||
report_to="none",
|
||||
save_total_limit=3,
|
||||
)
|
||||
|
||||
# ---- Trainer ----
|
||||
collator = DataCollatorForSeq2Seq(tokenizer, model=model, padding=True)
|
||||
trainer = Trainer(
|
||||
model=model,
|
||||
args=args,
|
||||
train_dataset=tok_ds["train"],
|
||||
eval_dataset=tok_ds["test"],
|
||||
data_collator=collator,
|
||||
)
|
||||
|
||||
print(f"Starting SFT training: {total_steps} total steps")
|
||||
trainer.train()
|
||||
|
||||
# Save final model
|
||||
final_model = os.path.join(out_dir, "sft_model_final")
|
||||
model.save_pretrained(final_model)
|
||||
tokenizer.save_pretrained(final_model)
|
||||
print(f"Final SFT model saved to {final_model}")
|
||||
|
||||
# Save training metadata
|
||||
meta = {
|
||||
"model": hf_id,
|
||||
"method": "sft",
|
||||
"epochs": ${SFT_EPOCHS},
|
||||
"train_examples": len(tok_ds["train"]),
|
||||
"final_train_loss": trainer.state.log_history[-1].get("loss"),
|
||||
}
|
||||
with open(os.path.join(out_dir, "training_meta.json"), "w") as f:
|
||||
import json; json.dump(meta, f, indent=2)
|
||||
print("Training metadata saved.")
|
||||
PYEOF
|
||||
|
||||
local exit_code=${PIPESTATUS[0]}
|
||||
local t1
|
||||
t1="$(date +%s)"
|
||||
local elapsed=$(( t1 - t0 ))
|
||||
|
||||
if [[ $exit_code -eq 0 ]]; then
|
||||
log_ok "SFT $model done in ${elapsed}s (~$(( elapsed / 3600 ))h $(( (elapsed % 3600) / 60 ))m)"
|
||||
return 0
|
||||
else
|
||||
log_error "SFT $model FAILED (exit $exit_code) after ${elapsed}s"
|
||||
return $exit_code
|
||||
fi
|
||||
}
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Post-training eval: run fast benchmarks on a checkpoint
|
||||
# ---------------------------------------------------------------------------
|
||||
run_eval() {
|
||||
local method="$1" # lora or sft
|
||||
local model="$2"
|
||||
local out_dir="${RESULTS_BASE}/${method}/${model}"
|
||||
|
||||
if [[ "$SKIP_EVAL" == "true" ]]; then
|
||||
log_info "Skipping eval (--skip-eval)"
|
||||
return 0
|
||||
fi
|
||||
|
||||
# Locate the final checkpoint / adapter
|
||||
local checkpoint=""
|
||||
if [[ "$method" == "lora" ]]; then
|
||||
checkpoint="${out_dir}/lora_adapter_final"
|
||||
else
|
||||
checkpoint="${out_dir}/sft_model_final"
|
||||
fi
|
||||
|
||||
if [[ ! -d "$checkpoint" ]]; then
|
||||
log_warn "Checkpoint not found for $method/$model at $checkpoint — skipping eval"
|
||||
return 0
|
||||
fi
|
||||
|
||||
log_info "--- Post-training eval: $method/$model ---"
|
||||
log_info " Checkpoint: $checkpoint"
|
||||
|
||||
for bench in "${FAST_BENCHMARKS[@]}"; do
|
||||
local eval_out="${out_dir}/eval/${bench}"
|
||||
mkdir -p "$eval_out"
|
||||
log_info " Eval: $bench -> $eval_out"
|
||||
|
||||
uv run python - <<PYEOF 2>&1 | tee -a "$LOG_FILE"
|
||||
import subprocess, sys
|
||||
|
||||
cmd = [
|
||||
"uv", "run", "python", "-m", "openjarvis.evals", "run",
|
||||
"--model-path", "${checkpoint}",
|
||||
"--model-id", "${model}-${method}",
|
||||
"--benchmark", "${bench}",
|
||||
"--max-samples", "${EVAL_MAX_SAMPLES}",
|
||||
"--output", "${eval_out}",
|
||||
]
|
||||
print("Running:", " ".join(cmd))
|
||||
result = subprocess.run(cmd, capture_output=False)
|
||||
sys.exit(result.returncode)
|
||||
PYEOF
|
||||
|
||||
local eval_exit=${PIPESTATUS[0]}
|
||||
if [[ $eval_exit -eq 0 ]]; then
|
||||
log_ok " Eval $method/$model/$bench OK"
|
||||
else
|
||||
log_warn " Eval $method/$model/$bench returned exit $eval_exit (non-fatal)"
|
||||
fi
|
||||
done
|
||||
}
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Summary
|
||||
# ---------------------------------------------------------------------------
|
||||
print_summary() {
|
||||
echo ""
|
||||
log_info "=== Track D Complete ==="
|
||||
log_info "Results written to:"
|
||||
|
||||
if [[ "$RUN_LORA" == "true" ]]; then
|
||||
for model in "${ALL_LORA_MODELS[@]}"; do
|
||||
local out="${RESULTS_BASE}/lora/${model}"
|
||||
if [[ -d "$out" ]]; then
|
||||
log_ok " lora/$model -> $out"
|
||||
else
|
||||
log_warn " lora/$model -> MISSING ($out)"
|
||||
fi
|
||||
done
|
||||
fi
|
||||
|
||||
if [[ "$RUN_SFT" == "true" ]]; then
|
||||
for model in "${ALL_SFT_MODELS[@]}"; do
|
||||
local out="${RESULTS_BASE}/sft/${model}"
|
||||
if [[ -d "$out" ]]; then
|
||||
log_ok " sft/$model -> $out"
|
||||
else
|
||||
log_warn " sft/$model -> MISSING ($out)"
|
||||
fi
|
||||
done
|
||||
fi
|
||||
|
||||
log_info "Full log: $LOG_FILE"
|
||||
}
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Main
|
||||
# ---------------------------------------------------------------------------
|
||||
main() {
|
||||
setup_env
|
||||
|
||||
local failed=0
|
||||
|
||||
# ---- LoRA runs ----
|
||||
if [[ "$RUN_LORA" == "true" ]]; then
|
||||
log_info "=========================================="
|
||||
log_info "Starting LoRA training runs"
|
||||
log_info "Models: ${ALL_LORA_MODELS[*]}"
|
||||
log_info "=========================================="
|
||||
for model in "${ALL_LORA_MODELS[@]}"; do
|
||||
run_lora "$model" || { failed=$(( failed + 1 )); log_error "LoRA $model failed, continuing..."; }
|
||||
run_eval lora "$model"
|
||||
echo ""
|
||||
done
|
||||
fi
|
||||
|
||||
# ---- SFT runs ----
|
||||
if [[ "$RUN_SFT" == "true" ]]; then
|
||||
log_info "=========================================="
|
||||
log_info "Starting SFT training runs"
|
||||
log_info "Models: ${ALL_SFT_MODELS[*]}"
|
||||
log_info "=========================================="
|
||||
for model in "${ALL_SFT_MODELS[@]}"; do
|
||||
run_sft "$model" || { failed=$(( failed + 1 )); log_error "SFT $model failed, continuing..."; }
|
||||
run_eval sft "$model"
|
||||
echo ""
|
||||
done
|
||||
fi
|
||||
|
||||
print_summary
|
||||
|
||||
if [[ $failed -gt 0 ]]; then
|
||||
log_error "$failed training run(s) failed. Check log: $LOG_FILE"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
log_ok "All Track D runs completed successfully."
|
||||
}
|
||||
|
||||
main "$@"
|
||||
+28
-12
@@ -14,7 +14,7 @@ import os
|
||||
import webbrowser
|
||||
from http.server import BaseHTTPRequestHandler, HTTPServer
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, List, Optional
|
||||
from typing import Any, Dict, List
|
||||
from urllib.parse import parse_qs, urlencode, urlparse
|
||||
|
||||
import httpx
|
||||
@@ -44,16 +44,31 @@ def _load_creds(filename: str) -> Dict[str, str]:
|
||||
|
||||
|
||||
_google = _load_creds("google.json")
|
||||
GOOGLE_CLIENT_ID = os.environ.get("OPENJARVIS_GOOGLE_CLIENT_ID", "") or _google["client_id"]
|
||||
GOOGLE_CLIENT_SECRET = os.environ.get("OPENJARVIS_GOOGLE_CLIENT_SECRET", "") or _google["client_secret"]
|
||||
GOOGLE_CLIENT_ID = (
|
||||
os.environ.get("OPENJARVIS_GOOGLE_CLIENT_ID", "") or _google["client_id"]
|
||||
)
|
||||
GOOGLE_CLIENT_SECRET = (
|
||||
os.environ.get("OPENJARVIS_GOOGLE_CLIENT_SECRET", "")
|
||||
or _google["client_secret"]
|
||||
)
|
||||
|
||||
_strava = _load_creds("strava.json")
|
||||
STRAVA_CLIENT_ID = os.environ.get("OPENJARVIS_STRAVA_CLIENT_ID", "") or _strava["client_id"]
|
||||
STRAVA_CLIENT_SECRET = os.environ.get("OPENJARVIS_STRAVA_CLIENT_SECRET", "") or _strava["client_secret"]
|
||||
STRAVA_CLIENT_ID = (
|
||||
os.environ.get("OPENJARVIS_STRAVA_CLIENT_ID", "") or _strava["client_id"]
|
||||
)
|
||||
STRAVA_CLIENT_SECRET = (
|
||||
os.environ.get("OPENJARVIS_STRAVA_CLIENT_SECRET", "")
|
||||
or _strava["client_secret"]
|
||||
)
|
||||
|
||||
_spotify = _load_creds("spotify.json")
|
||||
SPOTIFY_CLIENT_ID = os.environ.get("OPENJARVIS_SPOTIFY_CLIENT_ID", "") or _spotify["client_id"]
|
||||
SPOTIFY_CLIENT_SECRET = os.environ.get("OPENJARVIS_SPOTIFY_CLIENT_SECRET", "") or _spotify["client_secret"]
|
||||
SPOTIFY_CLIENT_ID = (
|
||||
os.environ.get("OPENJARVIS_SPOTIFY_CLIENT_ID", "") or _spotify["client_id"]
|
||||
)
|
||||
SPOTIFY_CLIENT_SECRET = (
|
||||
os.environ.get("OPENJARVIS_SPOTIFY_CLIENT_SECRET", "")
|
||||
or _spotify["client_secret"]
|
||||
)
|
||||
|
||||
# ── Generic OAuth helpers ────────────────────────────────────────────────────
|
||||
|
||||
@@ -128,7 +143,7 @@ def do_google() -> None:
|
||||
"prompt": "consent",
|
||||
})
|
||||
)
|
||||
print(f" Opening browser...")
|
||||
print(" Opening browser...")
|
||||
webbrowser.open(url)
|
||||
code = _wait_for_code()
|
||||
|
||||
@@ -176,7 +191,7 @@ def do_strava() -> None:
|
||||
"scope": "activity:read_all",
|
||||
})
|
||||
)
|
||||
print(f" Opening browser...")
|
||||
print(" Opening browser...")
|
||||
webbrowser.open(url)
|
||||
code = _wait_for_code()
|
||||
|
||||
@@ -208,7 +223,6 @@ def do_strava() -> None:
|
||||
|
||||
def _make_self_signed_cert() -> tuple[str, str]:
|
||||
"""Generate a temporary self-signed cert for localhost HTTPS."""
|
||||
import ssl
|
||||
import subprocess
|
||||
import tempfile
|
||||
|
||||
@@ -288,7 +302,7 @@ def do_spotify() -> None:
|
||||
"scope": "user-read-recently-played",
|
||||
})
|
||||
)
|
||||
print(f" Opening browser...")
|
||||
print(" Opening browser...")
|
||||
webbrowser.open(url)
|
||||
code = _wait_for_code(port=spotify_port)
|
||||
|
||||
@@ -323,7 +337,9 @@ def do_spotify() -> None:
|
||||
# ── Main ─────────────────────────────────────────────────────────────────────
|
||||
|
||||
if __name__ == "__main__":
|
||||
parser = argparse.ArgumentParser(description="Run OAuth flows for OpenJarvis connectors")
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Run OAuth flows for OpenJarvis connectors"
|
||||
)
|
||||
parser.add_argument("--google", action="store_true", help="Only Google")
|
||||
parser.add_argument("--strava", action="store_true", help="Only Strava")
|
||||
parser.add_argument("--spotify", action="store_true", help="Only Spotify")
|
||||
|
||||
@@ -47,27 +47,45 @@ print(' all pins OK')
|
||||
|
||||
echo "==> Running one-task smoke per (framework, benchmark)"
|
||||
mkdir -p results/smoke
|
||||
CONFIG_DIR="results/smoke/configs"
|
||||
mkdir -p "$CONFIG_DIR"
|
||||
SMOKE_BENCHES=(toolcall15 pinchbench gaia)
|
||||
SMOKE_FRAMEWORKS=(hermes openclaw openjarvis)
|
||||
SMOKE_MODEL="qwen-9b"
|
||||
|
||||
for fwk in "${SMOKE_FRAMEWORKS[@]}"; do
|
||||
for bench in "${SMOKE_BENCHES[@]}"; do
|
||||
config="src/openjarvis/evals/configs/framework_comparison/${bench}-${fwk}-${SMOKE_MODEL}.toml"
|
||||
if [ ! -f "$config" ]; then
|
||||
echo " ! missing config: $config (run make_configs --all-tier1 (or materialize the configs you need))"
|
||||
continue
|
||||
fi
|
||||
uv run python -m openjarvis.evals.comparison.make_configs \
|
||||
--framework "$fwk" \
|
||||
--model "$SMOKE_MODEL" \
|
||||
--benchmark "$bench" \
|
||||
--output-dir "$CONFIG_DIR" >/dev/null
|
||||
config="${CONFIG_DIR}/${bench}-${fwk}-${SMOKE_MODEL}.toml"
|
||||
run_dir="results/smoke/${fwk}/${SMOKE_MODEL}/${bench}/"
|
||||
JARVIS_BACKEND_BASE_URL="$JARVIS_MOCK_LLM_URL" uv run python - "$config" "$run_dir" <<'PY'
|
||||
from pathlib import Path
|
||||
import re
|
||||
import sys
|
||||
|
||||
path = Path(sys.argv[1])
|
||||
run_dir = sys.argv[2]
|
||||
text = path.read_text()
|
||||
text = re.sub(r'^output_dir = ".*"$', f'output_dir = "{run_dir}"', text, flags=re.M)
|
||||
text = re.sub(r"^max_samples = \d+$", "max_samples = 1", text, flags=re.M)
|
||||
path.write_text(text)
|
||||
PY
|
||||
echo " ▸ $fwk × $bench"
|
||||
uv run python -m openjarvis.evals run --config "$config" --max-samples 1 \
|
||||
--output-dir "results/smoke/${fwk}/${SMOKE_MODEL}/${bench}/" \
|
||||
mkdir -p "$run_dir"
|
||||
JARVIS_BACKEND_BASE_URL="$JARVIS_MOCK_LLM_URL" \
|
||||
JARVIS_BACKEND_API_KEY="${JARVIS_BACKEND_API_KEY:-smoke}" \
|
||||
uv run python -m openjarvis.evals run --config "$config" \
|
||||
|| echo " FAILED (continuing)"
|
||||
done
|
||||
done
|
||||
|
||||
echo "==> Generating T1 from smoke results"
|
||||
uv run python -m openjarvis.evals.comparison.table_gen \
|
||||
--results-glob "results/smoke/**/summary.json" \
|
||||
--results-glob "results/smoke/**/*.summary.json" \
|
||||
--tables T1 \
|
||||
--output-dir results/smoke/tables/
|
||||
|
||||
|
||||
@@ -6,13 +6,14 @@
|
||||
// --task <prompt> --model <m> --base-url <url> --api-key <k> \
|
||||
// --output-json <path> [--workspace <path>]
|
||||
//
|
||||
// Loads OpenClaw from $OPENCLAW_PATH and runs `openclaw chat --message <task> --json`
|
||||
// (or equivalent non-interactive entry). Emits a JSON dict matching the
|
||||
// Loads OpenClaw from $OPENCLAW_PATH and runs `openclaw agent --local
|
||||
// --message <task> --json`. Emits a JSON dict matching the
|
||||
// _RunnerOutput Pydantic schema in _subprocess_runner.py.
|
||||
|
||||
import { writeFileSync, existsSync } from 'node:fs';
|
||||
import { mkdtempSync, writeFileSync, existsSync } from 'node:fs';
|
||||
import { spawn } from 'node:child_process';
|
||||
import { join } from 'node:path';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { argv, env, exit, chdir } from 'node:process';
|
||||
|
||||
function parseArgs(args) {
|
||||
@@ -47,12 +48,30 @@ async function main() {
|
||||
return 3;
|
||||
}
|
||||
|
||||
const requestedModel = String(args.model || '').includes('/')
|
||||
? String(args.model || '')
|
||||
: `ollama/${args.model}`;
|
||||
const runDir = mkdtempSync(join(tmpdir(), 'openjarvis-openclaw-'));
|
||||
const configPath = join(runDir, 'openclaw.json');
|
||||
writeFileSync(configPath, JSON.stringify({
|
||||
agents: {
|
||||
defaults: {
|
||||
model: { primary: requestedModel },
|
||||
models: { [requestedModel]: {} },
|
||||
},
|
||||
},
|
||||
}));
|
||||
|
||||
const childEnv = {
|
||||
...env,
|
||||
OPENCLAW_MODEL: args.model,
|
||||
OPENCLAW_BASE_URL: args.base_url,
|
||||
OPENCLAW_API_KEY: args.api_key,
|
||||
OPENCLAW_CONFIG_PATH: configPath,
|
||||
};
|
||||
if (requestedModel.startsWith('ollama/')) {
|
||||
childEnv.OLLAMA_API_KEY = args.api_key || 'ollama';
|
||||
}
|
||||
|
||||
// Use the SAME node executable that's running this script — picking up
|
||||
// 'node' from PATH can resolve to a system Node too old for OpenClaw
|
||||
@@ -95,11 +114,18 @@ async function main() {
|
||||
// { response: str, usage: {...}, messages: [{...}], tool_calls: int }
|
||||
try {
|
||||
const parsed = JSON.parse(stdout);
|
||||
output.content = parsed.response || '';
|
||||
output.usage = parsed.usage || {};
|
||||
output.trajectory = parsed.messages || [];
|
||||
const payloads = Array.isArray(parsed.payloads) ? parsed.payloads : [];
|
||||
const messages = Array.isArray(parsed.messages) ? parsed.messages : [];
|
||||
const usage = parsed.usage || parsed.meta?.agentMeta?.usage || {};
|
||||
output.content = parsed.response || payloads.map((p) => p.text || '').join('\n');
|
||||
output.usage = {
|
||||
prompt_tokens: usage.prompt_tokens ?? usage.input ?? 0,
|
||||
completion_tokens: usage.completion_tokens ?? usage.output ?? 0,
|
||||
total_tokens: usage.total_tokens ?? usage.total ?? 0,
|
||||
};
|
||||
output.trajectory = messages;
|
||||
output.tool_calls = parsed.tool_calls || 0;
|
||||
output.turn_count = (parsed.messages || []).filter(
|
||||
output.turn_count = messages.filter(
|
||||
(m) => m.role === 'assistant'
|
||||
).length;
|
||||
} catch (e) {
|
||||
|
||||
@@ -38,6 +38,10 @@ class MixedCommitError(TableGenError):
|
||||
"""Raised when one (framework, model, benchmark) cell has multiple commits."""
|
||||
|
||||
|
||||
class NoResultsError(TableGenError):
|
||||
"""Raised when no valid summary files were loaded."""
|
||||
|
||||
|
||||
class _MetricStats(BaseModel):
|
||||
mean: float
|
||||
std: float
|
||||
@@ -395,6 +399,11 @@ def main(results_glob: str, tables: str, output_dir: Optional[Path]) -> None:
|
||||
click.echo(
|
||||
f"Loaded {len(frame.df)} metric rows; {frame.unloadable_count} files skipped."
|
||||
)
|
||||
if frame.df.is_empty():
|
||||
raise click.ClickException(
|
||||
"No valid summary files matched --results-glob; refusing to emit empty "
|
||||
"tables."
|
||||
)
|
||||
|
||||
requested = [t.strip() for t in tables.split(",") if t.strip()]
|
||||
for name in requested:
|
||||
|
||||
@@ -307,6 +307,38 @@ class EvalRunner:
|
||||
def _process_one(self, record: EvalRecord) -> EvalResult:
|
||||
"""Process a single evaluation sample."""
|
||||
cfg = self._config
|
||||
|
||||
def _backend_error_result(full: dict, message: str) -> EvalResult:
|
||||
usage = full.get("usage", {}) or {}
|
||||
energy_j = full.get("energy_joules", 0.0) or 0.0
|
||||
power_w = full.get("power_watts") or full.get("peak_power_w") or 0.0
|
||||
return EvalResult(
|
||||
record_id=record.record_id,
|
||||
model_answer=full.get("content", "") or "",
|
||||
error=message,
|
||||
latency_seconds=full.get("latency_seconds", 0.0) or 0.0,
|
||||
prompt_tokens=usage.get("prompt_tokens", 0),
|
||||
completion_tokens=usage.get("completion_tokens", 0),
|
||||
cost_usd=full.get("cost_usd", 0.0) or 0.0,
|
||||
ttft=full.get("ttft", 0.0) or 0.0,
|
||||
energy_joules=energy_j,
|
||||
power_watts=power_w,
|
||||
gpu_utilization_pct=full.get("gpu_utilization_pct", 0.0) or 0.0,
|
||||
throughput_tok_per_sec=full.get("throughput_tok_per_sec", 0.0)
|
||||
or 0.0,
|
||||
trace_data=full.get("trace_data"),
|
||||
framework=full.get(
|
||||
"framework",
|
||||
getattr(self._backend, "framework_name", "openjarvis"),
|
||||
),
|
||||
framework_commit=full.get(
|
||||
"framework_commit",
|
||||
getattr(self._backend, "framework_commit_value", "") or "",
|
||||
),
|
||||
tool_calls=int(full.get("tool_calls", 0) or 0),
|
||||
turn_count=int(full.get("turn_count", 0) or 0),
|
||||
)
|
||||
|
||||
try:
|
||||
gen_kwargs: dict = dict(
|
||||
model=cfg.model,
|
||||
@@ -327,6 +359,8 @@ class EvalRunner:
|
||||
**gen_kwargs,
|
||||
)
|
||||
full = full or {}
|
||||
if full.get("error"):
|
||||
return _backend_error_result(full, str(full["error"]))
|
||||
all_tool_results = list(
|
||||
full.get(
|
||||
"tool_results",
|
||||
@@ -346,6 +380,8 @@ class EvalRunner:
|
||||
**gen_kwargs,
|
||||
)
|
||||
sfull = sfull or {}
|
||||
if sfull.get("error"):
|
||||
return _backend_error_result(sfull, str(sfull["error"]))
|
||||
all_tool_results.extend(
|
||||
sfull.get("tool_results", []),
|
||||
)
|
||||
@@ -363,6 +399,8 @@ class EvalRunner:
|
||||
**gen_kwargs,
|
||||
)
|
||||
full = full or {}
|
||||
if full.get("error"):
|
||||
return _backend_error_result(full, str(full["error"]))
|
||||
content = full.get("content", "")
|
||||
is_correct, scoring_meta = self._scorer.score(
|
||||
record,
|
||||
@@ -378,7 +416,7 @@ class EvalRunner:
|
||||
# 'or 0.0' handles the case where the key is present but the
|
||||
# value is None -- .get(default) only kicks in when missing.
|
||||
energy_j = full.get("energy_joules", 0.0) or 0.0
|
||||
power_w = full.get("power_watts", 0.0) or 0.0
|
||||
power_w = full.get("power_watts") or full.get("peak_power_w") or 0.0
|
||||
throughput = full.get("throughput_tok_per_sec", 0.0) or 0.0
|
||||
accuracy_score = 1.0 if is_correct else 0.0
|
||||
|
||||
|
||||
@@ -122,7 +122,7 @@ class TestSummaryToDictEmitsRequiredFields:
|
||||
completion_tokens=50,
|
||||
cost_usd=0.001,
|
||||
energy_joules=10.0,
|
||||
power_watts=20.0,
|
||||
power_watts=30.0,
|
||||
framework="hermes",
|
||||
framework_commit="5d3be898a",
|
||||
tool_calls=2,
|
||||
@@ -156,6 +156,7 @@ class TestSummaryToDictEmitsRequiredFields:
|
||||
assert d["n_tasks"] == 5
|
||||
assert "accuracy" in d["metrics"]
|
||||
assert "latency_seconds" in d["metrics"]
|
||||
assert d["metrics"]["peak_power_w"]["mean"] == 30.0
|
||||
assert d["metrics"]["accuracy"]["n"] == 5
|
||||
assert d["metrics"]["latency_seconds"]["n"] == 5
|
||||
# Existing rich schema also present (unchanged)
|
||||
|
||||
@@ -0,0 +1,72 @@
|
||||
"""Contract tests for the OpenClaw subprocess runner."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import shutil
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
def test_openclaw_runner_parses_real_agent_json_shape(tmp_path: Path) -> None:
|
||||
node = shutil.which("node")
|
||||
if node is None:
|
||||
pytest.skip("node not available")
|
||||
|
||||
fake_openclaw = tmp_path / "openclaw.mjs"
|
||||
fake_openclaw.write_text(
|
||||
"\n".join(
|
||||
[
|
||||
"console.log(JSON.stringify({",
|
||||
" payloads: [{ text: 'hello from openclaw', mediaUrl: null }],",
|
||||
" meta: {",
|
||||
" agentMeta: { usage: { input: 11, output: 7, total: 18 } },",
|
||||
" },",
|
||||
"}));",
|
||||
]
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
runner = (
|
||||
Path(__file__).resolve().parents[3]
|
||||
/ "src/openjarvis/evals/backends/external/_runners/openclaw_runner.mjs"
|
||||
)
|
||||
out_json = tmp_path / "out.json"
|
||||
env = {
|
||||
"OPENCLAW_PATH": str(tmp_path),
|
||||
"HOME": str(tmp_path / "home"),
|
||||
}
|
||||
result = subprocess.run(
|
||||
[
|
||||
node,
|
||||
str(runner),
|
||||
"--task",
|
||||
"Say hello.",
|
||||
"--model",
|
||||
"qwen3:0.6b",
|
||||
"--base-url",
|
||||
"http://127.0.0.1:11434/v1",
|
||||
"--api-key",
|
||||
"ollama",
|
||||
"--output-json",
|
||||
str(out_json),
|
||||
],
|
||||
env=env,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=30,
|
||||
check=False,
|
||||
)
|
||||
|
||||
assert result.returncode == 0, result.stderr
|
||||
data = json.loads(out_json.read_text(encoding="utf-8"))
|
||||
assert data["content"] == "hello from openclaw"
|
||||
assert data["usage"] == {
|
||||
"prompt_tokens": 11,
|
||||
"completion_tokens": 7,
|
||||
"total_tokens": 18,
|
||||
}
|
||||
assert data["error"] is None
|
||||
@@ -9,6 +9,7 @@ happy-path EvalResult construction.
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from typing import Any, Dict, Iterable, List, Optional, Tuple
|
||||
|
||||
from openjarvis.evals.core.dataset import DatasetProvider
|
||||
@@ -33,6 +34,57 @@ class _FailingBackend:
|
||||
pass
|
||||
|
||||
|
||||
class _PeakOnlyBackend:
|
||||
"""Mock external-style backend that reports peak_power_w but no power_watts."""
|
||||
|
||||
backend_id = "hermes"
|
||||
framework_name = "hermes"
|
||||
framework_commit_value = "abc12345"
|
||||
|
||||
def generate(self, *args: Any, **kwargs: Any) -> str:
|
||||
return "hi"
|
||||
|
||||
def generate_full(self, *args: Any, **kwargs: Any) -> Dict[str, Any]:
|
||||
return {
|
||||
"content": "hi",
|
||||
"usage": {},
|
||||
"latency_seconds": 1.0,
|
||||
"energy_joules": 10.0,
|
||||
"peak_power_w": 42.0,
|
||||
"framework": "hermes",
|
||||
"framework_commit": "abc12345",
|
||||
}
|
||||
|
||||
def close(self) -> None:
|
||||
pass
|
||||
|
||||
|
||||
class _BackendErrorPayload:
|
||||
"""Mock external backend that returns an error payload instead of raising."""
|
||||
|
||||
backend_id = "hermes"
|
||||
framework_name = "hermes"
|
||||
framework_commit_value = "abc12345"
|
||||
|
||||
def generate(self, *args: Any, **kwargs: Any) -> str:
|
||||
return ""
|
||||
|
||||
def generate_full(self, *args: Any, **kwargs: Any) -> Dict[str, Any]:
|
||||
return {
|
||||
"content": "",
|
||||
"usage": {"prompt_tokens": 3, "completion_tokens": 0},
|
||||
"latency_seconds": 1.5,
|
||||
"energy_joules": None,
|
||||
"peak_power_w": 17.0,
|
||||
"framework": "hermes",
|
||||
"framework_commit": "abc12345",
|
||||
"error": "backend reported failure",
|
||||
}
|
||||
|
||||
def close(self) -> None:
|
||||
pass
|
||||
|
||||
|
||||
class _SingleRecordDataset(DatasetProvider):
|
||||
dataset_id = "mock"
|
||||
dataset_name = "mock"
|
||||
@@ -141,6 +193,70 @@ class TestErrorPathFrameworkPropagation:
|
||||
assert results[0].error is not None
|
||||
|
||||
|
||||
class TestExternalTelemetryPropagation:
|
||||
def test_peak_power_falls_back_to_power_watts(self, tmp_path: Any) -> None:
|
||||
"""External backends report peak_power_w; runner should preserve it
|
||||
in EvalResult.power_watts so summary/table metrics are populated."""
|
||||
cfg = RunConfig(
|
||||
benchmark="mock",
|
||||
backend="hermes",
|
||||
model="mock-model",
|
||||
max_samples=1,
|
||||
max_workers=1,
|
||||
output_path=str(tmp_path / "out.jsonl"),
|
||||
)
|
||||
runner = EvalRunner(
|
||||
config=cfg,
|
||||
dataset=_SingleRecordDataset(),
|
||||
backend=_PeakOnlyBackend(), # type: ignore[arg-type]
|
||||
scorer=_NoopScorer(),
|
||||
)
|
||||
runner.run()
|
||||
|
||||
assert runner.results[0].power_watts == 42.0
|
||||
summary = json.loads((tmp_path / "out.summary.json").read_text())
|
||||
assert summary["metrics"]["peak_power_w"]["mean"] == 42.0
|
||||
|
||||
|
||||
class TestBackendErrorPayloadPropagation:
|
||||
def test_backend_error_payload_marks_result_error(self, tmp_path: Any) -> None:
|
||||
"""Backends can return structured error payloads without raising;
|
||||
those must count as errors rather than scored empty answers."""
|
||||
cfg = RunConfig(
|
||||
benchmark="mock",
|
||||
backend="hermes",
|
||||
model="mock-model",
|
||||
max_samples=1,
|
||||
max_workers=1,
|
||||
output_path=str(tmp_path / "out.jsonl"),
|
||||
)
|
||||
runner = EvalRunner(
|
||||
config=cfg,
|
||||
dataset=_SingleRecordDataset(),
|
||||
backend=_BackendErrorPayload(), # type: ignore[arg-type]
|
||||
scorer=_NoopScorer(),
|
||||
)
|
||||
summary = runner.run()
|
||||
|
||||
result = runner.results[0]
|
||||
assert result.error == "backend reported failure"
|
||||
assert result.is_correct is None
|
||||
assert result.framework == "hermes"
|
||||
assert result.framework_commit == "abc12345"
|
||||
assert result.power_watts == 17.0
|
||||
assert result.prompt_tokens == 3
|
||||
assert summary.errors == 1
|
||||
assert summary.scored_samples == 0
|
||||
|
||||
summary_data = json.loads((tmp_path / "out.summary.json").read_text())
|
||||
assert summary_data["errors"] == 1
|
||||
assert summary_data["metrics"]["accuracy"] == {
|
||||
"mean": 0.0,
|
||||
"std": 0.0,
|
||||
"n": 0,
|
||||
}
|
||||
|
||||
|
||||
class _MockBackendWithCommit:
|
||||
"""Mock backend that exposes framework_name AND framework_commit_value."""
|
||||
|
||||
|
||||
@@ -13,6 +13,7 @@ from openjarvis.evals.comparison.table_gen import ( # noqa: E402
|
||||
MixedCommitError,
|
||||
ResultsFrame,
|
||||
load_results,
|
||||
main,
|
||||
)
|
||||
|
||||
|
||||
@@ -70,6 +71,23 @@ class TestLoadResults:
|
||||
with pytest.raises(MixedCommitError, match="abc123.*zzz999"):
|
||||
load_results(str(tmp_path / "**" / "summary.json"))
|
||||
|
||||
def test_cli_refuses_empty_result_set(self, tmp_path: Path) -> None:
|
||||
from click.testing import CliRunner
|
||||
|
||||
result = CliRunner().invoke(
|
||||
main,
|
||||
[
|
||||
"--results-glob",
|
||||
str(tmp_path / "**" / "summary.json"),
|
||||
"--tables",
|
||||
"T1",
|
||||
"--output-dir",
|
||||
str(tmp_path / "tables"),
|
||||
],
|
||||
)
|
||||
assert result.exit_code != 0
|
||||
assert "No valid summary files matched" in result.output
|
||||
|
||||
|
||||
class TestRenderBooktabs:
|
||||
def test_emits_valid_tabular(self) -> None:
|
||||
|
||||
Reference in New Issue
Block a user