orchestrator: give every generation run its own raw/ dir

--out was being used as a path, so runs overwrote each other and a curated
file couldn't be traced back to the rollout that produced it. It is now a
tag: every run lands in <OJ_DATA_ROOT>/raw/{label}_{MMDD}[_{tag}]/data.jsonl
and keeps the file name fixed.

raw/ sits above the sft/rl fork deliberately. SFT keeps only correct and
well-formed rows, but GRPO needs the failures, so the unfiltered rollouts
have to survive reject-sampling. Naming matches across stages -- only the
stage word differs -- so raw/qwen_0707/ pairs with sft/qwen_clean_0707.jsonl
by eye.

OJ_DATA_ROOT defaults to a repo-relative data/orchestrator so a fresh clone
works, and points outside the checkout in a real workspace.

scripts/orchestrator/build_orchestrator_sft.py

@@ -24,7 +24,7 @@ call.
 
 Example (v1, base self-sampling):
     .venv/bin/python scripts/orchestrator/build_orchestrator_sft.py \
-        --out data/orchestrator_sft_v1.jsonl \
+        --orchestrator-label qwen \
         --orchestrator-endpoint http://localhost:8001/v1 \
         --orchestrator-model qwen3-8b \
         --samples-per-task 8 --max-keep-per-task 1
@@ -67,11 +67,12 @@ def main(argv: Optional[list[str]] = None) -> int:
     p = argparse.ArgumentParser(
         description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter
     )
-    # Default output is timestamped (e.g. data/orchestrator_sft_06-22-1230am.jsonl)
-    # so every run lands in its own file and concurrent runs never collide. Pass
-    # --out explicitly to override (sharded runs do this, one file per shard).
+    # --out is a TAG, not a path. The run dir is always
+    # data/orchestrator/raw/{label}_{MMDD}[_{tag}]/ and the file inside is always
+    # data.jsonl — see the run_dir block below. Use --out only to distinguish two
+    # runs of the same orchestrator on the same day (e.g. --out balanced50).
     p.add_argument("--out", default=None,
-                   help="Output JSONL. Default: data/orchestrator_sft_<MM-DD-HHMMam>.jsonl")
+                   help="Optional tag appended to the run dir name (not a path).")
     # The orchestrator == the local Qwen3-8B self-sampling over its own rollouts.
     p.add_argument("--orchestrator-endpoint", default="http://localhost:8001/v1",
                    help="OpenAI-compatible vLLM base URL serving the orchestrator.")
@@ -163,20 +164,32 @@ def main(argv: Optional[list[str]] = None) -> int:
                    "huggingface_hub", "openai"):
         logging.getLogger(_noisy).setLevel(logging.WARNING)
 
-    # Every run lands in its OWN datetimed folder, all collected flat under
-    # data/runs/ — so runs sort by time, never collide, and never litter data/.
-    # --out is treated as an optional LABEL for the folder, not a path; the data
-    # file inside is always data.jsonl (+ .stats.json/.pretty.json/.txt/.html).
-    # Folder name: <MM-DD-HHMMam/pm>_<label> (local PT) e.g. 06-24-0349pm_gemma_20_anon
-    stamp = time.strftime("%m-%d-%I%M%p").lower()
-    label = Path(args.out).stem if args.out else "orchestrator_sft"
-    run_dir = (Path("data/runs") / f"{stamp}_{label}").resolve()
-    if run_dir.exists():  # parallel runs, same label+minute -> disambiguate
-        run_dir = run_dir.with_name(f"{run_dir.name}-{os.getpid()}")
+    # Every run lands in its OWN folder under data/orchestrator/raw/, named to
+    # MATCH the sft/ file it will eventually produce — only the stage word differs:
+    #
+    #   data/orchestrator/raw/qwen_0707/data.jsonl     <- every rollout, incl. failures
+    #   data/orchestrator/sft/qwen_clean_0707.jsonl    <- reject-sampled from it
+    #
+    # so any curated file traces back to its generation run by eye. raw/ sits ABOVE
+    # the sft/rl fork on purpose: SFT keeps only correct+clean rows, but GRPO needs
+    # the failures, and both read the same pool. Scheme is {name}_{MMDD}, name =
+    # --orchestrator-label (qwen/gemma/...). Same name twice in a day -> -2, -3
+    # suffix (creation order is preserved, and a MM-DD-HHMMpm stamp is redundant
+    # with mtime). --out is an optional extra tag, NOT a path; the data file inside
+    # is always data.jsonl.
+    prefix = args.orchestrator_label or "orch"
+    tag = f"_{Path(args.out).stem}" if args.out else ""
+    base = Path("data/orchestrator/raw") / f"{prefix}_{time.strftime('%m%d')}{tag}"
+    run_dir = base.resolve()
+    n = 1
+    while run_dir.exists():  # same name+day (incl. parallel shards) -> disambiguate
+        n += 1
+        run_dir = base.with_name(f"{base.name}-{n}").resolve()
+    label = run_dir.name
     out_p = run_dir / "data.jsonl"
     lock_p = out_p.with_suffix(out_p.suffix + ".lock")
     out_p.parent.mkdir(parents=True, exist_ok=True)
-    lock_p.write_text(f"{stamp} pid={os.getpid()}")
+    lock_p.write_text(f"{time.strftime('%m-%d-%I%M%p').lower()} pid={os.getpid()}")
     args.out = str(out_p)
     import atexit
     atexit.register(lambda: lock_p.exists() and lock_p.unlink())

scripts/orchestrator/make_splits.py

@@ -5,24 +5,27 @@ Replaces make_tranches.py + make_gemma_tranches.py, which carved 1k/2k/4k/8k
 data-scaling tranches from the June pool. That lineage is dead: the tranches it
 produced trained checkpoints that evaluated at base ~= 1k ~= 2k (the pool taught
 the orchestrator to re-derive answers itself instead of routing), and the pool
-was deleted. See data/DELETED_ERA1_MANIFEST.md.
+was deleted. The lesson is in data/orchestrator/README.md — it was a data-QUALITY
+ceiling, so re-running the old scaling ladder buys nothing.
 
-Naming is uniform: ``{name}_orch_{split}_{date}.jsonl`` with name in
-{qwen, gemma, pooled}. Pass several pools to merge-and-restratify into `pooled`.
+Reads a clean pool (reject-sampled from data/orchestrator/raw/) and writes the
+SFT splits to data/orchestrator/sft/. Naming is uniform — ``{name}_{split}_{date}``
+with name in {qwen, gemma, pooled}; pass several pools to merge-and-restratify
+into `pooled`.
 
 The holdout is domain-stratified so val-loss is leak-free, and `overfit100` is a
 strict prefix of train (it is a memorisation sanity check — the model must be
 able to fit 100 rows it *has* seen, else the format/masking is broken).
 Deterministic (seed 42).
 
-    # one pool -> qwen_orch_{train,holdout,overfit100}_0711.jsonl
+    # one pool -> qwen_{train,holdout,overfit100}_0711.jsonl
     python scripts/orchestrator/make_splits.py --name qwen \
-        --pool data/sft_variants/qwen_orch_clean_0711.jsonl
+        --pool data/orchestrator/sft/qwen_clean_0711.jsonl
 
-    # merge both -> pooled_orch_{train,holdout,overfit100}_0711.jsonl
+    # merge both -> pooled_{train,holdout,overfit100}_0711.jsonl
     python scripts/orchestrator/make_splits.py --name pooled \
-        --pool data/sft_variants/qwen_orch_clean_0711.jsonl \
-        --pool data/sft_variants/gemma_orch_clean_0711.jsonl
+        --pool data/orchestrator/sft/qwen_clean_0711.jsonl \
+        --pool data/orchestrator/sft/gemma_clean_0711.jsonl
 """
 import argparse
 import json
@@ -33,7 +36,7 @@ from collections import defaultdict
 from datetime import datetime
 from pathlib import Path
 
-OUT = Path("data/sft_variants")
+OUT = Path("data/orchestrator/sft")
 SEED = 42
 OVERFIT_N = 100
 # Orchestrator that generated each pool — stamped onto every row so provenance
@@ -93,7 +96,7 @@ def main() -> int:
     written = {}
     for split, data in (("train", train), ("holdout", holdout),
                         (f"overfit{OVERFIT_N}", overfit)):
-        f = args.out / f"{args.name}_orch_{split}_{args.date}.jsonl"
+        f = args.out / f"{args.name}_{split}_{args.date}.jsonl"
         f.write_text("".join(json.dumps(r) + "\n" for r in data))
         written[split] = f
         print(f"wrote {f} ({len(data)})")

scripts/orchestrator/run_sft_fsdp.py

@@ -6,7 +6,7 @@ fine-tune won't fit one L40S otherwise):
 
     accelerate launch --config_file <fsdp.yaml> \
         scripts/orchestrator/run_sft_fsdp.py \
-        --data data/runs/<...>/data.jsonl [--data <more> ...] \
+        --data data/orchestrator/sft/qwen_train_0707.jsonl [--data <more> ...] \
         --variant correct --model Qwen/Qwen3.5-9B \
         --out checkpoints/sft_correct --epochs 3 --batch-size 8 --grad-accum 8 \
         --wandb-project orchestrator-sft

scripts/orchestrator/upload_to_braintrust.py

@@ -22,16 +22,16 @@ distinguishable in the UI.
 
 Usage (CLI):
   .venv/bin/python scripts/orchestrator/upload_to_braintrust.py \
-      data/sft_variants/qwen_orch_train_0707.jsonl=qwen_train_0707 \
-      data/sft_variants/qwen_orch_holdout_0707.jsonl=qwen_holdout_0707
+      data/orchestrator/sft/qwen_train_0707.jsonl=qwen_train_0707 \
+      data/orchestrator/sft/qwen_holdout_0707.jsonl=qwen_holdout_0707
 
   # name is optional; defaults to {model_short}_{split}_{date}
   .venv/bin/python scripts/orchestrator/upload_to_braintrust.py \
-      data/sft_variants/qwen_orch_holdout_0707.jsonl
+      data/orchestrator/sft/qwen_holdout_0707.jsonl
 
 Programmatic (used by the pipeline auto-upload hook in make_splits.py):
   from upload_to_braintrust import autoupload
-  autoupload(["data/sft_variants/qwen_orch_train_0707.jsonl=qwen_train_0707"],
+  autoupload(["data/orchestrator/sft/qwen_train_0707.jsonl=qwen_train_0707"],
              run_label="...")
 """
 import argparse

tests/test_orchestrator_learning/test_build_orchestrator_sft.py

@@ -111,9 +111,9 @@ def test_driver_runs_end_to_end_with_fakes(tmp_path, monkeypatch):
     # make_call_orchestrator builds an OpenAI client lazily, so it never connects
     # here (run_unified_rollout is stubbed out).
 
-    # The driver now treats --out as a LABEL and always writes to
-    # data/runs/<stamp>_<label>/data.jsonl (relative to cwd). chdir into tmp_path
-    # so the run folder is created under the test's temp dir, not the real repo.
+    # The driver treats --out as a TAG and always writes to
+    # data/orchestrator/raw/<label>_<MMDD>[_<tag>]/data.jsonl (relative to cwd).
+    # chdir into tmp_path so the run folder lands in the test's temp dir.
     monkeypatch.chdir(tmp_path)
     rc = drv.main([
         "--out", "v1",
@@ -121,7 +121,7 @@ def test_driver_runs_end_to_end_with_fakes(tmp_path, monkeypatch):
         "--max-tasks", "2",
     ])
     assert rc == 0
-    produced = list((tmp_path / "data" / "runs").glob("*_v1/data.jsonl"))
+    produced = list((tmp_path / "data" / "orchestrator" / "raw").glob("*_v1/data.jsonl"))
     assert len(produced) == 1
     lines = produced[0].read_text().strip().splitlines()
     assert len(lines) == 2