refactor(session): flatten session_raw/, switch md to YYYY_MM_DD (#1098)

CLAUDE.md

@@ -212,6 +212,7 @@ Tauri/Rust in the shell is a **delivery vehicle** (windowing, process lifecycle,
 - Issues and PRs on upstream **[tinyhumansai/openhuman](https://github.com/tinyhumansai/openhuman)** — not a fork — unless explicitly told otherwise.
 - Issue templates: [`.github/ISSUE_TEMPLATE/feature.md`](.github/ISSUE_TEMPLATE/feature.md), [`.github/ISSUE_TEMPLATE/bug.md`](.github/ISSUE_TEMPLATE/bug.md). PR template: [`.github/PULL_REQUEST_TEMPLATE.md`](.github/PULL_REQUEST_TEMPLATE.md). AI-authored text should follow them verbatim.
 - PRs target **`main`**.
+- **Push branches to `origin` (the user's fork — `senamakel/openhuman`), never to `upstream` (`tinyhumansai/openhuman`).** PRs are still opened against `tinyhumansai/openhuman:main`, but with `--head senamakel:<branch>` so the source is the fork. Direct pushes to upstream pollute its branch list and skip code-review boundaries. Treat the `upstream` remote as fetch-only.
 - **When the user asks you to push or open a PR, resolve blockers and push — don't prompt for permission.** If a pre-push hook fails on something unrelated to your changes (e.g. pre-existing breakage on `main` in code you didn't touch), push with `--no-verify` and call it out in the PR body. If the hook fails on your own changes, fix them and push again. Don't ask the user whether to bypass — just do the right thing and tell them what you did.
 
 ---

Cargo.lock

@@ -4568,7 +4568,7 @@ checksum = "c08d65885ee38876c4f86fa503fb49d7b507c2b62552df7c70b2fce627e06381"
 
 [[package]]
 name = "openhuman"
-version = "0.53.4"
+version = "0.53.8"
 dependencies = [
  "aes-gcm",
  "anyhow",

src/core/jsonrpc.rs

@@ -919,6 +919,41 @@ pub async fn bootstrap_skill_runtime() {
         );
     }
 
+    // --- Session storage layout migration -------------------------------
+    // One-shot move from `session_raw/{DDMMYYYY}/` (≤ 0.53.4) to the new
+    // flat `session_raw/{stem}.jsonl` layout, plus DDMMYYYY → YYYY_MM_DD
+    // for the human-readable `sessions/` companions. Idempotent via a
+    // marker file at `state/migrations/session_layout_v1.done`, so this
+    // costs one stat() on every subsequent boot.
+    match crate::openhuman::agent::harness::session::migrate_session_layout_if_needed(
+        &workspace_dir,
+    ) {
+        Ok(outcome) if outcome.already_done => {
+            log::debug!("[runtime] session_layout migration already applied");
+        }
+        Ok(outcome) => {
+            log::info!(
+                "[runtime] session_layout migration applied: jsonl_moved={} md_moved={} pruned_dirs={} warnings={}",
+                outcome.jsonl_moved,
+                outcome.md_moved,
+                outcome.legacy_dirs_pruned,
+                outcome.warnings.len(),
+            );
+            for w in &outcome.warnings {
+                log::warn!("[runtime] session_layout migration warning: {w}");
+            }
+        }
+        Err(err) => {
+            // Don't bring down startup over a transcript-storage migration.
+            // The transcript module's legacy fallback covers the unmigrated
+            // case for one release window.
+            log::warn!(
+                "[runtime] session_layout migration failed: {err} — \
+                 falling back to in-place legacy reads"
+            );
+        }
+    }
+
     // --- Socket manager bootstrap ---
     let socket_mgr = Arc::new(SocketManager::new());
     set_global_socket_manager(socket_mgr.clone());

src/openhuman/agent/harness/session/migration.rs

@@ -0,0 +1,373 @@
+//! Session storage layout migration: date-grouped → flat `session_raw/`.
+//!
+//! Older releases (≤ 0.53.4) wrote transcripts to:
+//!
+//! ```text
+//! {workspace}/session_raw/{DDMMYYYY}/{stem}.jsonl
+//! {workspace}/sessions/{DDMMYYYY}/{stem}.md
+//! ```
+//!
+//! From 0.53.5 onwards the source of truth is the *flat*
+//! `session_raw/{stem}.jsonl` and the human-readable companion is
+//! `sessions/{YYYY_MM_DD}/{stem}.md` — see
+//! [`super::transcript`] for the rationale (idle-thread resume
+//! becomes date-independent).
+//!
+//! `find_latest_transcript` ships a fallback that reads the legacy
+//! layout when the flat dir is empty, so users upgrading don't lose
+//! resume even before this migration runs. This module performs the
+//! one-shot move so files end up in their canonical location and the
+//! transitional fallback can eventually be removed.
+//!
+//! ## Idempotency
+//!
+//! After a successful migration we write a marker at
+//! `{workspace}/state/migrations/session_layout_v1.done`. Subsequent
+//! starts read the marker and skip the scan entirely. If the workspace
+//! has no legacy layout (fresh install or already migrated by an
+//! external sync) we still write the marker so the scan stays
+//! single-cost.
+//!
+//! ## Version gate
+//!
+//! The marker doubles as the "have we already migrated past 0.53.4?"
+//! flag. A bare workspace with no legacy dirs and no marker is treated
+//! as "fresh — nothing to do, write the marker." A workspace with
+//! legacy dirs is treated as "upgrading from ≤ 0.53.4 — migrate then
+//! write the marker."
+//!
+//! Failures are surfaced as warnings (logged) and **never panic**:
+//! transcript files are valuable but not strictly required for
+//! continued operation, and an uncatchable migration error would brick
+//! every startup.
+
+use anyhow::{Context, Result};
+use std::fs;
+use std::path::{Path, PathBuf};
+
+/// Marker file that signals "the v1 session-layout migration ran
+/// successfully on this workspace at least once". Written under
+/// `state/migrations/` to keep the workspace root tidy.
+const MIGRATION_MARKER: &str = "state/migrations/session_layout_v1.done";
+
+#[derive(Debug, Default, Clone)]
+pub struct MigrationOutcome {
+    pub jsonl_moved: usize,
+    pub jsonl_skipped: usize,
+    pub md_moved: usize,
+    pub md_skipped: usize,
+    pub legacy_dirs_pruned: usize,
+    pub already_done: bool,
+    pub warnings: Vec<String>,
+}
+
+/// Migrate the session storage layout for `workspace_dir` if needed.
+///
+/// * Detects legacy `session_raw/{DDMMYYYY}/...jsonl` and
+///   `sessions/{DDMMYYYY}/...md` layouts (i.e. an upgrade from
+///   ≤ 0.53.4).
+/// * Moves jsonl files to flat `session_raw/{stem}.jsonl`.
+/// * Renames `DDMMYYYY` md dirs to ISO-style `YYYY_MM_DD` so the
+///   listing sorts lexicographically.
+/// * Writes the migration marker on success.
+///
+/// Idempotent: returns immediately with `already_done = true` if the
+/// marker already exists. Best-effort on individual file moves —
+/// failures are logged and surfaced via `warnings`, not propagated, so
+/// one bad rename can't brick startup.
+pub fn migrate_session_layout_if_needed(workspace_dir: &Path) -> Result<MigrationOutcome> {
+    let marker_path = workspace_dir.join(MIGRATION_MARKER);
+    if marker_path.exists() {
+        log::debug!(
+            "[session-migration] marker present at {} — skipping",
+            marker_path.display()
+        );
+        return Ok(MigrationOutcome {
+            already_done: true,
+            ..Default::default()
+        });
+    }
+
+    let mut outcome = MigrationOutcome::default();
+
+    let raw_root = workspace_dir.join("session_raw");
+    if raw_root.is_dir() {
+        migrate_raw_jsonl(&raw_root, &mut outcome)?;
+    }
+
+    let sessions_root = workspace_dir.join("sessions");
+    if sessions_root.is_dir() {
+        migrate_md_directories(&sessions_root, &mut outcome)?;
+    }
+
+    write_marker(&marker_path, &outcome).context("write session-migration marker")?;
+
+    log::info!(
+        "[session-migration] complete: jsonl moved={} skipped={}, md moved={} skipped={}, legacy dirs pruned={}, warnings={}",
+        outcome.jsonl_moved,
+        outcome.jsonl_skipped,
+        outcome.md_moved,
+        outcome.md_skipped,
+        outcome.legacy_dirs_pruned,
+        outcome.warnings.len(),
+    );
+
+    Ok(outcome)
+}
+
+/// Walk `session_raw/`, find direct subdirectories whose names look
+/// like `DDMMYYYY` (8 ascii digits), and move every `*.jsonl` file
+/// inside up to the flat `session_raw/` parent. Empty legacy dirs
+/// are removed; non-empty ones are left in place with a warning so a
+/// human can decide what to do.
+fn migrate_raw_jsonl(raw_root: &Path, outcome: &mut MigrationOutcome) -> Result<()> {
+    let entries = match fs::read_dir(raw_root) {
+        Ok(it) => it,
+        Err(err) => {
+            outcome
+                .warnings
+                .push(format!("read_dir({}) failed: {err}", raw_root.display()));
+            return Ok(());
+        }
+    };
+
+    for entry in entries.flatten() {
+        let path = entry.path();
+        if !path.is_dir() {
+            continue;
+        }
+        let Some(name) = path.file_name().and_then(|s| s.to_str()) else {
+            continue;
+        };
+        if !is_ddmmyyyy(name) {
+            // Not a legacy date dir — leave alone (could be the new
+            // flat layout's own files which would never be a dir, or
+            // a user-created subdirectory we shouldn't touch).
+            continue;
+        }
+        move_jsonl_files_up(&path, raw_root, outcome);
+        prune_if_empty(&path, outcome);
+    }
+
+    Ok(())
+}
+
+fn move_jsonl_files_up(legacy_dir: &Path, flat_dir: &Path, outcome: &mut MigrationOutcome) {
+    let entries = match fs::read_dir(legacy_dir) {
+        Ok(it) => it,
+        Err(err) => {
+            outcome
+                .warnings
+                .push(format!("read_dir({}) failed: {err}", legacy_dir.display()));
+            return;
+        }
+    };
+
+    for entry in entries.flatten() {
+        let path = entry.path();
+        if !path.is_file() {
+            continue;
+        }
+        if path.extension().and_then(|s| s.to_str()) != Some("jsonl") {
+            continue;
+        }
+        let Some(file_name) = path.file_name() else {
+            continue;
+        };
+        let dest = flat_dir.join(file_name);
+        if dest.exists() {
+            // Same stem already lives in the flat dir — the new layout
+            // is authoritative for current sessions, so leave the
+            // legacy copy in place and surface a warning instead of
+            // overwriting newer data.
+            outcome.jsonl_skipped += 1;
+            outcome.warnings.push(format!(
+                "skip move: destination already exists at {} (legacy file kept at {})",
+                dest.display(),
+                path.display()
+            ));
+            continue;
+        }
+        match fs::rename(&path, &dest) {
+            Ok(()) => {
+                outcome.jsonl_moved += 1;
+                log::debug!(
+                    "[session-migration] moved {} → {}",
+                    path.display(),
+                    dest.display()
+                );
+            }
+            Err(err) => {
+                outcome.warnings.push(format!(
+                    "rename({} → {}) failed: {err}",
+                    path.display(),
+                    dest.display()
+                ));
+            }
+        }
+    }
+}
+
+/// Walk `sessions/`, rename each `DDMMYYYY` subdirectory to its
+/// `YYYY_MM_DD` equivalent. We rename the dir wholesale rather than
+/// copying file-by-file: the contents are human-readable companions
+/// and don't need re-indexing.
+fn migrate_md_directories(sessions_root: &Path, outcome: &mut MigrationOutcome) -> Result<()> {
+    let entries = match fs::read_dir(sessions_root) {
+        Ok(it) => it,
+        Err(err) => {
+            outcome.warnings.push(format!(
+                "read_dir({}) failed: {err}",
+                sessions_root.display()
+            ));
+            return Ok(());
+        }
+    };
+
+    for entry in entries.flatten() {
+        let path = entry.path();
+        if !path.is_dir() {
+            continue;
+        }
+        let Some(name) = path.file_name().and_then(|s| s.to_str()) else {
+            continue;
+        };
+        let Some(iso) = ddmmyyyy_to_yyyy_mm_dd(name) else {
+            continue;
+        };
+        let dest = sessions_root.join(&iso);
+        if dest.exists() {
+            // ISO dir already exists — merge file-by-file, never
+            // overwrite. A user could have legitimately produced both
+            // names (e.g. by manual workflow) so we don't blindly
+            // discard either side.
+            merge_md_dirs(&path, &dest, outcome);
+            prune_if_empty(&path, outcome);
+            continue;
+        }
+        match fs::rename(&path, &dest) {
+            Ok(()) => {
+                outcome.md_moved += 1;
+                log::debug!(
+                    "[session-migration] renamed md dir {} → {}",
+                    path.display(),
+                    dest.display()
+                );
+            }
+            Err(err) => {
+                outcome.warnings.push(format!(
+                    "rename({} → {}) failed: {err}",
+                    path.display(),
+                    dest.display()
+                ));
+            }
+        }
+    }
+
+    Ok(())
+}
+
+fn merge_md_dirs(legacy: &Path, dest: &Path, outcome: &mut MigrationOutcome) {
+    let entries = match fs::read_dir(legacy) {
+        Ok(it) => it,
+        Err(err) => {
+            outcome
+                .warnings
+                .push(format!("read_dir({}) failed: {err}", legacy.display()));
+            return;
+        }
+    };
+    for entry in entries.flatten() {
+        let src = entry.path();
+        if !src.is_file() {
+            continue;
+        }
+        let Some(file_name) = src.file_name() else {
+            continue;
+        };
+        let target = dest.join(file_name);
+        if target.exists() {
+            outcome.md_skipped += 1;
+            outcome.warnings.push(format!(
+                "skip md merge: {} already exists (legacy at {} kept)",
+                target.display(),
+                src.display()
+            ));
+            continue;
+        }
+        match fs::rename(&src, &target) {
+            Ok(()) => outcome.md_moved += 1,
+            Err(err) => outcome.warnings.push(format!(
+                "rename({} → {}) failed: {err}",
+                src.display(),
+                target.display()
+            )),
+        }
+    }
+}
+
+fn prune_if_empty(dir: &Path, outcome: &mut MigrationOutcome) {
+    match fs::read_dir(dir) {
+        Ok(mut it) => {
+            if it.next().is_some() {
+                // Non-empty — leave for human inspection.
+                return;
+            }
+        }
+        Err(_) => return,
+    }
+    if fs::remove_dir(dir).is_ok() {
+        outcome.legacy_dirs_pruned += 1;
+    }
+}
+
+fn write_marker(marker_path: &Path, outcome: &MigrationOutcome) -> Result<()> {
+    if let Some(parent) = marker_path.parent() {
+        fs::create_dir_all(parent)
+            .with_context(|| format!("create marker dir {}", parent.display()))?;
+    }
+    let body = format!(
+        "openhuman session_layout migration v1\nrun_at: {}\njsonl_moved: {}\nmd_moved: {}\nlegacy_dirs_pruned: {}\nwarnings: {}\n",
+        chrono::Utc::now().to_rfc3339(),
+        outcome.jsonl_moved,
+        outcome.md_moved,
+        outcome.legacy_dirs_pruned,
+        outcome.warnings.len(),
+    );
+    fs::write(marker_path, body)
+        .with_context(|| format!("write marker {}", marker_path.display()))?;
+    Ok(())
+}
+
+/// Returns true iff `name` is exactly 8 ASCII digits — the legacy
+/// `DDMMYYYY` shape. We don't validate the date range (1–31, 1–12,
+/// 1900–2100) because chrono printed the value originally, so any
+/// real on-disk dir is well-formed; the digit shape is a sufficient
+/// fingerprint to distinguish from user-created subdirectories.
+fn is_ddmmyyyy(name: &str) -> bool {
+    name.len() == 8 && name.chars().all(|c| c.is_ascii_digit())
+}
+
+/// Convert `DDMMYYYY` → `YYYY_MM_DD`. Returns `None` if the input
+/// isn't 8 digits.
+fn ddmmyyyy_to_yyyy_mm_dd(name: &str) -> Option<String> {
+    if !is_ddmmyyyy(name) {
+        return None;
+    }
+    let dd = &name[0..2];
+    let mm = &name[2..4];
+    let yyyy = &name[4..8];
+    Some(format!("{yyyy}_{mm}_{dd}"))
+}
+
+/// Returns the path of the migration marker for `workspace_dir`.
+/// Exposed for tests and CLI tooling that wants to manually re-run
+/// the migration (delete the marker, then call
+/// [`migrate_session_layout_if_needed`] again).
+pub fn marker_path_for(workspace_dir: &Path) -> PathBuf {
+    workspace_dir.join(MIGRATION_MARKER)
+}
+
+#[cfg(test)]
+#[path = "migration_tests.rs"]
+mod tests;

src/openhuman/agent/harness/session/migration_tests.rs

@@ -0,0 +1,170 @@
+use super::*;
+use std::fs;
+use tempfile::TempDir;
+
+fn write_file(path: &std::path::Path, body: &str) {
+    fs::create_dir_all(path.parent().unwrap()).unwrap();
+    fs::write(path, body).unwrap();
+}
+
+#[test]
+fn fresh_workspace_writes_marker_with_no_moves() {
+    let dir = TempDir::new().unwrap();
+    let outcome = migrate_session_layout_if_needed(dir.path()).unwrap();
+    assert!(!outcome.already_done);
+    assert_eq!(outcome.jsonl_moved, 0);
+    assert_eq!(outcome.md_moved, 0);
+    assert!(marker_path_for(dir.path()).exists());
+}
+
+#[test]
+fn second_run_is_a_noop() {
+    let dir = TempDir::new().unwrap();
+    let _first = migrate_session_layout_if_needed(dir.path()).unwrap();
+    let second = migrate_session_layout_if_needed(dir.path()).unwrap();
+    assert!(second.already_done);
+    assert_eq!(second.jsonl_moved, 0);
+    assert_eq!(second.warnings.len(), 0);
+}
+
+#[test]
+fn moves_legacy_jsonl_files_up_to_flat_session_raw() {
+    let dir = TempDir::new().unwrap();
+    let ws = dir.path();
+    let legacy_a = ws.join("session_raw").join("01052026");
+    let legacy_b = ws.join("session_raw").join("02052026");
+    write_file(&legacy_a.join("1714000000_main.jsonl"), "a");
+    write_file(&legacy_a.join("1714000001_welcome.jsonl"), "b");
+    write_file(&legacy_b.join("1714999999_orchestrator.jsonl"), "c");
+
+    let outcome = migrate_session_layout_if_needed(ws).unwrap();
+    assert_eq!(outcome.jsonl_moved, 3);
+    assert_eq!(outcome.legacy_dirs_pruned, 2);
+
+    let raw_root = ws.join("session_raw");
+    assert!(raw_root.join("1714000000_main.jsonl").exists());
+    assert!(raw_root.join("1714000001_welcome.jsonl").exists());
+    assert!(raw_root.join("1714999999_orchestrator.jsonl").exists());
+    // Empty legacy date dirs should have been pruned.
+    assert!(!legacy_a.exists(), "legacy date dir should be removed");
+    assert!(!legacy_b.exists(), "legacy date dir should be removed");
+}
+
+#[test]
+fn jsonl_destination_collision_is_skipped_with_warning() {
+    // If a flat `session_raw/{stem}.jsonl` already exists for the
+    // same stem we don't overwrite — the flat copy is authoritative
+    // (the user may have already started a fresh session with the
+    // same key after a clock reset). Surface a warning instead.
+    let dir = TempDir::new().unwrap();
+    let ws = dir.path();
+    let raw_root = ws.join("session_raw");
+    write_file(&raw_root.join("1714000000_main.jsonl"), "new");
+    write_file(
+        &raw_root.join("01052026").join("1714000000_main.jsonl"),
+        "old",
+    );
+
+    let outcome = migrate_session_layout_if_needed(ws).unwrap();
+    assert_eq!(outcome.jsonl_moved, 0);
+    assert_eq!(outcome.jsonl_skipped, 1);
+    assert!(outcome
+        .warnings
+        .iter()
+        .any(|w| w.contains("already exists")));
+    // Both files still exist — nothing was overwritten.
+    assert_eq!(
+        fs::read_to_string(raw_root.join("1714000000_main.jsonl")).unwrap(),
+        "new"
+    );
+    assert_eq!(
+        fs::read_to_string(raw_root.join("01052026").join("1714000000_main.jsonl")).unwrap(),
+        "old"
+    );
+}
+
+#[test]
+fn renames_md_ddmmyyyy_dirs_to_iso() {
+    let dir = TempDir::new().unwrap();
+    let ws = dir.path();
+    let legacy_md = ws.join("sessions").join("01052026");
+    write_file(&legacy_md.join("main_0.md"), "x");
+    write_file(&legacy_md.join("main_1.md"), "y");
+
+    let outcome = migrate_session_layout_if_needed(ws).unwrap();
+    assert_eq!(outcome.md_moved, 1, "one rename of the dir as a whole");
+    let iso = ws.join("sessions").join("2026_05_01");
+    assert!(iso.is_dir());
+    assert!(iso.join("main_0.md").exists());
+    assert!(iso.join("main_1.md").exists());
+    assert!(
+        !legacy_md.exists(),
+        "DDMMYYYY md dir should be gone after rename"
+    );
+}
+
+#[test]
+fn merges_md_when_iso_dir_already_exists() {
+    let dir = TempDir::new().unwrap();
+    let ws = dir.path();
+    // Both layouts coexist for the same calendar date — e.g. user
+    // ran a hand-edited build that produced ISO dirs alongside the
+    // legacy DDMMYYYY ones.
+    let legacy = ws.join("sessions").join("01052026");
+    let iso = ws.join("sessions").join("2026_05_01");
+    write_file(&legacy.join("main_0.md"), "legacy");
+    write_file(&legacy.join("main_1.md"), "legacy");
+    write_file(&iso.join("main_1.md"), "newer");
+
+    let outcome = migrate_session_layout_if_needed(ws).unwrap();
+    // main_0.md moves over (no collision); main_1.md collides and is
+    // skipped without overwriting the newer copy.
+    assert_eq!(outcome.md_moved, 1);
+    assert_eq!(outcome.md_skipped, 1);
+    assert_eq!(fs::read_to_string(iso.join("main_0.md")).unwrap(), "legacy");
+    assert_eq!(fs::read_to_string(iso.join("main_1.md")).unwrap(), "newer");
+}
+
+#[test]
+fn ignores_non_date_subdirectories_in_session_raw() {
+    // Defensive: a user (or some other tool) might have created a
+    // sibling dir under session_raw/. We must not touch it — only
+    // 8-digit names are recognised as legacy date dirs.
+    let dir = TempDir::new().unwrap();
+    let ws = dir.path();
+    let weird = ws.join("session_raw").join("my_notes");
+    write_file(&weird.join("random.jsonl"), "keep me");
+
+    let outcome = migrate_session_layout_if_needed(ws).unwrap();
+    assert_eq!(outcome.jsonl_moved, 0);
+    assert!(weird.is_dir(), "non-date subdir must be left alone");
+    assert!(weird.join("random.jsonl").exists());
+}
+
+#[test]
+fn ddmmyyyy_to_iso_handles_boundary_dates() {
+    assert_eq!(
+        ddmmyyyy_to_yyyy_mm_dd("01012026").as_deref(),
+        Some("2026_01_01")
+    );
+    assert_eq!(
+        ddmmyyyy_to_yyyy_mm_dd("31122099").as_deref(),
+        Some("2099_12_31")
+    );
+    assert!(ddmmyyyy_to_yyyy_mm_dd("abc12345").is_none());
+    assert!(ddmmyyyy_to_yyyy_mm_dd("1234567").is_none(), "7 digits");
+    assert!(ddmmyyyy_to_yyyy_mm_dd("123456789").is_none(), "9 digits");
+}
+
+#[test]
+fn marker_persists_run_metadata() {
+    let dir = TempDir::new().unwrap();
+    let ws = dir.path();
+    let legacy = ws.join("session_raw").join("01052026");
+    write_file(&legacy.join("1714000000_main.jsonl"), "a");
+
+    migrate_session_layout_if_needed(ws).unwrap();
+    let marker = fs::read_to_string(marker_path_for(ws)).unwrap();
+    assert!(marker.contains("jsonl_moved: 1"));
+    assert!(marker.contains("openhuman session_layout migration v1"));
+}

src/openhuman/agent/harness/session/mod.rs

@@ -21,11 +21,14 @@
 //! The child files are an implementation detail.
 
 mod builder;
+pub mod migration;
 mod runtime;
 pub(crate) mod transcript;
 mod turn;
 mod types;
 
+pub use migration::{migrate_session_layout_if_needed, MigrationOutcome};
+
 #[cfg(test)]
 mod tests;
 

src/openhuman/agent/harness/session/transcript.rs

@@ -1,12 +1,34 @@
 //! Session transcript persistence for KV cache stability.
 //!
-//! **Source of truth**: `session_raw/{DDMMYYYY}/{agent}_{index}.jsonl`
+//! **Source of truth**: `session_raw/{stem}.jsonl` — a *flat* directory.
 //!
 //! Each JSONL file starts with a single metadata line (identified by an
 //! `_meta` key) followed by one JSON object per `ChatMessage`. On every
 //! write the companion `.md` file is re-rendered for human readability
-//! under `sessions/{DDMMYYYY}/{agent}_{index}.md`; it is **never** read
-//! back — all round-trip / resume logic uses the JSONL.
+//! under `sessions/{YYYY_MM_DD}/{stem}.md`; it is **never** read back —
+//! all round-trip / resume logic uses the JSONL.
+//!
+//! ## Storage layout
+//!
+//! ```text
+//! {workspace}/session_raw/{stem}.jsonl              ← source of truth (flat)
+//! {workspace}/sessions/YYYY_MM_DD/{stem}.md         ← human-readable view
+//! ```
+//!
+//! `stem` is `{unix_ts}_{agent_id}` for a root session, or
+//! `{parent_chain}__{unix_ts}_{agent_id}` for a sub-agent. Because the
+//! stem starts with the unix timestamp at agent-build time, a directory
+//! listing of `session_raw/` is naturally sorted by creation time and
+//! `find_latest_transcript` becomes O(scan one dir, filter by suffix)
+//! — it does not depend on the calendar date, so a session that's been
+//! idle for weeks resumes the same way as one from yesterday.
+//!
+//! ## Backward compatibility
+//!
+//! Older releases wrote into `session_raw/DDMMYYYY/{stem}.jsonl` (and
+//! the legacy `sessions/DDMMYYYY/{stem}.md`). [`find_latest_transcript`]
+//! falls back to scanning those date-grouped dirs when the flat
+//! directory yields nothing, so users upgrading don't lose resume.
 //!
 //! ## JSONL schema
 //!
@@ -25,13 +47,6 @@
 //!
 //! Only `role` and `content` are required. All other fields are optional.
 //! Unknown fields on read are ignored (forward-compat).
-//!
-//! ## Storage layout
-//!
-//! ```text
-//! {workspace}/session_raw/DDMMYYYY/{agent}_{index}.jsonl   ← source of truth
-//! {workspace}/sessions/DDMMYYYY/{agent}_{index}.md         ← human-readable view
-//! ```
 
 use crate::openhuman::providers::ChatMessage;
 use anyhow::{Context, Result};
@@ -363,25 +378,19 @@ fn read_transcript_jsonl(path: &Path) -> Result<SessionTranscript> {
 
 // ── Path resolution ──────────────────────────────────────────────────
 
-/// Resolve a new transcript path under
-/// `{workspace}/session_raw/DDMMYYYY/{agent}_{index}.jsonl`.
+/// Resolve a transcript path under `session_raw/{stem}.jsonl` — a
+/// *flat* directory keyed only by stem. Used by the session-key flow:
+/// the stem is `"{unix_ts}_{agent_id}"` for a root session, or
+/// `"{parent_chain}__{session_key}"` for a sub-agent, so nested
+/// delegations still produce a single flat filename that encodes the
+/// parent → child path.
 ///
-/// Creates the date directory if needed. Index = max existing + 1.
-/// Scans both the new `session_raw/` dir (for `.jsonl`) **and** the legacy
-/// `sessions/` dir (for `.md`) so indices stay unique across migration.
-/// Resolve a transcript path under `session_raw/DDMMYYYY/{stem}.jsonl`
-/// where `stem` is deterministic (no auto-indexing). Used by the new
-/// session-key flow: the session-key stem is `"{unix_ts}_{agent_id}"`
-/// for a root session, or `"{parent_chain}__{session_key}"` for a
-/// sub-agent — so nested delegations produce a single flat filename
-/// that encodes the parent → child path.
-///
-/// Creates the date directory if needed. Overwrites are intentional:
-/// the Agent persists the same transcript file across every turn of a
+/// Creates the directory if needed. Overwrites are intentional: the
+/// `Agent` persists the same transcript file across every turn of a
 /// session, and every sub-agent spawn gets a unique timestamp in its
 /// own key so collisions are effectively impossible.
 pub fn resolve_keyed_transcript_path(workspace_dir: &Path, stem: &str) -> Result<PathBuf> {
-    let raw_dir = today_raw_session_dir(workspace_dir);
+    let raw_dir = raw_session_dir(workspace_dir);
     fs::create_dir_all(&raw_dir)
         .with_context(|| format!("create session_raw dir {}", raw_dir.display()))?;
     let sanitized = sanitize_stem(stem);
@@ -389,9 +398,9 @@ pub fn resolve_keyed_transcript_path(workspace_dir: &Path, stem: &str) -> Result
 }
 
 /// Sanitize a user-supplied transcript stem so it never escapes the
-/// `session_raw/DDMMYYYY/` directory. Allows ASCII alphanumerics plus
-/// a small punctuation set (`_`, `-`, `.`); every other byte is
-/// replaced with `_`. Empty inputs fall back to `"session"`.
+/// `session_raw/` directory. Allows ASCII alphanumerics plus a small
+/// punctuation set (`_`, `-`, `.`); every other byte is replaced with
+/// `_`. Empty inputs fall back to `"session"`.
 fn sanitize_stem(stem: &str) -> String {
     let cleaned: String = stem
         .chars()
@@ -411,17 +420,17 @@ fn sanitize_stem(stem: &str) -> String {
 }
 
 pub fn resolve_new_transcript_path(workspace_dir: &Path, agent_name: &str) -> Result<PathBuf> {
-    let raw_dir = today_raw_session_dir(workspace_dir);
+    let raw_dir = raw_session_dir(workspace_dir);
     fs::create_dir_all(&raw_dir)
         .with_context(|| format!("create session_raw dir {}", raw_dir.display()))?;
 
     let sanitized = sanitize_agent_name(agent_name);
     let idx_raw = next_index(&raw_dir, &sanitized)?;
-    // Also consider the legacy sessions/ dir so companion .md (or legacy
-    // standalone .md) files don't cause index collisions.
-    let legacy_dir = today_session_dir(workspace_dir);
-    let idx_legacy = next_index(&legacy_dir, &sanitized)?;
-    let next_idx = idx_raw.max(idx_legacy);
+    // Also consider today's md companion dir so a stale .md from this
+    // session doesn't cause an index collision when only .md exists.
+    let md_dir = today_md_session_dir(workspace_dir);
+    let idx_md = next_index(&md_dir, &sanitized)?;
+    let next_idx = idx_raw.max(idx_md);
     let filename = format!("{}_{}.jsonl", sanitized, next_idx);
 
     Ok(raw_dir.join(filename))
@@ -429,28 +438,43 @@ pub fn resolve_new_transcript_path(workspace_dir: &Path, agent_name: &str) -> Re
 
 /// Find the most recent transcript for `agent_name`.
 ///
-/// Searches today's directory first, then yesterday's. Prefers `.jsonl`
-/// under `session_raw/` and falls back to `.md` under `sessions/` when
-/// only legacy files exist.
+/// **Primary**: scan the flat `session_raw/` directory and pick the
+/// newest matching stem (root sessions only — sub-agents are skipped).
+/// **Fallback**: scan the legacy `session_raw/DDMMYYYY/` dirs (today
+/// and yesterday) and the legacy `sessions/DDMMYYYY/` markdown dirs so
+/// users upgrading from the date-grouped layout don't lose resume.
+/// The fallback is one-release transitional and can be removed once
+/// existing transcripts have rolled forward.
 pub fn find_latest_transcript(workspace_dir: &Path, agent_name: &str) -> Option<PathBuf> {
     let sanitized = sanitize_agent_name(agent_name);
     let raw_root = workspace_dir.join("session_raw");
     let sessions_root = workspace_dir.join("sessions");
 
+    // Primary path: flat session_raw/ directory. The stem-suffix scan
+    // is naturally date-independent, so an idle thread resumes the same
+    // way today as it did weeks ago.
+    if raw_root.is_dir() {
+        if let Some(path) = latest_in_dir(&raw_root, &sanitized) {
+            return Some(path);
+        }
+    }
+
+    // Fallback: legacy date-grouped layout (one-release migration
+    // window). Today first, then yesterday — matches the previous
+    // behaviour so we don't regress while users still have files in
+    // the old structure.
     let today = chrono::Local::now().format("%d%m%Y").to_string();
     let yesterday = (chrono::Local::now() - chrono::Duration::days(1))
         .format("%d%m%Y")
         .to_string();
 
     for date_str in [&today, &yesterday] {
-        // Prefer the new session_raw/ location.
         let raw_dir = raw_root.join(date_str);
         if raw_dir.is_dir() {
             if let Some(path) = latest_in_dir(&raw_dir, &sanitized) {
                 return Some(path);
             }
         }
-        // Fall back to legacy sessions/ for pre-migration .md only.
         let legacy_dir = sessions_root.join(date_str);
         if legacy_dir.is_dir() {
             if let Some(path) = latest_in_dir(&legacy_dir, &sanitized) {
@@ -645,38 +669,62 @@ fn parse_legacy_messages(raw: &str) -> Result<Vec<ChatMessage>> {
 
 // ── Private helpers ───────────────────────────────────────────────────
 
-fn today_session_dir(workspace_dir: &Path) -> PathBuf {
-    let date = chrono::Local::now().format("%d%m%Y").to_string();
+/// Date-grouped directory for human-readable `.md` companions, e.g.
+/// `{workspace}/sessions/2026_05_02`. ISO-style `YYYY_MM_DD` so the
+/// listing sorts lexicographically by date.
+fn today_md_session_dir(workspace_dir: &Path) -> PathBuf {
+    let date = chrono::Local::now().format("%Y_%m_%d").to_string();
     workspace_dir.join("sessions").join(date)
 }
 
-fn today_raw_session_dir(workspace_dir: &Path) -> PathBuf {
-    let date = chrono::Local::now().format("%d%m%Y").to_string();
-    workspace_dir.join("session_raw").join(date)
+/// Flat directory for the JSONL source of truth, e.g.
+/// `{workspace}/session_raw`. Stems start with `{unix_ts}` so the
+/// listing is naturally time-ordered without a date subdirectory.
+fn raw_session_dir(workspace_dir: &Path) -> PathBuf {
+    workspace_dir.join("session_raw")
 }
 
-/// Given a `session_raw/DDMMYYYY/agent_N.jsonl` path, derive the companion
-/// `sessions/DDMMYYYY/agent_N.md` path by swapping the `session_raw`
-/// path component with `sessions` and the extension with `.md`.
+/// Given a `session_raw/{stem}.jsonl` path, derive the companion
+/// `sessions/YYYY_MM_DD/{stem}.md` path. The date is taken from the
+/// local clock at write time — fine for browsing because the source
+/// of truth lives in the flat raw dir; the `.md` is purely a view.
 ///
-/// If no `session_raw` component is present (e.g. tests using a flat
-/// tempdir), just swap the extension so the companion lives alongside.
+/// Legacy `session_raw/DDMMYYYY/{stem}.jsonl` paths (still on disk
+/// from older releases until they roll forward) keep their date
+/// component when generating the companion so we don't accidentally
+/// stamp old transcripts with today's date.
+///
+/// If no `session_raw` component is present (tests using a flat
+/// tempdir), the companion sits alongside as a sibling `.md`.
 fn md_companion_path(jsonl_path: &Path) -> PathBuf {
-    let mut out = PathBuf::new();
-    let mut swapped = false;
-    for comp in jsonl_path.components() {
-        match comp {
-            std::path::Component::Normal(s) if s == "session_raw" => {
-                out.push("sessions");
-                swapped = true;
-            }
-            other => out.push(other.as_os_str()),
-        }
-    }
-    if !swapped {
-        // Fall back to sibling .md in the same directory.
+    let components: Vec<_> = jsonl_path.components().collect();
+
+    let raw_idx = components
+        .iter()
+        .position(|comp| matches!(comp, std::path::Component::Normal(s) if *s == "session_raw"));
+
+    let Some(raw_idx) = raw_idx else {
         return jsonl_path.with_extension("md");
+    };
+
+    let mut out = PathBuf::new();
+    for comp in &components[..raw_idx] {
+        out.push(comp.as_os_str());
     }
+    out.push("sessions");
+
+    // Tail after `session_raw`:
+    //   * Flat: ["{stem}.jsonl"] — prepend today's YYYY_MM_DD.
+    //   * Legacy: ["DDMMYYYY", "{stem}.jsonl"] — keep the existing
+    //     date dir so we don't relabel old transcripts.
+    let tail = &components[raw_idx + 1..];
+    if tail.len() <= 1 {
+        out.push(chrono::Local::now().format("%Y_%m_%d").to_string());
+    }
+    for comp in tail {
+        out.push(comp.as_os_str());
+    }
+
     out.with_extension("md")
 }
 

src/openhuman/agent/harness/session/transcript_tests.rs

@@ -105,32 +105,59 @@ fn meta_round_trip() {
 }
 
 #[test]
-fn path_resolution_creates_dir_and_increments_index() {
+fn path_resolution_creates_flat_session_raw_dir_and_increments_index() {
     let dir = TempDir::new().unwrap();
     let workspace = dir.path();
 
     let path0 = resolve_new_transcript_path(workspace, "main").unwrap();
     assert!(path0.to_string_lossy().contains("main_0.jsonl"));
+    // Flat layout: jsonl lives directly under session_raw/, no date dir.
+    let parent = path0.parent().unwrap();
     assert!(
-        path0.to_string_lossy().contains("session_raw"),
-        "jsonl should live under session_raw/, got {}",
-        path0.display()
+        parent.ends_with("session_raw"),
+        "jsonl parent should be session_raw/ (flat layout), got {}",
+        parent.display()
     );
-    // Write something so the next call sees it.
     fs::write(&path0, "placeholder").unwrap();
 
     let path1 = resolve_new_transcript_path(workspace, "main").unwrap();
     assert!(path1.to_string_lossy().contains("main_1.jsonl"));
+    assert!(path1.parent().unwrap().ends_with("session_raw"));
 }
 
 #[test]
-fn md_companion_path_swaps_session_raw_to_sessions() {
+fn resolve_keyed_writes_to_flat_session_raw() {
+    let dir = TempDir::new().unwrap();
+    let path = resolve_keyed_transcript_path(dir.path(), "1714000000_orchestrator").unwrap();
+    assert_eq!(path.parent().unwrap(), dir.path().join("session_raw"));
+    assert!(path
+        .to_string_lossy()
+        .ends_with("1714000000_orchestrator.jsonl"));
+}
+
+#[test]
+fn md_companion_path_for_flat_jsonl_uses_iso_date_dir() {
+    let jsonl = PathBuf::from("/tmp/ws/session_raw/1714000000_main.jsonl");
+    let md = md_companion_path(&jsonl);
+    let today = chrono::Local::now().format("%Y_%m_%d").to_string();
+    assert_eq!(
+        md,
+        PathBuf::from(format!("/tmp/ws/sessions/{today}/1714000000_main.md")),
+        "flat session_raw should map to sessions/YYYY_MM_DD/ on the md side"
+    );
+}
+
+#[test]
+fn md_companion_path_preserves_legacy_ddmmyyyy_dir() {
+    // A pre-migration jsonl at session_raw/DDMMYYYY/{stem}.jsonl should
+    // keep its date component so old transcripts aren't relabeled with
+    // today's date.
     let jsonl = PathBuf::from("/tmp/ws/session_raw/17042026/main_0.jsonl");
     let md = md_companion_path(&jsonl);
     assert_eq!(
         md,
         PathBuf::from("/tmp/ws/sessions/17042026/main_0.md"),
-        "md companion should live under sessions/ with .md extension"
+        "legacy date-grouped raw paths must keep their original date dir"
     );
 }
 
@@ -142,19 +169,19 @@ fn md_companion_path_falls_back_to_sibling_when_no_session_raw_component() {
 }
 
 #[test]
-fn resolve_avoids_index_collision_with_legacy_md_in_sessions_dir() {
+fn resolve_avoids_index_collision_with_md_in_iso_date_dir() {
     let dir = TempDir::new().unwrap();
     let workspace = dir.path();
-    let date = chrono::Local::now().format("%d%m%Y").to_string();
-    let legacy = workspace.join("sessions").join(&date);
-    fs::create_dir_all(&legacy).unwrap();
-    fs::write(legacy.join("main_0.md"), "legacy").unwrap();
-    fs::write(legacy.join("main_1.md"), "legacy").unwrap();
+    let date = chrono::Local::now().format("%Y_%m_%d").to_string();
+    let md_dir = workspace.join("sessions").join(&date);
+    fs::create_dir_all(&md_dir).unwrap();
+    fs::write(md_dir.join("main_0.md"), "x").unwrap();
+    fs::write(md_dir.join("main_1.md"), "x").unwrap();
 
     let path = resolve_new_transcript_path(workspace, "main").unwrap();
     assert!(
         path.to_string_lossy().contains("main_2.jsonl"),
-        "should advance past legacy indices, got {}",
+        "should advance past md indices in today's YYYY_MM_DD dir, got {}",
         path.display()
     );
 }
@@ -167,10 +194,9 @@ fn sanitize_agent_name_strips_special_chars() {
 }
 
 #[test]
-fn find_latest_returns_highest_index() {
+fn find_latest_scans_flat_session_raw_dir() {
     let dir = TempDir::new().unwrap();
-    let date = chrono::Local::now().format("%d%m%Y").to_string();
-    let raw_dir = dir.path().join("session_raw").join(&date);
+    let raw_dir = dir.path().join("session_raw");
     fs::create_dir_all(&raw_dir).unwrap();
 
     fs::write(raw_dir.join("main_0.jsonl"), "a").unwrap();
@@ -178,11 +204,66 @@ fn find_latest_returns_highest_index() {
     fs::write(raw_dir.join("main_1.jsonl"), "b").unwrap();
     fs::write(raw_dir.join("other_0.jsonl"), "x").unwrap();
 
-    let latest = find_latest_transcript(dir.path(), "main");
-    assert!(latest.is_some());
-    let latest = latest.unwrap();
-    assert!(latest.to_string_lossy().contains("main_2.jsonl"));
-    assert!(latest.to_string_lossy().contains("session_raw"));
+    let latest = find_latest_transcript(dir.path(), "main").unwrap();
+    assert!(latest.to_string_lossy().ends_with("main_2.jsonl"));
+    assert_eq!(latest.parent().unwrap(), raw_dir);
+}
+
+#[test]
+fn find_latest_picks_newest_keyed_stem_in_flat_dir() {
+    let dir = TempDir::new().unwrap();
+    let raw_dir = dir.path().join("session_raw");
+    fs::create_dir_all(&raw_dir).unwrap();
+
+    // Keyed stem layout: `{unix_ts}_{agent_id}.jsonl`.
+    fs::write(raw_dir.join("1714000000_main.jsonl"), "old").unwrap();
+    fs::write(raw_dir.join("1714999999_main.jsonl"), "new").unwrap();
+    // Sub-agent transcripts (contain `__`) must be skipped.
+    fs::write(
+        raw_dir.join("1714000000_orchestrator__1714500000_planner.jsonl"),
+        "sub",
+    )
+    .unwrap();
+
+    let latest = find_latest_transcript(dir.path(), "main").unwrap();
+    assert!(latest.to_string_lossy().ends_with("1714999999_main.jsonl"));
+}
+
+#[test]
+fn find_latest_falls_back_to_legacy_ddmmyyyy_raw_dir() {
+    // Pre-migration transcript at session_raw/DDMMYYYY/main_*.jsonl
+    // must still resolve via the legacy fallback when the flat dir is
+    // empty.
+    let dir = TempDir::new().unwrap();
+    let date = chrono::Local::now().format("%d%m%Y").to_string();
+    let legacy_raw = dir.path().join("session_raw").join(&date);
+    fs::create_dir_all(&legacy_raw).unwrap();
+    fs::write(legacy_raw.join("main_5.jsonl"), "legacy").unwrap();
+
+    let latest = find_latest_transcript(dir.path(), "main").unwrap();
+    assert!(latest.to_string_lossy().ends_with("main_5.jsonl"));
+    assert!(latest.to_string_lossy().contains(&date));
+}
+
+#[test]
+fn find_latest_prefers_flat_over_legacy_ddmmyyyy() {
+    let dir = TempDir::new().unwrap();
+    let raw_root = dir.path().join("session_raw");
+    fs::create_dir_all(&raw_root).unwrap();
+    fs::write(raw_root.join("main_9.jsonl"), "flat").unwrap();
+
+    let date = chrono::Local::now().format("%d%m%Y").to_string();
+    let legacy_raw = raw_root.join(&date);
+    fs::create_dir_all(&legacy_raw).unwrap();
+    fs::write(legacy_raw.join("main_99.jsonl"), "legacy").unwrap();
+
+    let latest = find_latest_transcript(dir.path(), "main").unwrap();
+    // Flat dir takes precedence so newly-created sessions always win
+    // over stale legacy files — even when a legacy file has a higher
+    // numeric index. The flat dir is the canonical layout going
+    // forward.
+    assert_eq!(latest.parent().unwrap(), raw_root);
+    assert!(latest.to_string_lossy().ends_with("main_9.jsonl"));
 }
 
 #[test]