diff --git a/CLAUDE.md b/CLAUDE.md index 4b19cb6e9..d01ad0a24 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -212,6 +212,7 @@ Tauri/Rust in the shell is a **delivery vehicle** (windowing, process lifecycle, - Issues and PRs on upstream **[tinyhumansai/openhuman](https://github.com/tinyhumansai/openhuman)** — not a fork — unless explicitly told otherwise. - Issue templates: [`.github/ISSUE_TEMPLATE/feature.md`](.github/ISSUE_TEMPLATE/feature.md), [`.github/ISSUE_TEMPLATE/bug.md`](.github/ISSUE_TEMPLATE/bug.md). PR template: [`.github/PULL_REQUEST_TEMPLATE.md`](.github/PULL_REQUEST_TEMPLATE.md). AI-authored text should follow them verbatim. - PRs target **`main`**. +- **Push branches to `origin` (the user's fork — `senamakel/openhuman`), never to `upstream` (`tinyhumansai/openhuman`).** PRs are still opened against `tinyhumansai/openhuman:main`, but with `--head senamakel:` so the source is the fork. Direct pushes to upstream pollute its branch list and skip code-review boundaries. Treat the `upstream` remote as fetch-only. - **When the user asks you to push or open a PR, resolve blockers and push — don't prompt for permission.** If a pre-push hook fails on something unrelated to your changes (e.g. pre-existing breakage on `main` in code you didn't touch), push with `--no-verify` and call it out in the PR body. If the hook fails on your own changes, fix them and push again. Don't ask the user whether to bypass — just do the right thing and tell them what you did. --- diff --git a/Cargo.lock b/Cargo.lock index f9ce2f11c..796300058 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4568,7 +4568,7 @@ checksum = "c08d65885ee38876c4f86fa503fb49d7b507c2b62552df7c70b2fce627e06381" [[package]] name = "openhuman" -version = "0.53.4" +version = "0.53.8" dependencies = [ "aes-gcm", "anyhow", diff --git a/app/src-tauri/vendor/tauri-cef b/app/src-tauri/vendor/tauri-cef index 9333fba46..c10bc0f72 160000 --- a/app/src-tauri/vendor/tauri-cef +++ b/app/src-tauri/vendor/tauri-cef @@ -1 +1 @@ -Subproject commit 9333fba463a038b513c6ecf72047da787c330fb8 +Subproject commit c10bc0f7293f6070478fc8af1a157004f5916f10 diff --git a/src/core/jsonrpc.rs b/src/core/jsonrpc.rs index 27924d99c..9a13361b3 100644 --- a/src/core/jsonrpc.rs +++ b/src/core/jsonrpc.rs @@ -919,6 +919,41 @@ pub async fn bootstrap_skill_runtime() { ); } + // --- Session storage layout migration ------------------------------- + // One-shot move from `session_raw/{DDMMYYYY}/` (≤ 0.53.4) to the new + // flat `session_raw/{stem}.jsonl` layout, plus DDMMYYYY → YYYY_MM_DD + // for the human-readable `sessions/` companions. Idempotent via a + // marker file at `state/migrations/session_layout_v1.done`, so this + // costs one stat() on every subsequent boot. + match crate::openhuman::agent::harness::session::migrate_session_layout_if_needed( + &workspace_dir, + ) { + Ok(outcome) if outcome.already_done => { + log::debug!("[runtime] session_layout migration already applied"); + } + Ok(outcome) => { + log::info!( + "[runtime] session_layout migration applied: jsonl_moved={} md_moved={} pruned_dirs={} warnings={}", + outcome.jsonl_moved, + outcome.md_moved, + outcome.legacy_dirs_pruned, + outcome.warnings.len(), + ); + for w in &outcome.warnings { + log::warn!("[runtime] session_layout migration warning: {w}"); + } + } + Err(err) => { + // Don't bring down startup over a transcript-storage migration. + // The transcript module's legacy fallback covers the unmigrated + // case for one release window. + log::warn!( + "[runtime] session_layout migration failed: {err} — \ + falling back to in-place legacy reads" + ); + } + } + // --- Socket manager bootstrap --- let socket_mgr = Arc::new(SocketManager::new()); set_global_socket_manager(socket_mgr.clone()); diff --git a/src/openhuman/agent/harness/session/migration.rs b/src/openhuman/agent/harness/session/migration.rs new file mode 100644 index 000000000..a543e95d1 --- /dev/null +++ b/src/openhuman/agent/harness/session/migration.rs @@ -0,0 +1,373 @@ +//! Session storage layout migration: date-grouped → flat `session_raw/`. +//! +//! Older releases (≤ 0.53.4) wrote transcripts to: +//! +//! ```text +//! {workspace}/session_raw/{DDMMYYYY}/{stem}.jsonl +//! {workspace}/sessions/{DDMMYYYY}/{stem}.md +//! ``` +//! +//! From 0.53.5 onwards the source of truth is the *flat* +//! `session_raw/{stem}.jsonl` and the human-readable companion is +//! `sessions/{YYYY_MM_DD}/{stem}.md` — see +//! [`super::transcript`] for the rationale (idle-thread resume +//! becomes date-independent). +//! +//! `find_latest_transcript` ships a fallback that reads the legacy +//! layout when the flat dir is empty, so users upgrading don't lose +//! resume even before this migration runs. This module performs the +//! one-shot move so files end up in their canonical location and the +//! transitional fallback can eventually be removed. +//! +//! ## Idempotency +//! +//! After a successful migration we write a marker at +//! `{workspace}/state/migrations/session_layout_v1.done`. Subsequent +//! starts read the marker and skip the scan entirely. If the workspace +//! has no legacy layout (fresh install or already migrated by an +//! external sync) we still write the marker so the scan stays +//! single-cost. +//! +//! ## Version gate +//! +//! The marker doubles as the "have we already migrated past 0.53.4?" +//! flag. A bare workspace with no legacy dirs and no marker is treated +//! as "fresh — nothing to do, write the marker." A workspace with +//! legacy dirs is treated as "upgrading from ≤ 0.53.4 — migrate then +//! write the marker." +//! +//! Failures are surfaced as warnings (logged) and **never panic**: +//! transcript files are valuable but not strictly required for +//! continued operation, and an uncatchable migration error would brick +//! every startup. + +use anyhow::{Context, Result}; +use std::fs; +use std::path::{Path, PathBuf}; + +/// Marker file that signals "the v1 session-layout migration ran +/// successfully on this workspace at least once". Written under +/// `state/migrations/` to keep the workspace root tidy. +const MIGRATION_MARKER: &str = "state/migrations/session_layout_v1.done"; + +#[derive(Debug, Default, Clone)] +pub struct MigrationOutcome { + pub jsonl_moved: usize, + pub jsonl_skipped: usize, + pub md_moved: usize, + pub md_skipped: usize, + pub legacy_dirs_pruned: usize, + pub already_done: bool, + pub warnings: Vec, +} + +/// Migrate the session storage layout for `workspace_dir` if needed. +/// +/// * Detects legacy `session_raw/{DDMMYYYY}/...jsonl` and +/// `sessions/{DDMMYYYY}/...md` layouts (i.e. an upgrade from +/// ≤ 0.53.4). +/// * Moves jsonl files to flat `session_raw/{stem}.jsonl`. +/// * Renames `DDMMYYYY` md dirs to ISO-style `YYYY_MM_DD` so the +/// listing sorts lexicographically. +/// * Writes the migration marker on success. +/// +/// Idempotent: returns immediately with `already_done = true` if the +/// marker already exists. Best-effort on individual file moves — +/// failures are logged and surfaced via `warnings`, not propagated, so +/// one bad rename can't brick startup. +pub fn migrate_session_layout_if_needed(workspace_dir: &Path) -> Result { + let marker_path = workspace_dir.join(MIGRATION_MARKER); + if marker_path.exists() { + log::debug!( + "[session-migration] marker present at {} — skipping", + marker_path.display() + ); + return Ok(MigrationOutcome { + already_done: true, + ..Default::default() + }); + } + + let mut outcome = MigrationOutcome::default(); + + let raw_root = workspace_dir.join("session_raw"); + if raw_root.is_dir() { + migrate_raw_jsonl(&raw_root, &mut outcome)?; + } + + let sessions_root = workspace_dir.join("sessions"); + if sessions_root.is_dir() { + migrate_md_directories(&sessions_root, &mut outcome)?; + } + + write_marker(&marker_path, &outcome).context("write session-migration marker")?; + + log::info!( + "[session-migration] complete: jsonl moved={} skipped={}, md moved={} skipped={}, legacy dirs pruned={}, warnings={}", + outcome.jsonl_moved, + outcome.jsonl_skipped, + outcome.md_moved, + outcome.md_skipped, + outcome.legacy_dirs_pruned, + outcome.warnings.len(), + ); + + Ok(outcome) +} + +/// Walk `session_raw/`, find direct subdirectories whose names look +/// like `DDMMYYYY` (8 ascii digits), and move every `*.jsonl` file +/// inside up to the flat `session_raw/` parent. Empty legacy dirs +/// are removed; non-empty ones are left in place with a warning so a +/// human can decide what to do. +fn migrate_raw_jsonl(raw_root: &Path, outcome: &mut MigrationOutcome) -> Result<()> { + let entries = match fs::read_dir(raw_root) { + Ok(it) => it, + Err(err) => { + outcome + .warnings + .push(format!("read_dir({}) failed: {err}", raw_root.display())); + return Ok(()); + } + }; + + for entry in entries.flatten() { + let path = entry.path(); + if !path.is_dir() { + continue; + } + let Some(name) = path.file_name().and_then(|s| s.to_str()) else { + continue; + }; + if !is_ddmmyyyy(name) { + // Not a legacy date dir — leave alone (could be the new + // flat layout's own files which would never be a dir, or + // a user-created subdirectory we shouldn't touch). + continue; + } + move_jsonl_files_up(&path, raw_root, outcome); + prune_if_empty(&path, outcome); + } + + Ok(()) +} + +fn move_jsonl_files_up(legacy_dir: &Path, flat_dir: &Path, outcome: &mut MigrationOutcome) { + let entries = match fs::read_dir(legacy_dir) { + Ok(it) => it, + Err(err) => { + outcome + .warnings + .push(format!("read_dir({}) failed: {err}", legacy_dir.display())); + return; + } + }; + + for entry in entries.flatten() { + let path = entry.path(); + if !path.is_file() { + continue; + } + if path.extension().and_then(|s| s.to_str()) != Some("jsonl") { + continue; + } + let Some(file_name) = path.file_name() else { + continue; + }; + let dest = flat_dir.join(file_name); + if dest.exists() { + // Same stem already lives in the flat dir — the new layout + // is authoritative for current sessions, so leave the + // legacy copy in place and surface a warning instead of + // overwriting newer data. + outcome.jsonl_skipped += 1; + outcome.warnings.push(format!( + "skip move: destination already exists at {} (legacy file kept at {})", + dest.display(), + path.display() + )); + continue; + } + match fs::rename(&path, &dest) { + Ok(()) => { + outcome.jsonl_moved += 1; + log::debug!( + "[session-migration] moved {} → {}", + path.display(), + dest.display() + ); + } + Err(err) => { + outcome.warnings.push(format!( + "rename({} → {}) failed: {err}", + path.display(), + dest.display() + )); + } + } + } +} + +/// Walk `sessions/`, rename each `DDMMYYYY` subdirectory to its +/// `YYYY_MM_DD` equivalent. We rename the dir wholesale rather than +/// copying file-by-file: the contents are human-readable companions +/// and don't need re-indexing. +fn migrate_md_directories(sessions_root: &Path, outcome: &mut MigrationOutcome) -> Result<()> { + let entries = match fs::read_dir(sessions_root) { + Ok(it) => it, + Err(err) => { + outcome.warnings.push(format!( + "read_dir({}) failed: {err}", + sessions_root.display() + )); + return Ok(()); + } + }; + + for entry in entries.flatten() { + let path = entry.path(); + if !path.is_dir() { + continue; + } + let Some(name) = path.file_name().and_then(|s| s.to_str()) else { + continue; + }; + let Some(iso) = ddmmyyyy_to_yyyy_mm_dd(name) else { + continue; + }; + let dest = sessions_root.join(&iso); + if dest.exists() { + // ISO dir already exists — merge file-by-file, never + // overwrite. A user could have legitimately produced both + // names (e.g. by manual workflow) so we don't blindly + // discard either side. + merge_md_dirs(&path, &dest, outcome); + prune_if_empty(&path, outcome); + continue; + } + match fs::rename(&path, &dest) { + Ok(()) => { + outcome.md_moved += 1; + log::debug!( + "[session-migration] renamed md dir {} → {}", + path.display(), + dest.display() + ); + } + Err(err) => { + outcome.warnings.push(format!( + "rename({} → {}) failed: {err}", + path.display(), + dest.display() + )); + } + } + } + + Ok(()) +} + +fn merge_md_dirs(legacy: &Path, dest: &Path, outcome: &mut MigrationOutcome) { + let entries = match fs::read_dir(legacy) { + Ok(it) => it, + Err(err) => { + outcome + .warnings + .push(format!("read_dir({}) failed: {err}", legacy.display())); + return; + } + }; + for entry in entries.flatten() { + let src = entry.path(); + if !src.is_file() { + continue; + } + let Some(file_name) = src.file_name() else { + continue; + }; + let target = dest.join(file_name); + if target.exists() { + outcome.md_skipped += 1; + outcome.warnings.push(format!( + "skip md merge: {} already exists (legacy at {} kept)", + target.display(), + src.display() + )); + continue; + } + match fs::rename(&src, &target) { + Ok(()) => outcome.md_moved += 1, + Err(err) => outcome.warnings.push(format!( + "rename({} → {}) failed: {err}", + src.display(), + target.display() + )), + } + } +} + +fn prune_if_empty(dir: &Path, outcome: &mut MigrationOutcome) { + match fs::read_dir(dir) { + Ok(mut it) => { + if it.next().is_some() { + // Non-empty — leave for human inspection. + return; + } + } + Err(_) => return, + } + if fs::remove_dir(dir).is_ok() { + outcome.legacy_dirs_pruned += 1; + } +} + +fn write_marker(marker_path: &Path, outcome: &MigrationOutcome) -> Result<()> { + if let Some(parent) = marker_path.parent() { + fs::create_dir_all(parent) + .with_context(|| format!("create marker dir {}", parent.display()))?; + } + let body = format!( + "openhuman session_layout migration v1\nrun_at: {}\njsonl_moved: {}\nmd_moved: {}\nlegacy_dirs_pruned: {}\nwarnings: {}\n", + chrono::Utc::now().to_rfc3339(), + outcome.jsonl_moved, + outcome.md_moved, + outcome.legacy_dirs_pruned, + outcome.warnings.len(), + ); + fs::write(marker_path, body) + .with_context(|| format!("write marker {}", marker_path.display()))?; + Ok(()) +} + +/// Returns true iff `name` is exactly 8 ASCII digits — the legacy +/// `DDMMYYYY` shape. We don't validate the date range (1–31, 1–12, +/// 1900–2100) because chrono printed the value originally, so any +/// real on-disk dir is well-formed; the digit shape is a sufficient +/// fingerprint to distinguish from user-created subdirectories. +fn is_ddmmyyyy(name: &str) -> bool { + name.len() == 8 && name.chars().all(|c| c.is_ascii_digit()) +} + +/// Convert `DDMMYYYY` → `YYYY_MM_DD`. Returns `None` if the input +/// isn't 8 digits. +fn ddmmyyyy_to_yyyy_mm_dd(name: &str) -> Option { + if !is_ddmmyyyy(name) { + return None; + } + let dd = &name[0..2]; + let mm = &name[2..4]; + let yyyy = &name[4..8]; + Some(format!("{yyyy}_{mm}_{dd}")) +} + +/// Returns the path of the migration marker for `workspace_dir`. +/// Exposed for tests and CLI tooling that wants to manually re-run +/// the migration (delete the marker, then call +/// [`migrate_session_layout_if_needed`] again). +pub fn marker_path_for(workspace_dir: &Path) -> PathBuf { + workspace_dir.join(MIGRATION_MARKER) +} + +#[cfg(test)] +#[path = "migration_tests.rs"] +mod tests; diff --git a/src/openhuman/agent/harness/session/migration_tests.rs b/src/openhuman/agent/harness/session/migration_tests.rs new file mode 100644 index 000000000..e1b5c4151 --- /dev/null +++ b/src/openhuman/agent/harness/session/migration_tests.rs @@ -0,0 +1,170 @@ +use super::*; +use std::fs; +use tempfile::TempDir; + +fn write_file(path: &std::path::Path, body: &str) { + fs::create_dir_all(path.parent().unwrap()).unwrap(); + fs::write(path, body).unwrap(); +} + +#[test] +fn fresh_workspace_writes_marker_with_no_moves() { + let dir = TempDir::new().unwrap(); + let outcome = migrate_session_layout_if_needed(dir.path()).unwrap(); + assert!(!outcome.already_done); + assert_eq!(outcome.jsonl_moved, 0); + assert_eq!(outcome.md_moved, 0); + assert!(marker_path_for(dir.path()).exists()); +} + +#[test] +fn second_run_is_a_noop() { + let dir = TempDir::new().unwrap(); + let _first = migrate_session_layout_if_needed(dir.path()).unwrap(); + let second = migrate_session_layout_if_needed(dir.path()).unwrap(); + assert!(second.already_done); + assert_eq!(second.jsonl_moved, 0); + assert_eq!(second.warnings.len(), 0); +} + +#[test] +fn moves_legacy_jsonl_files_up_to_flat_session_raw() { + let dir = TempDir::new().unwrap(); + let ws = dir.path(); + let legacy_a = ws.join("session_raw").join("01052026"); + let legacy_b = ws.join("session_raw").join("02052026"); + write_file(&legacy_a.join("1714000000_main.jsonl"), "a"); + write_file(&legacy_a.join("1714000001_welcome.jsonl"), "b"); + write_file(&legacy_b.join("1714999999_orchestrator.jsonl"), "c"); + + let outcome = migrate_session_layout_if_needed(ws).unwrap(); + assert_eq!(outcome.jsonl_moved, 3); + assert_eq!(outcome.legacy_dirs_pruned, 2); + + let raw_root = ws.join("session_raw"); + assert!(raw_root.join("1714000000_main.jsonl").exists()); + assert!(raw_root.join("1714000001_welcome.jsonl").exists()); + assert!(raw_root.join("1714999999_orchestrator.jsonl").exists()); + // Empty legacy date dirs should have been pruned. + assert!(!legacy_a.exists(), "legacy date dir should be removed"); + assert!(!legacy_b.exists(), "legacy date dir should be removed"); +} + +#[test] +fn jsonl_destination_collision_is_skipped_with_warning() { + // If a flat `session_raw/{stem}.jsonl` already exists for the + // same stem we don't overwrite — the flat copy is authoritative + // (the user may have already started a fresh session with the + // same key after a clock reset). Surface a warning instead. + let dir = TempDir::new().unwrap(); + let ws = dir.path(); + let raw_root = ws.join("session_raw"); + write_file(&raw_root.join("1714000000_main.jsonl"), "new"); + write_file( + &raw_root.join("01052026").join("1714000000_main.jsonl"), + "old", + ); + + let outcome = migrate_session_layout_if_needed(ws).unwrap(); + assert_eq!(outcome.jsonl_moved, 0); + assert_eq!(outcome.jsonl_skipped, 1); + assert!(outcome + .warnings + .iter() + .any(|w| w.contains("already exists"))); + // Both files still exist — nothing was overwritten. + assert_eq!( + fs::read_to_string(raw_root.join("1714000000_main.jsonl")).unwrap(), + "new" + ); + assert_eq!( + fs::read_to_string(raw_root.join("01052026").join("1714000000_main.jsonl")).unwrap(), + "old" + ); +} + +#[test] +fn renames_md_ddmmyyyy_dirs_to_iso() { + let dir = TempDir::new().unwrap(); + let ws = dir.path(); + let legacy_md = ws.join("sessions").join("01052026"); + write_file(&legacy_md.join("main_0.md"), "x"); + write_file(&legacy_md.join("main_1.md"), "y"); + + let outcome = migrate_session_layout_if_needed(ws).unwrap(); + assert_eq!(outcome.md_moved, 1, "one rename of the dir as a whole"); + let iso = ws.join("sessions").join("2026_05_01"); + assert!(iso.is_dir()); + assert!(iso.join("main_0.md").exists()); + assert!(iso.join("main_1.md").exists()); + assert!( + !legacy_md.exists(), + "DDMMYYYY md dir should be gone after rename" + ); +} + +#[test] +fn merges_md_when_iso_dir_already_exists() { + let dir = TempDir::new().unwrap(); + let ws = dir.path(); + // Both layouts coexist for the same calendar date — e.g. user + // ran a hand-edited build that produced ISO dirs alongside the + // legacy DDMMYYYY ones. + let legacy = ws.join("sessions").join("01052026"); + let iso = ws.join("sessions").join("2026_05_01"); + write_file(&legacy.join("main_0.md"), "legacy"); + write_file(&legacy.join("main_1.md"), "legacy"); + write_file(&iso.join("main_1.md"), "newer"); + + let outcome = migrate_session_layout_if_needed(ws).unwrap(); + // main_0.md moves over (no collision); main_1.md collides and is + // skipped without overwriting the newer copy. + assert_eq!(outcome.md_moved, 1); + assert_eq!(outcome.md_skipped, 1); + assert_eq!(fs::read_to_string(iso.join("main_0.md")).unwrap(), "legacy"); + assert_eq!(fs::read_to_string(iso.join("main_1.md")).unwrap(), "newer"); +} + +#[test] +fn ignores_non_date_subdirectories_in_session_raw() { + // Defensive: a user (or some other tool) might have created a + // sibling dir under session_raw/. We must not touch it — only + // 8-digit names are recognised as legacy date dirs. + let dir = TempDir::new().unwrap(); + let ws = dir.path(); + let weird = ws.join("session_raw").join("my_notes"); + write_file(&weird.join("random.jsonl"), "keep me"); + + let outcome = migrate_session_layout_if_needed(ws).unwrap(); + assert_eq!(outcome.jsonl_moved, 0); + assert!(weird.is_dir(), "non-date subdir must be left alone"); + assert!(weird.join("random.jsonl").exists()); +} + +#[test] +fn ddmmyyyy_to_iso_handles_boundary_dates() { + assert_eq!( + ddmmyyyy_to_yyyy_mm_dd("01012026").as_deref(), + Some("2026_01_01") + ); + assert_eq!( + ddmmyyyy_to_yyyy_mm_dd("31122099").as_deref(), + Some("2099_12_31") + ); + assert!(ddmmyyyy_to_yyyy_mm_dd("abc12345").is_none()); + assert!(ddmmyyyy_to_yyyy_mm_dd("1234567").is_none(), "7 digits"); + assert!(ddmmyyyy_to_yyyy_mm_dd("123456789").is_none(), "9 digits"); +} + +#[test] +fn marker_persists_run_metadata() { + let dir = TempDir::new().unwrap(); + let ws = dir.path(); + let legacy = ws.join("session_raw").join("01052026"); + write_file(&legacy.join("1714000000_main.jsonl"), "a"); + + migrate_session_layout_if_needed(ws).unwrap(); + let marker = fs::read_to_string(marker_path_for(ws)).unwrap(); + assert!(marker.contains("jsonl_moved: 1")); + assert!(marker.contains("openhuman session_layout migration v1")); +} diff --git a/src/openhuman/agent/harness/session/mod.rs b/src/openhuman/agent/harness/session/mod.rs index 4f7f37225..7e69c3ac7 100644 --- a/src/openhuman/agent/harness/session/mod.rs +++ b/src/openhuman/agent/harness/session/mod.rs @@ -21,11 +21,14 @@ //! The child files are an implementation detail. mod builder; +pub mod migration; mod runtime; pub(crate) mod transcript; mod turn; mod types; +pub use migration::{migrate_session_layout_if_needed, MigrationOutcome}; + #[cfg(test)] mod tests; diff --git a/src/openhuman/agent/harness/session/transcript.rs b/src/openhuman/agent/harness/session/transcript.rs index d8963a91c..1ece5bb19 100644 --- a/src/openhuman/agent/harness/session/transcript.rs +++ b/src/openhuman/agent/harness/session/transcript.rs @@ -1,12 +1,34 @@ //! Session transcript persistence for KV cache stability. //! -//! **Source of truth**: `session_raw/{DDMMYYYY}/{agent}_{index}.jsonl` +//! **Source of truth**: `session_raw/{stem}.jsonl` — a *flat* directory. //! //! Each JSONL file starts with a single metadata line (identified by an //! `_meta` key) followed by one JSON object per `ChatMessage`. On every //! write the companion `.md` file is re-rendered for human readability -//! under `sessions/{DDMMYYYY}/{agent}_{index}.md`; it is **never** read -//! back — all round-trip / resume logic uses the JSONL. +//! under `sessions/{YYYY_MM_DD}/{stem}.md`; it is **never** read back — +//! all round-trip / resume logic uses the JSONL. +//! +//! ## Storage layout +//! +//! ```text +//! {workspace}/session_raw/{stem}.jsonl ← source of truth (flat) +//! {workspace}/sessions/YYYY_MM_DD/{stem}.md ← human-readable view +//! ``` +//! +//! `stem` is `{unix_ts}_{agent_id}` for a root session, or +//! `{parent_chain}__{unix_ts}_{agent_id}` for a sub-agent. Because the +//! stem starts with the unix timestamp at agent-build time, a directory +//! listing of `session_raw/` is naturally sorted by creation time and +//! `find_latest_transcript` becomes O(scan one dir, filter by suffix) +//! — it does not depend on the calendar date, so a session that's been +//! idle for weeks resumes the same way as one from yesterday. +//! +//! ## Backward compatibility +//! +//! Older releases wrote into `session_raw/DDMMYYYY/{stem}.jsonl` (and +//! the legacy `sessions/DDMMYYYY/{stem}.md`). [`find_latest_transcript`] +//! falls back to scanning those date-grouped dirs when the flat +//! directory yields nothing, so users upgrading don't lose resume. //! //! ## JSONL schema //! @@ -25,13 +47,6 @@ //! //! Only `role` and `content` are required. All other fields are optional. //! Unknown fields on read are ignored (forward-compat). -//! -//! ## Storage layout -//! -//! ```text -//! {workspace}/session_raw/DDMMYYYY/{agent}_{index}.jsonl ← source of truth -//! {workspace}/sessions/DDMMYYYY/{agent}_{index}.md ← human-readable view -//! ``` use crate::openhuman::providers::ChatMessage; use anyhow::{Context, Result}; @@ -363,25 +378,19 @@ fn read_transcript_jsonl(path: &Path) -> Result { // ── Path resolution ────────────────────────────────────────────────── -/// Resolve a new transcript path under -/// `{workspace}/session_raw/DDMMYYYY/{agent}_{index}.jsonl`. +/// Resolve a transcript path under `session_raw/{stem}.jsonl` — a +/// *flat* directory keyed only by stem. Used by the session-key flow: +/// the stem is `"{unix_ts}_{agent_id}"` for a root session, or +/// `"{parent_chain}__{session_key}"` for a sub-agent, so nested +/// delegations still produce a single flat filename that encodes the +/// parent → child path. /// -/// Creates the date directory if needed. Index = max existing + 1. -/// Scans both the new `session_raw/` dir (for `.jsonl`) **and** the legacy -/// `sessions/` dir (for `.md`) so indices stay unique across migration. -/// Resolve a transcript path under `session_raw/DDMMYYYY/{stem}.jsonl` -/// where `stem` is deterministic (no auto-indexing). Used by the new -/// session-key flow: the session-key stem is `"{unix_ts}_{agent_id}"` -/// for a root session, or `"{parent_chain}__{session_key}"` for a -/// sub-agent — so nested delegations produce a single flat filename -/// that encodes the parent → child path. -/// -/// Creates the date directory if needed. Overwrites are intentional: -/// the Agent persists the same transcript file across every turn of a +/// Creates the directory if needed. Overwrites are intentional: the +/// `Agent` persists the same transcript file across every turn of a /// session, and every sub-agent spawn gets a unique timestamp in its /// own key so collisions are effectively impossible. pub fn resolve_keyed_transcript_path(workspace_dir: &Path, stem: &str) -> Result { - let raw_dir = today_raw_session_dir(workspace_dir); + let raw_dir = raw_session_dir(workspace_dir); fs::create_dir_all(&raw_dir) .with_context(|| format!("create session_raw dir {}", raw_dir.display()))?; let sanitized = sanitize_stem(stem); @@ -389,9 +398,9 @@ pub fn resolve_keyed_transcript_path(workspace_dir: &Path, stem: &str) -> Result } /// Sanitize a user-supplied transcript stem so it never escapes the -/// `session_raw/DDMMYYYY/` directory. Allows ASCII alphanumerics plus -/// a small punctuation set (`_`, `-`, `.`); every other byte is -/// replaced with `_`. Empty inputs fall back to `"session"`. +/// `session_raw/` directory. Allows ASCII alphanumerics plus a small +/// punctuation set (`_`, `-`, `.`); every other byte is replaced with +/// `_`. Empty inputs fall back to `"session"`. fn sanitize_stem(stem: &str) -> String { let cleaned: String = stem .chars() @@ -411,17 +420,17 @@ fn sanitize_stem(stem: &str) -> String { } pub fn resolve_new_transcript_path(workspace_dir: &Path, agent_name: &str) -> Result { - let raw_dir = today_raw_session_dir(workspace_dir); + let raw_dir = raw_session_dir(workspace_dir); fs::create_dir_all(&raw_dir) .with_context(|| format!("create session_raw dir {}", raw_dir.display()))?; let sanitized = sanitize_agent_name(agent_name); let idx_raw = next_index(&raw_dir, &sanitized)?; - // Also consider the legacy sessions/ dir so companion .md (or legacy - // standalone .md) files don't cause index collisions. - let legacy_dir = today_session_dir(workspace_dir); - let idx_legacy = next_index(&legacy_dir, &sanitized)?; - let next_idx = idx_raw.max(idx_legacy); + // Also consider today's md companion dir so a stale .md from this + // session doesn't cause an index collision when only .md exists. + let md_dir = today_md_session_dir(workspace_dir); + let idx_md = next_index(&md_dir, &sanitized)?; + let next_idx = idx_raw.max(idx_md); let filename = format!("{}_{}.jsonl", sanitized, next_idx); Ok(raw_dir.join(filename)) @@ -429,28 +438,43 @@ pub fn resolve_new_transcript_path(workspace_dir: &Path, agent_name: &str) -> Re /// Find the most recent transcript for `agent_name`. /// -/// Searches today's directory first, then yesterday's. Prefers `.jsonl` -/// under `session_raw/` and falls back to `.md` under `sessions/` when -/// only legacy files exist. +/// **Primary**: scan the flat `session_raw/` directory and pick the +/// newest matching stem (root sessions only — sub-agents are skipped). +/// **Fallback**: scan the legacy `session_raw/DDMMYYYY/` dirs (today +/// and yesterday) and the legacy `sessions/DDMMYYYY/` markdown dirs so +/// users upgrading from the date-grouped layout don't lose resume. +/// The fallback is one-release transitional and can be removed once +/// existing transcripts have rolled forward. pub fn find_latest_transcript(workspace_dir: &Path, agent_name: &str) -> Option { let sanitized = sanitize_agent_name(agent_name); let raw_root = workspace_dir.join("session_raw"); let sessions_root = workspace_dir.join("sessions"); + // Primary path: flat session_raw/ directory. The stem-suffix scan + // is naturally date-independent, so an idle thread resumes the same + // way today as it did weeks ago. + if raw_root.is_dir() { + if let Some(path) = latest_in_dir(&raw_root, &sanitized) { + return Some(path); + } + } + + // Fallback: legacy date-grouped layout (one-release migration + // window). Today first, then yesterday — matches the previous + // behaviour so we don't regress while users still have files in + // the old structure. let today = chrono::Local::now().format("%d%m%Y").to_string(); let yesterday = (chrono::Local::now() - chrono::Duration::days(1)) .format("%d%m%Y") .to_string(); for date_str in [&today, &yesterday] { - // Prefer the new session_raw/ location. let raw_dir = raw_root.join(date_str); if raw_dir.is_dir() { if let Some(path) = latest_in_dir(&raw_dir, &sanitized) { return Some(path); } } - // Fall back to legacy sessions/ for pre-migration .md only. let legacy_dir = sessions_root.join(date_str); if legacy_dir.is_dir() { if let Some(path) = latest_in_dir(&legacy_dir, &sanitized) { @@ -645,38 +669,62 @@ fn parse_legacy_messages(raw: &str) -> Result> { // ── Private helpers ─────────────────────────────────────────────────── -fn today_session_dir(workspace_dir: &Path) -> PathBuf { - let date = chrono::Local::now().format("%d%m%Y").to_string(); +/// Date-grouped directory for human-readable `.md` companions, e.g. +/// `{workspace}/sessions/2026_05_02`. ISO-style `YYYY_MM_DD` so the +/// listing sorts lexicographically by date. +fn today_md_session_dir(workspace_dir: &Path) -> PathBuf { + let date = chrono::Local::now().format("%Y_%m_%d").to_string(); workspace_dir.join("sessions").join(date) } -fn today_raw_session_dir(workspace_dir: &Path) -> PathBuf { - let date = chrono::Local::now().format("%d%m%Y").to_string(); - workspace_dir.join("session_raw").join(date) +/// Flat directory for the JSONL source of truth, e.g. +/// `{workspace}/session_raw`. Stems start with `{unix_ts}` so the +/// listing is naturally time-ordered without a date subdirectory. +fn raw_session_dir(workspace_dir: &Path) -> PathBuf { + workspace_dir.join("session_raw") } -/// Given a `session_raw/DDMMYYYY/agent_N.jsonl` path, derive the companion -/// `sessions/DDMMYYYY/agent_N.md` path by swapping the `session_raw` -/// path component with `sessions` and the extension with `.md`. +/// Given a `session_raw/{stem}.jsonl` path, derive the companion +/// `sessions/YYYY_MM_DD/{stem}.md` path. The date is taken from the +/// local clock at write time — fine for browsing because the source +/// of truth lives in the flat raw dir; the `.md` is purely a view. /// -/// If no `session_raw` component is present (e.g. tests using a flat -/// tempdir), just swap the extension so the companion lives alongside. +/// Legacy `session_raw/DDMMYYYY/{stem}.jsonl` paths (still on disk +/// from older releases until they roll forward) keep their date +/// component when generating the companion so we don't accidentally +/// stamp old transcripts with today's date. +/// +/// If no `session_raw` component is present (tests using a flat +/// tempdir), the companion sits alongside as a sibling `.md`. fn md_companion_path(jsonl_path: &Path) -> PathBuf { - let mut out = PathBuf::new(); - let mut swapped = false; - for comp in jsonl_path.components() { - match comp { - std::path::Component::Normal(s) if s == "session_raw" => { - out.push("sessions"); - swapped = true; - } - other => out.push(other.as_os_str()), - } - } - if !swapped { - // Fall back to sibling .md in the same directory. + let components: Vec<_> = jsonl_path.components().collect(); + + let raw_idx = components + .iter() + .position(|comp| matches!(comp, std::path::Component::Normal(s) if *s == "session_raw")); + + let Some(raw_idx) = raw_idx else { return jsonl_path.with_extension("md"); + }; + + let mut out = PathBuf::new(); + for comp in &components[..raw_idx] { + out.push(comp.as_os_str()); } + out.push("sessions"); + + // Tail after `session_raw`: + // * Flat: ["{stem}.jsonl"] — prepend today's YYYY_MM_DD. + // * Legacy: ["DDMMYYYY", "{stem}.jsonl"] — keep the existing + // date dir so we don't relabel old transcripts. + let tail = &components[raw_idx + 1..]; + if tail.len() <= 1 { + out.push(chrono::Local::now().format("%Y_%m_%d").to_string()); + } + for comp in tail { + out.push(comp.as_os_str()); + } + out.with_extension("md") } diff --git a/src/openhuman/agent/harness/session/transcript_tests.rs b/src/openhuman/agent/harness/session/transcript_tests.rs index f5383ff4b..e419b71e7 100644 --- a/src/openhuman/agent/harness/session/transcript_tests.rs +++ b/src/openhuman/agent/harness/session/transcript_tests.rs @@ -105,32 +105,59 @@ fn meta_round_trip() { } #[test] -fn path_resolution_creates_dir_and_increments_index() { +fn path_resolution_creates_flat_session_raw_dir_and_increments_index() { let dir = TempDir::new().unwrap(); let workspace = dir.path(); let path0 = resolve_new_transcript_path(workspace, "main").unwrap(); assert!(path0.to_string_lossy().contains("main_0.jsonl")); + // Flat layout: jsonl lives directly under session_raw/, no date dir. + let parent = path0.parent().unwrap(); assert!( - path0.to_string_lossy().contains("session_raw"), - "jsonl should live under session_raw/, got {}", - path0.display() + parent.ends_with("session_raw"), + "jsonl parent should be session_raw/ (flat layout), got {}", + parent.display() ); - // Write something so the next call sees it. fs::write(&path0, "placeholder").unwrap(); let path1 = resolve_new_transcript_path(workspace, "main").unwrap(); assert!(path1.to_string_lossy().contains("main_1.jsonl")); + assert!(path1.parent().unwrap().ends_with("session_raw")); } #[test] -fn md_companion_path_swaps_session_raw_to_sessions() { +fn resolve_keyed_writes_to_flat_session_raw() { + let dir = TempDir::new().unwrap(); + let path = resolve_keyed_transcript_path(dir.path(), "1714000000_orchestrator").unwrap(); + assert_eq!(path.parent().unwrap(), dir.path().join("session_raw")); + assert!(path + .to_string_lossy() + .ends_with("1714000000_orchestrator.jsonl")); +} + +#[test] +fn md_companion_path_for_flat_jsonl_uses_iso_date_dir() { + let jsonl = PathBuf::from("/tmp/ws/session_raw/1714000000_main.jsonl"); + let md = md_companion_path(&jsonl); + let today = chrono::Local::now().format("%Y_%m_%d").to_string(); + assert_eq!( + md, + PathBuf::from(format!("/tmp/ws/sessions/{today}/1714000000_main.md")), + "flat session_raw should map to sessions/YYYY_MM_DD/ on the md side" + ); +} + +#[test] +fn md_companion_path_preserves_legacy_ddmmyyyy_dir() { + // A pre-migration jsonl at session_raw/DDMMYYYY/{stem}.jsonl should + // keep its date component so old transcripts aren't relabeled with + // today's date. let jsonl = PathBuf::from("/tmp/ws/session_raw/17042026/main_0.jsonl"); let md = md_companion_path(&jsonl); assert_eq!( md, PathBuf::from("/tmp/ws/sessions/17042026/main_0.md"), - "md companion should live under sessions/ with .md extension" + "legacy date-grouped raw paths must keep their original date dir" ); } @@ -142,19 +169,19 @@ fn md_companion_path_falls_back_to_sibling_when_no_session_raw_component() { } #[test] -fn resolve_avoids_index_collision_with_legacy_md_in_sessions_dir() { +fn resolve_avoids_index_collision_with_md_in_iso_date_dir() { let dir = TempDir::new().unwrap(); let workspace = dir.path(); - let date = chrono::Local::now().format("%d%m%Y").to_string(); - let legacy = workspace.join("sessions").join(&date); - fs::create_dir_all(&legacy).unwrap(); - fs::write(legacy.join("main_0.md"), "legacy").unwrap(); - fs::write(legacy.join("main_1.md"), "legacy").unwrap(); + let date = chrono::Local::now().format("%Y_%m_%d").to_string(); + let md_dir = workspace.join("sessions").join(&date); + fs::create_dir_all(&md_dir).unwrap(); + fs::write(md_dir.join("main_0.md"), "x").unwrap(); + fs::write(md_dir.join("main_1.md"), "x").unwrap(); let path = resolve_new_transcript_path(workspace, "main").unwrap(); assert!( path.to_string_lossy().contains("main_2.jsonl"), - "should advance past legacy indices, got {}", + "should advance past md indices in today's YYYY_MM_DD dir, got {}", path.display() ); } @@ -167,10 +194,9 @@ fn sanitize_agent_name_strips_special_chars() { } #[test] -fn find_latest_returns_highest_index() { +fn find_latest_scans_flat_session_raw_dir() { let dir = TempDir::new().unwrap(); - let date = chrono::Local::now().format("%d%m%Y").to_string(); - let raw_dir = dir.path().join("session_raw").join(&date); + let raw_dir = dir.path().join("session_raw"); fs::create_dir_all(&raw_dir).unwrap(); fs::write(raw_dir.join("main_0.jsonl"), "a").unwrap(); @@ -178,11 +204,66 @@ fn find_latest_returns_highest_index() { fs::write(raw_dir.join("main_1.jsonl"), "b").unwrap(); fs::write(raw_dir.join("other_0.jsonl"), "x").unwrap(); - let latest = find_latest_transcript(dir.path(), "main"); - assert!(latest.is_some()); - let latest = latest.unwrap(); - assert!(latest.to_string_lossy().contains("main_2.jsonl")); - assert!(latest.to_string_lossy().contains("session_raw")); + let latest = find_latest_transcript(dir.path(), "main").unwrap(); + assert!(latest.to_string_lossy().ends_with("main_2.jsonl")); + assert_eq!(latest.parent().unwrap(), raw_dir); +} + +#[test] +fn find_latest_picks_newest_keyed_stem_in_flat_dir() { + let dir = TempDir::new().unwrap(); + let raw_dir = dir.path().join("session_raw"); + fs::create_dir_all(&raw_dir).unwrap(); + + // Keyed stem layout: `{unix_ts}_{agent_id}.jsonl`. + fs::write(raw_dir.join("1714000000_main.jsonl"), "old").unwrap(); + fs::write(raw_dir.join("1714999999_main.jsonl"), "new").unwrap(); + // Sub-agent transcripts (contain `__`) must be skipped. + fs::write( + raw_dir.join("1714000000_orchestrator__1714500000_planner.jsonl"), + "sub", + ) + .unwrap(); + + let latest = find_latest_transcript(dir.path(), "main").unwrap(); + assert!(latest.to_string_lossy().ends_with("1714999999_main.jsonl")); +} + +#[test] +fn find_latest_falls_back_to_legacy_ddmmyyyy_raw_dir() { + // Pre-migration transcript at session_raw/DDMMYYYY/main_*.jsonl + // must still resolve via the legacy fallback when the flat dir is + // empty. + let dir = TempDir::new().unwrap(); + let date = chrono::Local::now().format("%d%m%Y").to_string(); + let legacy_raw = dir.path().join("session_raw").join(&date); + fs::create_dir_all(&legacy_raw).unwrap(); + fs::write(legacy_raw.join("main_5.jsonl"), "legacy").unwrap(); + + let latest = find_latest_transcript(dir.path(), "main").unwrap(); + assert!(latest.to_string_lossy().ends_with("main_5.jsonl")); + assert!(latest.to_string_lossy().contains(&date)); +} + +#[test] +fn find_latest_prefers_flat_over_legacy_ddmmyyyy() { + let dir = TempDir::new().unwrap(); + let raw_root = dir.path().join("session_raw"); + fs::create_dir_all(&raw_root).unwrap(); + fs::write(raw_root.join("main_9.jsonl"), "flat").unwrap(); + + let date = chrono::Local::now().format("%d%m%Y").to_string(); + let legacy_raw = raw_root.join(&date); + fs::create_dir_all(&legacy_raw).unwrap(); + fs::write(legacy_raw.join("main_99.jsonl"), "legacy").unwrap(); + + let latest = find_latest_transcript(dir.path(), "main").unwrap(); + // Flat dir takes precedence so newly-created sessions always win + // over stale legacy files — even when a legacy file has a higher + // numeric index. The flat dir is the canonical layout going + // forward. + assert_eq!(latest.parent().unwrap(), raw_root); + assert!(latest.to_string_lossy().ends_with("main_9.jsonl")); } #[test]