From 5fcde25320b6ef86ab7535bf79276826fdbc5460 Mon Sep 17 00:00:00 2001 From: Steven Enamakel <31011319+senamakel@users.noreply.github.com> Date: Wed, 22 Jul 2026 08:27:22 +0300 Subject: [PATCH] feat(dev): library-mode benchmark environment, fleet/budget gates, and profiling optimizations (#5107) --- Cargo.lock | 41 +- Cargo.toml | 16 + app/src-tauri/profiling/.gitignore | 1 + app/src-tauri/profiling/Cargo.lock | 292 +++++++ app/src-tauri/profiling/Cargo.toml | 14 + app/src-tauri/profiling/README.md | 54 ++ app/src-tauri/profiling/src/main.rs | 813 ++++++++++++++++++ docs/harness-comparison-2026-07-22.md | 101 +++ docs/library-benchmarking.md | 345 ++++++++ docs/library-minimal-recipe.md | 246 ++++++ docs/resource-profiling-session-2026-07-21.md | 503 +++++++++++ docs/tauri-live-profiling-brief.md | 85 ++ package.json | 1 + scripts/profile/README.md | 113 +++ scripts/profile/library-bench.sh | 310 +++++++ scripts/profile/library-cpu.sh | 104 +++ scripts/profile/library-fleet.sh | 371 ++++++++ scripts/profile/library-heap.sh | 93 ++ scripts/profile/library-instances.sh | 548 ++++++++++++ src/bin/library_profile/harness.rs | 495 +++++++++++ src/bin/library_profile/main.rs | 132 +++ src/bin/library_profile/mock.rs | 570 ++++++++++++ .../library_profile/scenarios/agent_turn.rs | 36 + .../library_profile/scenarios/cold_phases.rs | 79 ++ src/bin/library_profile/scenarios/fleet.rs | 300 +++++++ .../library_profile/scenarios/long_agent.rs | 58 ++ .../scenarios/memory_ingest.rs | 56 ++ src/bin/library_profile/scenarios/mod.rs | 13 + .../library_profile/scenarios/skill_run.rs | 149 ++++ .../scenarios/subagent_storm.rs | 191 ++++ .../library_profile/scenarios/subagents.rs | 56 ++ .../library_profile/scenarios/subconscious.rs | 56 ++ src/bin/library_profile/scenarios/workflow.rs | 75 ++ src/openhuman/agent/prompts/render_helpers.rs | 10 + src/openhuman/inference/provider/factory.rs | 22 +- .../provider/ops/provider_factory.rs | 4 +- src/openhuman/proc_metrics/mod.rs | 218 ++++- src/openhuman/proc_metrics/tree.rs | 419 +++++++++ vendor/tinycortex | 2 +- 39 files changed, 6965 insertions(+), 27 deletions(-) create mode 100644 app/src-tauri/profiling/.gitignore create mode 100644 app/src-tauri/profiling/Cargo.lock create mode 100644 app/src-tauri/profiling/Cargo.toml create mode 100644 app/src-tauri/profiling/README.md create mode 100644 app/src-tauri/profiling/src/main.rs create mode 100644 docs/harness-comparison-2026-07-22.md create mode 100644 docs/library-benchmarking.md create mode 100644 docs/library-minimal-recipe.md create mode 100644 docs/resource-profiling-session-2026-07-21.md create mode 100644 docs/tauri-live-profiling-brief.md create mode 100644 scripts/profile/README.md create mode 100755 scripts/profile/library-bench.sh create mode 100755 scripts/profile/library-cpu.sh create mode 100755 scripts/profile/library-fleet.sh create mode 100755 scripts/profile/library-heap.sh create mode 100755 scripts/profile/library-instances.sh create mode 100644 src/bin/library_profile/harness.rs create mode 100644 src/bin/library_profile/main.rs create mode 100644 src/bin/library_profile/mock.rs create mode 100644 src/bin/library_profile/scenarios/agent_turn.rs create mode 100644 src/bin/library_profile/scenarios/cold_phases.rs create mode 100644 src/bin/library_profile/scenarios/fleet.rs create mode 100644 src/bin/library_profile/scenarios/long_agent.rs create mode 100644 src/bin/library_profile/scenarios/memory_ingest.rs create mode 100644 src/bin/library_profile/scenarios/mod.rs create mode 100644 src/bin/library_profile/scenarios/skill_run.rs create mode 100644 src/bin/library_profile/scenarios/subagent_storm.rs create mode 100644 src/bin/library_profile/scenarios/subagents.rs create mode 100644 src/bin/library_profile/scenarios/subconscious.rs create mode 100644 src/bin/library_profile/scenarios/workflow.rs create mode 100644 src/openhuman/proc_metrics/tree.rs diff --git a/Cargo.lock b/Cargo.lock index d2e9023c6..ea9bc745a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -533,7 +533,7 @@ dependencies = [ "proc-macro2", "quote", "regex", - "rustc-hash", + "rustc-hash 2.1.2", "shlex", "syn 2.0.117", ] @@ -1738,6 +1738,22 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "dhat" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "98cd11d84628e233de0ce467de10b8633f4ddaecafadefc86e13b84b8739b827" +dependencies = [ + "backtrace", + "lazy_static", + "mintex", + "parking_lot", + "rustc-hash 1.1.0", + "serde", + "serde_json", + "thousands", +] + [[package]] name = "digest" version = "0.10.7" @@ -4003,6 +4019,12 @@ dependencies = [ "simd-adler32", ] +[[package]] +name = "mintex" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c505b3e17ed6b70a7ed2e67fbb2c560ee327353556120d6e72f5232b6880d536" + [[package]] name = "mio" version = "1.2.0" @@ -4659,6 +4681,7 @@ dependencies = [ "cron", "crossterm", "curve25519-dalek", + "dhat", "directories", "dirs 5.0.1", "docx-rs", @@ -5499,7 +5522,7 @@ dependencies = [ "pin-project-lite", "quinn-proto", "quinn-udp", - "rustc-hash", + "rustc-hash 2.1.2", "rustls", "socket2", "thiserror 2.0.18", @@ -5520,7 +5543,7 @@ dependencies = [ "lru-slab", "rand 0.9.4", "ring", - "rustc-hash", + "rustc-hash 2.1.2", "rustls", "rustls-pki-types", "slab", @@ -6084,6 +6107,12 @@ version = "0.1.27" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b50b8869d9fc858ce7266cce0194bd74df58b9d0e3f6df3a9fc8eb470d95c09d" +[[package]] +name = "rustc-hash" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" + [[package]] name = "rustc-hash" version = "2.1.2" @@ -7232,6 +7261,12 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "thousands" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3bf63baf9f5039dadc247375c29eb13706706cfde997d0330d05aa63a77d8820" + [[package]] name = "thread_local" version = "1.1.9" diff --git a/Cargo.toml b/Cargo.toml index 70d41d6c0..5c092ad5d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -54,6 +54,13 @@ name = "rss-bench" path = "src/bin/rss_bench.rs" required-features = ["rss-bench"] +# Stateful library profiling workloads (memory ingestion + real sub-agent +# delegation under a hermetic mock provider). Local/dev only. +[[bin]] +name = "library-profile" +path = "src/bin/library_profile/main.rs" +required-features = ["rss-bench"] + [lib] name = "openhuman_core" crate-type = ["rlib"] @@ -112,6 +119,10 @@ serde = { version = "1", features = ["derive"] } serde_json = "1" serde_repr = "0.1" serde_yaml = "0.9" +# dhat — heap profiler for the offline `library-profile` benchmark binary only. +# Default-OFF, dev-only: pulled in solely by the `rss-bench-dhat` feature so it +# never enters the shipped desktop/library build (same posture as `rss-bench`). +dhat = { version = "0.3", optional = true } # (Removed `html2md` dep. dhat-rs profiling on real Gmail inboxes # showed `html2md::walk` and `html2md::tables::handle` allocating # ~894 MB peak heap on a 10 KB HTML input from Otter.ai-style emails @@ -601,6 +612,11 @@ e2e-test-support = [] # part of the shipped desktop/library build and the feature-forwarding gate # (which only inspects the `default` list) never requires forwarding it. rss-bench = [] +# Adds dhat heap profiling on top of `rss-bench` for the `library-profile` +# binary. Default-OFF, dev-only: installs dhat's global allocator, which +# perturbs RSS/timing numbers, so it is a separate opt-in feature rather than +# folded into `rss-bench`. Never forwarded to the shipped build. +rss-bench-dhat = ["rss-bench", "dep:dhat"] [lints.rust] unexpected_cfgs = { level = "warn", check-cfg = ['cfg(coverage)'] } diff --git a/app/src-tauri/profiling/.gitignore b/app/src-tauri/profiling/.gitignore new file mode 100644 index 000000000..b83d22266 --- /dev/null +++ b/app/src-tauri/profiling/.gitignore @@ -0,0 +1 @@ +/target/ diff --git a/app/src-tauri/profiling/Cargo.lock b/app/src-tauri/profiling/Cargo.lock new file mode 100644 index 000000000..d2b289f01 --- /dev/null +++ b/app/src-tauri/profiling/Cargo.lock @@ -0,0 +1,292 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "core-foundation-sys" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "libc" +version = "0.2.188" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22053b6a34f84abc97f9129e61334f40174659a1b9bd18c970b83db6a9a6348b" + +[[package]] +name = "memchr" +version = "2.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" + +[[package]] +name = "ntapi" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3b335231dfd352ffb0f8017f3b6027a4917f7df785ea2143d8af2adc66980ae" +dependencies = [ + "winapi", +] + +[[package]] +name = "openhuman-tauri-resource-profiler" +version = "0.1.0" +dependencies = [ + "serde", + "serde_json", + "sysinfo", +] + +[[package]] +name = "proc-macro2" +version = "1.0.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "serde" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_core" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.2", +] + +[[package]] +name = "serde_json" +version = "1.0.151" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "syn" +version = "2.0.119" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "3.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a207d6d6a2b7fc470b80443726053f18a2481b7e1eee970597051596567987a3" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "sysinfo" +version = "0.33.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fc858248ea01b66f19d8e8a6d55f41deaf91e9d495246fd01368d99935c6c01" +dependencies = [ + "core-foundation-sys", + "libc", + "memchr", + "ntapi", + "windows", +] + +[[package]] +name = "unicode-ident" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" + +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + +[[package]] +name = "windows" +version = "0.57.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "12342cb4d8e3b046f3d80effd474a7a02447231330ef77d71daa6fbc40681143" +dependencies = [ + "windows-core", + "windows-targets", +] + +[[package]] +name = "windows-core" +version = "0.57.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2ed2439a290666cd67ecce2b0ffaad89c2a56b976b736e6ece670297897832d" +dependencies = [ + "windows-implement", + "windows-interface", + "windows-result", + "windows-targets", +] + +[[package]] +name = "windows-implement" +version = "0.57.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9107ddc059d5b6fbfbffdfa7a7fe3e22a226def0b2608f72e9d552763d3e1ad7" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "windows-interface" +version = "0.57.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29bee4b38ea3cde66011baa44dba677c432a78593e202392d1e9070cf2a7fca7" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "windows-result" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e383302e8ec8515204254685643de10811af0ed97ea37210dc26fb0032647f8" +dependencies = [ + "windows-targets", +] + +[[package]] +name = "windows-targets" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" +dependencies = [ + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_gnullvm", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" + +[[package]] +name = "zmij" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" diff --git a/app/src-tauri/profiling/Cargo.toml b/app/src-tauri/profiling/Cargo.toml new file mode 100644 index 000000000..c79fecf38 --- /dev/null +++ b/app/src-tauri/profiling/Cargo.toml @@ -0,0 +1,14 @@ +[package] +name = "openhuman-tauri-resource-profiler" +version = "0.1.0" +edition = "2021" +publish = false + +[dependencies] +serde = { version = "1", features = ["derive"] } +serde_json = "1" +sysinfo = { version = "0.33", default-features = false, features = ["system"] } + +# Keep this offline dev tool independent from both the root core crate and the +# Tauri package so profiling never changes the shipped dependency graph. +[workspace] diff --git a/app/src-tauri/profiling/README.md b/app/src-tauri/profiling/README.md new file mode 100644 index 000000000..61b78889e --- /dev/null +++ b/app/src-tauri/profiling/README.md @@ -0,0 +1,54 @@ +# Tauri resource profiler + +Offline developer tool for measuring a locally running OpenHuman desktop app. +It is a standalone Cargo crate and is not linked into, registered with, or +shipped in the Tauri app. + +## Capture + +Start the current checkout: + + pnpm dev:app + +Find the main host PID, excluding OpenHuman Helper processes: + + pgrep -fl '/OpenHuman$' + +Then capture a representative workload: + + pnpm profile:tauri --pid --duration 15 + +Useful options: + + --interval-ms 250 + --out target/profile/my-scenario + --no-stacks + +The default output directory is target/profile/tauri-resources-. +Each run writes: + +- resources.md: summary table for the Rust host, CEF process roles, and total + desktop process tree. +- resources.json: raw time series plus mean and peak values. +- cpu-stacks.txt: macOS sample report for the host process. This is enabled by + default on macOS and can be disabled with --no-stacks. + +## Attribution boundary + +The Rust core runs inside the Tauri host process. Operating-system CPU and RAM +metrics therefore cannot split the shell from openhuman_core, or assign heap +pages to individual Rust modules. The profiler reports that combined process +honestly as Tauri host + embedded Rust core. + +CEF renderer, GPU, utility, and other helper processes are separate and are +reported independently. On macOS, the tool also parses the recursive stack +counts from cpu-stacks.txt and groups OpenHuman symbols by Rust domain, such as +openhuman_core::openhuman::agent or openhuman::core_process. + +CPU percentages are percentages of one logical CPU and may exceed 100 when a +component uses multiple cores. RAM is resident memory reported by sysinfo. + +For the smaller embedded-core-only Linux RSS/PSS benchmark, use the existing +root rss-bench harness: + + cargo build --release --features rss-bench --bin rss-bench diff --git a/app/src-tauri/profiling/src/main.rs b/app/src-tauri/profiling/src/main.rs new file mode 100644 index 000000000..782c7b70e --- /dev/null +++ b/app/src-tauri/profiling/src/main.rs @@ -0,0 +1,813 @@ +//! Offline CPU/RAM profiler for a locally running OpenHuman Tauri process. +//! +//! This binary is gated by the default-OFF dev-resource-profiler feature and +//! is never linked into the shipped app. It samples the Tauri host, which also +//! embeds openhuman_core, and its CEF descendants. On macOS it also captures an +//! Apple sample report for Rust module-level CPU attribution. +//! +//! Run from the repository root: +//! pnpm profile:tauri --pid PID --duration 15 + +use std::collections::{BTreeMap, HashMap, HashSet}; +use std::env; +use std::fs; +use std::path::{Path, PathBuf}; +use std::process::{Child, Command}; +use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; + +use serde::Serialize; +use sysinfo::{Pid, ProcessesToUpdate, System, MINIMUM_CPU_UPDATE_INTERVAL}; + +const DEFAULT_DURATION_SECS: u64 = 15; +const DEFAULT_INTERVAL_MS: u64 = 250; + +#[derive(Debug)] +struct Args { + pid: u32, + duration: Duration, + interval: Duration, + out_dir: PathBuf, + capture_stacks: bool, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Serialize)] +#[serde(rename_all = "snake_case")] +enum Component { + TauriHostAndEmbeddedCore, + CefRenderer, + CefGpu, + CefUtility, + CefOther, + OtherChild, +} + +impl Component { + fn label(self) -> &'static str { + match self { + Self::TauriHostAndEmbeddedCore => "Tauri host + embedded Rust core", + Self::CefRenderer => "CEF renderer", + Self::CefGpu => "CEF GPU", + Self::CefUtility => "CEF utility", + Self::CefOther => "CEF other", + Self::OtherChild => "Other child process", + } + } +} + +#[derive(Debug, Clone)] +struct ProcessSample { + pid: u32, + parent_pid: Option, + name: String, + command: String, + memory_bytes: u64, + cpu_percent: f32, +} + +#[derive(Debug, Clone, Serialize)] +struct ComponentSample { + component: Component, + process_count: usize, + memory_bytes: u64, + /// Percentage of one logical CPU. Above 100 means multiple logical CPUs. + cpu_percent: f32, +} + +#[derive(Debug, Clone, Serialize)] +struct TimeSample { + elapsed_ms: u64, + components: Vec, +} + +#[derive(Debug, Clone, Default, Serialize)] +struct ComponentSummary { + component: Option, + sample_count: usize, + peak_process_count: usize, + mean_memory_bytes: u64, + peak_memory_bytes: u64, + mean_cpu_percent: f32, + peak_cpu_percent: f32, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +struct RustModuleCpu { + module: String, + recursive_samples: u64, +} + +#[derive(Debug, Serialize)] +struct ProfileReport { + schema_version: u32, + host_pid: u32, + duration_ms: u64, + interval_ms: u64, + logical_cpu_count: usize, + sampled_at_unix_ms: u128, + attribution_note: &'static str, + rust_binary: ComponentSummary, + desktop_total: ComponentSummary, + components: Vec, + samples: Vec, + rust_cpu_modules: Vec, + cpu_stack_report: Option, + cpu_stack_error: Option, +} + +struct ProfileCapture { + host_pid: u32, + duration: Duration, + interval: Duration, + logical_cpu_count: usize, + samples: Vec, + rust_cpu_modules: Vec, + cpu_stack_report: Option, + cpu_stack_error: Option, +} + +fn main() { + let arguments = env::args().skip(1).collect::>(); + if arguments + .iter() + .any(|argument| argument == "-h" || argument == "--help") + { + println!("{}", usage()); + return; + } + if let Err(err) = run(arguments) { + eprintln!("tauri-resource-profiler: {err}"); + std::process::exit(1); + } +} + +fn run(arguments: Vec) -> Result<(), String> { + let args = parse_args(arguments)?; + fs::create_dir_all(&args.out_dir) + .map_err(|err| format!("create output directory {}: {err}", args.out_dir.display()))?; + + let stack_path = args.out_dir.join("cpu-stacks.txt"); + let (mut stack_child, initial_stack_error) = if args.capture_stacks { + start_stack_capture(args.pid, args.duration, &stack_path) + } else { + (None, None) + }; + + let (logical_cpu_count, samples) = capture_samples(&args)?; + let final_stack_error = finish_stack_capture(stack_child.as_mut(), initial_stack_error); + let stack_report = stack_path.exists().then(|| { + stack_path + .file_name() + .unwrap_or_default() + .to_string_lossy() + .into_owned() + }); + let rust_cpu_modules = fs::read_to_string(&stack_path) + .map(|contents| parse_rust_module_cpu(&contents)) + .unwrap_or_default(); + + let report = build_report(ProfileCapture { + host_pid: args.pid, + duration: args.duration, + interval: args.interval, + logical_cpu_count, + samples, + rust_cpu_modules, + cpu_stack_report: stack_report, + cpu_stack_error: final_stack_error, + })?; + let json_path = args.out_dir.join("resources.json"); + let markdown_path = args.out_dir.join("resources.md"); + fs::write( + &json_path, + serde_json::to_string_pretty(&report).map_err(|err| format!("serialize report: {err}"))?, + ) + .map_err(|err| format!("write {}: {err}", json_path.display()))?; + let markdown = render_markdown(&report); + fs::write(&markdown_path, &markdown) + .map_err(|err| format!("write {}: {err}", markdown_path.display()))?; + + println!("{markdown}"); + println!("Raw report: {}", json_path.display()); + if stack_path.exists() { + println!("CPU stacks: {}", stack_path.display()); + } + Ok(()) +} + +fn parse_args(arguments: impl IntoIterator) -> Result { + let mut pid = None; + let mut duration_secs = DEFAULT_DURATION_SECS; + let mut interval_ms = DEFAULT_INTERVAL_MS; + let mut out_dir = None; + let mut capture_stacks = cfg!(target_os = "macos"); + let mut arguments = arguments.into_iter(); + + while let Some(argument) = arguments.next() { + match argument.as_str() { + "--pid" => pid = Some(parse_value::("--pid", arguments.next())?), + "--duration" => { + duration_secs = parse_value::("--duration", arguments.next())?; + } + "--interval-ms" => { + interval_ms = parse_value::("--interval-ms", arguments.next())?; + } + "--out" => { + out_dir = Some(PathBuf::from( + arguments + .next() + .ok_or_else(|| "--out requires a path".to_string())?, + )); + } + "--no-stacks" => capture_stacks = false, + "--stacks" => capture_stacks = true, + "--" => {} + "-h" | "--help" => unreachable!("main handles help before parsing"), + unknown => return Err(format!("unknown argument {unknown}\n\n{}", usage())), + } + } + + let pid = pid.ok_or_else(|| format!("--pid is required\n\n{}", usage()))?; + if duration_secs == 0 { + return Err("--duration must be greater than zero".into()); + } + if interval_ms == 0 { + return Err("--interval-ms must be greater than zero".into()); + } + let interval = Duration::from_millis(interval_ms).max(MINIMUM_CPU_UPDATE_INTERVAL); + + Ok(Args { + pid, + duration: Duration::from_secs(duration_secs), + interval, + out_dir: out_dir.unwrap_or_else(default_out_dir), + capture_stacks, + }) +} + +fn parse_value(flag: &str, value: Option) -> Result +where + T: std::str::FromStr, +{ + let raw = value.ok_or_else(|| format!("{flag} requires a value"))?; + raw.parse() + .map_err(|_| format!("invalid {flag} value {raw}")) +} + +fn usage() -> &'static str { + "Usage: pnpm profile:tauri --pid PID [--duration SECONDS] [--interval-ms MS] [--out PATH] [--stacks|--no-stacks]\n\nAttach to the main OpenHuman Tauri PID, not a CEF helper PID. On macOS, CPU stack sampling is enabled by default." +} + +fn default_out_dir() -> PathBuf { + let timestamp = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap_or_default() + .as_secs(); + PathBuf::from("target") + .join("profile") + .join(format!("tauri-resources-{timestamp}")) +} + +fn capture_samples(args: &Args) -> Result<(usize, Vec), String> { + let root_pid = Pid::from_u32(args.pid); + let mut system = System::new_all(); + if system.process(root_pid).is_none() { + return Err(format!("pid {} is not running", args.pid)); + } + + let started = Instant::now(); + let mut samples = Vec::new(); + while started.elapsed() < args.duration { + std::thread::sleep(args.interval); + system.refresh_processes(ProcessesToUpdate::All, true); + if system.process(root_pid).is_none() { + return Err(format!("pid {} exited during profiling", args.pid)); + } + let processes = collect_processes(&system); + samples.push(TimeSample { + elapsed_ms: started.elapsed().as_millis() as u64, + components: group_process_tree(args.pid, &processes)?, + }); + } + Ok((system.cpus().len(), samples)) +} + +fn collect_processes(system: &System) -> Vec { + system + .processes() + .iter() + .map(|(pid, process)| ProcessSample { + pid: pid.as_u32(), + parent_pid: process.parent().map(Pid::as_u32), + name: process.name().to_string_lossy().into_owned(), + command: process + .cmd() + .iter() + .map(|part| part.to_string_lossy()) + .collect::>() + .join(" "), + memory_bytes: process.memory(), + cpu_percent: process.cpu_usage(), + }) + .collect() +} + +fn group_process_tree( + root_pid: u32, + samples: &[ProcessSample], +) -> Result, String> { + let by_pid = samples + .iter() + .map(|sample| (sample.pid, sample)) + .collect::>(); + if !by_pid.contains_key(&root_pid) { + return Err(format!( + "host pid {root_pid} disappeared from the process table" + )); + } + + let mut grouped = BTreeMap::::new(); + for sample in samples + .iter() + .filter(|sample| belongs_to_tree(sample.pid, root_pid, &by_pid)) + { + let component = classify_process(sample, root_pid); + let entry = grouped.entry(component).or_insert(ComponentSample { + component, + process_count: 0, + memory_bytes: 0, + cpu_percent: 0.0, + }); + entry.process_count += 1; + entry.memory_bytes = entry.memory_bytes.saturating_add(sample.memory_bytes); + entry.cpu_percent += sample.cpu_percent; + } + Ok(grouped.into_values().collect()) +} + +fn belongs_to_tree(pid: u32, root_pid: u32, by_pid: &HashMap) -> bool { + let mut current = Some(pid); + let mut seen = HashSet::new(); + while let Some(candidate) = current { + if candidate == root_pid { + return true; + } + if !seen.insert(candidate) { + return false; + } + current = by_pid.get(&candidate).and_then(|sample| sample.parent_pid); + } + false +} + +fn classify_process(sample: &ProcessSample, root_pid: u32) -> Component { + if sample.pid == root_pid { + return Component::TauriHostAndEmbeddedCore; + } + let identity = format!("{} {}", sample.name, sample.command).to_ascii_lowercase(); + if identity.contains("--type=renderer") || identity.contains("renderer") { + Component::CefRenderer + } else if identity.contains("--type=gpu-process") + || identity.contains("gpu process") + || identity.contains("gpu-process") + { + Component::CefGpu + } else if identity.contains("--type=utility") || identity.contains("utility") { + Component::CefUtility + } else if identity.contains("--type=zygote") + || identity.contains("--type=broker") + || identity.contains("crashpad") + || identity.contains("cef") + { + Component::CefOther + } else { + Component::OtherChild + } +} + +fn build_report(capture: ProfileCapture) -> Result { + if capture.samples.is_empty() { + return Err("profiling produced no samples".into()); + } + + let mut by_component = BTreeMap::>::new(); + let mut totals = Vec::new(); + for sample in &capture.samples { + for component in &sample.components { + by_component + .entry(component.component) + .or_default() + .push(component.clone()); + } + totals.push(ComponentSample { + component: Component::TauriHostAndEmbeddedCore, + process_count: sample + .components + .iter() + .map(|value| value.process_count) + .sum(), + memory_bytes: sample + .components + .iter() + .map(|value| value.memory_bytes) + .sum(), + cpu_percent: sample + .components + .iter() + .map(|value| value.cpu_percent) + .sum(), + }); + } + let components = by_component + .iter() + .map(|(component, values)| summarize(Some(*component), values)) + .collect::>(); + let rust_binary = components + .iter() + .find(|summary| summary.component == Some(Component::TauriHostAndEmbeddedCore)) + .cloned() + .ok_or_else(|| "host process was not present in samples".to_string())?; + + Ok(ProfileReport { + schema_version: 1, + host_pid: capture.host_pid, + duration_ms: capture.duration.as_millis() as u64, + interval_ms: capture.interval.as_millis() as u64, + logical_cpu_count: capture.logical_cpu_count, + sampled_at_unix_ms: SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap_or_default() + .as_millis(), + attribution_note: "The Rust core is embedded in the Tauri host, so OS process metrics report them together. CEF roles are separate child processes. Use cpu-stacks.txt to attribute host CPU samples to Rust modules.", + rust_binary, + desktop_total: summarize(None, &totals), + components, + samples: capture.samples, + rust_cpu_modules: capture.rust_cpu_modules, + cpu_stack_report: capture.cpu_stack_report, + cpu_stack_error: capture.cpu_stack_error, + }) +} + +fn summarize(component: Option, values: &[ComponentSample]) -> ComponentSummary { + let count = values.len().max(1); + ComponentSummary { + component, + sample_count: values.len(), + peak_process_count: values + .iter() + .map(|value| value.process_count) + .max() + .unwrap_or(0), + mean_memory_bytes: values.iter().map(|value| value.memory_bytes).sum::() + / count as u64, + peak_memory_bytes: values + .iter() + .map(|value| value.memory_bytes) + .max() + .unwrap_or(0), + mean_cpu_percent: values.iter().map(|value| value.cpu_percent).sum::() / count as f32, + peak_cpu_percent: values + .iter() + .map(|value| value.cpu_percent) + .fold(0.0, f32::max), + } +} + +fn render_markdown(report: &ProfileReport) -> String { + use std::fmt::Write as _; + let mut output = String::new(); + let _ = writeln!(output, "### Tauri resource profile"); + let _ = writeln!(output); + let _ = writeln!( + output, + "PID {} - {:.1}s - {}ms interval - {} logical CPUs", + report.host_pid, + report.duration_ms as f64 / 1000.0, + report.interval_ms, + report.logical_cpu_count + ); + let _ = writeln!(output); + let _ = writeln!( + output, + "| component | processes (peak) | RAM mean | RAM peak | CPU mean | CPU peak |" + ); + let _ = writeln!( + output, + "| --------- | ---------------- | -------- | -------- | -------- | -------- |" + ); + for summary in &report.components { + let label = summary + .component + .map(Component::label) + .unwrap_or("Desktop total"); + let _ = writeln!( + output, + "| {label} | {} | {:.1} MiB | {:.1} MiB | {:.1}% | {:.1}% |", + summary.peak_process_count, + to_mib(summary.mean_memory_bytes), + to_mib(summary.peak_memory_bytes), + summary.mean_cpu_percent, + summary.peak_cpu_percent, + ); + } + let total = &report.desktop_total; + let _ = writeln!( + output, + "| **Desktop total** | **{}** | **{:.1} MiB** | **{:.1} MiB** | **{:.1}%** | **{:.1}%** |", + total.peak_process_count, + to_mib(total.mean_memory_bytes), + to_mib(total.peak_memory_bytes), + total.mean_cpu_percent, + total.peak_cpu_percent, + ); + let _ = writeln!(output); + let _ = writeln!(output, "> {}", report.attribution_note); + if !report.rust_cpu_modules.is_empty() { + let _ = writeln!(output); + let _ = writeln!(output, "| Rust module | recursive CPU samples |"); + let _ = writeln!(output, "| ----------- | --------------------- |"); + for module in report.rust_cpu_modules.iter().take(20) { + let _ = writeln!( + output, + "| {} | {} |", + module.module, module.recursive_samples + ); + } + } + if let Some(error) = &report.cpu_stack_error { + let _ = writeln!(output); + let _ = writeln!(output, "CPU stack capture unavailable: {error}"); + } + output +} + +fn to_mib(bytes: u64) -> f64 { + bytes as f64 / (1024.0 * 1024.0) +} + +fn parse_rust_module_cpu(contents: &str) -> Vec { + let recursive_section = contents + .split("Total number in stack (recursive counted multiple") + .nth(1) + .and_then(|tail| tail.split("Sort by top of stack").next()) + .unwrap_or(""); + let mut modules = BTreeMap::::new(); + for line in recursive_section.lines() { + let trimmed = line.trim(); + let Some((count, symbol_line)) = trimmed.split_once(char::is_whitespace) else { + continue; + }; + let Ok(count) = count.parse::() else { + continue; + }; + let symbol = symbol_line + .split(" (in ") + .next() + .unwrap_or(symbol_line) + .trim(); + let Some(module) = own_rust_module(symbol) else { + continue; + }; + *modules.entry(module).or_default() += count; + } + let mut modules = modules + .into_iter() + .map(|(module, recursive_samples)| RustModuleCpu { + module, + recursive_samples, + }) + .collect::>(); + modules.sort_by(|left, right| { + right + .recursive_samples + .cmp(&left.recursive_samples) + .then_with(|| left.module.cmp(&right.module)) + }); + modules +} + +fn own_rust_module(symbol: &str) -> Option { + const CORE_PREFIX: &str = "openhuman_core::openhuman::"; + const TAURI_PREFIX: &str = "openhuman::"; + if let Some(start) = symbol.find(CORE_PREFIX) { + let tail = &symbol[start + CORE_PREFIX.len()..]; + let domain = tail + .split("::") + .next()? + .trim_matches(|value| value == '<' || value == '>'); + if !domain.is_empty() { + return Some(format!("{CORE_PREFIX}{domain}")); + } + } + if let Some(start) = symbol.find(TAURI_PREFIX) { + let tail = &symbol[start + TAURI_PREFIX.len()..]; + let module = tail + .split("::") + .next()? + .trim_matches(|value| value == '<' || value == '>'); + if !module.is_empty() { + return Some(format!("{TAURI_PREFIX}{module}")); + } + } + None +} + +#[cfg(target_os = "macos")] +fn start_stack_capture( + pid: u32, + duration: Duration, + output: &Path, +) -> (Option, Option) { + match Command::new("/usr/bin/sample") + .arg(pid.to_string()) + .arg(duration.as_secs().max(1).to_string()) + .arg("10") + .arg("-mayDie") + .arg("-file") + .arg(output) + .spawn() + { + Ok(child) => (Some(child), None), + Err(err) => (None, Some(format!("start /usr/bin/sample: {err}"))), + } +} + +#[cfg(not(target_os = "macos"))] +fn start_stack_capture( + _pid: u32, + _duration: Duration, + _output: &Path, +) -> (Option, Option) { + ( + None, + Some("automatic stack capture is currently available on macOS only".into()), + ) +} + +fn finish_stack_capture(child: Option<&mut Child>, prior_error: Option) -> Option { + if prior_error.is_some() { + return prior_error; + } + let child = child?; + match child.wait() { + Ok(status) if status.success() => None, + Ok(status) => Some(format!("stack sampler exited with {status}")), + Err(err) => Some(format!("wait for stack sampler: {err}")), + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn process( + pid: u32, + parent_pid: Option, + name: &str, + command: &str, + memory_bytes: u64, + cpu_percent: f32, + ) -> ProcessSample { + ProcessSample { + pid, + parent_pid, + name: name.into(), + command: command.into(), + memory_bytes, + cpu_percent, + } + } + + #[test] + fn classifies_cef_roles() { + assert_eq!( + classify_process( + &process(2, Some(1), "OpenHuman Helper", "--type=renderer", 1, 1.0), + 1 + ), + Component::CefRenderer + ); + assert_eq!( + classify_process( + &process(3, Some(1), "OpenHuman Helper", "--type=gpu-process", 1, 1.0), + 1 + ), + Component::CefGpu + ); + assert_eq!( + classify_process( + &process(4, Some(1), "OpenHuman Helper", "--type=utility", 1, 1.0), + 1 + ), + Component::CefUtility + ); + } + + #[test] + fn groups_only_host_process_tree() { + let processes = vec![ + process(10, Some(1), "OpenHuman", "OpenHuman", 100, 20.0), + process(11, Some(10), "Helper", "--type=renderer", 40, 30.0), + process(12, Some(11), "Helper", "--type=utility", 10, 5.0), + process(99, Some(1), "unrelated", "unrelated", 1_000, 100.0), + ]; + + let grouped = group_process_tree(10, &processes).unwrap(); + assert_eq!(grouped.len(), 3); + assert_eq!( + grouped.iter().map(|value| value.memory_bytes).sum::(), + 150 + ); + assert_eq!( + grouped.iter().map(|value| value.cpu_percent).sum::(), + 55.0 + ); + } + + #[test] + fn report_separates_rust_binary_from_desktop_total() { + let samples = vec![TimeSample { + elapsed_ms: 250, + components: vec![ + ComponentSample { + component: Component::TauriHostAndEmbeddedCore, + process_count: 1, + memory_bytes: 100, + cpu_percent: 20.0, + }, + ComponentSample { + component: Component::CefRenderer, + process_count: 2, + memory_bytes: 50, + cpu_percent: 30.0, + }, + ], + }]; + let report = build_report(ProfileCapture { + host_pid: 10, + duration: Duration::from_secs(1), + interval: Duration::from_millis(250), + logical_cpu_count: 8, + samples, + rust_cpu_modules: Vec::new(), + cpu_stack_report: None, + cpu_stack_error: None, + }) + .unwrap(); + + assert_eq!(report.rust_binary.mean_memory_bytes, 100); + assert_eq!(report.desktop_total.mean_memory_bytes, 150); + assert_eq!(report.desktop_total.peak_process_count, 3); + assert!(render_markdown(&report).contains("Tauri host + embedded Rust core")); + } + + #[test] + fn parser_requires_pid_and_clamps_cpu_interval() { + assert!(parse_args(Vec::::new()).is_err()); + let args = parse_args([ + "--pid".into(), + "123".into(), + "--duration".into(), + "2".into(), + "--interval-ms".into(), + "1".into(), + "--no-stacks".into(), + ]) + .unwrap(); + assert_eq!(args.pid, 123); + assert_eq!(args.duration, Duration::from_secs(2)); + assert_eq!(args.interval, MINIMUM_CPU_UPDATE_INTERVAL); + assert!(!args.capture_stacks); + } + + #[test] + fn parses_recursive_stack_counts_into_openhuman_modules() { + let sample = r#" +Total number in stack (recursive counted multiple, when >=5): + 81 openhuman_core::openhuman::agent::run (in OpenHuman) + 10 + 34 ::poll (in OpenHuman) + 2 + 17 openhuman_core::openhuman::memory::search (in OpenHuman) + 4 + 9 openhuman::core_process::ensure_running (in OpenHuman) + 1 + 200 tokio::runtime::park (in OpenHuman) + 3 + +Sort by top of stack, same collapsed (when >= 5): +"#; + assert_eq!( + parse_rust_module_cpu(sample), + vec![ + RustModuleCpu { + module: "openhuman_core::openhuman::agent".into(), + recursive_samples: 115, + }, + RustModuleCpu { + module: "openhuman_core::openhuman::memory".into(), + recursive_samples: 17, + }, + RustModuleCpu { + module: "openhuman::core_process".into(), + recursive_samples: 9, + }, + ] + ); + } +} diff --git a/docs/harness-comparison-2026-07-22.md b/docs/harness-comparison-2026-07-22.md new file mode 100644 index 000000000..f3844562c --- /dev/null +++ b/docs/harness-comparison-2026-07-22.md @@ -0,0 +1,101 @@ +# Agent harness resource-footprint comparison + +Date: 2026-07-22 +Method: web research against primary sources where they exist (GitHub repos and +issue trackers, official docs), with every weakly-sourced figure flagged. Our +own numbers come from the measured benchmarks in +[`library-benchmarking.md`](library-benchmarking.md) and +[`resource-profiling-session-2026-07-21.md`](resource-profiling-session-2026-07-21.md). + +The single most important source-quality finding: **the only fully measured, +reproducible numbers in this comparison are ours.** Codex publishes binary +size but no RSS; ZeroClaw's numbers are vendor marketing with no third-party +verification; Claude Code's dramatic figures are leak bugs, not steady state; +Hermes' figure is self-reported documentation. + +## Comparison table + +| Harness | Language / runtime | Deployment shape | RAM idle | RAM under load | Startup | Binary / install | N-agent scaling | Source quality | +|---|---|---|---|---|---|---|---|---| +| **OpenHuman core** (ours) | Rust, embeddable library | Library or one RPC process; agents share the process | 44-51 MiB settled (default); 35-44 MiB slim | Cold turn +26-31 MiB (first-use); warm turn +0.5-1.9 MiB | ~100-140 ms cold turn; ~0 idle CPU | 116 MiB default / 81 MiB library-minimal / 60 MiB stripped | **In-process**: ~0.4 MiB/agent cold roster, ~1.8 MiB warm marginal | Measured, reproducible (this repo) | +| OpenAI Codex CLI (codex-rs) | Rust, single native binary | CLI process per session | no published RSS | no published RSS (qualitative claims only) | "milliseconds" (qualitative) | **80 MB** (macOS arm64, primary: issue #13091) | N independent processes | Binary size primary; RSS unpublished | +| Codex CLI (old Node/TS) | Node.js / V8 | CLI process per session | no published data | no published data | Node startup | npm + Node runtime | N processes | none published | +| ZeroClaw | Rust, static binary | CLI + optional daemon | **< 5 MB (self-reported, unverified)** | **no verified figure** (the oft-quoted "7.8-12 MiB" has no locatable primary source) | "< 10 ms" (self-reported) | 3.4 MB (one page says ~8.8 MB — internally inconsistent) | "multiple concurrently", no numbers | Marketing only; provenance suspect (SEO domain cluster) | +| OpenClaw (Clawdbot → Moltbot → OpenClaw) | TypeScript / Node.js | Local daemon + channel bridge | "> 1 GB" claimed only by competitor marketing | no neutral figure | slow (Node + heavy deps) | ~28 MB (per competitor comparison) | N processes | Rebrand history primary (TechCrunch/CNBC/Forbes); RAM figure biased | +| Claude Code | Node.js / V8 CLI | CLI process per session | ~500 MB claimed (weak SEO source) | documented **leak bugs**: 400-500 MB/min idle growth, multi-GB, extremes 14-93 GB | Node startup | npm + Node runtime | N processes | Leak bugs primary (issues #67433, #28731, #22188); baseline weak | +| Hermes Agent (Nous Research) | **Python 81% / TS 16%** (not Rust; it bundles the Rust-written `uv`) | CLI + gateway daemon; subagents are isolated subprocesses | no granular RSS; **4 GB RAM minimum** system req | "< 500 MB without a local LLM" (self-reported docs) | not published | Python 3.11 env | N subprocesses | Repo/languages primary; RAM self-reported | + +## Per-harness notes + +**OpenAI Codex CLI.** Confirmed Rust rewrite (~June 2025) shipping one +self-contained binary. The only hard number is 80 MB binary size on macOS +arm64, from OpenAI's own tracker (openai/codex#13091) — which proposes +feature-gating heavy dependencies to reach ~55-60 MB, directly analogous to +our Cargo domain gates. Memory claims are qualitative ("no unbounded Node heap +growth"). Scope: coding agent only — no persistent curated cross-session +memory core, no multi-agent orchestration, no channels, no workflow engine. + +**ZeroClaw.** Rust single-binary positioned against OpenClaw. All numbers are +vendor self-reported (`/usr/bin/time -l` on their own build) with zero +third-party verification, promoted across a cluster of lookalike SEO domains. +The "7.8-12 MiB under load" figure previously cited in our docs could not be +found in any source and has been downgraded to unverified. ZeroClaw is a +separate project from OpenClaw, not a rebrand. + +**OpenClaw lineage.** The Clawdbot → Moltbot → OpenClaw rebrand chain is +well-sourced (TechCrunch, CNBC, Forbes, Jan 2026). The ">1 GB RAM" figure +appears only in ZeroClaw's competitive marketing; plausible for a Node daemon +with browser automation, but there is no neutral benchmark. + +**Claude Code.** Node/V8 CLI. No clean published idle baseline; ~500 MB comes +from a third-party SEO article and leak-report starting points. What is +well-documented (primary GitHub issues) is a family of off-heap RSS leak bugs: +400-500 MB/min growth while idle (#67433), 14 GB OOM (#28731), 93 GB heap +(#22188), idle CPU thrash (#18280). Those are bugs, not steady state — but +they are a cautionary tale about native-buffer discipline in long-running +Node agent processes. + +**Hermes Agent.** The closest scope match to OpenHuman (SQLite + FTS5 + WAL +curated memory, parent/child subagent lineage, cron, unified +Telegram/Discord/Slack/Signal/WhatsApp/WeChat gateway) — and it is Python 81% / +TypeScript 16%, not Rust. Subagents run as isolated subprocesses, so it pays +its base footprint per agent. Self-reported "under 500 MB without a local +LLM", 4 GB RAM minimum. + +## What this means for OpenHuman + +**Today.** Against honest scope-matched peers we are clearly leaner: Hermes at +similar capability self-reports ~10x our settled RSS and requires 4 GB +minimum; Claude Code starts around a claimed ~500 MB with documented multi-GB +leaks; Codex's binary (80 MB) is larger than our stripped library-minimal +build (60 MB). The only harness claiming to be dramatically smaller — +ZeroClaw at "<5 MB" — is unverified marketing carrying far less capability. + +**End-state.** The library-minimal + shared-services target (~15 MiB private +footprint + ~2 MiB per in-process agent) is not a stretch goal: today's +~42 MiB slim RSS already decomposes to 15.2 MiB private / 3.2 MiB live heap, +the rest being reclaimable executable text and allocator high-water. State +that with the RSS-vs-private-footprint caveat attached. + +**Worth borrowing / leaning into:** + +1. Feature-gating heavy deps is now industry practice (Codex #13091) — + external validation of our domain-gate investment. +2. Rust + single self-contained binary is the market direction; the + Node-based peers are the ones with RSS horror stories. +3. **In-process shared-services scaling is our differentiator.** Every + scope-matched peer scales agents as N OS processes, paying the fixed base + N times. Our ~0.4-1.8 MiB marginal per in-process agent is the entire + basis of the 1000-agents-in-2-GB story; nobody else has it. +4. Internalize (not borrow) Claude Code's leak history: keep the warmed + repeated-turn plateau benchmark as a standing regression gate. + +## Sources + +- OpenAI codex#13091 — 80 MB binary / feature-gating proposal +- devclass (2025-06) — Codex Rust rewrite announcement coverage +- anthropics/claude-code#67433, #28731, #22188, #18280 — leak/idle-CPU bugs +- zeroclaw.net; openclawconsult.com "lab" comparison (self-reported marketing) +- TechCrunch / Forbes (2026-01) — OpenClaw rebrand lineage +- github.com/nousresearch/hermes-agent — language split, architecture +- hermes-agent.nousresearch.com docs — memory features, footprint claim diff --git a/docs/library-benchmarking.md b/docs/library-benchmarking.md new file mode 100644 index 000000000..d82a38c26 --- /dev/null +++ b/docs/library-benchmarking.md @@ -0,0 +1,345 @@ +# Library benchmarking environment + +## Purpose + +"opencompany" wants to embed the OpenHuman Rust core as a library: no always-on +RPC server, no Tauri shell, just the core linked in-process and driven +directly. That changes what "resource usage" means. There is no single steady +process to profile; there are per-use-case workloads (a long-running agent +loop, a delegated multi-agent turn, a saved workflow run, a background +subconscious pass, a memory ingest, a bare embed) that each have their own +startup cost, steady-state footprint, and growth curve. + +This document describes the benchmark environment built to measure that: a +pinned `library-profile` binary with eight scenarios, four driver scripts +under `scripts/profile/`, and the comparison point the team cares about +(ZeroClaw). It builds on the manual investigation in +[`docs/resource-profiling-session-2026-07-21.md`](resource-profiling-session-2026-07-21.md); +read that document for the deep memory/CPU attribution work. This document is +about running repeatable benchmarks, not re-deriving those findings. + +## The eight scenarios + +All scenarios run in `target/release/library-profile `, replace +network inference with a deterministic provider (`rss-bench` feature), and +print one pretty-printed JSON result object to stdout (diagnostics go to +stderr). Each models a distinct embedding use case: + +| Scenario | Models | +| --- | --- | +| `memory-ingest` | Canonicalizing and ingesting a batch of chat messages through the real extraction/admission/tree-queue pipeline. | +| `subagents` | A delegation turn: an orchestrator session spawns real subagents via `spawn_parallel_agents` and merges their findings. | +| `agent-turn` | The minimal embed case: one agent, one turn, no delegation, no workflow. The smallest useful "hello world" for a host that just wants a single reply. | +| `long-agent` | A long-running agent loop (`OPENHUMAN_PROFILE_TURNS`, default 25) in one process, to see whether RSS plateaus or grows per turn. | +| `workflow` | A saved automation run (`flows_create` + `flows_run`), representing the flows/automation embedding path rather than ad hoc chat. | +| `subconscious` | A background subconscious turn (the always-on reflective pass), distinct from an interactive chat turn. | +| `cold-phases` | Bootstrap attribution: per-phase checkpoints (config load, registry init, agent build, memory construction, first turn) so cold-start cost can be attributed to a phase instead of one lump sum. | +| `fleet` | N concurrent live agents with latency-realistic mock inference — the "100-1000 agents in a 2 GB / 2 vCPU server" question. See [below](#the-2-gb--2-vcpu-server-budget). | + +## How to run + +Five scripts under `scripts/profile/` (each has `-h`/`--help`): + +- **`library-bench.sh`** — the primary RSS/duration benchmark. Builds the + binaries, runs each scenario N fresh-process repeats (default 5), and + aggregates median/min/max into `summary.json` + `summary.md`. + + ```bash + ./scripts/profile/library-bench.sh # default build, all scenarios + ./scripts/profile/library-bench.sh --slim # --no-default-features recipe + ./scripts/profile/library-bench.sh --scenarios "long-agent,subagents" --turns 50 --warm + ``` + +- **`library-cpu.sh`** — a `samply` wrapper for one scenario's CPU profile, + isolated from persistence/timezone noise by default. + + ```bash + ./scripts/profile/library-cpu.sh subagents + samply load target/profile/rust-library/subagents-cpu.json.gz + ``` + +- **`library-heap.sh`** — builds the `rss-bench-dhat` variant and runs a + scenario under dhat for live-heap attribution (allocation sites, retained + bytes). RSS/timing under dhat are perturbed; don't compare those numbers to + `library-bench.sh` output. + + ```bash + ./scripts/profile/library-heap.sh memory-ingest + # load target/profile/rust-library/dhat-memory-ingest.json at + # https://nnethercote.github.io/dh_view/dh_view.html + ``` + +- **`library-fleet.sh`** — sweeps the `fleet` scenario across a list of agent + counts and gates the result against the 2 GB / 2 vCPU server budget (see + below). + + ```bash + ./scripts/profile/library-fleet.sh --agents 100 --latency-ms 200 + ./scripts/profile/library-fleet.sh --agents "50,100,500" --target 1000 --budget-mib 2048 + ``` + +- **`library-instances.sh`** — sweeps N independent *processes* (not agents + in one process) of a scenario, held alive via `OPENHUMAN_PROFILE_HOLD_SECS`, + and measures per-instance/aggregate cost — the many-processes counterpart + to `library-fleet.sh`'s one-process model (see + [below](#fleet-one-process-vs-instances-many-processes)). + + ```bash + ./scripts/profile/library-instances.sh --instances "10,25,50" --hold-secs 30 + ``` + +### Default vs slim builds + +Default-feature builds link every compile-time domain gate (`voice`, `web3`, +`media`, `meet`, `skills`, `flows`, `mcp`, `tui`) — the byte-identical desktop +recipe. The slim recipe drops everything not required by the harness: + +```bash +GGML_NATIVE=OFF cargo build --release \ + --no-default-features --features rss-bench \ + --bin library-profile --bin rss-bench +``` + +`--slim` on `library-bench.sh` builds this recipe. Per the prior session, +compile-time gates shrink the binary substantially but only move settled RSS +by a few MiB — most of the RSS story is initialization and allocator +behavior, not linked code size. + +### Useful env knobs + +| Variable | Effect | +| --- | --- | +| `OPENHUMAN_PROFILE_TURNS` | Turn count for `long-agent` (default 25). | +| `OPENHUMAN_PROFILE_PREWARM_SUBAGENTS=1` | Run one warm-up turn before measuring (`subagents`/`subconscious`), isolating first-use cost from steady state. | +| `OPENHUMAN_PROFILE_DISABLE_MEMORY_WRITES=1` | Disable `memory.auto_save` and episodic capture, isolating orchestration from persistence. | +| `OPENHUMAN_PROFILE_FORCE_UTC=1` | Skip `iana_time_zone`/CoreFoundation timezone resolution. | +| `OPENHUMAN_PROFILE_HOLD_SECS` / `HOLD_BEFORE_SECS` | Pause the process at settled/baseline state for external inspection (`vmmap`, `heap`, `malloc_history`, Instruments). | +| `OPENHUMAN_PROFILE_DHAT_OUT` | Output path for dhat JSON (set by `library-heap.sh`). | + +## Metrics and interpretation + +Every run reports `baseline`/`settled`/`peak_rss_kib` (macOS: `proc_pid_rusage` +RSS, `getrusage` peak, `proc_pidinfo` thread count), `retained_delta_kib` +(settled minus baseline), `peak_delta_kib`, and `duration_ms`. `long-agent` +additionally reports `checkpoints[]` so first-turn vs. last-turn growth is +visible directly. + +**RSS is not private heap.** The prior session's deep attribution found a +~42 MiB slim-build snapshot broken down as roughly 15.2 MiB private physical +footprint, 3.18 MiB live heap, 18.7 MiB resident executable text, and ~9.4 MiB +of resident-but-mostly-inactive malloc pages (allocator high-water +retention). See +[`docs/resource-profiling-session-2026-07-21.md`](resource-profiling-session-2026-07-21.md#deep-memory-attribution) +for the full breakdown, the executable-paging finding (a cold turn faults in +~15 MiB of previously nonresident OpenHuman code), and the warmed-process +control showing steady-state turns cost ~0.5-1.9 MiB once warm rather than +the ~26-31 MiB a cold turn costs. Use `library-bench.sh` for the RSS/duration +headline numbers, `library-cpu.sh` when CPU attribution is the question, and +`library-heap.sh` only when RSS numbers need live-allocation attribution +(accepting the dhat perturbation). + +## The ZeroClaw comparison + +ZeroClaw self-reports idling under 5 MiB RAM; the "7.8-12 MiB under load" +figure sometimes quoted alongside it has no locatable primary source, and even +the idle figure is vendor marketing with no third-party verification (see +[`docs/harness-comparison-2026-07-22.md`](harness-comparison-2026-07-22.md)). +OpenHuman's Rust core currently settles around 35-50 MiB depending on +scenario and feature set (see the baseline table below). + +Treat this as a **north star, not an apples-to-apples benchmark**. ZeroClaw's +scope and feature set differ substantially from the OpenHuman core: OpenHuman +links a full agent/memory/tool/orchestration stack (SQLite-backed unified +memory, TinyCortex PII detection, prompt-injection detection, a builtin-agent +registry, tool catalogs, provider routing) that a narrower harness may not +carry at all. A closer gap is a meaningful signal that the initialization +graph is leaner; it is not evidence of feature parity, and a wider gap is not +automatically a regression if it comes from carrying more capability. Every +`library-bench.sh` summary includes a labeled comparison row/note for exactly +this reason: visible, but explicitly called out as external. + +## The 2 GB / 2 vCPU server budget + +"opencompany" wants a single server to host 100-1000 live agents inside a 2 GB +RAM / 2 vCPU box. That is a budget question, not a per-scenario RSS question: +2048 MiB / 1000 agents is roughly 2 MiB per agent all-in, but the fixed +per-process base (allocator high water, code paging, registries, detectors — +the same ~30 MiB every scenario above pays once) amortizes across however many +agents share the process. What actually determines whether 1000 agents fit is +the **marginal** cost per additional agent once that base is paid, not the +per-agent average. `library-fleet.sh` runs the `fleet` scenario (N concurrent +live agents, latency-realistic mock inference so idle time looks like real +network waits rather than a busy loop) across a sweep of N and reports that +marginal cost directly (`marginal_rss_kib_per_agent`), alongside idle CPU over +a parked 10s window, thread count, and open FD count — all of which should +stay roughly flat as N grows if per-agent state is cheap and idle agents cost +~zero CPU. + +Working targets: marginal cost ≤ 1.5 MiB/agent, threads and FDs flat (not +linear) in N, and idle CPU low regardless of N — an agent that isn't mid-turn +should not be spending cycles. `OPENHUMAN_PROFILE_WORKER_THREADS=2` pins the +scenario's tokio runtime to 2 worker threads to simulate the 2 vCPU box rather +than scaling with the host's actual core count. The `budget` block in each +run's JSON (`target_agents`, `ram_budget_mib`, `projected_rss_mib_at_target`, +`fits`) projects the swept marginal cost out to the real target (default 1000 +agents / 2048 MiB); `library-fleet.sh` aggregates medians per N into +`summary.md` and exits nonzero if any swept N projects `fits: false`, making +it usable as a CI-style regression gate (`--no-gate` to disable). + +**Caveats, stated plainly:** these numbers are gathered on macOS, which has no +cgroup memory limit to enforce or observe locally — the budget check is a +projection from measured marginal cost, not a live "did it actually get +OOM-killed at N agents" test. macOS also lacks Linux's `/proc//smaps_rollup`, +which would give true PSS (proportional shared memory) instead of RSS; RSS +overcounts shared pages (executable text, shared library mappings) in a way +that matters more as agent count grows and more of the process footprint is +genuinely shared. Treat the macOS numbers as an approximation of the target +Linux server, not a substitute for it. The JSON schema already has a Linux +path — `proc_metrics` reads `/proc//status` and `/proc//stat` on +Linux — so true validation should eventually mean running the same +`library-profile fleet` binary on a cgroup-limited Linux box (matching the 2 +vCPU / 2 GB target) rather than trusting the macOS projection alone. + +### Fleet (one process) vs instances (many processes) + +The budget section above measures one deployment shape: N agents sharing a +single process. But "opencompany" may instead run OpenHuman as **N +independent processes or containers** — one per tenant — rather than N +agents inside one process. Those are different cost models and the fleet +number does not answer the second one. + +- **Fleet (`library-fleet.sh`)** pays the ~30-50 MiB fixed base (allocator + high water, code paging, registries, detectors) **once**, and amortizes it + across however many agents share that process. Marginal cost per agent is + what matters, and it can be well under 1 MiB once the base is paid. +- **Instances (`library-instances.sh`)** pays that same fixed base **N + times**, once per process — minus whatever the OS actually shares across + processes (resident executable text, shared library mappings). Summed RSS + across instances therefore **double-counts** those shared pages; it is an + upper bound, not the true footprint. True per-instance marginal cost is + better read from summed PSS (Linux only — macOS has no PSS-equivalent + metric), which divides shared pages across the processes that share them. + +`library-instances.sh` spawns N held `library-profile` processes staggered +on startup, samples aggregate sum-RSS every 2s while they hold at settled +state, and reports median settled RSS/instance, mean and peak aggregate +sum-RSS, and summed PSS when available, plus a labeled 2 GB-box +extrapolation estimate: + +```bash +./scripts/profile/library-instances.sh --instances "10,25,50" --hold-secs 30 +``` + +**This is still a macOS proxy, not container validation.** True validation +means running the same binary under real `cgroup` memory limits (e.g. +`docker run --memory=2g`) on a Linux host and observing whether it survives +or gets OOM-killed at the target instance count — not projecting from local +sum-RSS. That is follow-up work, and it belongs on a Linux box: this repo's +own `openhuman-core` Docker build is currently blocked on Apple Silicon (the +`whisper-rs-sys`/whisper.cpp NEON fp16 intrinsics fail to compile under +arm64-Linux emulation with GCC 12 — see the umbrella repo's root `CLAUDE.md` +gotchas and `docs/resource-profiling-session-2026-07-21.md`). The path +around that blocker is either building for `linux/amd64` under emulation (the +whisper AVX path has no NEON bug) or running the validation on a native Linux +host rather than macOS Docker Desktop. + +## Profiling escalation path + +Start cheap, escalate only as needed: + +1. **`library-bench.sh`** — RSS/duration medians across fresh processes. Answers "did this change move the needle" for most changes. +2. **`library-cpu.sh` (samply)** — symbolized CPU profile when a scenario is slower than expected, or to attribute cold-path CPU to a specific phase (registry init, agent build, memory construction, SQLite init, TinyAgents turn runner were the top contributors in the prior session). +3. **`library-heap.sh` (dhat)** — live-heap allocation sites and retained bytes when RSS is high but the cause isn't obvious from CPU alone (e.g. the TinyCortex PII `RegexSet` finding came from stack-logged allocation attribution, not CPU sampling). +4. **Instruments / `vmmap` / `heap` / `malloc_history`** — deepest macOS-native attribution, using the `OPENHUMAN_PROFILE_HOLD_SECS` / `HOLD_BEFORE_SECS` hooks to pause the process at baseline or settled state: + + ```bash + OPENHUMAN_PROFILE_HOLD_SECS=120 target/release/library-profile subagents & + vmmap -summary + heap -sH + + MallocStackLogging=1 OPENHUMAN_PROFILE_HOLD_SECS=120 \ + target/release/library-profile subagents & + malloc_history -allBySize + ``` + + This is what surfaced the PII-sanitizer regex cache and the first-turn + executable-paging finding in the prior session; reach for it only once + `library-bench.sh`/`library-cpu.sh`/`library-heap.sh` have narrowed the + question to a specific scenario and metric. + +## Current baseline numbers + +From the 2026-07-21 profiling session (medians over five fresh processes +unless noted; see that document for methodology and caveats): + +| Scenario | Build | Median settled RSS | Median retained Δ | +| --- | --- | ---: | ---: | +| 1-agent roster | default | 38.7 MiB | - | +| 8-agent roster | default | 41.5 MiB | +2.8 MiB total (~0.40 MiB/agent) | +| 1-agent roster | slim | 35.5 MiB | - | +| 8-agent roster | slim | 38.7 MiB | +3.2 MiB total | +| `memory-ingest` (100 msgs) | default | 25.5 MiB | 9.31 MiB | +| `memory-ingest` (100 msgs) | slim | 23.8 MiB | 8.58 MiB | +| `subagents` (cold, 2 children) | default | 48.5 MiB | 30.8 MiB | +| `subagents` (cold, 2 children) | slim | 42.4 MiB | 25.7 MiB | +| `subagents` (warmed repeat, persistence off) | default | - | 0.52 MiB | +| `subagents` (warmed repeat, normal capture) | default | - | 1.84 MiB | +| ZeroClaw (external, idle, self-reported/unverified) | - | < 5 MiB | - | + +First full `library-bench.sh` run of the new scenarios (default build, 5 +fresh-process repeats, 2026-07-21, Apple Silicon macOS): + +| Scenario | Build | Median settled RSS | Median retained Δ | Median duration | +| --- | --- | ---: | ---: | ---: | +| `agent-turn` (cold, 1 turn) | default | 47.6 MiB | 29.5 MiB | 102 ms | +| `subconscious` (cold, no delegation) | default | 47.9 MiB | 29.8 MiB | 138 ms | +| `subagents` (cold, 2 children) | default | 48.0 MiB | 29.9 MiB | 142 ms | +| `workflow` (`flows_create` + `flows_run`) | default | 50.9 MiB | 29.9 MiB | 110 ms | +| `long-agent` (25 warmed turns) | default | 65.8 MiB | 18.5 MiB | 1,361 ms | +| `cold-phases` (9 bootstrap phases) | default | 51.2 MiB | 36.5 MiB | 476 ms | +| `memory-ingest` (100 msgs) | default | 25.8 MiB | 9.3 MiB | 2,099 ms | + +Notable structure behind these medians: + +- The `long-agent` per-turn series plateaus: typical turns add 30-150 KiB, + and the 25-turn total (~16.8 MiB first-to-last) is dominated by two async + persistence/compaction bursts of 6-8 MiB each, matching the prior session's + warmed-repeat outlier observation. Steady-state growth is not linear. +- Cold `agent-turn`, `subconscious`, `subagents`, and `workflow` all retain + approximately the same ~29-30 MiB, confirming the cost is shared bootstrap + (code paging, registries, detectors, allocator high water), not the + specific workload on top of it. +- A dhat run of `agent-turn` measured 33.4 MB total allocated across 135,756 + blocks, but only 5.0 MB peak live heap and 3.1 MB live at exit, again + showing RSS is mostly not live heap data. + +### Fleet, instances, and runtime baselines (2026-07-22, post-PII-prefilter) + +`library-fleet.sh` sweep (default build, 3 repeats, 3 turns/agent, 200 ms +mock latency, 2 worker threads, target 1000 agents / 2048 MiB): + +| N agents | Marginal KiB/agent | Settled MiB | Idle CPU ms/10s | Threads | fds | p95 turn ms | Projected MiB @1000 | Fits | +| ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | :---: | +| 50 | 1,985 | 223 | 3 | 71 | 420 | 2,848 | 1,956 | yes | +| 100 | 1,866 | 356 | 3 | 123 | 820 | 5,402 | 1,840 | yes | +| 500 | 1,770 | 1,393 | 3 | 211 | 3,220 | 25,484 | 1,747 | yes | + +`library-instances.sh` (many-processes model, `agent-turn` held 20 s): +~47.8-48.2 MiB per instance flat at N=10/25/50 → roughly 42 instances per +2 GiB by sum-RSS (upper bound; macOS has no PSS). The one-process fleet +model is ~25x denser than the per-process model. + +Runtime/subagent scenarios: `skill-run` measures a real `node` child at +~72-75 MB RSS (~121 MB process tree vs ~51 MB self) — the basis of the +runtime-pooling issue (tinyhumansai/openhuman#5106); `subagent-storm` shows +~0.78 MiB marginal per additional parallel subagent (K=8→32 cross-width). + +Watch-items from the sweep: thread count grows ~0.35/agent (needs +attribution + cap before real 1000-agent runs), and p95 latency at N=500 on +2 workers shows CPU saturation is the load constraint, not memory. + +## See also + +- [`docs/resource-profiling-session-2026-07-21.md`](resource-profiling-session-2026-07-21.md) — the full manual investigation (deep attribution, cold-path CPU, library-design implications, recommended optimization order). +- [`scripts/profile/README.md`](../scripts/profile/README.md) — script quick reference. +- `src/bin/library_profile/main.rs` — the scenario implementations. diff --git a/docs/library-minimal-recipe.md b/docs/library-minimal-recipe.md new file mode 100644 index 000000000..e6fff3743 --- /dev/null +++ b/docs/library-minimal-recipe.md @@ -0,0 +1,246 @@ +# Library-minimal feature recipe + +A **supported, measured** compile-time feature recipe for embedding the OpenHuman +Rust core as a library in "opencompany" — headless, no RPC server, no Tauri +shell, targeting 100-1000 live agents in a 2 GB RAM / 2 vCPU box. + +It follows the repo's existing slim convention (`cargo build --no-default-features +--features ""`, see AGENTS.md "Compile-time domain gates") and keeps +only the domains the opencompany use cases actually exercise: **agent turns, +subagent delegation, memory ingest, workflow (flows) runs, and python/js skill +execution.** + +## The build command + +Opencompany recipe (production embed — no benchmark/harness code): + +```bash +GGML_NATIVE=OFF cargo build --release \ + --no-default-features --features "skills,flows" +``` + +- `GGML_NATIVE=OFF` is the Apple-Silicon dev workaround for whisper-rs/llama; on + the x86-64 Linux target it is unnecessary (the always-on whisper build uses the + AVX path). Keep it in the command for macOS developers. +- To build the profiling harness against the same recipe, add the dev-only + `rss-bench` feature and the two bench bins: + + ```bash + GGML_NATIVE=OFF cargo build --release \ + --no-default-features --features "rss-bench,skills,flows" \ + --bin library-profile --bin rss-bench + ``` + +There is **no** `library-minimal` meta-feature in `Cargo.toml`, on purpose — see +[Why no alias](#why-no-cargotoml-alias) below. + +## Keep / drop table + +`default = ["tokenjuice-treesitter","voice","web3","media","meet","skills","flows","mcp","desktop-automation","tui"]` + +| Gate | Default | Decision | Why | Deps shed | +| --- | :---: | :---: | --- | --- | +| `skills` | ON | **KEEP** | python/js `SKILL.md` execution is a stated opencompany use case | none (surface/prompt/startup only) | +| `flows` | ON | **KEEP** | saved-workflow (`flows_create`+`flows_run`) runs are a stated use case | — (adds `tinyflows`, `jaq-*`, `rhai`; see cost note) | +| `tokenjuice-treesitter` | ON | **DROP** | AST-aware code compression → gracefully falls back to the brace-depth heuristic. Functional, only compression *quality* degrades. | tree-sitter Rust/TS/Python grammars + their C build | +| `voice` | ON | **DROP** | STT/TTS/dictation/podcast — a headless host does no audio I/O | `hound`, `lettre` | +| `web3` | ON | **DROP** | crypto wallet / swap / x402 machine payments — not an opencompany path | `bitcoin`, `curve25519-dalek` | +| `media` | ON | **DROP** | `media_generate_*` image/video tools — surface-only | none (backend-proxied) | +| `meet` | ON | **DROP** | Google-Meet join/live-STT/TTS bot — no headless use | none | +| `mcp` | ON | **DROP** | MCP stdio/HTTP server + Smithery registry (~20k LOC, ~19 tools) — a library host is not an MCP host | none (hand-rolled over tokio/reqwest/axum) | +| `desktop-automation` | ON | **DROP** | AX / screen-capture / `computer` tool family drives a **local desktop UI** — meaningless headless | `uiautomation` | +| `tui` | ON | **DROP** | `openhuman tui`/`chat` terminal UI — no terminal in a library host | `ratatui`, `crossterm`, `unicode-width` | + +**Non-default optional features** (`sandbox-landlock`, `sandbox-bubblewrap`, +`peripheral-rpi`, `browser-native`/`fantoccini`, `landlock`, `whatsapp-web`, +`e2e-test-support`, `rss-bench`, `rss-bench-dhat`) are all default-OFF, so a +`--no-default-features` build never links them unless explicitly added. None are +needed for opencompany; `rss-bench`/`rss-bench-dhat` are dev/benchmark-only. + +### On `tokenjuice-treesitter` + +This is the one judgment call. Dropping it removes the largest *native C build* +in the domain gates (three tree-sitter grammars) and shrinks the binary, at the +cost of coarser code-context compression (brace-depth heuristic instead of AST). +For a memory/binary-minimal library host it is dropped here. **If token budget +per agent turn matters more than binary size, add `tokenjuice-treesitter` back** +— it sheds no runtime behavior beyond compression fidelity. + +## Measured results + +All numbers gathered on this branch, Apple-Silicon macOS, `--release` profile +(`optimized + debuginfo`). "default" = the prior 2026-07-21 session baselines in +[`docs/library-benchmarking.md`](library-benchmarking.md); "pure slim" = +`--no-default-features --features rss-bench` (drops everything). Both slim numbers +were reproduced on this machine and match the prior doc exactly (68.4 MiB). + +### Binary size + +| Build | Features | Unstripped | Stripped | +| --- | --- | ---: | ---: | +| default | (all gates) | 115.9 MiB¹ | — | +| **library-minimal** | `skills,flows` | **~81.1 MiB** | **~60.4 MiB** | +| pure slim | (none) | 68.4 MiB | 51.0 MiB | + +¹ from the prior session (unstripped, same profile). library-minimal bins measured +directly: `rss-bench` 81.1 MiB, `library-profile` 83.0 MiB unstripped (the extra +~2 MiB is the harness itself). The domain recipe (`skills,flows`, no `rss-bench`) +matches the `rss-bench` figure — the bench feature adds negligible code. + +- **library-minimal vs default: -34.8 MiB (~30% smaller)**, and a correspondingly + narrower code-paging surface (the dominant cold-turn RSS factor per the prior + session's executable-paging finding). +- **library-minimal vs pure slim: +12.7 MiB unstripped / +9.4 MiB stripped — all + of it `flows`.** `cargo tree` confirms the delta is `rhai 1.25` + `rhai_codegen` + + `jaq-core/std/json` + `tinyflows`; `skills` sheds **zero** deps (its value is + tool-surface/prompt/startup, not size). `flows` is by far the most expensive + domain we *keep* — see follow-up #2. + +### Per-scenario RSS (5 fresh-process repeats, median, `OPENHUMAN_PROFILE_FORCE_UTC=1`) + +| Scenario | minimal settled | minimal retained Δ | default settled² | default retained² | Δ settled | +| --- | ---: | ---: | ---: | ---: | ---: | +| `agent-turn` (cold, 1 turn) | 44.0 MiB | 26.6 MiB | 47.6 MiB | 29.5 MiB | **-3.6 MiB** | +| `subagents` (cold, 2 children) | 44.5 MiB | 27.1 MiB | 48.0 MiB | 29.9 MiB | **-3.5 MiB** | +| `workflow` (`flows_create`+`flows_run`) | 46.2 MiB | 26.0 MiB | 50.9 MiB | 29.9 MiB | **-4.7 MiB** | +| `memory-ingest` (100 msgs) | 24.7 MiB | 8.8 MiB | 25.8 MiB | 9.3 MiB | **-1.1 MiB** | +| `long-agent` (10 turns) | 46.4 MiB | 2.9 MiB | — (25-turn: 65.8 MiB) | — | n/a³ | + +² default column from `docs/library-benchmarking.md` (2026-07-21). Those medians +may not have used `OPENHUMAN_PROFILE_FORCE_UTC=1`, so treat the Δ as approximate +(±~1 MiB). The direction and magnitude match the prior session's "slim saves +~3.2 MiB settled RSS" finding. + +³ `long-agent` was run at 10 turns here vs 25 in the default baseline, so the +absolute settled figures aren't comparable. The low 2.9 MiB retained Δ confirms +per-turn growth plateaus (matches the prior "not linear" observation). + +**Takeaway (consistent with the prior session):** compile-time gates shrink the +*binary* substantially (-30%) but move *settled RSS* by only ~3-5 MiB per +scenario. Most of the RSS story is initialization + allocator high-water, not +linked code size. The binary/code-paging win is the primary reason to prefer this +recipe; the RSS win is real but secondary. + +## What is functionally absent in this build + +Summarized from the per-gate behavior notes in AGENTS.md. Dropped domains fail +*closed and cleanly* — controllers become unknown-method, tools are simply absent +from the tool list (not degraded to runtime errors), CLI subcommands report a +build-fact error: + +- **voice/audio:** voice + audio controllers unregistered (unknown-method over + RPC, absent from `/schema`); `audio_generate_podcast` tools absent; `openhuman + voice` returns "voice disabled". +- **web3:** wallet / web3 / x402 controllers unregistered; swap/bridge/dapp agent + tools absent; the x402 402-retry path returns unpaid; tinyplace on-chain + payments + Polymarket *writes* degrade to graceful "wallet disabled" errors + (tinyplace comms + ed25519 signing are unaffected). +- **media:** `media_generate_*` agent tools absent. +- **meet:** meet controllers unregistered; live Meet bot / STT-LLM-TTS loop absent. +- **mcp:** `mcp_server` / `mcp_registry` (`mcp_clients` namespace) / `mcp_audit` + controllers unknown-method; ~19 MCP agent tools absent; `openhuman mcp` CLI + returns a "rebuild with --features mcp" build-fact error. (`McpHttpClient` + + `sanitize` stay compiled — the gitbooks docs tool and the orchestrator prompt + sanitizer still work.) +- **desktop-automation:** `accessibility` / `screen_intelligence` / `autocomplete` + / `desktop_companion` domains + the `computer` tool family (`ax_interact`, + `automate`, mouse/keyboard) absent. +- **tui:** `openhuman tui` / `chat` returns "tui feature disabled at compile time". +- **tokenjuice-treesitter:** code compression falls back to the brace-depth + heuristic — degraded fidelity, not absent. + +Everything the opencompany use cases need remains: the agent harness + turn +runner, subagent delegation (`spawn_parallel_agents`), the full memory stack +(TinyCortex store/tree/queue/ingest + PII/injection detectors), threads, config, +security policy, provider routing/inference, `skills` (SKILL.md discovery/install ++ node/python execution + `run_workflow`/`await_workflow`), and `flows` (saved +graph create/run/schedule + `workflow_builder`/`flow_discovery` agents). + +## Test verification + +The disabled-build test gotcha (AGENTS.md: CI's smoke lane runs `cargo check` +only and never compiles `--no-default-features` test code) was checked directly: + +```bash +GGML_NATIVE=OFF cargo test --lib --no-default-features --features "skills,flows" core:: +# result: ok. 660 passed; 0 failed; 1 ignored; 10513 filtered out +``` + +The both-ways gate tests in `src/core/all_tests.rs` (which assert dropped domains +become unknown-method) pass under this recipe. No pre-existing failures. + +## CI note + +Nothing is added to the `default` feature list — this is a **subtractive** +`--no-default-features` recipe, not a new default-ON gate. The **Feature +Forwarding Gate** (`scripts/ci/check-feature-forwarding.mjs`) only inspects the +`default` list and its forwarding into the desktop shell's `Cargo.toml`, so it +**does not apply** here: there is nothing to forward. This recipe carries no CI +risk and needs no `INTENTIONALLY_NOT_FORWARDED` entry. + +## Why no `Cargo.toml` alias + +The repo convention (AGENTS.md "Slim-profile convention") is deliberate: **no +`full` meta-feature; build slim variants with an explicit feature list.** A +`library-minimal = ["skills","flows"]` alias would be convenient, but it: + +- duplicates the `default` list's maintenance burden — a new default-ON gate that + opencompany *should* pick up would silently be missing from a frozen alias + (the exact failure mode the "no meta-feature" rule exists to avoid), and +- hides the subtractive intent behind a name, making the drop set invisible at + the call site. + +**Recommendation: document the explicit list (this file), do not add the alias.** +If maintainers later decide an alias is worth it, the minimal-drift option is to +express it *subtractively* in tooling rather than as a frozen additive list — +but that is a follow-up decision, not part of this recipe. + +## Follow-up shed list (ranked) + +Largest remaining always-on costs a headless library host does not need. These +are **not implemented here** — they require new gates/refactors — listed for +prioritization. + +1. **`inference` gate → shed `whisper-rs` + `whisper-rs-sys` (+ `cpal`/`coreaudio`).** + `whisper-rs-sys` statically links the whisper.cpp + GGML C++ inference library + — the single largest always-on *native* chunk in the binary and the reason for + the `GGML_NATIVE=OFF` build dance. `cargo tree` confirms `whisper-rs 0.16` is a + **direct always-on dependency of `openhuman`** (not gated by `voice`, per the + AGENTS.md scope note), pulling `whisper-rs-sys 0.15`; `cpal 0.15` + `coreaudio` + ride alongside for audio capture. A headless library host does no local STT, so + an `inference` gate would shed all of this — the biggest remaining binary + + native-build win by far. Bonus: `cpal` is shared only with `accessibility`, + which `desktop-automation` (already dropped here) owns — so with this recipe, + `cpal` becomes sheddable the moment the inference gate lands. + *(No `llama`/`candle`/`tokenizers`/`onnx` crates appear in the recipe's tree, so + the local-LLM path is either already optional or absent — whisper is the target.)* + +2. **Split `rhai` out of the `flows` gate.** `flows` is the most expensive domain + we *keep* (+12.7 MiB, dominated by `rhai 1.25` — a full scripting engine). + `rhai` arrives only via `tinyagents/repl`, which powers the `.ragsh` + language-workflow tool (`rhai_workflows`). If opencompany needs `tinyflows` + saved-graph runs but **not** the `.ragsh` rhai tool, splitting `rhai_workflows` + into its own sub-gate would reclaim most of that 12.7 MiB while keeping the + flows graph engine. Currently all-or-nothing. + +3. **`git2` (vendored libgit2).** Always-on native dependency of the `memory_diff` + change-ledger (git-backed snapshots/checkpoints/diffs). A large vendored C lib. + If a library host does not need git-backed memory diffs, this is a candidate for + a future gate. + +4. **`reqwest` dual TLS backends.** The root `reqwest` enables both `rustls-tls` + **and** `native-tls` — two full TLS stacks linked simultaneously. A headless + host on a known target could pick one, shedding the other. + +5. **Node/Python runtime bootstrap deps** (`tar`, `xz2`+liblzma, `zip`, `flate2`). + Only needed if `skills`/`flows` actually execute node/python workloads; kept + here because `skills` is on. If a deployment runs only pure-LLM skills, these + archive/decompression deps become sheddable. + +## See also + +- [`docs/library-benchmarking.md`](library-benchmarking.md) — the benchmark + environment, scenario definitions, and default/slim baselines. +- [`docs/resource-profiling-session-2026-07-21.md`](resource-profiling-session-2026-07-21.md) + — deep memory/CPU attribution (why RSS is mostly not live heap). +- AGENTS.md "Compile-time domain gates" — the per-gate behavior and dependency notes. diff --git a/docs/resource-profiling-session-2026-07-21.md b/docs/resource-profiling-session-2026-07-21.md new file mode 100644 index 000000000..4706b3736 --- /dev/null +++ b/docs/resource-profiling-session-2026-07-21.md @@ -0,0 +1,503 @@ +# OpenHuman resource profiling session + +Date: 2026-07-21 +Platform: Apple Silicon macOS 26.5.1 +Worktree: `worktrees/tauri-resource-profiler` +Primary question: What CPU and RAM does OpenHuman consume, which components account for it, and what would it take to use the Rust core as an efficient embedded library? + +## Executive summary + +The desktop application's large footprint is primarily outside the Rust core. The full Tauri/CEF process family measured about 1.2-1.4 GiB depending on CEF prewarming and spaCy. Disabling CEF prewarming saved about 86 MiB, and disabling spaCy after that saved another 146 MiB. + +The Rust core is much smaller: + +- A warmed one-agent roster used 38.7 MiB RSS with default features and 35.5 MiB in a slim build. +- Growing from one to eight warmed agents cost only about 0.40 MiB per additional agent. +- Ingesting 100 representative chat messages retained about 9.3 MiB and completed in about 2.25 seconds. +- A cold chat turn that spawned two real subagents increased RSS by about 26-31 MiB, depending on feature selection and run conditions. +- That subagent increase is overwhelmingly a first-use cost. After one complete warm-up turn, another equivalent turn added only 0.52 MiB with persistence disabled or 1.84 MiB with normal memory capture enabled. + +There is no single Rust module holding 45 MiB of live data. In the clean slim-build snapshot at approximately 42 MiB total RSS: + +- only 15.2 MiB was private physical footprint; +- only 3.18 MiB was active heap allocation; +- 18.7 MiB was resident executable code from the OpenHuman binary; +- the malloc zones retained 9.4 MiB despite only about 3.2 MiB being live, indicating substantial allocator high-water retention/fragmentation. + +The most actionable module-level findings are: + +1. Parent and child agents each initialize full memory/SQLite infrastructure. +2. TinyCortex's multilingual PII `RegexSet` and its regex caches dominate the identified live Rust heap growth during normal memory capture. +3. The first turn touches about 15 MiB of previously nonresident OpenHuman executable code. +4. Built-in agent TOML parsing, agent construction, unified-memory construction, and SQLite initialization dominate cold-path CPU. +5. Compile-time feature selection reduces binary size dramatically but reduces live RSS only moderately. + +## Scope and methodology + +The session used three progressively narrower boundaries: + +1. The complete Tauri desktop process family, including CEF and helper processes. +2. The standalone Rust core with no Tauri host. +3. Direct Rust library paths for agent construction, memory ingestion, chat orchestration, and subagent delegation. + +All Rust measurements used release builds. Network inference was replaced by a deterministic provider behind the default-off `rss-bench` feature. The subagent scenario still used the real `LongLivedSession`, built-in agent registry, agent builder, `spawn_parallel_agents` tool, two researcher agents, memory infrastructure, prompt enforcement, and post-turn hooks. + +RSS was sampled every 5 ms during measured workloads. On macOS, the profiler obtains: + +- current RSS from `proc_pid_rusage`; +- peak RSS from `getrusage`; +- thread count from `proc_pidinfo`; +- binary size from the running executable. + +macOS does not expose Linux `/proc`-style PSS and private clean/dirty page fields through the same interface, so those JSON fields remain zero. `vmmap`, `heap`, `malloc_history`, Instruments Allocations, and Samply were used for deeper attribution. + +Unless stated otherwise, summarized Rust results are medians from five fresh processes. CPU samples from `/usr/bin/time` are representative runs rather than five-run medians. + +## Desktop/Tauri measurements + +These measurements aggregate the desktop process family rather than only the Rust process. + +| Configuration | Mean RAM | Change | +| ------------------------------ | ----------: | ---------------: | +| Default CEF prewarm and spaCy | 1,439.8 MiB | baseline | +| CEF prewarm disabled | 1,353.5 MiB | -86.3 MiB | +| CEF prewarm and spaCy disabled | 1,207.6 MiB | -232.2 MiB total | + +The clearest desktop optimizations are therefore: + +- initialize spaCy lazily only when memory-tree operations require it; +- avoid permanent CEF prewarming, or make the prewarm process short-lived; +- keep accessibility snapshots and `osascript`-based probes event-driven and rate-limited rather than continuously polling. + +These desktop results motivated isolating the Rust core: most of the shipped application's memory is not explained by core agent objects. + +## Bare Rust agent roster + +The existing `rss-bench` binary constructs real OpenHuman agents without Tauri and measures stable RSS in fresh child processes. + +### Default feature build + +| Roster | Median RSS | Threads | Binary size | +| -------- | --------------------: | ------: | ----------: | +| 1 agent | 39,616 KiB / 38.7 MiB | 22 | 115.9 MiB | +| 8 agents | 42,480 KiB / 41.5 MiB | 24 | 115.9 MiB | + +The seven additional agents added 2,864 KiB in total, or approximately 409 KiB per agent. The fixed runtime and linked-code cost is much larger than the marginal agent object cost. + +### Slim feature build + +The slim binaries were built with: + +```bash +GGML_NATIVE=OFF cargo build --release \ + --no-default-features --features rss-bench \ + --bin rss-bench --bin library-profile +``` + +| Roster | Median RSS | Binary size | +| -------- | --------------------: | ----------: | +| 1 agent | 36,368 KiB / 35.5 MiB | 68.4 MiB | +| 8 agents | 39,584 KiB / 38.7 MiB | 68.4 MiB | + +Feature selection reduced the profiling binary by about 40%, but reduced one-agent RSS by only 3.2 MiB. Compile-time gates are very valuable for download and embedding size, but they are not sufficient by themselves to minimize the active working set. + +## Rust-only memory ingestion + +The `memory-ingest` scenario creates an isolated workspace, disables local inference, Python, spaCy, and embeddings, then sends 100 representative chat messages through the real canonicalization, ingestion, admission, persistence, and memory-queue drain paths. + +### Default features + +| Metric | Median | +| ---------------------- | -----------------------: | +| Duration | 2,251 ms | +| Baseline RSS | 16,624 KiB / 16.2 MiB | +| Settled RSS | 26,144 KiB / 25.5 MiB | +| Retained/peak increase | 9,536 KiB / 9.31 MiB | +| Throughput | about 44 messages/second | + +### Slim features + +| Metric | Median | +| ---------------------- | --------------------: | +| Duration | 2,313 ms | +| Baseline RSS | 15,536 KiB / 15.2 MiB | +| Settled RSS | 24,320 KiB / 23.8 MiB | +| Retained/peak increase | 8,784 KiB / 8.58 MiB | + +The feature change reduced settled RSS by about 1.8 MiB and had no meaningful throughput benefit. + +A representative default-feature `/usr/bin/time -l` run reported 0.18 seconds user CPU and 0.69 seconds system CPU. This should be treated as an upper bound: sampling RSS every 5 ms adds macOS process-inspection system calls, and the workload also performs real temporary-workspace I/O. + +## Rust-only subagent chat + +The `subagents` scenario uses a deterministic local provider but otherwise follows the real orchestration path: + +1. Build a long-lived subconscious session. +2. Submit a promoted chat message. +3. Call the real `spawn_parallel_agents` tool. +4. Spawn two real `researcher` agents with separate ownership scopes. +5. Verify that both child prompts were executed. +6. Measure baseline, peak, and settled RSS. + +### Cold default-feature result + +| Metric | Median | +| ---------------------- | --------------------: | +| Duration | 163 ms | +| Baseline RSS | 18,192 KiB / 17.8 MiB | +| Settled RSS | 49,712 KiB / 48.5 MiB | +| Retained/peak increase | 31,520 KiB / 30.8 MiB | + +A representative process used 0.04 seconds user CPU and 0.06 seconds system CPU. Model and network latency are deliberately excluded. + +### Cold slim-feature result + +| Metric | Median | +| ---------------------- | --------------------: | +| Duration | 158 ms | +| Baseline RSS | 17,072 KiB / 16.7 MiB | +| Settled RSS | 43,392 KiB / 42.4 MiB | +| Retained/peak increase | 26,304 KiB / 25.7 MiB | + +The slim build saved about 6.2 MiB of settled RSS in this richer workload. + +## Deep memory attribution + +### Clean normal snapshot + +A non-instrumented slim-build run with normal memory capture settled at 43,104 KiB RSS, or 42.1 MiB. + +| VM/heap measurement | Result | +| ------------------------------------- | ------------: | +| Total RSS | 42.1 MiB | +| Private physical footprint | 15.2 MiB | +| Approximate clean/file-backed portion | 26.9 MiB | +| Live heap allocations | 3.18 MiB | +| Resident malloc regions | about 9.4 MiB | +| Resident stacks | 0.97 MiB | +| Resident OpenHuman executable text | 18.7 MiB | + +These categories overlap and must not be added together. For example, active heap and stacks are part of the private footprint, while executable text is part of clean/file-backed RSS. + +The key interpretation is that RSS is not equivalent to private heap. The process reports roughly 42 MiB RSS while holding only about 3.2 MiB of live heap allocations. + +### First-use executable paging + +Before the chat turn, only about 3.3 MiB of the profiling executable's `__TEXT` segment was resident. After the turn, 18.7 MiB was resident. The first turn therefore faulted in approximately 15.4 MiB of OpenHuman's own executable code. + +CoreFoundation, Foundation, ICU, Security, CoreAudio, and other macOS framework `__TEXT` residency was effectively identical in the controlled baseline and post-turn snapshots. The increase came from the OpenHuman executable, not from a single newly loaded macOS framework. + +Executable pages are clean, file-backed, reclaimable under pressure, and shareable between identical processes. They count toward RSS but are not the same as permanently retained private data. + +### Allocator retention + +The normal snapshot had about 3.2 MiB of active allocations inside approximately 9.4 MiB of resident malloc pages. Roughly 6 MiB was therefore allocator slack, fragmentation, size-class pages, or high-water retention after the burst of first-turn allocations. + +This does not prove a leak. A repeated-turn plateau test is more meaningful than expecting macOS RSS to immediately return to its pre-turn value. + +### Memory capture and PII cost + +A controlled profiler switch disabled both: + +- `memory.auto_save`; +- `learning.episodic_capture_enabled` and its archivist hook. + +With those writes disabled, the median cold-turn increase fell from about 25.9 MiB to 22.0 MiB, saving approximately 3.8 MiB. + +Stack-logged allocations identified TinyCortex's multilingual PII sanitizer as the dominant live Rust allocation family during normal memory capture. Important allocations originated from: + +- `tinycortex::memory::store::safety::pii::SCREEN`; +- the combined `RegexSet` NFA; +- per-thread hybrid-DFA regex caches; +- calls through `sanitize_text`, document upsert, FTS5 episodic insertion, autosave, and the archivist hook. + +The `SCREEN` implementation is described as a cheap prefilter, but its large multilingual, Unicode-aware combined automaton is not cheap in retained memory. A byte-oriented candidate scan followed by targeted regex evaluation is a promising replacement. + +### Prompt-injection detector + +With memory writes disabled, the next visible regex allocation family came from `prompt_injection::detector::DETECTION_RULE_SET`. It was materially smaller than the TinyCortex PII machinery, on the order of a few hundred KiB in this workload. + +This is not currently a first-priority RAM optimization, but its scratch/cache strategy should be considered if agent turns become highly parallel. + +### Timezone control + +`current_datetime_line` normally calls `iana_time_zone`, which uses CoreFoundation on macOS. Forcing the profiling build to use UTC saved only about 0.5-0.6 MiB in the isolated subagent scenario. This is measurable but not a primary explanation for the 42-49 MiB working set. + +## Cold-path CPU attribution + +A symbolized Samply profile was recorded across repeated no-network, no-memory-write, UTC-controlled runs. Inclusive percentages overlap because a sample contributes to every parent frame in its call stack. + +| Cold-path component | Approximate inclusive CPU | +| -------------------------------------- | ------------------------: | +| Built-in agent registry initialization | 17% | +| `LongLivedSession::build_agent` | 16% | +| `Agent::from_config_for_agent` | 15% | +| Built-in agent TOML parsing/loading | 13% | +| Unified-memory construction | 10% | +| SQLite/unified-memory initialization | 7% | +| Actual TinyAgents turn runner | 5% | + +The profile also showed `Config::load_or_init`, config serialization/migrations, filesystem synchronization, tool-policy cloning, prompt-injection initialization, and runtime task scheduling. + +The built-in registry is initialized before the profiler's RSS baseline, so its CPU appears in the whole-process profile but its already-resident pages are part of the baseline rather than the measured turn delta. + +## Warmed-process control + +The most important experiment prewarmed one complete two-subagent turn, dropped that warm-up session, then measured a new session performing the same workload in the same process. + +| Subsequent equivalent turn | Median duration | Median added RSS | +| ----------------------------------- | --------------: | -------------------: | +| Persistence disabled and UTC forced | 46 ms | 528 KiB / 0.52 MiB | +| Normal memory capture | 60 ms | 1,888 KiB / 1.84 MiB | + +One normal-memory repetition was an 8.3 MiB outlier, consistent with asynchronous persistence or allocator behavior; the other four were between about 1.4 and 1.9 MiB. + +This result changes the interpretation of the cold 26-31 MiB increase. It is overwhelmingly initialization, code paging, global regex/cache construction, and allocator high-water behavior. It is not a linear 26 MiB cost per chat turn. + +## Library-design implications + +OpenHuman is feasible as a Rust library, but the current construction path behaves like an application bootstrap rather than a lightweight per-instance library API. + +### Share services between agents + +`Agent::build_session_agent_inner` constructs session memory and obtains a SQLite connection for the agent. Parent and child agents should instead receive shared instance services such as: + +```text +OpenHumanLibrary + Arc + Arc + Arc + Arc + Arc + Arc +``` + +Subagents should borrow or clone these `Arc` handles rather than rebuilding configuration, memory stores, schema state, providers, and tool catalogs. + +### Offer explicit warm-up + +A library API should expose an optional warm-up method that initializes predictable first-use costs: + +- built-in agent definitions; +- prompt and PII detectors; +- memory schemas and connection pools; +- tool catalogs and policy snapshots; +- provider routing; +- commonly used prompt fragments. + +This lets latency-sensitive hosts choose between low startup work and predictable first-message latency. + +### Separate runtime and compile-time slimness + +`DomainSet` is useful for runtime surface selection, but runtime-disabled code remains linked. Library consumers need documented compile-time feature recipes or a dedicated library/harness feature set so unrelated domains never enter the binary. + +### Avoid mandatory globals + +The profiling harness currently has to initialize a global event bus, a global built-in registry, environment-derived workspace selection, and a global provider override. Instance-owned state would make it safer to embed multiple OpenHuman instances in one process and would improve deterministic testing. + +## Recommended optimization order + +1. **Share unified-memory and SQLite services between parent and child agents.** This is the clearest architectural duplication in the cold path. +2. **Replace the TinyCortex PII `RegexSet` screen with a lightweight candidate scan.** Preserve the strict regex/checksum validation after a candidate is found. +3. **Add a warmed repeated-turn benchmark to CI or the local profiling suite.** Track both cold-start and steady-state behavior so one does not obscure the other. +4. **Provide a supported library-minimal feature recipe.** Measure binary size, cold code paging, and steady RSS for that exact recipe. +5. **Reuse temporary prompt, tool-schema, and sanitizer buffers.** Reduce burst allocation and malloc-zone fragmentation. +6. **Benchmark an alternate allocator only in the profiling binary.** This can reveal how much of the 6 MiB malloc slack is allocator-specific, but the library should not impose a global allocator on consumers. +7. **Add per-phase checkpoints.** Measure config load, agent build, memory construction, prompt render, delegation, child execution, merge, hooks, and teardown separately. + +## Functional observation + +In the deterministic `LongLivedSession` workload, both parallel child agents executed, but the session returned the last subagent result rather than performing the prepared parent synthesis response. Direct `Agent::turn` tests cover synthesis behavior elsewhere, so the long-lived-session boundary deserves a focused correctness audit. This is separate from the resource findings but was exposed by the same harness. + +## Reproducing the measurements + +Build the default-feature profiling binaries: + +```bash +GGML_NATIVE=OFF cargo build --release --features rss-bench \ + --bin rss-bench --bin library-profile +``` + +Build the slim versions: + +```bash +GGML_NATIVE=OFF cargo build --release \ + --no-default-features --features rss-bench \ + --bin rss-bench --bin library-profile +``` + +Run the bare roster benchmark: + +```bash +target/release/rss-bench --repeat 5 \ + --out target/profile/rust-library/bare-agents.json +``` + +Run stateful library workloads: + +```bash +target/release/library-profile memory-ingest +target/release/library-profile subagents +``` + +Measure a warmed subagent turn: + +```bash +OPENHUMAN_PROFILE_PREWARM_SUBAGENTS=1 \ + target/release/library-profile subagents +``` + +Isolate orchestration from persistence and local timezone initialization: + +```bash +OPENHUMAN_PROFILE_DISABLE_MEMORY_WRITES=1 \ +OPENHUMAN_PROFILE_FORCE_UTC=1 \ + target/release/library-profile subagents +``` + +Hold a process at its baseline or settled state for `vmmap`, `heap`, or `malloc_history`: + +```bash +OPENHUMAN_PROFILE_HOLD_BEFORE_SECS=120 \ + target/release/library-profile subagents + +OPENHUMAN_PROFILE_HOLD_SECS=120 \ + target/release/library-profile subagents +``` + +Example live inspection: + +```bash +vmmap -summary +heap -sH + +MallocStackLogging=1 OPENHUMAN_PROFILE_HOLD_SECS=120 \ + target/release/library-profile subagents +malloc_history -allBySize +``` + +Record a symbolized CPU profile: + +```bash +OPENHUMAN_PROFILE_DISABLE_MEMORY_WRITES=1 \ +OPENHUMAN_PROFILE_FORCE_UTC=1 \ +samply record --save-only --unstable-presymbolicate \ + --rate 1000 --iteration-count 5 \ + --output target/profile/rust-library/subagents-cpu.json.gz \ + -- target/release/library-profile subagents +``` + +## Artifacts and code added during the session + +- `src/bin/library_profile/main.rs`: hermetic memory-ingestion and subagent workloads, peak RSS sampler, isolation controls, warm-up control, and debugger hold points. +- `src/openhuman/proc_metrics/mod.rs`: macOS RSS, peak RSS, thread-count, and binary-size sampling. +- `src/openhuman/inference/provider/factory.rs`: deterministic provider override enabled under the default-off profiling feature. +- `src/openhuman/inference/provider/ops/provider_factory.rs`: routed-provider support for the same profiling override. +- `src/openhuman/agent/prompts/render_helpers.rs`: profiling-only UTC control under `rss-bench`. +- `target/profile/rust-library/`: ignored JSON, Instruments, and Samply artifacts from local runs. + +None of the profiling binaries or provider overrides enter normal shipped builds because they require the default-off `rss-bench` feature. + +## Validation completed + +- Default-feature release builds of `rss-bench` and `library-profile`. +- Slim release builds using `--no-default-features --features rss-bench`. +- Five-run fresh-process repetitions for the main scenarios. +- macOS process-metric unit test. +- Existing datetime prompt test with `rss-bench` enabled. +- `cargo fmt --check`. +- `git diff --check`. + +Repository warnings observed during builds were pre-existing unused-import/dead-code and future-incompatibility warnings; no new build failure was introduced by the profiling harness. + +## Bottom line + +OpenHuman's Rust core is not holding 45 MiB of agent objects. The steady process is mostly executable working set plus runtime/allocator pages, with a small live heap. The cold first turn is expensive because it initializes and touches a broad application-oriented path. Once warmed, subagent turns are inexpensive and appear to plateau rather than grow linearly. + +The best route to an efficient library is therefore not micro-optimizing every agent struct. It is narrowing and sharing the initialization graph: reuse memory and SQLite services, avoid rebuilding agent infrastructure for children, simplify the PII prefilter, expose an explicit warm-up lifecycle, and provide a compile-time library-minimal profile. + +--- + +## Addendum: library benchmarking session (2026-07-22) + +The follow-up session turned the manual investigation above into a permanent +benchmark environment, executed two of the recommended optimizations, and +answered the deployment-density question. Full detail lives in +[`library-benchmarking.md`](library-benchmarking.md); this addendum records +the deltas against this document. + +### What was built + +- **Ten hermetic scenarios** in `library-profile` (was two): `agent-turn`, + `long-agent`, `workflow`, `subconscious`, `cold-phases`, `fleet`, + `skill-run`, `subagent-storm` joined `memory-ingest`/`subagents`. All + offline, mock-provider, JSON schema v2 with per-phase/per-turn checkpoints. +- **Five driver scripts** under `scripts/profile/`: `library-bench.sh` + (medians + summary), `library-fleet.sh` (agent-count sweep with a + 2 GB/2 vCPU pass/fail gate), `library-instances.sh` (many-processes model), + `library-cpu.sh` (samply), `library-heap.sh` (dhat). +- **Professional profilers wired in**: dhat behind the default-off + `rss-bench-dhat` feature; samply scripted; `proc_metrics` extended with + CPU-time, fd counts, and a descendant process-tree sampler (`tree.rs`). + +### Headline results (Apple Silicon, default build unless noted) + +| Question | Answer | +| --- | --- | +| Cold turn cost, any shape (chat/subconscious/delegation/workflow) | ~29-30 MiB retained — shared bootstrap, not workload | +| Warmed long-agent growth | 30-150 KiB/turn plateau; occasional 6-8 MiB async persistence bursts | +| Live heap vs RSS (dhat, agent-turn) | 33.4 MB total allocated, 5.0 MB peak live, 3.1 MB at exit | +| Fleet marginal per in-process agent | ~1.7-2.0 MiB (N=50/100/500 sweep) | +| Idle CPU, parked fleet | ~3 ms per 10 s at every N | +| **1000 agents in 2 GiB (one process)** | **PASS — projected ~1747 MiB @ 1000; real 500-agent run settled 1393 MiB** | +| Same workload as N processes | ~48 MiB/instance flat → only ~42 instances per 2 GiB; one-process model is ~25x denser | +| True cost of a JS skill run | node child ~72-75 MB RSS (tree ~121 MB vs ~51 MB self) | +| Marginal per parallel subagent (storm K=8→32) | ~0.78 MiB | +| Library-minimal build (`--no-default-features --features skills,flows`) | 81 MiB binary (60 stripped) vs 116; RSS -3.5 to -4.7 MiB | + +### Changes landed + +- **PII prefilter replaced upstream** (recommendation 2 above): + tinycortex#119 swaps the resident 17-pattern `RegexSet` + per-thread DFA + caches for a single-pass byte candidate scan gating lazily-compiled + per-class regexes; superset-verified against the old set, no common-path + heap regression, wins scale with thread/agent concurrency. Merged with + #120 (unrelated rustdoc fix); submodule bumped. +- **Library-minimal recipe** documented and measured + ([`library-minimal-recipe.md`](library-minimal-recipe.md)); ranked + follow-up sheds identified (an `inference` gate for whisper/GGML first). +- **Harness comparison** ([`harness-comparison-2026-07-22.md`](harness-comparison-2026-07-22.md)): + the scope-matched peer (Hermes, Python) self-reports ~10x our RSS; the + ZeroClaw "7.8-12 MiB under load" figure has no locatable primary source and + is now flagged unverified wherever cited. Our in-process ~2 MiB marginal + scaling has no equivalent among the surveyed harnesses. + +### New watch-items surfaced by the fleet sweep + +1. **Thread growth ~0.35/agent** (71 @ N=50 → 211 @ N=500, →~420 projected at + 1000). Attribute (SQLite? blocking pool?) and cap before 1000-agent runs. +2. **CPU, not memory, is the load constraint on 2 workers**: p95 turn latency + 25.5 s at N=500 under 200 ms mock latency. Scheduling/backpressure design + matters more than RAM once the fleet is dense. +3. **Interpreter children break the budget**: pooling/sharing the node and + python runtimes is filed as tinyhumansai/openhuman#5106 — without it, + ~25 concurrent JS skill runs exhaust the whole 2 GiB box. + +### Updated optimization order + +1. Shared services between parent/child agents (unchanged, still first; the + fleet benchmark is its regression instrument). +2. ~~PII prefilter~~ — done, merged upstream. +3. Runtime pooling for node/python (#5106) — new, promoted to near-top by the + skill-run measurement. +4. Thread-growth attribution and cap (new). +5. Warm-up API, allocator experiment, per-phase checkpoints in CI — the + checkpoints now exist (`cold-phases`); CI wiring remains. +6. `inference` compile gate (whisper/GGML) for the library-minimal profile. + +### Next: the live desktop app + +The same rigor now needs to reach the shipped Tauri/CEF app (the ~1.2-1.4 GiB +family this document opened with). The prepared brief for that session — +scenarios, reusable assets, gates — is +[`tauri-live-profiling-brief.md`](tauri-live-profiling-brief.md). diff --git a/docs/tauri-live-profiling-brief.md b/docs/tauri-live-profiling-brief.md new file mode 100644 index 000000000..dacb818dc --- /dev/null +++ b/docs/tauri-live-profiling-brief.md @@ -0,0 +1,85 @@ +# Brief: Tauri live-app benchmark and profiling + +Audience: the next agent/session taking on desktop-app (live Tauri + CEF) +profiling. The Rust-core library side is done and documented in +[`library-benchmarking.md`](library-benchmarking.md); this brief covers what +to build for the shipped desktop app, what already exists, and what to reuse. + +## Goal + +Bring the desktop app to the same standard the core now has: named, +repeatable scenarios; fresh-process repeats with median aggregation; JSON +results; pass/fail budget gates; and an escalation path for attribution. The +app is a process *family* (Tauri host + CEF helpers + utility/GPU/renderer +processes + node/python runtime children), so every measurement must be +family-wide, not single-pid. + +## What already exists (reuse, do not rebuild) + +- **`app/src-tauri/profiling/`** — an offline Tauri process-family profiler + (own small crate: `src/main.rs` ~813 lines, README) added in commit + `ce6cd2291`. Start here; extend it rather than writing a new sampler. +- **`src/openhuman/proc_metrics/`** — RSS/peak/threads/CPU-ms/fds sampling + plus `tree.rs` (descendant process-tree walker, macOS `proc_listchildpids` + + Linux `/proc` ppid walk). The tree sampler is exactly what family-wide + measurement needs. +- **`scripts/profile/`** — the driver-script pattern (build → N fresh runs → + jq medians → summary.md + gate exit code). Copy `library-bench.sh`'s shape + for an `app-bench.sh`. +- **JSON schema v2** (`library-profile` output) — reuse the field names + (`baseline`/`settled`/`peak_rss_kib`/`checkpoints[]`/`budget`) so existing + aggregation and future CI tooling work on both suites. +- **Env-var conventions** — `OPENHUMAN_PROFILE_*` knobs, `HOLD_SECS`-style + inspection points. + +## Prior findings to build on (2026-07-21 session) + +| Finding | Number | +| --- | ---: | +| Full desktop process family | 1,207-1,440 MiB | +| CEF prewarm cost | ~86 MiB (disable/short-lived candidate) | +| spaCy cost | ~146 MiB (lazy-init candidate) | +| Rust core share of the family | ~40-50 MiB | + +The gap between the ~50 MiB core and the ~1.2-1.4 GiB family is the entire +story: CEF/renderer processes, prewarm policy, spaCy, and shell-side polling. + +## Suggested scenarios + +1. **cold-boot** — launch to interactive UI; family RSS + wall time, + checkpointed (host start, core ready, CEF first frame, UI route mounted). +2. **idle-drift** — 10-30 min idle; family RSS + CPU sampled continuously. + This is where scanner polling, heartbeat, accessibility probes, and + `osascript` probes show up (keep them event-driven per prior findings). +3. **chat-turn-e2e** — one full chat turn through the real UI (drive via CDP + on the CEF debug port or the Appium harness in `openhuman/e2e/`); compare + against the core-only `agent-turn` baseline to attribute shell overhead. +4. **webview-cycle** — open/close provider webviews (N cycles); CEF child + process lifecycle, leak check on repeat. +5. **prewarm-matrix** — CEF prewarm on/off × spaCy on/off, reproducing and + pinning the prior session's ~86/~146 MiB findings as a regression gate. +6. **overlay-surfaces** — mascot/notch/companion windows up vs down. + +## Method notes + +- Family enumeration: union of the Tauri host's descendant tree (use + `proc_metrics::tree`) plus CEF helper processes, which may re-parent — + match by bundle path/name as `app/src-tauri/profiling` already does. +- Sum-RSS double-counts shared CEF framework pages across helpers; on macOS + record `footprint` output alongside ps-style sums (the instances driver + in `scripts/profile/library-instances.sh` shows the pattern and caveat + wording); on Linux use PSS. +- Drive the UI mechanically, not by hand: CDP against CEF (`:19222` per the + e2e harness) or the WDIO/Appium specs. Every scenario must run + unattended. +- Gate suggestion: family budget per scenario (e.g. cold-boot ≤ X MiB, + idle-drift slope ≈ 0), same PASS/FAIL summary style as + `library-fleet.sh`. + +## Definition of done + +- `scripts/profile/app-bench.sh` (or equivalent) running the scenarios + above unattended with median aggregation and gates. +- Baselines recorded in a doc table (like `library-benchmarking.md`). +- The prewarm/spaCy findings converted from one-off observations into + standing regression gates. diff --git a/package.json b/package.json index f38a97b6a..913758c99 100644 --- a/package.json +++ b/package.json @@ -51,6 +51,7 @@ "agent-batch": "node scripts/agent-batch/cli.mjs", "agent-batch:test": "node --test scripts/agent-batch/__tests__/lib.test.mjs scripts/agent-batch/__tests__/cli.test.mjs", "debug": "bash scripts/debug/cli.sh", + "profile:tauri": "cargo run --manifest-path app/src-tauri/profiling/Cargo.toml --", "test:install-ps1": "pwsh -NoProfile -File scripts/tests/OpenHumanWindowsInstall.Tests.ps1", "rust:check": "pnpm --filter openhuman-app rust:check", "rust:clippy": "cargo clippy -p openhuman -- -D warnings && pnpm --filter openhuman-app rust:clippy", diff --git a/scripts/profile/README.md b/scripts/profile/README.md new file mode 100644 index 000000000..1cfc7a435 --- /dev/null +++ b/scripts/profile/README.md @@ -0,0 +1,113 @@ +# `scripts/profile/` + +Reproducible benchmarking scripts for the OpenHuman Rust core as an embedded +library (no RPC server), built around the `library-profile` and `rss-bench` +binaries (see `src/bin/library_profile/main.rs`). Full write-up: +[`docs/library-benchmarking.md`](../../docs/library-benchmarking.md). Prior +findings: [`docs/resource-profiling-session-2026-07-21.md`](../../docs/resource-profiling-session-2026-07-21.md). + +Five driver scripts: `library-bench.sh` (per-scenario RSS/duration), +`library-cpu.sh` (samply), `library-heap.sh` (dhat), `library-fleet.sh` +(fleet-scale sweep + budget gate), and `library-instances.sh` (multi-process +instance sweep). + +## Scripts + +### `library-bench.sh` — RSS/duration benchmark + +Builds `library-profile` + `rss-bench`, runs each scenario N times as a fresh +process, and aggregates median/min/max duration, settled RSS, retained delta, +and peak delta into `summary.json` + `summary.md`. + +```bash +./scripts/profile/library-bench.sh # all 7 scenarios, default build, 5 repeats +./scripts/profile/library-bench.sh --slim --repeat 7 # slim (no-default-features) build +./scripts/profile/library-bench.sh --scenarios "long-agent,subagents" --turns 50 --warm +``` + +Results land in `target/profile/rust-library/bench-/` (or `--out DIR`). + +### `library-cpu.sh` — CPU profile via samply + +Wraps `samply record` around one scenario, isolated from persistence/timezone +noise by default (matching the documented cold-path CPU recipe). + +```bash +./scripts/profile/library-cpu.sh subagents +./scripts/profile/library-cpu.sh long-agent -- OPENHUMAN_PROFILE_TURNS=50 +samply load target/profile/rust-library/subagents-cpu.json.gz +``` + +### `library-heap.sh` — live heap attribution via dhat + +Builds the `rss-bench-dhat` variant and runs one scenario under dhat. RSS and +timing numbers from this build are perturbed by instrumentation; use it only +for allocation-site/retained-bytes attribution, not for RSS comparisons. + +```bash +./scripts/profile/library-heap.sh memory-ingest +# open https://nnethercote.github.io/dh_view/dh_view.html and load +# target/profile/rust-library/dhat-memory-ingest.json +``` + +### `library-fleet.sh` — fleet sweep + 2 GB / 2 vCPU budget gate + +Builds `library-profile` + `rss-bench`, sweeps the `fleet` scenario (N +concurrent live agents with latency-realistic mock inference) across a list +of agent counts, aggregates medians per N, and gates on whether the +projected footprint at the target agent count fits the RAM budget. + +```bash +./scripts/profile/library-fleet.sh --agents 100 --latency-ms 200 +./scripts/profile/library-fleet.sh --agents "50,100,500" --target 1000 --budget-mib 2048 +``` + +Results land in `target/profile/rust-library/fleet-/` (or +`--out DIR`). Exits nonzero if any swept N reports `fits: false` (use +`--no-gate` to report only). See +[`docs/library-benchmarking.md`](../../docs/library-benchmarking.md#the-2-gb--2-vcpu-server-budget) +for the budget math. + +### `library-instances.sh` — multi-instance (many-processes) sweep + +Spawns N independent `library-profile` processes (each a live instance held +alive via `OPENHUMAN_PROFILE_HOLD_SECS`), staggered on startup, and measures +**per-process** cost and box survivability — the opencompany "N independent +processes/containers" deployment model, as opposed to `library-fleet.sh`'s +"N agents in one process" model. Samples aggregate sum-RSS + live count every +2s while instances hold, captures a `vm_stat` snapshot at peak (and a +best-effort `footprint` sample if that macOS tool is present), then +aggregates per swept N: launched/ok counts, median settled RSS per instance, +mean and peak aggregate sum-RSS, and — on Linux, where it's meaningful — +summed PSS. + +```bash +./scripts/profile/library-instances.sh --instances "10,50" --hold-secs 30 +./scripts/profile/library-instances.sh --instances "100,500" --max-instances 500 --gate +``` + +Results land in `target/profile/rust-library/instances-/` (or +`--out DIR`). Refuses to spawn more than `--max-instances` (default 200) +without an explicit raise — see the script's `--help` for the RAM math. Exits +nonzero with `--gate` if any instance failed to complete cleanly (nonzero +exit or missing/invalid JSON); default is report-only. See +[`docs/library-benchmarking.md`](../../docs/library-benchmarking.md#fleet-one-process-vs-instances-many-processes) +for the fleet-vs-instances framing. + +## Quick start + +```bash +# 1. Baseline RSS/duration across all scenarios +./scripts/profile/library-bench.sh + +# 2. CPU attribution for the slowest/most interesting scenario +./scripts/profile/library-cpu.sh subagents + +# 3. If a scenario's RSS looks off, drill into live heap +./scripts/profile/library-heap.sh subagents +``` + +All scripts require `jq` for JSON parsing/aggregation; `library-cpu.sh` also +requires `samply` (`cargo install samply`). Build commands use +`GGML_NATIVE=OFF` to work around the Apple Silicon whisper-rs/llama.cpp NEON +fp16 build issue. diff --git a/scripts/profile/library-bench.sh b/scripts/profile/library-bench.sh new file mode 100755 index 000000000..e56912e13 --- /dev/null +++ b/scripts/profile/library-bench.sh @@ -0,0 +1,310 @@ +#!/usr/bin/env bash +# library-bench.sh — reproducible RSS/duration benchmark for the OpenHuman +# core as an embedded library, using the `library-profile` binary. +# +# Runs each scenario N times as a FRESH process (no state shared across +# repeats), captures each run's JSON, and aggregates medians/min/max into a +# markdown + JSON summary. Companion scripts: library-cpu.sh (samply), +# library-heap.sh (dhat). +# +# Usage: +# ./scripts/profile/library-bench.sh [options] +# +# Options: +# --slim Build with --no-default-features (slim library recipe) +# --repeat N Fresh-process repeats per scenario (default: 5) +# --scenarios "a,b,c" Comma-separated scenario list (default: all seven) +# --turns N OPENHUMAN_PROFILE_TURNS for long-agent (default binary default: 25) +# --skip-build Reuse the existing target/release binaries +# --warm Also run PREWARM_SUBAGENTS=1 variants for subagents + subconscious +# --out DIR Output directory (default: target/profile/rust-library/bench-) +# -h, --help Show this help +# +# Examples: +# ./scripts/profile/library-bench.sh +# ./scripts/profile/library-bench.sh --slim --repeat 7 +# ./scripts/profile/library-bench.sh --scenarios "long-agent,subagents" --turns 50 --warm + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" + +ALL_SCENARIOS="memory-ingest,subagents,agent-turn,long-agent,workflow,subconscious,cold-phases" +WARM_ELIGIBLE=("subagents" "subconscious") + +SLIM=0 +REPEAT=5 +SCENARIOS="$ALL_SCENARIOS" +TURNS="" +SKIP_BUILD=0 +WARM=0 +OUT_DIR="" + +usage() { + sed -n '2,26p' "${BASH_SOURCE[0]}" | sed 's/^# \{0,1\}//' + exit "${1:-0}" +} + +while [[ $# -gt 0 ]]; do + case "$1" in + --slim) SLIM=1; shift ;; + --repeat) + REPEAT="${2:?--repeat requires a value}"; shift 2 ;; + --scenarios) + SCENARIOS="${2:?--scenarios requires a value}"; shift 2 ;; + --turns) + TURNS="${2:?--turns requires a value}"; shift 2 ;; + --skip-build) SKIP_BUILD=1; shift ;; + --warm) WARM=1; shift ;; + --out) + OUT_DIR="${2:?--out requires a value}"; shift 2 ;; + -h|--help) usage 0 ;; + *) + echo "ERROR: unknown argument: $1" >&2 + usage 1 ;; + esac +done + +if ! command -v jq >/dev/null 2>&1; then + echo "ERROR: jq is required (aggregation over run JSON). Install it (e.g. 'brew install jq')." >&2 + exit 1 +fi + +if [[ -z "$OUT_DIR" ]]; then + OUT_DIR="$REPO_ROOT/target/profile/rust-library/bench-$(date +%Y%m%d-%H%M%S)" +fi +mkdir -p "$OUT_DIR" + +BIN="$REPO_ROOT/target/release/library-profile" + +log() { echo "[library-bench] $*" >&2; } + +build_binaries() { + if [[ "$SKIP_BUILD" -eq 1 ]]; then + log "skipping build (--skip-build)" + return + fi + local feature_args=(--features rss-bench) + if [[ "$SLIM" -eq 1 ]]; then + feature_args=(--no-default-features --features rss-bench) + log "building slim library-profile + rss-bench (GGML_NATIVE=OFF)" + else + log "building default-feature library-profile + rss-bench (GGML_NATIVE=OFF)" + fi + ( + cd "$REPO_ROOT" + GGML_NATIVE=OFF cargo build --release \ + "${feature_args[@]}" \ + --bin library-profile --bin rss-bench + ) +} + +run_scenario() { + local scenario="$1" + local variant="$2" # "" or "warm" + local label="$scenario" + [[ -n "$variant" ]] && label="${scenario}-${variant}" + + local scenario_dir="$OUT_DIR/$label" + mkdir -p "$scenario_dir" + + log "running scenario '$label' x$REPEAT (fresh process each run)" + + local i + for ((i = 1; i <= REPEAT; i++)); do + local run_file="$scenario_dir/run-$i.json" + local env_args=() + if [[ "$scenario" == "long-agent" && -n "$TURNS" ]]; then + env_args+=(env "OPENHUMAN_PROFILE_TURNS=$TURNS") + fi + if [[ "$variant" == "warm" ]]; then + env_args+=(env "OPENHUMAN_PROFILE_PREWARM_SUBAGENTS=1") + fi + + if [[ ${#env_args[@]} -gt 0 ]]; then + "${env_args[@]}" "$BIN" "$scenario" >"$run_file" + else + "$BIN" "$scenario" >"$run_file" + fi + + if ! jq empty "$run_file" >/dev/null 2>&1; then + echo "ERROR: run $i for scenario '$label' did not produce valid JSON: $run_file" >&2 + exit 1 + fi + + # library-heap.sh clobbers target/release/library-profile with the + # rss-bench-dhat build, whose allocator perturbs RSS/timing. The dhat + # binary marks its output, so refuse to benchmark it. + if [[ "$(jq -r '.dhat // false' "$run_file")" == "true" ]]; then + echo "ERROR: $BIN was built with rss-bench-dhat (library-heap.sh clobbered it)." >&2 + echo " Re-run without --skip-build to rebuild the plain rss-bench binary." >&2 + exit 1 + fi + done +} + +# Emit the median (lower-middle for even N), min, and max of a jq numeric +# field across the run files for one scenario, as a JSON object on stdout. +aggregate_field() { + local scenario_dir="$1" + local jq_path="$2" + jq -s " + [ .[] | $jq_path | select(. != null) ] as \$vals | + (\$vals | sort) as \$sorted | + (\$sorted | length) as \$n | + { + median: (if \$n == 0 then null else \$sorted[(( \$n - 1) / 2 | floor)] end), + min: (\$sorted[0] // null), + max: (\$sorted[-1] // null), + n: \$n + } + " "$scenario_dir"/run-*.json +} + +aggregate_scenario() { + local label="$1" + local scenario_dir="$OUT_DIR/$label" + + local duration settled_rss retained_delta peak_delta + duration=$(aggregate_field "$scenario_dir" ".duration_ms") + settled_rss=$(aggregate_field "$scenario_dir" ".settled.rss_kib") + retained_delta=$(aggregate_field "$scenario_dir" ".retained_delta_kib") + peak_delta=$(aggregate_field "$scenario_dir" ".peak_delta_kib") + + local turn_growth="null" + if [[ "$label" == long-agent* ]]; then + # first-turn vs last-turn checkpoint rss delta (steady-state growth). + turn_growth=$(jq -s ' + [ .[] | + (.checkpoints // []) as $cps | + if ($cps | length) >= 2 then + ($cps[-1].rss_kib - $cps[0].rss_kib) + else empty end + ] as $vals | + ($vals | sort) as $sorted | + ($sorted | length) as $n | + if $n == 0 then null + else $sorted[((($n - 1) / 2) | floor)] + end + ' "$scenario_dir"/run-*.json) + fi + + jq -n \ + --arg scenario "$label" \ + --argjson duration "$duration" \ + --argjson settled_rss "$settled_rss" \ + --argjson retained_delta "$retained_delta" \ + --argjson peak_delta "$peak_delta" \ + --argjson turn_growth_kib "$turn_growth" \ + '{ + scenario: $scenario, + duration_ms: $duration, + settled_rss_kib: $settled_rss, + retained_delta_kib: $retained_delta, + peak_delta_kib: $peak_delta, + long_agent_turn_growth_kib: $turn_growth_kib + }' +} + +kib_to_mib() { + # $1: kib value (may be null). Prints "n/a" for null. + local kib="$1" + if [[ "$kib" == "null" || -z "$kib" ]]; then + echo "n/a" + return + fi + jq -n --argjson kib "$kib" '($kib / 1024 * 100 | round) / 100' +} + +write_summary() { + local summary_json="$OUT_DIR/summary.json" + local summary_md="$OUT_DIR/summary.md" + + jq -s '{ generated_at: (now | todate), build: ($ENV.LIBRARY_BENCH_BUILD // "default"), repeat: ($ENV.LIBRARY_BENCH_REPEAT | tonumber), scenarios: . }' \ + "$OUT_DIR"/*.scenario.json >"$summary_json" + + { + echo "# Library benchmark summary" + echo + echo "Build: \`${LIBRARY_BENCH_BUILD}\` " + echo "Repeats per scenario: ${LIBRARY_BENCH_REPEAT} " + echo "Generated: $(date)" + echo + echo "| Scenario | Median settled RSS (MiB) | Median retained Δ (MiB) | Median peak Δ (MiB) | Median duration (ms) |" + echo "| --- | ---: | ---: | ---: | ---: |" + local f + for f in "$OUT_DIR"/*.scenario.json; do + local scenario settled retained peak duration + scenario=$(jq -r '.scenario' "$f") + settled=$(kib_to_mib "$(jq -r '.settled_rss_kib.median' "$f")") + retained=$(kib_to_mib "$(jq -r '.retained_delta_kib.median' "$f")") + peak=$(kib_to_mib "$(jq -r '.peak_delta_kib.median' "$f")") + duration=$(jq -r '.duration_ms.median // "n/a"' "$f") + echo "| $scenario | $settled | $retained | $peak | $duration |" + done + echo + local long_growth_file + for long_growth_file in "$OUT_DIR"/long-agent*.scenario.json; do + [[ -e "$long_growth_file" ]] || continue + local growth + growth=$(jq -r '.long_agent_turn_growth_kib // "n/a"' "$long_growth_file") + if [[ "$growth" != "n/a" && "$growth" != "null" ]]; then + local growth_mib + growth_mib=$(kib_to_mib "$growth") + echo "**$(jq -r '.scenario' "$long_growth_file") steady-state growth** (first-turn to last-turn checkpoint delta, median across runs): ${growth_mib} MiB." + echo + fi + done + cat <<'EOF' +## External comparison point (not apples-to-apples) + +ZeroClaw self-reports (unverified) idling under 5 MiB RAM and roughly 8-12 MiB +under load. OpenHuman's Rust core currently settles around 35-50 MiB depending +on scenario and feature set (see docs/library-benchmarking.md and +docs/resource-profiling-session-2026-07-21.md for scope/caveats). Treat this as +a north star, not a like-for-like comparison: ZeroClaw's feature surface and +scope differ substantially from the OpenHuman core. +EOF + } >"$summary_md" +} + +main() { + build_binaries + + if [[ ! -x "$BIN" ]]; then + echo "ERROR: $BIN not found or not executable. Build it or drop --skip-build." >&2 + exit 1 + fi + + IFS=',' read -r -a scenario_list <<<"$SCENARIOS" + + for scenario in "${scenario_list[@]}"; do + run_scenario "$scenario" "" + if [[ "$WARM" -eq 1 ]]; then + for eligible in "${WARM_ELIGIBLE[@]}"; do + if [[ "$scenario" == "$eligible" ]]; then + run_scenario "$scenario" "warm" + fi + done + fi + done + + # Build one *.scenario.json per label for the summary step. + for scenario_dir in "$OUT_DIR"/*/; do + [[ -d "$scenario_dir" ]] || continue + local_label="$(basename "$scenario_dir")" + aggregate_scenario "$local_label" >"$OUT_DIR/${local_label}.scenario.json" + done + + export LIBRARY_BENCH_BUILD + LIBRARY_BENCH_BUILD=$([[ "$SLIM" -eq 1 ]] && echo "slim" || echo "default") + export LIBRARY_BENCH_REPEAT="$REPEAT" + + write_summary + + log "results: $OUT_DIR" + cat "$OUT_DIR/summary.md" +} + +main "$@" diff --git a/scripts/profile/library-cpu.sh b/scripts/profile/library-cpu.sh new file mode 100755 index 000000000..fb8d908af --- /dev/null +++ b/scripts/profile/library-cpu.sh @@ -0,0 +1,104 @@ +#!/usr/bin/env bash +# library-cpu.sh — samply wrapper for a CPU profile of one library-profile +# scenario, following the recipe used in docs/resource-profiling-session-2026-07-21.md. +# +# Usage: +# ./scripts/profile/library-cpu.sh [-- ...] +# ./scripts/profile/library-cpu.sh --no-isolate +# +# Options: +# --no-isolate Do not force OPENHUMAN_PROFILE_DISABLE_MEMORY_WRITES=1 / +# OPENHUMAN_PROFILE_FORCE_UTC=1 (default: isolated, matching +# the documented cold-path CPU recipe) +# --skip-build Reuse the existing target/release/library-profile binary +# -h, --help Show this help +# +# Extra environment variables can be passed after `--`, e.g.: +# ./scripts/profile/library-cpu.sh long-agent -- OPENHUMAN_PROFILE_TURNS=50 +# +# Output: target/profile/rust-library/-cpu.json.gz +# View it with: samply load target/profile/rust-library/-cpu.json.gz + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" + +ISOLATE=1 +SKIP_BUILD=0 +SCENARIO="" +EXTRA_ENV=() + +usage() { + sed -n '2,20p' "${BASH_SOURCE[0]}" | sed 's/^# \{0,1\}//' + exit "${1:-0}" +} + +while [[ $# -gt 0 ]]; do + case "$1" in + --no-isolate) ISOLATE=0; shift ;; + --skip-build) SKIP_BUILD=1; shift ;; + -h|--help) usage 0 ;; + --) + shift + EXTRA_ENV=("$@") + break + ;; + *) + if [[ -z "$SCENARIO" ]]; then + SCENARIO="$1" + shift + else + echo "ERROR: unexpected argument: $1" >&2 + usage 1 + fi + ;; + esac +done + +if [[ -z "$SCENARIO" ]]; then + echo "ERROR: scenario is required" >&2 + usage 1 +fi + +if ! command -v samply >/dev/null 2>&1; then + echo "ERROR: samply is required. Install it with 'cargo install samply'." >&2 + exit 1 +fi + +BIN="$REPO_ROOT/target/release/library-profile" + +if [[ "$SKIP_BUILD" -eq 0 ]]; then + echo "[library-cpu] building library-profile (GGML_NATIVE=OFF)" >&2 + ( + cd "$REPO_ROOT" + GGML_NATIVE=OFF cargo build --release --features rss-bench --bin library-profile + ) +fi + +if [[ ! -x "$BIN" ]]; then + echo "ERROR: $BIN not found or not executable. Build it or drop --skip-build." >&2 + exit 1 +fi + +OUT_DIR="$REPO_ROOT/target/profile/rust-library" +mkdir -p "$OUT_DIR" +OUT_FILE="$OUT_DIR/${SCENARIO}-cpu.json.gz" + +ENV_ARGS=() +if [[ "$ISOLATE" -eq 1 ]]; then + ENV_ARGS+=(OPENHUMAN_PROFILE_DISABLE_MEMORY_WRITES=1 OPENHUMAN_PROFILE_FORCE_UTC=1) +fi +ENV_ARGS+=("${EXTRA_ENV[@]+"${EXTRA_ENV[@]}"}") + +echo "[library-cpu] recording scenario '$SCENARIO' -> $OUT_FILE" >&2 +env "${ENV_ARGS[@]+"${ENV_ARGS[@]}"}" samply record \ + --save-only \ + --unstable-presymbolicate \ + --rate 1000 \ + --iteration-count 5 \ + --output "$OUT_FILE" \ + -- "$BIN" "$SCENARIO" + +echo "[library-cpu] done: $OUT_FILE" >&2 +echo "[library-cpu] view it with: samply load $OUT_FILE" >&2 diff --git a/scripts/profile/library-fleet.sh b/scripts/profile/library-fleet.sh new file mode 100755 index 000000000..a55189f97 --- /dev/null +++ b/scripts/profile/library-fleet.sh @@ -0,0 +1,371 @@ +#!/usr/bin/env bash +# library-fleet.sh — sweep the `fleet` scenario of library-profile across +# agent counts and gate the result against the 2 GB RAM / 2 vCPU server +# budget (100-1000 live agents). Companion scripts: library-bench.sh +# (per-scenario RSS/duration), library-cpu.sh (samply), library-heap.sh +# (dhat). +# +# Usage: +# ./scripts/profile/library-fleet.sh [options] +# +# Options: +# --agents "50,100,500" Comma-separated agent-count sweep (default: "50,100,500") +# --turns N OPENHUMAN_PROFILE_TURNS per agent (default: 3) +# --latency-ms N OPENHUMAN_PROFILE_MOCK_LATENCY_MS (default: 200) +# --workers N OPENHUMAN_PROFILE_WORKER_THREADS, simulates the +# 2 vCPU box (default: 2) +# --repeat N Fresh-process repeats per agent count (default: 3) +# --target N OPENHUMAN_PROFILE_TARGET_AGENTS (default: 1000) +# --budget-mib N OPENHUMAN_PROFILE_RAM_BUDGET_MIB (default: 2048) +# --skip-build Reuse the existing target/release binaries +# --slim Build with --no-default-features (slim library recipe) +# --out DIR Output directory (default: target/profile/rust-library/fleet-) +# --no-gate Do not fail the exit code on fits==false (report only) +# -h, --help Show this help +# +# Examples: +# ./scripts/profile/library-fleet.sh +# ./scripts/profile/library-fleet.sh --agents 100 --latency-ms 200 +# ./scripts/profile/library-fleet.sh --agents "100,1000" --target 1000 --budget-mib 2048 + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" + +AGENTS="50,100,500" +TURNS=3 +LATENCY_MS=200 +WORKERS=2 +REPEAT=3 +TARGET=1000 +BUDGET_MIB=2048 +SKIP_BUILD=0 +SLIM=0 +OUT_DIR="" +GATE=1 + +# Idle CPU is measured over a 10s parked window and should be ~flat +# regardless of agent count (idle agents cost ~zero CPU). 500ms of CPU +# across that window (5% of one core) is the "low" threshold used in the +# PASS/FAIL verdict line; see docs/library-benchmarking.md. +IDLE_CPU_MS_MAX=500 + +usage() { + sed -n '2,27p' "${BASH_SOURCE[0]}" | sed 's/^# \{0,1\}//' + exit "${1:-0}" +} + +while [[ $# -gt 0 ]]; do + case "$1" in + --agents) + AGENTS="${2:?--agents requires a value}"; shift 2 ;; + --turns) + TURNS="${2:?--turns requires a value}"; shift 2 ;; + --latency-ms) + LATENCY_MS="${2:?--latency-ms requires a value}"; shift 2 ;; + --workers) + WORKERS="${2:?--workers requires a value}"; shift 2 ;; + --repeat) + REPEAT="${2:?--repeat requires a value}"; shift 2 ;; + --target) + TARGET="${2:?--target requires a value}"; shift 2 ;; + --budget-mib) + BUDGET_MIB="${2:?--budget-mib requires a value}"; shift 2 ;; + --skip-build) SKIP_BUILD=1; shift ;; + --slim) SLIM=1; shift ;; + --out) + OUT_DIR="${2:?--out requires a value}"; shift 2 ;; + --no-gate) GATE=0; shift ;; + -h|--help) usage 0 ;; + *) + echo "ERROR: unknown argument: $1" >&2 + usage 1 ;; + esac +done + +if ! command -v jq >/dev/null 2>&1; then + echo "ERROR: jq is required (aggregation over run JSON). Install it (e.g. 'brew install jq')." >&2 + exit 1 +fi + +if [[ -z "$OUT_DIR" ]]; then + OUT_DIR="$REPO_ROOT/target/profile/rust-library/fleet-$(date +%Y%m%d-%H%M%S)" +fi +mkdir -p "$OUT_DIR" + +BIN="$REPO_ROOT/target/release/library-profile" + +log() { echo "[library-fleet] $*" >&2; } + +build_binaries() { + if [[ "$SKIP_BUILD" -eq 1 ]]; then + log "skipping build (--skip-build)" + return + fi + local feature_args=(--features rss-bench) + if [[ "$SLIM" -eq 1 ]]; then + feature_args=(--no-default-features --features rss-bench) + log "building slim library-profile + rss-bench (GGML_NATIVE=OFF)" + else + log "building default-feature library-profile + rss-bench (GGML_NATIVE=OFF)" + fi + ( + cd "$REPO_ROOT" + GGML_NATIVE=OFF cargo build --release \ + "${feature_args[@]}" \ + --bin library-profile --bin rss-bench + ) +} + +run_sweep_point() { + local n="$1" + local point_dir="$OUT_DIR/fleet-$n" + mkdir -p "$point_dir" + + log "running fleet scenario with $n agents x$REPEAT (fresh process each run)" + + local i + for ((i = 1; i <= REPEAT; i++)); do + local run_file="$point_dir/run-$i.json" + + env \ + "OPENHUMAN_PROFILE_AGENTS=$n" \ + "OPENHUMAN_PROFILE_TURNS=$TURNS" \ + "OPENHUMAN_PROFILE_MOCK_LATENCY_MS=$LATENCY_MS" \ + "OPENHUMAN_PROFILE_WORKER_THREADS=$WORKERS" \ + "OPENHUMAN_PROFILE_TARGET_AGENTS=$TARGET" \ + "OPENHUMAN_PROFILE_RAM_BUDGET_MIB=$BUDGET_MIB" \ + "$BIN" fleet >"$run_file" + + if ! jq empty "$run_file" >/dev/null 2>&1; then + echo "ERROR: run $i for agents=$n did not produce valid JSON: $run_file" >&2 + exit 1 + fi + + # library-heap.sh clobbers target/release/library-profile with the + # rss-bench-dhat build, whose allocator perturbs RSS/timing. The dhat + # binary marks its output, so refuse to benchmark it. + if [[ "$(jq -r '.dhat // false' "$run_file")" == "true" ]]; then + echo "ERROR: $BIN was built with rss-bench-dhat (library-heap.sh clobbered it)." >&2 + echo " Re-run without --skip-build to rebuild the plain rss-bench binary." >&2 + exit 1 + fi + done +} + +# Emit the median (lower-middle for even N), min, and max of a jq numeric +# field across the run files for one sweep point, as a JSON object on stdout. +aggregate_field() { + local point_dir="$1" + local jq_path="$2" + jq -s " + [ .[] | $jq_path | select(. != null) ] as \$vals | + (\$vals | sort) as \$sorted | + (\$sorted | length) as \$n | + { + median: (if \$n == 0 then null else \$sorted[(( \$n - 1) / 2 | floor)] end), + min: (\$sorted[0] // null), + max: (\$sorted[-1] // null), + n: \$n + } + " "$point_dir"/run-*.json +} + +# fits is a boolean per run; report it as "fits" only when every repeat +# agreed it fits (a single unlucky repeat should not hide a marginal case). +aggregate_fits() { + local point_dir="$1" + jq -s ' + [ .[] | .budget.fits // false ] as $vals | + ($vals | length) as $n | + ($vals | map(select(. == true)) | length) as $true_n | + { all_fit: ($n > 0 and $true_n == $n), true_n: $true_n, n: $n } + ' "$point_dir"/run-*.json +} + +aggregate_point() { + local n="$1" + local point_dir="$OUT_DIR/fleet-$n" + + local marginal settled_rss idle_cpu threads open_fds p50 p95 p99 projected fits + marginal=$(aggregate_field "$point_dir" ".marginal_rss_kib_per_agent") + settled_rss=$(aggregate_field "$point_dir" ".settled.rss_kib") + idle_cpu=$(aggregate_field "$point_dir" ".idle_cpu_ms") + threads=$(aggregate_field "$point_dir" ".settled.threads") + open_fds=$(aggregate_field "$point_dir" ".settled.open_fds") + p50=$(aggregate_field "$point_dir" ".turn_latency_ms.p50") + p95=$(aggregate_field "$point_dir" ".turn_latency_ms.p95") + p99=$(aggregate_field "$point_dir" ".turn_latency_ms.p99") + projected=$(aggregate_field "$point_dir" ".budget.projected_rss_mib_at_target") + fits=$(aggregate_fits "$point_dir") + + jq -n \ + --argjson agents "$n" \ + --argjson marginal_rss_kib_per_agent "$marginal" \ + --argjson settled_rss_kib "$settled_rss" \ + --argjson idle_cpu_ms "$idle_cpu" \ + --argjson threads "$threads" \ + --argjson open_fds "$open_fds" \ + --argjson p50 "$p50" \ + --argjson p95 "$p95" \ + --argjson p99 "$p99" \ + --argjson projected_rss_mib_at_target "$projected" \ + --argjson fits "$fits" \ + '{ + agents: $agents, + marginal_rss_kib_per_agent: $marginal_rss_kib_per_agent, + settled_rss_kib: $settled_rss_kib, + idle_cpu_ms: $idle_cpu_ms, + threads: $threads, + open_fds: $open_fds, + turn_latency_ms: { p50: $p50, p95: $p95, p99: $p99 }, + projected_rss_mib_at_target: $projected_rss_mib_at_target, + fits: $fits + }' +} + +kib_to_mib() { + # $1: kib value (may be null/"null"). Prints "n/a" for null. + local kib="$1" + if [[ "$kib" == "null" || -z "$kib" ]]; then + echo "n/a" + return + fi + jq -n --argjson kib "$kib" '($kib / 1024 * 100 | round) / 100' +} + +num_or_na() { + local v="$1" + [[ "$v" == "null" || -z "$v" ]] && echo "n/a" || echo "$v" +} + +write_summary() { + local summary_json="$OUT_DIR/summary.json" + local summary_md="$OUT_DIR/summary.md" + + jq -s \ + --argjson turns "$TURNS" \ + --argjson latency_ms "$LATENCY_MS" \ + --argjson workers "$WORKERS" \ + --argjson repeat "$REPEAT" \ + --argjson target "$TARGET" \ + --argjson budget_mib "$BUDGET_MIB" \ + --arg build "$([[ "$SLIM" -eq 1 ]] && echo "slim" || echo "default")" \ + '{ + generated_at: (now | todate), + build: $build, + config: { + turns: $turns, + mock_latency_ms: $latency_ms, + worker_threads: $workers, + repeat: $repeat, + target_agents: $target, + ram_budget_mib: $budget_mib + }, + sweep: . + }' \ + "$OUT_DIR"/*.point.json >"$summary_json" + + local any_fail=0 + + { + echo "# Fleet benchmark summary" + echo + echo "Build: \`$([[ "$SLIM" -eq 1 ]] && echo "slim" || echo "default")\` " + echo "Repeats per agent count: ${REPEAT} " + echo "Turns/agent: ${TURNS}, mock latency: ${LATENCY_MS}ms, worker threads: ${WORKERS} " + echo "Target: ${TARGET} agents, budget: ${BUDGET_MIB} MiB " + echo "Generated: $(date)" + echo + echo "| N | marginal KiB/agent | settled MiB | idle CPU ms/10s | threads | fds | p95 ms | projected MiB @ target | fits |" + echo "| ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | :---: |" + + local f + for f in "$OUT_DIR"/*.point.json; do + local n marginal settled idle threads fds p95 projected fits_all + n=$(jq -r '.agents' "$f") + marginal=$(num_or_na "$(jq -r '.marginal_rss_kib_per_agent.median' "$f")") + settled=$(kib_to_mib "$(jq -r '.settled_rss_kib.median' "$f")") + idle=$(num_or_na "$(jq -r '.idle_cpu_ms.median' "$f")") + threads=$(num_or_na "$(jq -r '.threads.median' "$f")") + fds=$(num_or_na "$(jq -r '.open_fds.median' "$f")") + p95=$(num_or_na "$(jq -r '.turn_latency_ms.p95.median' "$f")") + projected=$(num_or_na "$(jq -r '.projected_rss_mib_at_target.median' "$f")") + fits_all=$(jq -r '.fits.all_fit' "$f") + echo "| $n | $marginal | $settled | $idle | $threads | $fds | $p95 | $projected | $fits_all |" + done + + echo + echo "## Verdict" + echo + + for f in "$OUT_DIR"/*.point.json; do + local n fits_all idle_median verdict reasons + n=$(jq -r '.agents' "$f") + fits_all=$(jq -r '.fits.all_fit' "$f") + idle_median=$(jq -r '.idle_cpu_ms.median // "null"' "$f") + + reasons="" + verdict="PASS" + if [[ "$fits_all" != "true" ]]; then + verdict="FAIL" + reasons="${reasons}projected RSS at target does not fit ${BUDGET_MIB} MiB budget; " + any_fail=1 + fi + local idle_over + idle_over=0 + if [[ "$idle_median" != "null" ]]; then + idle_over=$(jq -n --argjson v "$idle_median" --argjson max "$IDLE_CPU_MS_MAX" 'if $v > $max then 1 else 0 end') + fi + if [[ "$idle_over" -eq 1 ]]; then + verdict="FAIL" + reasons="${reasons}idle CPU ${idle_median}ms/10s exceeds ${IDLE_CPU_MS_MAX}ms threshold; " + fi + + if [[ "$verdict" == "PASS" ]]; then + echo "- **N=$n: PASS** — fits budget, idle CPU low." + else + echo "- **N=$n: FAIL** — ${reasons}" + fi + done + } >"$summary_md" + + if [[ "$any_fail" -eq 1 ]]; then + return 1 + fi + return 0 +} + +main() { + build_binaries + + if [[ ! -x "$BIN" ]]; then + echo "ERROR: $BIN not found or not executable. Build it or drop --skip-build." >&2 + exit 1 + fi + + IFS=',' read -r -a agent_list <<<"$AGENTS" + + local n + for n in "${agent_list[@]}"; do + run_sweep_point "$n" + done + + for n in "${agent_list[@]}"; do + aggregate_point "$n" >"$OUT_DIR/fleet-$n.point.json" + done + + local gate_status=0 + write_summary || gate_status=1 + + log "results: $OUT_DIR" + cat "$OUT_DIR/summary.md" + + if [[ "$GATE" -eq 1 && "$gate_status" -ne 0 ]]; then + log "GATE FAILED: at least one swept N does not fit the ${BUDGET_MIB} MiB / ${TARGET}-agent budget" + exit 1 + fi +} + +main "$@" diff --git a/scripts/profile/library-heap.sh b/scripts/profile/library-heap.sh new file mode 100755 index 000000000..a62b7de03 --- /dev/null +++ b/scripts/profile/library-heap.sh @@ -0,0 +1,93 @@ +#!/usr/bin/env bash +# library-heap.sh — builds the rss-bench-dhat variant of library-profile and +# runs one scenario under dhat heap profiling. +# +# Note: dhat instrumentation perturbs RSS/timing measurements. Use +# library-bench.sh / library-cpu.sh for RSS and CPU numbers; use this script +# only for live-heap attribution (allocation sites, retained bytes). +# +# This build REPLACES target/release/library-profile with the dhat variant. +# A later `library-bench.sh --skip-build` would pick it up; the bench script +# detects the dhat marker in the output JSON and refuses, but the clean fix +# is to rerun library-bench.sh without --skip-build afterwards. +# +# Usage: +# ./scripts/profile/library-heap.sh [-- ...] +# +# Options: +# --skip-build Reuse the existing target/release/library-profile binary +# -h, --help Show this help +# +# Output: target/profile/rust-library/dhat-.json +# View it at: https://nnethercote.github.io/dh_view/dh_view.html + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" + +SKIP_BUILD=0 +SCENARIO="" +EXTRA_ENV=() + +usage() { + sed -n '2,17p' "${BASH_SOURCE[0]}" | sed 's/^# \{0,1\}//' + exit "${1:-0}" +} + +while [[ $# -gt 0 ]]; do + case "$1" in + --skip-build) SKIP_BUILD=1; shift ;; + -h|--help) usage 0 ;; + --) + shift + EXTRA_ENV=("$@") + break + ;; + *) + if [[ -z "$SCENARIO" ]]; then + SCENARIO="$1" + shift + else + echo "ERROR: unexpected argument: $1" >&2 + usage 1 + fi + ;; + esac +done + +if [[ -z "$SCENARIO" ]]; then + echo "ERROR: scenario is required" >&2 + usage 1 +fi + +BIN="$REPO_ROOT/target/release/library-profile" +OUT_DIR="$REPO_ROOT/target/profile/rust-library" +mkdir -p "$OUT_DIR" +OUT_FILE="$OUT_DIR/dhat-${SCENARIO}.json" + +if [[ "$SKIP_BUILD" -eq 0 ]]; then + echo "[library-heap] building library-profile with rss-bench-dhat (GGML_NATIVE=OFF)" >&2 + ( + cd "$REPO_ROOT" + GGML_NATIVE=OFF cargo build --release --features rss-bench-dhat --bin library-profile + ) +fi + +if [[ ! -x "$BIN" ]]; then + echo "ERROR: $BIN not found or not executable. Build it or drop --skip-build." >&2 + exit 1 +fi + +echo "[library-heap] running scenario '$SCENARIO' under dhat" >&2 +echo "[library-heap] WARNING: dhat instrumentation perturbs RSS and timing; do not compare these numbers against library-bench.sh output" >&2 + +env OPENHUMAN_PROFILE_DHAT_OUT="$OUT_FILE" "${EXTRA_ENV[@]+"${EXTRA_ENV[@]}"}" "$BIN" "$SCENARIO" >/dev/null + +if [[ ! -f "$OUT_FILE" ]]; then + echo "ERROR: expected dhat output was not written: $OUT_FILE" >&2 + exit 1 +fi + +echo "[library-heap] done: $OUT_FILE" >&2 +echo "[library-heap] view it at https://nnethercote.github.io/dh_view/dh_view.html (load the JSON file)" >&2 diff --git a/scripts/profile/library-instances.sh b/scripts/profile/library-instances.sh new file mode 100755 index 000000000..1c28e03a9 --- /dev/null +++ b/scripts/profile/library-instances.sh @@ -0,0 +1,548 @@ +#!/usr/bin/env bash +# library-instances.sh — multi-instance fuzz driver: spawn N independent +# `library-profile` processes (one live instance each, held alive at settled +# state via OPENHUMAN_PROFILE_HOLD_SECS) and measure per-INSTANCE cost and +# box survivability under the "N processes" deployment model, as opposed to +# the "N agents in one process" model that `library-fleet.sh` measures. +# +# Where library-fleet.sh answers "how many agents fit in one process", this +# answers "how many independent processes/containers fit on one box" — the +# opencompany per-tenant-instance question. Companion scripts: library-bench.sh +# (per-scenario RSS/duration), library-fleet.sh (one-process fleet sweep). +# +# Usage: +# ./scripts/profile/library-instances.sh [options] +# +# Options: +# --instances "10,25,50" Comma-separated instance-count sweep (default: "10,25,50") +# --hold-secs N Seconds each instance holds alive at settled state +# after finishing its workload (default: 30) +# --scenario NAME library-profile scenario to run per instance +# (default: agent-turn) +# --stagger-ms N Delay between spawning consecutive instances, in +# milliseconds (default: 100) +# --skip-build Reuse the existing target/release binary +# --slim Build with --no-default-features (slim library recipe) +# --max-instances N Hard safety cap on any swept N (default: 200). +# Each held instance settles ~47 MiB RSS on a +# default build, so sum-RSS scales roughly +# linearly: 1000 instances (at default settings) +# is on the order of 1000 x 47 MiB ~= 47 GB +# aggregate RSS. Raising this cap is an explicit +# "yes, I mean to spawn that many processes" — it +# will not happen implicitly. +# --out DIR Output directory (default: +# target/profile/rust-library/instances-) +# --gate Exit nonzero if any instance failed (nonzero exit +# or missing/invalid JSON). Default: report only, +# always exit 0 unless argument/build errors occur. +# -h, --help Show this help +# +# Examples: +# ./scripts/profile/library-instances.sh +# ./scripts/profile/library-instances.sh --instances "10,50" --hold-secs 30 +# ./scripts/profile/library-instances.sh --instances "500" --max-instances 500 --gate + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" + +INSTANCES="10,25,50" +HOLD_SECS=30 +SCENARIO="agent-turn" +STAGGER_MS=100 +SKIP_BUILD=0 +SLIM=0 +MAX_INSTANCES=200 +OUT_DIR="" +GATE=0 + +# Default-build settled RSS per held instance, used only for the safety-cap +# explanation message and the summary.md extrapolation fallback when PSS is +# unavailable (macOS). This is a rule-of-thumb constant, not a measurement — +# the script's own measured settled RSS supersedes it once real data exists. +ASSUMED_PER_INSTANCE_RSS_MIB=47 +BUDGET_MIB=2048 + +usage() { + sed -n '2,38p' "${BASH_SOURCE[0]}" | sed 's/^# \{0,1\}//' + exit "${1:-0}" +} + +while [[ $# -gt 0 ]]; do + case "$1" in + --instances) + INSTANCES="${2:?--instances requires a value}"; shift 2 ;; + --hold-secs) + HOLD_SECS="${2:?--hold-secs requires a value}"; shift 2 ;; + --scenario) + SCENARIO="${2:?--scenario requires a value}"; shift 2 ;; + --stagger-ms) + STAGGER_MS="${2:?--stagger-ms requires a value}"; shift 2 ;; + --skip-build) SKIP_BUILD=1; shift ;; + --slim) SLIM=1; shift ;; + --max-instances) + MAX_INSTANCES="${2:?--max-instances requires a value}"; shift 2 ;; + --out) + OUT_DIR="${2:?--out requires a value}"; shift 2 ;; + --gate) GATE=1; shift ;; + -h|--help) usage 0 ;; + *) + echo "ERROR: unknown argument: $1" >&2 + usage 1 ;; + esac +done + +if ! command -v jq >/dev/null 2>&1; then + echo "ERROR: jq is required (aggregation over run JSON). Install it (e.g. 'brew install jq')." >&2 + exit 1 +fi + +if [[ -z "$OUT_DIR" ]]; then + OUT_DIR="$REPO_ROOT/target/profile/rust-library/instances-$(date +%Y%m%d-%H%M%S)" +fi +mkdir -p "$OUT_DIR" + +BIN="$REPO_ROOT/target/release/library-profile" + +log() { echo "[library-instances] $*" >&2; } + +# --- safety cap ------------------------------------------------------------- + +check_cap() { + local n="$1" + if [[ "$n" -gt "$MAX_INSTANCES" ]]; then + local naive_gib + naive_gib=$(jq -n --argjson n "$n" --argjson mib "$ASSUMED_PER_INSTANCE_RSS_MIB" '(($n * $mib) / 1024 * 100 | round) / 100') + echo "ERROR: requested $n instances exceeds --max-instances cap ($MAX_INSTANCES)." >&2 + echo " RAM math: each held instance settles ~${ASSUMED_PER_INSTANCE_RSS_MIB} MiB RSS on a" >&2 + echo " default build (many-processes model pays that base N times, unlike the" >&2 + echo " one-process fleet model, which amortizes it once). $n instances is roughly" >&2 + echo " $n x ${ASSUMED_PER_INSTANCE_RSS_MIB} MiB ~= ${naive_gib} GiB of aggregate sum-RSS — likely more" >&2 + echo " than this machine has. Pass --max-instances $n (or higher) to confirm you" >&2 + echo " intend to spawn that many processes." >&2 + exit 1 + fi +} + +# --- build ------------------------------------------------------------------- + +build_binaries() { + if [[ "$SKIP_BUILD" -eq 1 ]]; then + log "skipping build (--skip-build)" + return + fi + local feature_args=(--features rss-bench) + if [[ "$SLIM" -eq 1 ]]; then + feature_args=(--no-default-features --features rss-bench) + log "building slim library-profile + rss-bench (GGML_NATIVE=OFF)" + else + log "building default-feature library-profile + rss-bench (GGML_NATIVE=OFF)" + fi + ( + cd "$REPO_ROOT" + GGML_NATIVE=OFF cargo build --release \ + "${feature_args[@]}" \ + --bin library-profile --bin rss-bench + ) +} + +# One quick, short-lived run before the sweep: confirms the binary emits +# valid schema JSON and is not the dhat-instrumented build (library-heap.sh +# clobbers target/release/library-profile with rss-bench-dhat, whose +# allocator perturbs RSS in a way that would corrupt every number below). +probe_binary() { + log "probe run: validating '$BIN $SCENARIO' before the sweep" + local probe_file="$OUT_DIR/probe.json" + local probe_log="$OUT_DIR/probe.log" + + OPENHUMAN_PROFILE_HOLD_SECS=0 "$BIN" "$SCENARIO" >"$probe_file" 2>"$probe_log" + + if ! jq empty "$probe_file" >/dev/null 2>&1; then + echo "ERROR: probe run did not produce valid JSON: $probe_file" >&2 + exit 1 + fi + if [[ "$(jq -r '.dhat // false' "$probe_file")" == "true" ]]; then + echo "ERROR: $BIN was built with rss-bench-dhat (library-heap.sh clobbered it)." >&2 + echo " Re-run without --skip-build to rebuild the plain rss-bench binary." >&2 + exit 1 + fi + log "probe OK: settled rss_kib=$(jq -r '.settled.rss_kib' "$probe_file")" +} + +# --- spawn + sample ---------------------------------------------------------- + +sleep_ms() { + local ms="$1" + [[ "$ms" -le 0 ]] && return 0 + local secs + secs=$(awk -v ms="$ms" 'BEGIN { printf "%.3f", ms / 1000 }') + sleep "$secs" +} + +# Best-effort per-process attribution via the macOS `footprint` tool, run +# against a handful of pids for one sweep point only (it is slow and +# invasive; ignore any failure — this is a bonus artifact, not load-bearing). +capture_footprint_sample() { + local point_dir="$1"; shift + local pids=("$@") + + if ! command -v footprint >/dev/null 2>&1; then + log "footprint tool not present, skipping footprint sample" + return 0 + fi + + local sampled=0 + local pid + for pid in "${pids[@]}"; do + [[ "$sampled" -ge 3 ]] && break + footprint "$pid" >"$point_dir/footprint-pid$pid.txt" 2>&1 || true + sampled=$((sampled + 1)) + done + log "footprint: sampled $sampled pid(s) (best-effort, see footprint-pid*.txt)" +} + +# Samples aggregate sum-RSS + live count every 2s via `ps -o pid=,rss=` until +# every pid in the list has exited, appending to samples.csv. Records the +# peak sum-RSS observed and takes one `vm_stat` system-memory snapshot at +# that peak. +sample_while_holding() { + local point_dir="$1"; shift + local -a pids=("$@") + + local samples_csv="$point_dir/samples.csv" + echo "elapsed_s,sum_rss_kib,live_count" >"$samples_csv" + + local pid_csv + pid_csv=$(IFS=,; echo "${pids[*]}") + + local start_ts + start_ts=$(date +%s) + local peak_sum=0 + + while true; do + local ps_out + ps_out=$(ps -o pid=,rss= -p "$pid_csv" 2>/dev/null || true) + + local live_count sum_rss + if [[ -z "$ps_out" ]]; then + live_count=0 + sum_rss=0 + else + read -r live_count sum_rss <<<"$(echo "$ps_out" | awk '{s += $2; c += 1} END { print c + 0, s + 0 }')" + fi + + local elapsed=$(( $(date +%s) - start_ts )) + echo "$elapsed,$sum_rss,$live_count" >>"$samples_csv" + + if [[ "$sum_rss" -gt "$peak_sum" ]]; then + peak_sum="$sum_rss" + vm_stat >"$point_dir/vm_stat-peak.txt" 2>/dev/null || true + fi + + if [[ "$live_count" -eq 0 ]]; then + break + fi + sleep 2 + done + + echo "$peak_sum" >"$point_dir/.peak_sum_rss_kib" +} + +run_sweep_point() { + local n="$1" + local do_footprint="$2" + local point_dir="$OUT_DIR/n$n" + mkdir -p "$point_dir" + + log "spawning $n instance(s) of scenario '$SCENARIO' (hold=${HOLD_SECS}s, stagger=${STAGGER_MS}ms)" + + local -a pids=() + local i + for ((i = 1; i <= n; i++)); do + local out_file="$point_dir/proc-$i.json" + local log_file="$point_dir/proc-$i.log" + + env "OPENHUMAN_PROFILE_HOLD_SECS=$HOLD_SECS" "$BIN" "$SCENARIO" \ + >"$out_file" 2>"$log_file" & + pids+=("$!") + + if [[ "$i" -lt "$n" ]]; then + sleep_ms "$STAGGER_MS" + fi + done + + if [[ "$do_footprint" -eq 1 ]]; then + capture_footprint_sample "$point_dir" "${pids[@]}" + fi + + sample_while_holding "$point_dir" "${pids[@]}" + + local ok=0 failed=0 + local pid_index=0 + for pid in "${pids[@]}"; do + pid_index=$((pid_index + 1)) + local out_file="$point_dir/proc-$pid_index.json" + local status=0 + wait "$pid" || status=$? + + if [[ "$status" -eq 0 && -s "$out_file" ]] && jq empty "$out_file" >/dev/null 2>&1; then + ok=$((ok + 1)) + else + failed=$((failed + 1)) + log "WARN: instance $pid_index (pid $pid, n=$n) failed (exit=$status)" + fi + done + + echo "$ok" >"$point_dir/.ok_count" + echo "$failed" >"$point_dir/.failed_count" + log "n=$n done: ok=$ok failed=$failed" +} + +# --- aggregate ---------------------------------------------------------------- + +kib_to_mib() { + local kib="$1" + if [[ "$kib" == "null" || -z "$kib" ]]; then + echo "n/a" + return + fi + jq -n --argjson kib "$kib" '($kib / 1024 * 100 | round) / 100' +} + +num_or_na() { + local v="$1" + [[ "$v" == "null" || -z "$v" ]] && echo "n/a" || echo "$v" +} + +aggregate_point() { + local n="$1" + local point_dir="$OUT_DIR/n$n" + + local ok failed peak_sum_rss_kib + ok=$(cat "$point_dir/.ok_count" 2>/dev/null || echo 0) + failed=$(cat "$point_dir/.failed_count" 2>/dev/null || echo 0) + peak_sum_rss_kib=$(cat "$point_dir/.peak_sum_rss_kib" 2>/dev/null || echo 0) + + local -a valid_files=() + local f + for f in "$point_dir"/proc-*.json; do + [[ -e "$f" ]] || continue + if jq empty "$f" >/dev/null 2>&1; then + valid_files+=("$f") + fi + done + + local settled_rss_median pss_max pss_sum_kib + if [[ "${#valid_files[@]}" -gt 0 ]]; then + settled_rss_median=$(jq -s ' + [ .[] | .settled.rss_kib | select(. != null) ] as $vals | + ($vals | sort) as $sorted | + ($sorted | length) as $n | + (if $n == 0 then null else $sorted[(($n - 1) / 2 | floor)] end) + ' "${valid_files[@]}") + pss_max=$(jq -s '[ .[] | .settled.pss_kib // 0 ] | max' "${valid_files[@]}") + pss_sum_kib=$(jq -s '[ .[] | .settled.pss_kib // 0 ] | add' "${valid_files[@]}") + else + settled_rss_median=null + pss_max=0 + pss_sum_kib=0 + fi + + local mean_sum_rss_kib_per_instance + if [[ "$ok" -gt 0 ]]; then + mean_sum_rss_kib_per_instance=$(jq -n --argjson total "$peak_sum_rss_kib" --argjson ok "$ok" '$total / $ok') + else + mean_sum_rss_kib_per_instance=null + fi + + local pss_available + pss_available=$(jq -n --argjson v "$pss_max" 'if $v > 0 then true else false end') + + jq -n \ + --argjson instances "$n" \ + --argjson launched "$n" \ + --argjson ok "$ok" \ + --argjson failed "$failed" \ + --argjson peak_sum_rss_kib "$peak_sum_rss_kib" \ + --argjson mean_sum_rss_kib_per_instance "$mean_sum_rss_kib_per_instance" \ + --argjson settled_rss_kib_median "$settled_rss_median" \ + --argjson pss_available "$pss_available" \ + --argjson pss_sum_kib "$pss_sum_kib" \ + '{ + instances: $instances, + launched: $launched, + ok: $ok, + failed: $failed, + peak_sum_rss_kib: $peak_sum_rss_kib, + mean_sum_rss_kib_per_instance: $mean_sum_rss_kib_per_instance, + settled_rss_kib_median: $settled_rss_kib_median, + pss_available: $pss_available, + pss_sum_kib: (if $pss_available then $pss_sum_kib else null end) + }' +} + +write_summary() { + local summary_json="$OUT_DIR/summary.json" + local summary_md="$OUT_DIR/summary.md" + + jq -s \ + --argjson hold_secs "$HOLD_SECS" \ + --arg scenario "$SCENARIO" \ + --argjson stagger_ms "$STAGGER_MS" \ + --arg build "$([[ "$SLIM" -eq 1 ]] && echo "slim" || echo "default")" \ + '{ + generated_at: (now | todate), + build: $build, + config: { scenario: $scenario, hold_secs: $hold_secs, stagger_ms: $stagger_ms }, + sweep: . + }' \ + "$OUT_DIR"/*.point.json >"$summary_json" + + local any_fail=0 + local last_point_file="" + + { + echo "# Multi-instance (many-processes) benchmark summary" + echo + echo "Build: \`$([[ "$SLIM" -eq 1 ]] && echo "slim" || echo "default")\` " + echo "Scenario: \`${SCENARIO}\`, hold: ${HOLD_SECS}s, stagger: ${STAGGER_MS}ms " + echo "Generated: $(date)" + echo + echo "This measures the **many-processes** deployment model: N independent" + echo "\`library-profile\` processes, each a live held instance, as opposed to" + echo "\`library-fleet.sh\`'s **one-process** model (N agents inside a single" + echo "process). **Correctness note:** summed RSS below double-counts shared" + echo "clean pages — all instances share one binary's resident executable text" + echo "and any shared library mappings — so sum-RSS is an **upper bound** on" + echo "real physical memory use, not the true cost. On Linux, summed PSS" + echo "(proportional set size, from each instance's \`settled.pss_kib\`) divides" + echo "shared pages across the processes that share them and is the honest" + echo "number; it is surfaced below whenever the field is nonzero. On macOS" + echo "there is no PSS equivalent, so only sum-RSS is available and the table" + echo "says so explicitly." + echo + echo "| N | ok/launched | median settled RSS/instance (MiB) | mean sum-RSS/instance (MiB) | peak aggregate sum-RSS (MiB) | summed PSS (MiB) |" + echo "| ---: | :---: | ---: | ---: | ---: | ---: |" + + local pf + for pf in "$OUT_DIR"/*.point.json; do + [[ -e "$pf" ]] || continue + last_point_file="$pf" + local n ok launched settled_rss mean_sum peak_sum pss_available pss_sum pss_cell + n=$(jq -r '.instances' "$pf") + ok=$(jq -r '.ok' "$pf") + launched=$(jq -r '.launched' "$pf") + settled_rss=$(kib_to_mib "$(jq -r '.settled_rss_kib_median' "$pf")") + mean_sum=$(kib_to_mib "$(jq -r '.mean_sum_rss_kib_per_instance' "$pf")") + peak_sum=$(kib_to_mib "$(jq -r '.peak_sum_rss_kib' "$pf")") + pss_available=$(jq -r '.pss_available' "$pf") + if [[ "$pss_available" == "true" ]]; then + pss_sum=$(kib_to_mib "$(jq -r '.pss_sum_kib' "$pf")") + pss_cell="$pss_sum" + else + pss_cell="n/a (macOS)" + fi + + if [[ "$ok" != "$launched" ]]; then + any_fail=1 + fi + + echo "| $n | $ok/$launched | $settled_rss | $mean_sum | $peak_sum | $pss_cell |" + done + + echo + echo "## Verdict (2 GB box extrapolation, estimate)" + echo + + if [[ -n "$last_point_file" ]]; then + local pss_available per_instance_mib per_instance_label fits_estimate + pss_available=$(jq -r '.pss_available' "$last_point_file") + if [[ "$pss_available" == "true" ]]; then + per_instance_mib=$(kib_to_mib "$(jq -r '(.pss_sum_kib / .ok)' "$last_point_file")") + per_instance_label="summed PSS/instance (honest, shared pages divided across sharers)" + else + per_instance_mib=$(kib_to_mib "$(jq -r '.mean_sum_rss_kib_per_instance' "$last_point_file")") + per_instance_label="mean sum-RSS/instance (macOS, no PSS — this OVERSTATES true cost by double-counting shared pages)" + fi + + if [[ "$per_instance_mib" != "n/a" ]]; then + fits_estimate=$(jq -n --argjson budget "$BUDGET_MIB" --argjson per "$per_instance_mib" '($budget / $per) | floor') + echo "Using the largest swept N's ${per_instance_label}: **${per_instance_mib} MiB/instance**." + echo + echo "\`instances_that_fit ~= ${BUDGET_MIB} / ${per_instance_mib} ~= ${fits_estimate}\` — a rough" + echo "**estimate**, not a measured limit. It assumes every additional instance" + echo "costs the same as the ones already measured (no host-level contention," + echo "no cgroup memory limit enforced here)." + if [[ "$pss_available" == "true" ]]; then + echo "PSS was available (Linux), so this divides shared pages across the" + echo "processes that share them rather than double-counting them — still an" + echo "estimate, but not skewed in a known direction the way the macOS" + echo "sum-RSS fallback is." + else + echo "No PSS was available (macOS), so this uses sum-RSS as a stand-in," + echo "which double-counts shared pages (executable text, shared library" + echo "mappings) across instances — treat this estimate as" + echo "conservative-in-the-wrong-direction (fewer instances would actually fit" + echo "than the sum-RSS math implies is safe, since real physical use is lower" + echo "than sum-RSS thanks to shared pages, but this is also unverified against" + echo "an actual memory-limited box)." + fi + else + echo "No successful instances at the largest swept N — cannot extrapolate." + any_fail=1 + fi + else + echo "No sweep points completed — nothing to extrapolate." + any_fail=1 + fi + } >"$summary_md" + + if [[ "$any_fail" -eq 1 ]]; then + return 1 + fi + return 0 +} + +main() { + IFS=',' read -r -a instance_list <<<"$INSTANCES" + + local n + for n in "${instance_list[@]}"; do + check_cap "$n" + done + + build_binaries + + if [[ ! -x "$BIN" ]]; then + echo "ERROR: $BIN not found or not executable. Build it or drop --skip-build." >&2 + exit 1 + fi + + probe_binary + + local first_n="${instance_list[0]}" + for n in "${instance_list[@]}"; do + local do_footprint=0 + [[ "$n" == "$first_n" ]] && do_footprint=1 + run_sweep_point "$n" "$do_footprint" + done + + for n in "${instance_list[@]}"; do + aggregate_point "$n" >"$OUT_DIR/n$n.point.json" + done + + local gate_status=0 + write_summary || gate_status=1 + + log "results: $OUT_DIR" + cat "$OUT_DIR/summary.md" + + if [[ "$GATE" -eq 1 && "$gate_status" -ne 0 ]]; then + log "GATE FAILED: at least one instance failed to complete cleanly" + exit 1 + fi +} + +main "$@" diff --git a/src/bin/library_profile/harness.rs b/src/bin/library_profile/harness.rs new file mode 100644 index 000000000..9ef1888b7 --- /dev/null +++ b/src/bin/library_profile/harness.rs @@ -0,0 +1,495 @@ +//! Shared profiling harness: hermetic fixture, RSS sampling, and the +//! `measure()` wrapper that every scenario runs its workload through. +//! +//! All diagnostics go to **stderr** (stdout must stay pure JSON) using the +//! stable `[library-profile]` prefix. + +use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::{Arc, Mutex}; +use std::time::{Duration, Instant}; + +use anyhow::{Context, Result}; +use openhuman_core::openhuman::config::Config; +use openhuman_core::openhuman::proc_metrics::{self, ProcSample, TreeSample}; +use serde::Serialize; + +/// One sampled point inside a measured workload. `delta_kib` is the RSS change +/// versus the *previous* checkpoint (or the baseline for the first one). +#[derive(Debug, Clone, Serialize)] +pub struct Checkpoint { + pub label: String, + pub at_ms: u128, + pub rss_kib: u64, + pub delta_kib: i64, +} + +/// Turn wall-latency percentiles (milliseconds) collected under overlapping +/// load by the `fleet` scenario. +#[derive(Debug, Clone, Copy, Serialize)] +pub struct TurnLatency { + pub p50: u128, + pub p95: u128, + pub p99: u128, + pub max: u128, +} + +/// Per-descendant RSS entry in a [`TreeReport`]. +#[derive(Debug, Clone, Serialize)] +pub struct TreeChild { + pub name: String, + pub rss_kib: u64, +} + +/// Process-*tree* RSS reporting: this process plus every descendant +/// (interpreter children such as `node` / `python` a skill run spawns). Folds a +/// [`TreeSample`] down to the reported shape. `tree_rss_kib` counts self + all +/// descendants, so it exceeds `settled.rss_kib` whenever a child was live at +/// sample time. +#[derive(Debug, Clone, Serialize)] +pub struct TreeReport { + pub tree_rss_kib: u64, + pub child_count: usize, + pub children: Vec, +} + +impl TreeReport { + pub fn from_sample(sample: &TreeSample) -> Self { + Self { + tree_rss_kib: sample.tree_rss_kib, + child_count: sample.children.len(), + children: sample + .children + .iter() + .map(|child| TreeChild { + name: child.name.clone(), + rss_kib: child.rss_kib, + }) + .collect(), + } + } +} + +/// Fleet capacity budget math (purely informational — scripts turn `fits` into +/// pass/fail). `projected_rss_mib_at_target = settled_base + marginal * target`. +#[derive(Debug, Clone, Copy, Serialize)] +pub struct FleetBudget { + pub target_agents: u64, + pub ram_budget_mib: u64, + pub projected_rss_mib_at_target: f64, + pub fits: bool, +} + +/// Pinned JSON output contract (schema_version = 2). New optional fields are +/// skipped when absent so the two original scenarios stay byte-identical. +#[derive(Debug, Serialize)] +pub struct ProfileResult { + pub schema_version: u32, + pub scenario: &'static str, + pub workload_units: usize, + pub duration_ms: u128, + pub baseline: ProcSample, + pub settled: ProcSample, + pub peak_rss_kib: u64, + pub retained_delta_kib: i64, + pub peak_delta_kib: u64, + #[serde(skip_serializing_if = "Option::is_none")] + pub turns: Option, + /// (`fleet`) requested live-agent count. + #[serde(skip_serializing_if = "Option::is_none")] + pub agents: Option, + /// (`fleet`) agents actually constructed (may be < `agents` on fd/OOM). + #[serde(skip_serializing_if = "Option::is_none")] + pub agents_built: Option, + /// (`fleet`) marginal RSS cost per agent (`baseline → constructed`), KiB. + #[serde(skip_serializing_if = "Option::is_none")] + pub marginal_rss_kib_per_agent: Option, + /// (`fleet`) user+system CPU delta over the 10 s idle window, ms. + #[serde(skip_serializing_if = "Option::is_none")] + pub idle_cpu_ms: Option, + /// (`fleet`) turn wall-latency percentiles under overlapping load. + #[serde(skip_serializing_if = "Option::is_none")] + pub turn_latency_ms: Option, + /// (`fleet`) capacity budget projection. + #[serde(skip_serializing_if = "Option::is_none")] + pub budget: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub checkpoints: Option>, + /// (`subagent-storm`) K parallel researcher subagents fanned out in the turn. + #[serde(skip_serializing_if = "Option::is_none")] + pub subagents: Option, + /// (`skill-run`) process-tree RSS including interpreter child processes. + /// Reports the richest tree seen — the peak-during-workload sample when a + /// short-lived child (e.g. `node`) has already exited by settle time. + #[serde(skip_serializing_if = "Option::is_none")] + pub tree: Option, + /// Present (and `true`) only when the dhat heap profiler is active, since + /// RSS/time numbers are perturbed by dhat's global allocator. + #[serde(skip_serializing_if = "Option::is_none")] + pub dhat: Option, +} + +/// Restores (or removes) an environment variable on drop. +pub struct EnvGuard { + key: &'static str, + old: Option, +} + +impl EnvGuard { + pub fn set(key: &'static str, value: &str) -> Self { + let old = std::env::var(key).ok(); + std::env::set_var(key, value); + Self { key, old } + } +} + +impl Drop for EnvGuard { + fn drop(&mut self) { + match &self.old { + Some(value) => std::env::set_var(self.key, value), + None => std::env::remove_var(self.key), + } + } +} + +/// A hermetic config rooted in a throwaway temp workspace. Local inference, +/// Python, spaCy, and embeddings are all disabled so runs are offline. +pub struct Fixture { + pub config: Config, + _workspace_env: EnvGuard, + _keyring_env: EnvGuard, + _action_env: EnvGuard, + _tmp: tempfile::TempDir, +} + +pub fn fixture() -> Result { + let tmp = tempfile::tempdir().context("create profile workspace")?; + let root = tmp.path(); + let workspace = root.join("workspace"); + std::fs::create_dir_all(&workspace)?; + // A real, writable action sandbox so acting tools (e.g. `node_exec`, which + // spawns `node` in the action dir) have a valid cwd. Harmless for scenarios + // that never act. + let action_dir = root.join("action"); + std::fs::create_dir_all(&action_dir)?; + + let mut config_toml = r#"api_url = "http://127.0.0.1:9" +default_model = "profile-mock" +default_temperature = 0.0 +chat_onboarding_completed = true + +[secrets] +encrypt = false + +[local_ai] +enabled = false +runtime_enabled = false + +[runtime_python] +enabled = false + +[memory_tree] +spacy_enabled = false +"# + .to_string(); + if std::env::var_os("OPENHUMAN_PROFILE_DISABLE_MEMORY_WRITES").is_some() { + config_toml.push_str( + r#" +[memory] +auto_save = false + +[learning] +episodic_capture_enabled = false +"#, + ); + } + // `skill-run` executes real acting tools (`node_exec`); those need the Full + // autonomy tier so the write-class gate does not park the turn on approval. + // Config::load_or_init inside the detached workflow run re-reads this file, + // so the tier must live in config.toml (not just the in-memory Config). + if std::env::var_os("OPENHUMAN_PROFILE_FULL_AUTONOMY").is_some() { + config_toml.push_str( + r#" +[autonomy] +level = "full" +"#, + ); + } + std::fs::write(root.join("config.toml"), &config_toml)?; + + let workspace_env = EnvGuard::set("OPENHUMAN_WORKSPACE", &root.to_string_lossy()); + let keyring_env = EnvGuard::set("OPENHUMAN_KEYRING_BACKEND", "file"); + let action_env = EnvGuard::set("OPENHUMAN_ACTION_DIR", &action_dir.to_string_lossy()); + let mut config: Config = toml::from_str(&config_toml)?; + config.workspace_dir = workspace; + config.action_dir = action_dir; + config.memory_tree.embedding_endpoint = None; + config.memory_tree.embedding_model = None; + config.memory_tree.embedding_strict = false; + + Ok(Fixture { + config, + _workspace_env: workspace_env, + _keyring_env: keyring_env, + _action_env: action_env, + _tmp: tmp, + }) +} + +/// Background task that polls `proc_metrics::sample_self()` every 5 ms and +/// keeps the running peak RSS. +pub struct PeakSampler { + peak: Arc, + stop: Arc, + task: tokio::task::JoinHandle<()>, +} + +impl PeakSampler { + pub fn start(initial_kib: u64) -> Self { + let peak = Arc::new(AtomicU64::new(initial_kib)); + let stop = Arc::new(AtomicU64::new(0)); + let task_peak = Arc::clone(&peak); + let task_stop = Arc::clone(&stop); + let task = tokio::spawn(async move { + while task_stop.load(Ordering::Relaxed) == 0 { + if let Ok(sample) = proc_metrics::sample_self() { + task_peak.fetch_max(sample.rss_kib, Ordering::Relaxed); + } + tokio::time::sleep(Duration::from_millis(5)).await; + } + }); + Self { peak, stop, task } + } + + pub async fn stop(self) -> u64 { + self.stop.store(1, Ordering::Relaxed); + let _ = self.task.await; + self.peak.load(Ordering::Relaxed) + } +} + +/// Background task that polls `proc_metrics::sample_tree()` and keeps the +/// [`TreeSample`] with the highest `tree_rss_kib` seen — capturing the moment a +/// short-lived interpreter child (e.g. a `node -e` step) is resident, which a +/// settle-time-only sample would miss because the child has already exited. +pub struct TreePeakSampler { + best: Arc>>, + stop: Arc, + task: tokio::task::JoinHandle<()>, +} + +impl TreePeakSampler { + pub fn start() -> Self { + let best: Arc>> = Arc::new(Mutex::new(None)); + let stop = Arc::new(AtomicU64::new(0)); + let task_best = Arc::clone(&best); + let task_stop = Arc::clone(&stop); + let task = tokio::spawn(async move { + while task_stop.load(Ordering::Relaxed) == 0 { + if let Ok(sample) = proc_metrics::sample_tree() { + let mut guard = task_best.lock().expect("tree peak lock"); + let replace = guard + .as_ref() + .map(|prev| sample.tree_rss_kib > prev.tree_rss_kib) + .unwrap_or(true); + if replace { + *guard = Some(sample); + } + } + tokio::time::sleep(Duration::from_millis(15)).await; + } + }); + Self { best, stop, task } + } + + pub async fn stop(self) -> Option { + self.stop.store(1, Ordering::Relaxed); + let _ = self.task.await; + self.best.lock().expect("tree peak lock").clone() + } +} + +/// Collects per-phase / per-turn checkpoints inside a measured workload. +/// Cloneable (shares one buffer) so it can be handed to closures freely. +#[derive(Clone)] +pub struct Recorder { + inner: Arc>, + started: Instant, +} + +struct RecorderState { + checkpoints: Vec, + last_rss_kib: u64, +} + +impl Recorder { + fn new(baseline_kib: u64, started: Instant) -> Self { + Self { + inner: Arc::new(Mutex::new(RecorderState { + checkpoints: Vec::new(), + last_rss_kib: baseline_kib, + })), + started, + } + } + + /// Sample RSS now and append a labelled checkpoint whose `delta_kib` is + /// relative to the previous checkpoint (baseline for the first). + pub fn checkpoint(&self, label: impl Into) -> Result<()> { + let label = label.into(); + let sample = proc_metrics::sample_self()?; + let at_ms = self.started.elapsed().as_millis(); + let mut state = self.inner.lock().expect("recorder lock"); + let delta_kib = sample.rss_kib as i64 - state.last_rss_kib as i64; + state.last_rss_kib = sample.rss_kib; + eprintln!( + "[library-profile] checkpoint label={label} at_ms={at_ms} rss_kib={} delta_kib={delta_kib}", + sample.rss_kib + ); + state.checkpoints.push(Checkpoint { + label, + at_ms, + rss_kib: sample.rss_kib, + delta_kib, + }); + Ok(()) + } + + fn take(self) -> Vec { + self.inner + .lock() + .expect("recorder lock") + .checkpoints + .clone() + } +} + +/// Run `workload` between a settled baseline and a settled post-run sample, +/// tracking peak RSS throughout. `turns` and any checkpoints pushed via the +/// `Recorder` are folded into the result (both `None`/omitted when unused). +pub async fn measure( + scenario: &'static str, + workload_units: usize, + turns: Option, + workload: F, +) -> Result +where + F: FnOnce(Recorder) -> Fut, + Fut: std::future::Future>, +{ + measure_impl(scenario, workload_units, turns, false, workload).await +} + +/// Like [`measure`], but also samples the process **tree** (self + descendant +/// interpreter processes). A [`TreePeakSampler`] runs alongside the RSS peak +/// sampler, and the settle-time tree is captured too; the reported `tree` is +/// whichever of the two has the higher `tree_rss_kib`, so a `node` child that +/// exits before settle is still attributed. Used by `skill-run`. +pub async fn measure_with_tree( + scenario: &'static str, + workload_units: usize, + turns: Option, + workload: F, +) -> Result +where + F: FnOnce(Recorder) -> Fut, + Fut: std::future::Future>, +{ + measure_impl(scenario, workload_units, turns, true, workload).await +} + +async fn measure_impl( + scenario: &'static str, + workload_units: usize, + turns: Option, + sample_tree: bool, + workload: F, +) -> Result +where + F: FnOnce(Recorder) -> Fut, + Fut: std::future::Future>, +{ + tokio::time::sleep(Duration::from_millis(250)).await; + let baseline = proc_metrics::sample_self()?; + if let Some(seconds) = std::env::var("OPENHUMAN_PROFILE_HOLD_BEFORE_SECS") + .ok() + .and_then(|value| value.parse::().ok()) + .filter(|seconds| *seconds > 0) + { + eprintln!( + "[library-profile] pid={} holding at baseline for {seconds}s", + std::process::id() + ); + tokio::time::sleep(Duration::from_secs(seconds)).await; + } + let sampler = PeakSampler::start(baseline.rss_kib); + let tree_sampler = if sample_tree { + Some(TreePeakSampler::start()) + } else { + None + }; + let started = Instant::now(); + let recorder = Recorder::new(baseline.rss_kib, started); + eprintln!("[library-profile] scenario={scenario} workload starting"); + workload(recorder.clone()).await?; + let duration_ms = started.elapsed().as_millis(); + eprintln!("[library-profile] scenario={scenario} workload done duration_ms={duration_ms}"); + tokio::time::sleep(Duration::from_millis(500)).await; + let settled = proc_metrics::sample_self()?; + let peak_rss_kib = sampler.stop().await.max(settled.rss_kib); + + // Fold the process-tree samples (peak-during-workload + settle-time) into a + // single report: whichever has the higher tree RSS wins, so a short-lived + // interpreter child that already exited by settle is still attributed. + let tree = if let Some(tree_sampler) = tree_sampler { + let peak_tree = tree_sampler.stop().await; + let settle_tree = proc_metrics::sample_tree().ok(); + let best = match (peak_tree, settle_tree) { + (Some(peak), Some(settle)) => { + if peak.tree_rss_kib >= settle.tree_rss_kib { + Some(peak) + } else { + Some(settle) + } + } + (peak, settle) => peak.or(settle), + }; + if let Some(best) = best.as_ref() { + eprintln!( + "[library-profile] scenario={scenario} tree_rss_kib={} child_count={}", + best.tree_rss_kib, + best.children.len() + ); + } + best.map(|sample| TreeReport::from_sample(&sample)) + } else { + None + }; + let checkpoints = recorder.take(); + let checkpoints = if checkpoints.is_empty() { + None + } else { + Some(checkpoints) + }; + Ok(ProfileResult { + schema_version: 2, + scenario, + workload_units, + duration_ms, + baseline, + settled, + peak_rss_kib, + retained_delta_kib: settled.rss_kib as i64 - baseline.rss_kib as i64, + peak_delta_kib: peak_rss_kib.saturating_sub(baseline.rss_kib), + turns, + agents: None, + agents_built: None, + marginal_rss_kib_per_agent: None, + idle_cpu_ms: None, + turn_latency_ms: None, + budget: None, + checkpoints, + subagents: None, + tree, + dhat: None, + }) +} diff --git a/src/bin/library_profile/main.rs b/src/bin/library_profile/main.rs new file mode 100644 index 000000000..79213d7bd --- /dev/null +++ b/src/bin/library_profile/main.rs @@ -0,0 +1,132 @@ +//! Hermetic, Rust-only library profiling workloads. +//! +//! This binary never enters shipped builds (it requires the default-off +//! `rss-bench` feature). It measures production code paths in fresh processes +//! with network inference replaced by a deterministic provider. +//! +//! Scenarios (`library-profile `): +//! - `memory-ingest` — ingest 100 chat messages, drain the memory queue. +//! - `subagents` — one orchestrator turn spawning two real researchers. +//! - `agent-turn` — a single cold agent turn (minimal library unit). +//! - `long-agent` — N warmed sequential turns with a per-turn checkpoint series. +//! - `workflow` — a real flows trigger->transform->agent graph, end to end. +//! - `subconscious` — one promoted subconscious turn WITHOUT delegation. +//! - `cold-phases` — per-phase checkpoints of the cold bootstrap in one region. +//! - `fleet` — N live agents: marginal RSS, idle CPU, fd/thread growth, turn latency. +//! - `skill-run` — a skill step executing on a real `node` child: process-tree RSS. +//! - `subagent-storm`— K parallel researcher subagents in one instance: marginal RSS per subagent. +//! +//! stdout is ALWAYS a single pretty JSON object (the pinned schema in +//! `harness::ProfileResult`); every diagnostic goes to stderr with the stable +//! `[library-profile]` prefix. +//! +//! With the `rss-bench-dhat` feature, dhat's global allocator + profiler are +//! active: RSS/time numbers are perturbed, the result carries `"dhat": true`, +//! and a `dhat-.json` heap profile is written under +//! `target/profile/rust-library/` (override via `OPENHUMAN_PROFILE_DHAT_OUT`). + +mod harness; +mod mock; +mod scenarios; + +use std::time::Duration; + +use anyhow::{Context, Result}; + +use harness::ProfileResult; + +#[cfg(feature = "rss-bench-dhat")] +#[global_allocator] +static ALLOC: dhat::Alloc = dhat::Alloc; + +/// Builds the dhat profiler (feature-gated), writing to the requested path. +/// Kept alive by the caller until after the JSON result is printed. +#[cfg(feature = "rss-bench-dhat")] +fn start_dhat(scenario: &str) -> Result { + let out = std::env::var("OPENHUMAN_PROFILE_DHAT_OUT") + .unwrap_or_else(|_| format!("target/profile/rust-library/dhat-{scenario}.json")); + if let Some(parent) = std::path::Path::new(&out).parent() { + std::fs::create_dir_all(parent).context("create dhat output directory")?; + } + eprintln!("[library-profile] dhat active — heap profile -> {out}"); + Ok(dhat::Profiler::builder().file_name(&out).build()) +} + +async fn dispatch(scenario: &str) -> Result { + match scenario { + "memory-ingest" => scenarios::memory_ingest::run().await, + "subagents" => scenarios::subagents::run().await, + "agent-turn" => scenarios::agent_turn::run().await, + "long-agent" => scenarios::long_agent::run().await, + "workflow" => scenarios::workflow::run().await, + "subconscious" => scenarios::subconscious::run().await, + "cold-phases" => scenarios::cold_phases::run().await, + "fleet" => scenarios::fleet::run().await, + "skill-run" => scenarios::skill_run::run().await, + "subagent-storm" => scenarios::subagent_storm::run().await, + other => anyhow::bail!("unknown scenario: {other}"), + } +} + +/// Build the tokio runtime. When `OPENHUMAN_PROFILE_WORKER_THREADS` is set the +/// multi-thread runtime is built manually with that worker count (set to `2` to +/// simulate the 2 vCPU box); otherwise the standard multi-thread default runs. +fn build_runtime() -> Result { + let mut builder = tokio::runtime::Builder::new_multi_thread(); + builder.enable_all(); + if let Some(workers) = std::env::var("OPENHUMAN_PROFILE_WORKER_THREADS") + .ok() + .and_then(|value| value.parse::().ok()) + .filter(|n| *n > 0) + { + eprintln!("[library-profile] tokio worker_threads={workers}"); + builder.worker_threads(workers); + } + builder.build().context("build tokio runtime") +} + +fn main() -> Result<()> { + // Parse args BEFORE building the runtime so `OPENHUMAN_PROFILE_WORKER_THREADS` + // can size the worker pool (the `fleet` scenario simulates the 2 vCPU box). + let scenario = std::env::args().nth(1).context( + "usage: library-profile \ + ", + )?; + + // Profiler must outlive the whole run + the JSON print so its Drop writes + // the complete heap profile last. + #[cfg(feature = "rss-bench-dhat")] + let _dhat = start_dhat(&scenario)?; + + eprintln!( + "[library-profile] pid={} scenario={scenario} start", + std::process::id() + ); + + let runtime = build_runtime()?; + runtime.block_on(async move { + #[cfg_attr(not(feature = "rss-bench-dhat"), allow(unused_mut))] + let mut result = dispatch(&scenario).await?; + + #[cfg(feature = "rss-bench-dhat")] + { + result.dhat = Some(true); + } + + println!("{}", serde_json::to_string_pretty(&result)?); + + if let Some(seconds) = std::env::var("OPENHUMAN_PROFILE_HOLD_SECS") + .ok() + .and_then(|value| value.parse::().ok()) + .filter(|seconds| *seconds > 0) + { + eprintln!( + "[library-profile] pid={} holding for {seconds}s", + std::process::id() + ); + tokio::time::sleep(Duration::from_secs(seconds)).await; + } + Ok(()) + }) +} diff --git a/src/bin/library_profile/mock.rs b/src/bin/library_profile/mock.rs new file mode 100644 index 000000000..3c75f6d33 --- /dev/null +++ b/src/bin/library_profile/mock.rs @@ -0,0 +1,570 @@ +//! Deterministic offline `Provider` mocks installed via +//! `test_provider_override` (honoured only under the `rss-bench` feature). + +use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::{Arc, Mutex}; +use std::time::Duration; + +use anyhow::Result; +use async_trait::async_trait; +use openhuman_core::openhuman::inference::provider::traits::{ + ChatRequest, ChatResponse, ProviderCapabilities, ToolCall, +}; +use openhuman_core::openhuman::inference::provider::Provider; + +/// A plain `ChatResponse` carrying only text (no tool calls). +pub fn response(text: &str) -> ChatResponse { + ChatResponse { + text: Some(text.into()), + tool_calls: Vec::new(), + usage: None, + reasoning_content: None, + } +} + +/// Records every prompt it sees so scenarios can assert what ran. +fn record(prompts: &Mutex>, joined: &str) { + prompts + .lock() + .expect("mock prompt lock") + .push(joined.into()); +} + +/// Read `key` as a `u64`, falling back to `default` when unset/unparsable. +fn env_u64(key: &str, default: u64) -> u64 { + std::env::var(key) + .ok() + .and_then(|value| value.parse::().ok()) + .unwrap_or(default) +} + +/// Stable per-researcher marker embedded in a delegated subagent's prompt. +/// Zero-padded so `..._001` is never a substring of `..._011` — the storm mock +/// routes by exact marker containment across K up to 32. +pub fn subagent_marker(index: usize) -> String { + format!("LIB_PROFILE_SUBAGENT_{index:03}") +} + +/// The finding text a delegated researcher returns for `index`. The merge turn +/// is detected by all K of these being present in the conversation. +pub fn finding_text(index: usize) -> String { + format!("Finding {index:03}: researcher {index} reports healthy.") +} + +/// Text the orchestrator returns once it has merged every researcher finding; +/// its arrival in the parent (subconscious) conversation ends the storm turn. +pub const MERGE_SENTINEL: &str = "STORM_MERGE_COMPLETE"; + +/// Shared, dependency-free latency sampler driven by the standard env knobs +/// (`OPENHUMAN_PROFILE_MOCK_LATENCY_MS` mean, `OPENHUMAN_PROFILE_MOCK_JITTER_MS` +/// jitter, default `mean / 4`). Reused by both [`LatencyMock`] and +/// [`SubagentMock`] so a delegated subagent turn can carry realistic latency. +pub struct LatencyKnobs { + mean_ms: u64, + jitter_ms: u64, + counter: AtomicU64, +} + +impl LatencyKnobs { + pub fn from_env() -> Self { + let mean_ms = env_u64("OPENHUMAN_PROFILE_MOCK_LATENCY_MS", 0); + let jitter_ms = env_u64("OPENHUMAN_PROFILE_MOCK_JITTER_MS", mean_ms / 4); + eprintln!("[library-profile] LatencyKnobs mean_ms={mean_ms} jitter_ms={jitter_ms}"); + Self { + mean_ms, + jitter_ms, + counter: AtomicU64::new(0), + } + } + + /// Sample `mean ± jitter` (clamped at zero) via a seeded xorshift step. The + /// seed advances per call so successive turns get distinct latencies. + pub fn sample_ms(&self) -> u64 { + if self.mean_ms == 0 && self.jitter_ms == 0 { + return 0; + } + let seed = self.counter.fetch_add(1, Ordering::Relaxed).wrapping_add(1); + // xorshift64 — deterministic, dependency-free pseudo-randomness. + let mut x = seed.wrapping_mul(0x9E37_79B9_7F4A_7C15); + x ^= x >> 12; + x ^= x << 25; + x ^= x >> 27; + let span = self.jitter_ms.saturating_mul(2).saturating_add(1); + let delta = (x % span) as i64 - self.jitter_ms as i64; + (self.mean_ms as i64 + delta).max(0) as u64 + } + + /// Sleep a sampled latency and return the ms slept (0 when disabled). + pub async fn sleep_sampled(&self) -> u64 { + let ms = self.sample_ms(); + if ms > 0 { + tokio::time::sleep(Duration::from_millis(ms)).await; + } + ms + } +} + +/// Orchestration mock: the first (orchestrator) turn emits a +/// `spawn_parallel_agents` tool call fanning out to **K** researchers; each +/// researcher turn returns its finding; the final merge turn returns plain +/// text once all K findings are present. +/// +/// `width` = K parallel researchers. `new()` keeps the original two-researcher +/// shape (K = 2, no injected latency, driven from the subconscious which has no +/// `spawn_parallel_agents` — the tool call is rejected and its markers echo +/// back, which is all the `subagents` scenario asserts). `with_width(k)` drives +/// the **orchestrator directly** and scripts the full delegation chain, so it +/// must first hand off via `delegate_orchestrator`-free direct fan-out — this is +/// the `subagent-storm` fuzz-width path. +pub struct SubagentMock { + pub prompts: Mutex>, + /// Actual wall-time (ms) of each *researcher* chat call, for percentiles. + pub researcher_latencies_ms: Mutex>, + width: usize, + latency: LatencyKnobs, + /// `true` for `with_width` (orchestrator-driven storm): route the full + /// agent-aware chain. `false` for `new` (subconscious-driven `subagents`): + /// the original simple routing that emits the fan-out directly. + orchestrator_driven: bool, + /// Increments per fan-out so successive `spawn_parallel_agents` calls carry + /// distinct task prompts. + spawn_nonce: AtomicU64, +} + +impl SubagentMock { + /// Two researchers, no injected latency (the original `subagents` shape). + pub fn new() -> Arc { + Arc::new(Self { + prompts: Mutex::new(Vec::new()), + researcher_latencies_ms: Mutex::new(Vec::new()), + width: 2, + latency: LatencyKnobs { + mean_ms: 0, + jitter_ms: 0, + counter: AtomicU64::new(0), + }, + orchestrator_driven: false, + spawn_nonce: AtomicU64::new(0), + }) + } + + /// K researchers with per-researcher latency drawn from the env knobs, driven + /// directly against the orchestrator agent (full agent-aware chain). + pub fn with_width(width: usize) -> Arc { + Arc::new(Self { + prompts: Mutex::new(Vec::new()), + researcher_latencies_ms: Mutex::new(Vec::new()), + width: width.max(1), + latency: LatencyKnobs::from_env(), + orchestrator_driven: true, + spawn_nonce: AtomicU64::new(0), + }) + } + + /// Which researcher (1-based) this prompt is for, if any. A researcher + /// prompt embeds exactly one `subagent_marker`. + fn researcher_index(&self, joined: &str) -> Option { + (1..=self.width).find(|&i| joined.contains(&subagent_marker(i))) + } + + /// True once every researcher's finding is present — the merge turn. + fn is_merge(&self, joined: &str) -> bool { + (1..=self.width).all(|i| joined.contains(&finding_text(i))) + } + + /// True when this call is a real researcher worker turn: exactly one task + /// marker is present and the executing agent is neither the orchestrator nor + /// the subconscious (their Tool Policy Boundary headers name them, and their + /// merge/echo turns also carry every marker). The researcher agent's system + /// prompt names it `Researcher`, so it matches neither header string. + fn is_researcher_turn(&self, joined: &str) -> bool { + self.researcher_index(joined).is_some() + && !joined.contains("Agent: orchestrator") + && !joined.contains("Agent: subconscious") + } + + /// Build the fan-out tool call delegating to K parallel researchers. Only + /// valid on an **orchestrator** turn — the subconscious has no + /// `spawn_parallel_agents` tool, so we `delegate_orchestrator` there first. + fn spawn_call(&self) -> ChatResponse { + let nonce = self.spawn_nonce.fetch_add(1, Ordering::Relaxed); + let tasks: Vec = (1..=self.width) + .map(|i| { + serde_json::json!({ + "agent_id": "researcher", + // The nonce keeps each fan-out's tasks byte-distinct so the + // parallel-graph result cache can't short-circuit a re-spawn. + "prompt": format!("{} [spawn {nonce}]: inspect subsystem {i}", subagent_marker(i)), + "ownership": format!("scope: subsystem-{i}-spawn-{nonce}") + }) + }) + .collect(); + ChatResponse { + text: Some(format!("Delegating to {} researchers.", self.width)), + tool_calls: vec![ToolCall { + id: "profile-parallel-call".into(), + name: "spawn_parallel_agents".into(), + arguments: serde_json::json!({ "tasks": tasks }).to_string(), + extra_content: None, + }], + usage: None, + reasoning_content: None, + } + } + + /// The subconscious's first turn: hand the task to the orchestrator (which + /// owns `spawn_parallel_agents` and allows the `researcher` subagent). + fn delegate_orchestrator_call(&self) -> ChatResponse { + ChatResponse { + text: Some("Delegating to the orchestrator for a parallel research fan-out.".into()), + tool_calls: vec![ToolCall { + id: "profile-delegate-orchestrator".into(), + name: "delegate_orchestrator".into(), + arguments: serde_json::json!({ + "prompt": "Research every subsystem in parallel and merge the findings." + }) + .to_string(), + extra_content: None, + }], + usage: None, + reasoning_content: None, + } + } + + /// Classify the turn by the *executing agent* (from the Tool Policy Boundary + /// header) and script the real delegation chain: + /// subconscious → `delegate_orchestrator` → orchestrator → + /// `spawn_parallel_agents(K)` → K researchers → orchestrator merge → + /// subconscious final. No latency/recording here — the async `chat` wrappers + /// handle sleeping + latency capture around this. + fn reply(&self, joined: &str) -> ChatResponse { + if self.orchestrator_driven { + return self.reply_orchestrator_driven(joined); + } + // Original `subagents` routing (subconscious-driven): merge once both + // findings are present, answer a researcher's marker with its finding, + // else emit the fan-out directly. The subconscious rejects the unknown + // `spawn_parallel_agents`, echoing the markers back — which is all the + // `subagents` scenario asserts. + if self.is_merge(joined) { + return response("Merged all researcher findings."); + } + if let Some(i) = self.researcher_index(joined) { + return response(&finding_text(i)); + } + self.spawn_call() + } + + /// Agent-aware routing for the orchestrator-driven storm. + fn reply_orchestrator_driven(&self, joined: &str) -> ChatResponse { + // Researcher worker: return its finding. + if self.is_researcher_turn(joined) { + let i = self + .researcher_index(joined) + .expect("researcher turn has a marker"); + return response(&finding_text(i)); + } + // Orchestrator: fan out, then merge once every finding is back. + if joined.contains("Agent: orchestrator") { + if self.is_merge(joined) { + return response(MERGE_SENTINEL); + } + return self.spawn_call(); + } + // Parent (subconscious / any other): finish once the orchestrator's + // merged result has flowed back; otherwise delegate to the orchestrator. + if joined.contains(MERGE_SENTINEL) { + return response("Storm complete: merged every researcher's finding."); + } + self.delegate_orchestrator_call() + } +} + +#[async_trait] +impl Provider for SubagentMock { + fn capabilities(&self) -> ProviderCapabilities { + ProviderCapabilities { + native_tool_calling: true, + vision: false, + } + } + + async fn chat_with_system( + &self, + system_prompt: Option<&str>, + message: &str, + _model: &str, + _temperature: f64, + ) -> Result { + let joined = format!("{}\n{message}", system_prompt.unwrap_or("")); + Ok(self.dispatch(&joined).await.text.unwrap_or_default()) + } + + async fn chat( + &self, + request: ChatRequest<'_>, + _model: &str, + _temperature: f64, + ) -> Result { + let joined = request + .messages + .iter() + .map(|message| format!("{}: {}", message.role, message.content)) + .collect::>() + .join("\n"); + Ok(self.dispatch(&joined).await) + } +} + +impl SubagentMock { + /// Record the prompt, sleep a sampled latency for *researcher* calls (and + /// capture their wall time), then return the classified response. + async fn dispatch(&self, joined: &str) -> ChatResponse { + record(&self.prompts, joined); + let is_researcher = self.is_researcher_turn(joined); + let started = std::time::Instant::now(); + if is_researcher { + self.latency.sleep_sampled().await; + } + let resp = self.reply(joined); + if is_researcher { + self.researcher_latencies_ms + .lock() + .expect("mock latency lock") + .push(started.elapsed().as_millis()); + } + resp + } +} + +/// Latency-configurable text-only mock used by the `fleet` scenario. Before +/// returning its fixed answer it sleeps a sampled latency: a mean from +/// `OPENHUMAN_PROFILE_MOCK_LATENCY_MS` (default `0` = no sleep) with jitter +/// `± OPENHUMAN_PROFILE_MOCK_JITTER_MS` (default `mean / 4`). Per-call jitter is +/// derived from a seeded xorshift counter — deterministic and dependency-free +/// (no `rand` crate). +pub struct LatencyMock { + text: String, + latency: LatencyKnobs, + pub prompts: Mutex>, +} + +impl LatencyMock { + /// Build from the standard env knobs. + pub fn from_env(text: impl Into) -> Arc { + Arc::new(Self { + text: text.into(), + latency: LatencyKnobs::from_env(), + prompts: Mutex::new(Vec::new()), + }) + } +} + +#[async_trait] +impl Provider for LatencyMock { + fn capabilities(&self) -> ProviderCapabilities { + ProviderCapabilities { + native_tool_calling: true, + vision: false, + } + } + + async fn chat_with_system( + &self, + system_prompt: Option<&str>, + message: &str, + _model: &str, + _temperature: f64, + ) -> Result { + self.latency.sleep_sampled().await; + record( + &self.prompts, + &format!("{}\n{message}", system_prompt.unwrap_or("")), + ); + Ok(self.text.clone()) + } + + async fn chat( + &self, + request: ChatRequest<'_>, + _model: &str, + _temperature: f64, + ) -> Result { + self.latency.sleep_sampled().await; + let joined = request + .messages + .iter() + .map(|message| format!("{}: {}", message.role, message.content)) + .collect::>() + .join("\n"); + record(&self.prompts, &joined); + Ok(response(&self.text)) + } +} + +/// Text-only mock: always returns a fixed direct answer, never a tool call. +/// Used by the single-turn / workflow scenarios that must NOT delegate. +pub struct PlainTextMock { + text: String, + pub prompts: Mutex>, +} + +impl PlainTextMock { + pub fn new(text: impl Into) -> Arc { + Arc::new(Self { + text: text.into(), + prompts: Mutex::new(Vec::new()), + }) + } +} + +#[async_trait] +impl Provider for PlainTextMock { + fn capabilities(&self) -> ProviderCapabilities { + ProviderCapabilities { + native_tool_calling: true, + vision: false, + } + } + + async fn chat_with_system( + &self, + system_prompt: Option<&str>, + message: &str, + _model: &str, + _temperature: f64, + ) -> Result { + record( + &self.prompts, + &format!("{}\n{message}", system_prompt.unwrap_or("")), + ); + Ok(self.text.clone()) + } + + async fn chat( + &self, + request: ChatRequest<'_>, + _model: &str, + _temperature: f64, + ) -> Result { + let joined = request + .messages + .iter() + .map(|message| format!("{}: {}", message.role, message.content)) + .collect::>() + .join("\n"); + record(&self.prompts, &joined); + Ok(response(&self.text)) + } +} + +/// The stdout marker the profiling `node` step prints. Its presence in the run +/// conversation proves the real interpreter child executed (not merely that a +/// tool call was emitted). +pub const NODE_MARKER: &str = "PROFILE_NODE_RESULT"; + +/// Skill-run mock: the first turn emits a `node_exec` tool call running a short, +/// real JavaScript computation (which spawns a genuine `node` child process); +/// once its printed JSON (carrying [`NODE_MARKER`]) rides back into the +/// conversation, the mock returns a plain final answer so the agent turn +/// completes. This scripts exactly the tool call the code-executor specialist +/// needs to spawn the language runtime whose cost we measure. +pub struct SkillRunMock { + code: String, + node_call_emitted: AtomicU64, + node_output_seen: AtomicU64, + pub prompts: Mutex>, +} + +impl SkillRunMock { + pub fn new() -> Arc { + // A real computation, a live allocation, and a ~1.2s busy-wait so the + // child stays resident long enough for the tree sampler (15 ms poll) to + // catch it. The JSON it prints carries NODE_MARKER. + let code = format!( + "const start = Date.now();\n\ + const buf = [];\n\ + let sum = 0;\n\ + for (let i = 0; i < 500000; i++) {{ buf.push(i % 97); sum += i; }}\n\ + while (Date.now() - start < 1200) {{ sum += buf.length; }}\n\ + console.log(JSON.stringify({{ marker: '{NODE_MARKER}', sum, kept: buf.length }}));\n" + ); + Arc::new(Self { + code, + node_call_emitted: AtomicU64::new(0), + node_output_seen: AtomicU64::new(0), + prompts: Mutex::new(Vec::new()), + }) + } + + /// True once the `node_exec` tool call has been emitted. + pub fn node_call_emitted(&self) -> bool { + self.node_call_emitted.load(Ordering::Relaxed) > 0 + } + + /// True once the node child's printed output flowed back into the turn — + /// i.e. the interpreter child actually ran and printed. + pub fn node_output_seen(&self) -> bool { + self.node_output_seen.load(Ordering::Relaxed) > 0 + } + + fn reply(&self, joined: &str) -> ChatResponse { + record(&self.prompts, joined); + if joined.contains(NODE_MARKER) { + self.node_output_seen.store(1, Ordering::Relaxed); + return response( + "Skill complete: the Node.js step computed the value and it checks out.", + ); + } + self.node_call_emitted.store(1, Ordering::Relaxed); + ChatResponse { + text: Some("Running the JavaScript computation step.".into()), + tool_calls: vec![ToolCall { + id: "profile-node-call".into(), + name: "node_exec".into(), + arguments: serde_json::json!({ "inline_code": self.code }).to_string(), + extra_content: None, + }], + usage: None, + reasoning_content: None, + } + } +} + +#[async_trait] +impl Provider for SkillRunMock { + fn capabilities(&self) -> ProviderCapabilities { + ProviderCapabilities { + native_tool_calling: true, + vision: false, + } + } + + async fn chat_with_system( + &self, + system_prompt: Option<&str>, + message: &str, + _model: &str, + _temperature: f64, + ) -> Result { + Ok(self + .reply(&format!("{}\n{message}", system_prompt.unwrap_or(""))) + .text + .unwrap_or_default()) + } + + async fn chat( + &self, + request: ChatRequest<'_>, + _model: &str, + _temperature: f64, + ) -> Result { + let joined = request + .messages + .iter() + .map(|message| format!("{}: {}", message.role, message.content)) + .collect::>() + .join("\n"); + Ok(self.reply(&joined)) + } +} diff --git a/src/bin/library_profile/scenarios/agent_turn.rs b/src/bin/library_profile/scenarios/agent_turn.rs new file mode 100644 index 000000000..6768f0a5b --- /dev/null +++ b/src/bin/library_profile/scenarios/agent_turn.rs @@ -0,0 +1,36 @@ +//! `agent-turn`: the minimal "embed OpenHuman as a library" unit — a single +//! cold agent turn built directly from config, with a plain-text mock provider +//! (no tool calls, no delegation). + +use std::sync::Arc; + +use anyhow::Result; +use openhuman_core::core::event_bus::init_global; +use openhuman_core::openhuman::agent::harness::AgentDefinitionRegistry; +use openhuman_core::openhuman::agent::Agent; +use openhuman_core::openhuman::inference::provider::factory::test_provider_override; +use openhuman_core::openhuman::inference::provider::Provider; + +use crate::harness::{fixture, measure, ProfileResult}; +use crate::mock::PlainTextMock; + +pub async fn run() -> Result { + let fixture = fixture()?; + let _ = init_global(256); + openhuman_core::openhuman::agent::bus::register_agent_handlers(); + let _ = AgentDefinitionRegistry::init_global_builtins(); + let mock = PlainTextMock::new("The Phoenix migration is healthy and on track."); + let provider: Arc = mock.clone(); + let _provider = test_provider_override::install(provider); + eprintln!("[library-profile] agent-turn: registries ready, mock installed"); + + measure("agent-turn", 1, None, |_rec| async { + let mut agent = Agent::from_config_for_agent(&fixture.config, "subconscious")?; + let reply = agent + .run_single("Give me a one-line status on the Phoenix migration.") + .await?; + anyhow::ensure!(!reply.trim().is_empty(), "empty agent reply"); + Ok(()) + }) + .await +} diff --git a/src/bin/library_profile/scenarios/cold_phases.rs b/src/bin/library_profile/scenarios/cold_phases.rs new file mode 100644 index 000000000..17b7f0809 --- /dev/null +++ b/src/bin/library_profile/scenarios/cold_phases.rs @@ -0,0 +1,79 @@ +//! `cold-phases`: sequential per-phase checkpoints of the cold bootstrap, all +//! inside one measured region. Each phase is sampled right after it completes +//! so the JSON `checkpoints` series attributes the cold-start cost per phase. + +use std::sync::Arc; +use std::time::Duration; + +use anyhow::Result; +use openhuman_core::core::event_bus::init_global; +use openhuman_core::openhuman::agent::harness::AgentDefinitionRegistry; +use openhuman_core::openhuman::agent::Agent; +use openhuman_core::openhuman::inference::provider::factory::test_provider_override; +use openhuman_core::openhuman::inference::provider::Provider; +use openhuman_core::openhuman::memory_store::MemoryClient; + +use crate::harness::{fixture, measure, ProfileResult}; +use crate::mock::PlainTextMock; + +/// config, event-bus, agent-registry, detectors, memory-store, agent-build, +/// first-turn, warm-turn, teardown. +const PHASE_COUNT: usize = 9; + +pub async fn run() -> Result { + measure("cold-phases", PHASE_COUNT, None, |rec| async move { + // a. config — hermetic fixture parse (see deviation note in the report: + // kept as fixture parsing rather than `Config::load_or_init` to + // guarantee we never touch the real ~/.openhuman). + let fixture = fixture()?; + rec.checkpoint("config-parse")?; + + // b. event-bus (plus agent-handler registration so turns can run). + let _ = init_global(256); + openhuman_core::openhuman::agent::bus::register_agent_handlers(); + rec.checkpoint("event-bus")?; + + // c. agent-registry. + let _ = AgentDefinitionRegistry::init_global_builtins(); + rec.checkpoint("agent-registry")?; + + // d. detectors — force the lazy PII + prompt-injection statics. + let _ = openhuman_core::openhuman::security::pii::scan(""); + let _ = openhuman_core::openhuman::prompt_injection::scan_tool_definition("x", ""); + rec.checkpoint("detectors")?; + + // e. memory-store — build and hold a unified-memory client until teardown. + let mem = MemoryClient::from_workspace_dir(fixture.config.workspace_dir.clone()) + .map_err(anyhow::Error::msg)?; + rec.checkpoint("memory-store")?; + + // Provider mock for the two turns below (not itself a phase). + let mock = PlainTextMock::new("Phoenix migration is healthy and on track."); + let provider: Arc = mock.clone(); + let _provider = test_provider_override::install(provider); + + // f. agent-build. + let mut agent = Agent::from_config_for_agent(&fixture.config, "subconscious")?; + rec.checkpoint("agent-build")?; + + // g. first-turn (cold). + let first = agent + .run_single("Give me a one-line status on the Phoenix migration.") + .await?; + anyhow::ensure!(!first.trim().is_empty(), "empty first-turn reply"); + rec.checkpoint("first-turn")?; + + // h. warm-turn (second, same agent). + let warm = agent.run_single("Any change since the last check?").await?; + anyhow::ensure!(!warm.trim().is_empty(), "empty warm-turn reply"); + rec.checkpoint("warm-turn")?; + + // i. teardown — drop the agent + memory client, settle, sample. + drop(agent); + drop(mem); + tokio::time::sleep(Duration::from_millis(300)).await; + rec.checkpoint("teardown")?; + Ok(()) + }) + .await +} diff --git a/src/bin/library_profile/scenarios/fleet.rs b/src/bin/library_profile/scenarios/fleet.rs new file mode 100644 index 000000000..6b65cb57c --- /dev/null +++ b/src/bin/library_profile/scenarios/fleet.rs @@ -0,0 +1,300 @@ +//! `fleet`: can OpenHuman host 100–1000 live agents on a 2 GB / 2 vCPU box? +//! +//! Answers four questions in one process: (1) marginal RSS per live agent +//! (`baseline → constructed`), (2) idle CPU of parked agents (CPU delta over a +//! 10 s do-nothing window), (3) thread + fd growth vs N (rides along in +//! `ProcSample`), and (4) turn latency percentiles under overlapping load. +//! +//! Env knobs: +//! - `OPENHUMAN_PROFILE_AGENTS` (default 100) — live agents to construct. +//! - `OPENHUMAN_PROFILE_TURNS` (default 3) — turns per agent under load. +//! - `OPENHUMAN_PROFILE_MOCK_LATENCY_MS` / `_JITTER_MS` — mock reply latency. +//! - `OPENHUMAN_PROFILE_TARGET_AGENTS` (default 1000) — budget projection target. +//! - `OPENHUMAN_PROFILE_RAM_BUDGET_MIB` (default 2048) — budget ceiling. +//! +//! `OPENHUMAN_PROFILE_WORKER_THREADS` is honoured in `main` (runtime built +//! manually) rather than here. + +use std::sync::{Arc, Mutex}; +use std::time::{Duration, Instant}; + +use anyhow::Result; +use openhuman_core::core::event_bus::init_global; +use openhuman_core::openhuman::agent::harness::AgentDefinitionRegistry; +use openhuman_core::openhuman::agent::Agent; +use openhuman_core::openhuman::inference::provider::factory::test_provider_override; +use openhuman_core::openhuman::inference::provider::Provider; +use openhuman_core::openhuman::proc_metrics; + +use crate::harness::{fixture, measure, FleetBudget, ProfileResult, Recorder, TurnLatency}; +use crate::mock::LatencyMock; + +const DEFAULT_AGENTS: usize = 100; +const DEFAULT_TURNS: usize = 3; +const DEFAULT_TARGET_AGENTS: u64 = 1000; +const DEFAULT_RAM_BUDGET_MIB: u64 = 2048; +const IDLE_WINDOW: Duration = Duration::from_secs(10); + +fn env_usize(key: &str, default: usize) -> usize { + std::env::var(key) + .ok() + .and_then(|v| v.parse::().ok()) + .filter(|n| *n > 0) + .unwrap_or(default) +} + +fn env_u64(key: &str, default: u64) -> u64 { + std::env::var(key) + .ok() + .and_then(|v| v.parse::().ok()) + .filter(|n| *n > 0) + .unwrap_or(default) +} + +/// Raise `RLIMIT_NOFILE` toward its hard cap so N agents (each opening SQLite +/// etc.) don't exhaust the default macOS 256 soft limit. Logs old/new to stderr. +fn raise_fd_limit() { + use std::mem::MaybeUninit; + let mut lim = MaybeUninit::::uninit(); + // SAFETY: `getrlimit` initialises `lim` on success. + if unsafe { libc::getrlimit(libc::RLIMIT_NOFILE, lim.as_mut_ptr()) } != 0 { + eprintln!("[library-profile] fleet: getrlimit(RLIMIT_NOFILE) failed"); + return; + } + // SAFETY: initialised by the successful `getrlimit`. + let mut lim = unsafe { lim.assume_init() }; + let old_soft = lim.rlim_cur; + lim.rlim_cur = lim.rlim_max; + // SAFETY: raising the soft limit to the existing hard limit is always valid. + let rc = unsafe { libc::setrlimit(libc::RLIMIT_NOFILE, &lim) }; + eprintln!( + "[library-profile] fleet: RLIMIT_NOFILE soft {old_soft} -> {} (hard {}) setrlimit_rc={rc}", + lim.rlim_cur, lim.rlim_max + ); +} + +/// Shared metrics captured inside the measured closure and read back after. +#[derive(Default)] +struct FleetMetrics { + agents_built: usize, + baseline_rss_kib: u64, + constructed_rss_kib: u64, + idle_cpu_ms: u64, + latency_ms: Vec, +} + +/// Percentile (nearest-rank) of an already-sorted slice. `p` in `[0,100]`. +fn percentile(sorted: &[u128], p: u128) -> u128 { + if sorted.is_empty() { + return 0; + } + let rank = ((p * sorted.len() as u128) + 99) / 100; // ceil(p% * n) + let idx = rank.saturating_sub(1).min(sorted.len() as u128 - 1) as usize; + sorted[idx] +} + +fn latency_summary(mut samples: Vec) -> Option { + if samples.is_empty() { + return None; + } + samples.sort_unstable(); + Some(TurnLatency { + p50: percentile(&samples, 50), + p95: percentile(&samples, 95), + p99: percentile(&samples, 99), + max: *samples.last().unwrap(), + }) +} + +/// Build agents sequentially, checkpointing the marginal-RSS curve. On a +/// mid-construction failure, records `construction-failed-` and returns +/// what was built rather than crashing. +fn build_agents( + config: &openhuman_core::openhuman::config::Config, + n: usize, + rec: &Recorder, +) -> Result> { + let stride = (n / 10).max(1); + let mut agents = Vec::with_capacity(n); + for i in 0..n { + match Agent::from_config_for_agent(config, "subconscious") { + Ok(agent) => agents.push(agent), + Err(err) => { + eprintln!( + "[library-profile] fleet: construction failed at agent {} — {err}", + i + 1 + ); + rec.checkpoint(format!("construction-failed-{}", agents.len()))?; + break; + } + } + let built = i + 1; + if built % stride == 0 || built == n { + rec.checkpoint(format!("built-{built}"))?; + } + } + Ok(agents) +} + +pub async fn run() -> Result { + let agents_requested = env_usize("OPENHUMAN_PROFILE_AGENTS", DEFAULT_AGENTS); + let turns = env_usize("OPENHUMAN_PROFILE_TURNS", DEFAULT_TURNS); + let target_agents = env_u64("OPENHUMAN_PROFILE_TARGET_AGENTS", DEFAULT_TARGET_AGENTS); + let ram_budget_mib = env_u64("OPENHUMAN_PROFILE_RAM_BUDGET_MIB", DEFAULT_RAM_BUDGET_MIB); + + raise_fd_limit(); + + let fixture = fixture()?; + let _ = init_global(256); + openhuman_core::openhuman::agent::bus::register_agent_handlers(); + let _ = AgentDefinitionRegistry::init_global_builtins(); + let mock = LatencyMock::from_env("Fleet agent: nothing needs your attention."); + let provider: Arc = mock.clone(); + let _provider = test_provider_override::install(provider); + eprintln!( + "[library-profile] fleet: agents={agents_requested} turns={turns} \ + target={target_agents} budget_mib={ram_budget_mib}" + ); + + let metrics = Arc::new(Mutex::new(FleetMetrics::default())); + let config = fixture.config.clone(); + let metrics_for_workload = Arc::clone(&metrics); + + let mut result = measure( + "fleet", + agents_requested, + Some(turns), + move |rec| async move { + rec.checkpoint("baseline")?; + let baseline_rss = proc_metrics::sample_self()?.rss_kib; + + // (b) construct N agents sequentially, curve visible via checkpoints. + let mut agents = build_agents(&config, agents_requested, &rec)?; + let agents_built = agents.len(); + rec.checkpoint("constructed")?; + let constructed_rss = proc_metrics::sample_self()?.rss_kib; + + { + let mut m = metrics_for_workload.lock().expect("metrics lock"); + m.agents_built = agents_built; + m.baseline_rss_kib = baseline_rss; + m.constructed_rss_kib = constructed_rss; + } + eprintln!( + "[library-profile] fleet: built {agents_built}/{agents_requested} agents; \ + baseline_rss={baseline_rss} constructed_rss={constructed_rss}" + ); + + // (d) idle phase — park all agents, measure CPU drift over 10 s. + rec.checkpoint("idle-start")?; + let idle_start = proc_metrics::sample_self()?; + tokio::time::sleep(IDLE_WINDOW).await; + rec.checkpoint("idle-end")?; + let idle_end = proc_metrics::sample_self()?; + let idle_cpu_ms = (idle_end.cpu_user_ms + idle_end.cpu_system_ms) + .saturating_sub(idle_start.cpu_user_ms + idle_start.cpu_system_ms); + metrics_for_workload + .lock() + .expect("metrics lock") + .idle_cpu_ms = idle_cpu_ms; + eprintln!("[library-profile] fleet: idle CPU over 10s = {idle_cpu_ms} ms"); + + // (e) load phase — TURNS turns per agent, one task each, staggered. + let mut handles = Vec::with_capacity(agents.len()); + for (idx, mut agent) in agents.drain(..).enumerate() { + let stagger = Duration::from_millis(((idx as u64) * 10).min(2000)); + handles.push(tokio::spawn(async move { + tokio::time::sleep(stagger).await; + let mut latencies = Vec::with_capacity(turns); + for _ in 0..turns { + let started = Instant::now(); + let reply = agent.run_single("Give me a one-line status update.").await; + let elapsed = started.elapsed().as_millis(); + match reply { + Ok(text) if !text.trim().is_empty() => latencies.push(elapsed), + Ok(_) => eprintln!("[library-profile] fleet: empty reply agent={idx}"), + Err(err) => { + eprintln!("[library-profile] fleet: turn error agent={idx} — {err}") + } + } + } + latencies + })); + } + + let mut all_latencies = Vec::new(); + for handle in handles { + match handle.await { + Ok(mut latencies) => all_latencies.append(&mut latencies), + Err(err) => eprintln!("[library-profile] fleet: task join error — {err}"), + } + } + metrics_for_workload + .lock() + .expect("metrics lock") + .latency_ms = all_latencies; + rec.checkpoint("load-done")?; + Ok(()) + }, + ) + .await?; + + // Fold fleet-specific metrics into the pinned result. + let metrics = Arc::try_unwrap(metrics) + .map(|m| m.into_inner().expect("metrics lock")) + .unwrap_or_default(); + + let agents_built = metrics.agents_built; + let marginal = if agents_built > 0 { + Some( + (metrics.constructed_rss_kib as f64 - metrics.baseline_rss_kib as f64) + / agents_built as f64, + ) + } else { + None + }; + + // (4) budget projection: settled base + marginal * target. + let base_mib = metrics.baseline_rss_kib as f64 / 1024.0; + let marginal_mib = marginal.unwrap_or(0.0) / 1024.0; + let projected = base_mib + marginal_mib * target_agents as f64; + let budget = FleetBudget { + target_agents, + ram_budget_mib, + projected_rss_mib_at_target: projected, + fits: projected <= ram_budget_mib as f64, + }; + + result.agents = Some(agents_requested); + result.agents_built = Some(agents_built); + result.marginal_rss_kib_per_agent = marginal; + result.idle_cpu_ms = Some(metrics.idle_cpu_ms); + result.turn_latency_ms = latency_summary(metrics.latency_ms); + result.budget = Some(budget); + Ok(result) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn percentiles_nearest_rank() { + let mut v: Vec = (1..=100).collect(); + v.sort_unstable(); + assert_eq!(percentile(&v, 50), 50); + assert_eq!(percentile(&v, 95), 95); + assert_eq!(percentile(&v, 99), 99); + assert_eq!(percentile(&v, 100), 100); + assert_eq!(percentile(&[], 50), 0); + assert_eq!(percentile(&[7], 99), 7); + } + + #[test] + fn latency_summary_none_when_empty() { + assert!(latency_summary(Vec::new()).is_none()); + let s = latency_summary(vec![10, 20, 30]).unwrap(); + assert_eq!(s.max, 30); + assert_eq!(s.p50, 20); + } +} diff --git a/src/bin/library_profile/scenarios/long_agent.rs b/src/bin/library_profile/scenarios/long_agent.rs new file mode 100644 index 000000000..a9628aef5 --- /dev/null +++ b/src/bin/library_profile/scenarios/long_agent.rs @@ -0,0 +1,58 @@ +//! `long-agent`: steady-state many-turn loop on ONE warmed agent — models a +//! long-running opencompany agent. Builds the agent and runs one warm-up turn +//! BEFORE the measured region, then runs N sequential turns inside it, pushing +//! a per-turn checkpoint so the plateau/leak curve is visible. + +use std::sync::Arc; + +use anyhow::Result; +use openhuman_core::core::event_bus::init_global; +use openhuman_core::openhuman::agent::harness::AgentDefinitionRegistry; +use openhuman_core::openhuman::agent::Agent; +use openhuman_core::openhuman::inference::provider::factory::test_provider_override; +use openhuman_core::openhuman::inference::provider::Provider; + +use crate::harness::{fixture, measure, ProfileResult}; +use crate::mock::PlainTextMock; + +const DEFAULT_TURNS: usize = 25; + +const PROMPTS: &[&str] = &[ + "Summarise today's Phoenix migration standup in one line.", + "What is the current staging p99 latency and error rate?", + "Who owns the rollback runbook and on-call coordination?", + "When does the phoenix_v2_enabled flag ramp, and what gates it?", + "Draft a one-sentence status update for the billing-ledger team.", +]; + +pub async fn run() -> Result { + let turns = std::env::var("OPENHUMAN_PROFILE_TURNS") + .ok() + .and_then(|value| value.parse::().ok()) + .filter(|n| *n > 0) + .unwrap_or(DEFAULT_TURNS); + + let fixture = fixture()?; + let _ = init_global(256); + openhuman_core::openhuman::agent::bus::register_agent_handlers(); + let _ = AgentDefinitionRegistry::init_global_builtins(); + let mock = PlainTextMock::new("Phoenix migration is healthy; no action needed."); + let provider: Arc = mock.clone(); + let _provider = test_provider_override::install(provider); + + let mut agent = Agent::from_config_for_agent(&fixture.config, "subconscious")?; + eprintln!("[library-profile] long-agent: warming agent with one pre-measure turn"); + let warm = agent.run_single("Warm-up: confirm you are ready.").await?; + anyhow::ensure!(!warm.trim().is_empty(), "empty warm-up reply"); + + measure("long-agent", turns, Some(turns), move |rec| async move { + for i in 0..turns { + let prompt = PROMPTS[i % PROMPTS.len()]; + let reply = agent.run_single(prompt).await?; + anyhow::ensure!(!reply.trim().is_empty(), "empty reply on turn {i}"); + rec.checkpoint(format!("turn-{i}"))?; + } + Ok(()) + }) + .await +} diff --git a/src/bin/library_profile/scenarios/memory_ingest.rs b/src/bin/library_profile/scenarios/memory_ingest.rs new file mode 100644 index 000000000..c568e8168 --- /dev/null +++ b/src/bin/library_profile/scenarios/memory_ingest.rs @@ -0,0 +1,56 @@ +//! `memory-ingest`: canonicalise and ingest 100 chat messages, then drain the +//! real extraction/admission/tree queue. + +use anyhow::Result; +use chrono::{TimeZone, Utc}; +use openhuman_core::core::event_bus::init_global; +use openhuman_core::openhuman::memory::ingest_pipeline::ingest_chat; +use openhuman_core::openhuman::memory_queue::drain_until_idle; +use openhuman_core::openhuman::memory_sync::canonicalize::chat::{ChatBatch, ChatMessage}; + +use crate::harness::{fixture, measure, ProfileResult}; + +const INGEST_MESSAGE_COUNT: usize = 100; + +fn ingestion_batch() -> ChatBatch { + let messages = (0..INGEST_MESSAGE_COUNT) + .map(|index| ChatMessage { + author: if index % 2 == 0 { "alice" } else { "bob" }.into(), + timestamp: Utc + .timestamp_millis_opt(1_700_000_000_000 + index as i64 * 60_000) + .single() + .expect("valid profile timestamp"), + text: format!( + "Phoenix migration update {index}: staging p99 is 12ms and error rate is 0.001%. \ + Alice owns the rollback runbook, Bob owns on-call coordination, and the \ + phoenix_v2_enabled flag ramps Friday after billing-ledger verification." + ), + source_ref: Some(format!("profile://message/{index}")), + }) + .collect(); + ChatBatch { + platform: "profile".into(), + channel_label: "library-benchmark".into(), + messages, + } +} + +pub async fn run() -> Result { + let fixture = fixture()?; + let _ = init_global(256); + eprintln!("[library-profile] memory-ingest: fixture + event bus ready"); + measure("memory-ingest", INGEST_MESSAGE_COUNT, None, |_rec| async { + let result = ingest_chat( + &fixture.config, + "profile:chat:100", + "profile-user", + vec!["profile".into()], + ingestion_batch(), + ) + .await?; + anyhow::ensure!(result.chunks_written > 0, "ingestion wrote no chunks"); + drain_until_idle(&fixture.config).await?; + Ok(()) + }) + .await +} diff --git a/src/bin/library_profile/scenarios/mod.rs b/src/bin/library_profile/scenarios/mod.rs new file mode 100644 index 000000000..cc889863f --- /dev/null +++ b/src/bin/library_profile/scenarios/mod.rs @@ -0,0 +1,13 @@ +//! One module per profiling scenario. Each exposes a single +//! `run() -> Result` entry point dispatched from `main`. + +pub mod agent_turn; +pub mod cold_phases; +pub mod fleet; +pub mod long_agent; +pub mod memory_ingest; +pub mod skill_run; +pub mod subagent_storm; +pub mod subagents; +pub mod subconscious; +pub mod workflow; diff --git a/src/bin/library_profile/scenarios/skill_run.rs b/src/bin/library_profile/scenarios/skill_run.rs new file mode 100644 index 000000000..6b1d3043f --- /dev/null +++ b/src/bin/library_profile/scenarios/skill_run.rs @@ -0,0 +1,149 @@ +//! `skill-run`: the true, process-*tree* cost of a skill step that executes on +//! a real language runtime — the interpreter child process included. +//! +//! ## What it actually runs +//! +//! A skill run's orchestrator (`spawn_workflow_run_background` → the +//! `orchestrator` agent) deliberately owns **no** `node_exec` tool: the +//! chat-tier orchestrator never executes code itself, it delegates every code +//! step to the `code_executor` specialist (the only builtin whose allow-list +//! carries `node_exec` / `npm_exec`). So the agent that genuinely spawns the +//! language runtime *is* `code_executor`. This scenario drives that specialist +//! directly — one turn, one scripted `node_exec` call — which is the real +//! node-executing path, not a bare `std::process` spawn. Measuring the +//! orchestrator→specialist delegation on top would add in-process agent cost +//! without changing the runtime-child cost this scenario exists to capture; +//! the compromise is documented here on purpose. +//! +//! The mock ([`SkillRunMock`]) emits a `node_exec` call whose inline JavaScript +//! does real work, allocates, and busy-waits ~1.2 s so the child stays resident +//! long enough for the harness tree sampler (15 ms poll) to attribute it, then +//! prints JSON carrying [`NODE_MARKER`]. When that output rides back into the +//! turn the mock returns a plain final answer and the turn completes. +//! +//! ## No interpreter download +//! +//! `node.prefer_system = true` (the default) means a host `node` whose **major** +//! matches the configured target is reused rather than downloaded. This +//! scenario **requires** a system `node` and bails with a clear stderr error + +//! nonzero exit if none is on `PATH` — it must never pull a runtime. +//! +//! The measured cost lands in `result.tree` (`tree_rss_kib`, `child_count`, +//! per-child RSS) — captured at the workload peak, since the `node` child has +//! already exited by settle time. + +use std::sync::Arc; + +use anyhow::Result; +use openhuman_core::core::event_bus::init_global; +use openhuman_core::openhuman::agent::harness::AgentDefinitionRegistry; +use openhuman_core::openhuman::agent::Agent; +use openhuman_core::openhuman::inference::provider::factory::test_provider_override; +use openhuman_core::openhuman::inference::provider::Provider; +use openhuman_core::openhuman::security::AutonomyLevel; + +use crate::harness::{fixture, measure_with_tree, EnvGuard, ProfileResult}; +use crate::mock::SkillRunMock; + +/// The specialist agent that owns `node_exec` and spawns the runtime child. +const CODE_AGENT: &str = "code_executor"; + +/// Probe for a usable system `node`. Returns its version on success; on failure +/// prints a clear `[library-profile]` stderr error and returns `Err` (which +/// propagates to a nonzero process exit). We must NOT download an interpreter. +fn require_system_node() -> Result { + match std::process::Command::new("node").arg("--version").output() { + Ok(output) if output.status.success() => { + let version = String::from_utf8_lossy(&output.stdout).trim().to_string(); + eprintln!("[library-profile] skill-run: using system node {version}"); + Ok(version) + } + Ok(output) => { + let stderr = String::from_utf8_lossy(&output.stderr); + eprintln!( + "[library-profile] skill-run: `node --version` failed (status {:?}): {stderr}", + output.status.code() + ); + anyhow::bail!("skill-run requires a working system `node`, but `node --version` failed") + } + Err(err) => { + eprintln!( + "[library-profile] skill-run: `node` not found on PATH: {err}. Install Node.js — \ + the profiler will NOT download an interpreter." + ); + anyhow::bail!("skill-run requires a system `node` on PATH; none found") + } + } +} + +pub async fn run() -> Result { + // Hard requirement: a system node must be present (no download). + require_system_node()?; + + let mut fixture = fixture()?; + // `node_exec` is a Write-class acting tool. Full autonomy keeps the gate + // from parking the turn on approval; the gate is also opted out explicitly. + fixture.config.autonomy.level = AutonomyLevel::Full; + let _approval_env = EnvGuard::set("OPENHUMAN_APPROVAL_GATE", "0"); + + let _ = init_global(256); + openhuman_core::openhuman::agent::bus::register_agent_handlers(); + let _ = AgentDefinitionRegistry::init_global_builtins(); + + let mock = SkillRunMock::new(); + let provider: Arc = mock.clone(); + let _provider = test_provider_override::install(provider); + eprintln!( + "[library-profile] skill-run: registries ready, node_exec mock installed \ + (agent={CODE_AGENT})" + ); + + let mock_for_workload = mock.clone(); + let config = fixture.config.clone(); + let mut result = measure_with_tree("skill-run", 1, None, move |rec| async move { + rec.checkpoint("turn-start")?; + let mut agent = Agent::from_config_for_agent(&config, CODE_AGENT)?; + let reply = agent + .run_single( + "Run a short JavaScript computation with node_exec and report the JSON it prints.", + ) + .await?; + rec.checkpoint("turn-done")?; + anyhow::ensure!(!reply.trim().is_empty(), "empty code_executor reply"); + anyhow::ensure!( + mock_for_workload.node_call_emitted(), + "the node_exec tool call was never emitted" + ); + anyhow::ensure!( + mock_for_workload.node_output_seen(), + "the node child's output never flowed back — the interpreter child did not run/print" + ); + Ok(()) + }) + .await?; + + match result.tree.as_ref() { + Some(tree) if tree.child_count >= 1 => { + eprintln!( + "[library-profile] skill-run: captured tree_rss_kib={} child_count={} \ + children={:?}", + tree.tree_rss_kib, tree.child_count, tree.children + ); + } + Some(tree) => { + eprintln!( + "[library-profile] skill-run: WARNING tree captured but no child was resident at \ + peak (tree_rss_kib={}). The node child may have been too short-lived; \ + the busy-wait should have kept it alive.", + tree.tree_rss_kib + ); + } + None => { + eprintln!("[library-profile] skill-run: WARNING no process-tree sample captured"); + } + } + + // Fold in scenario-visible fields (schema stays additive). + result.workload_units = 1; + Ok(result) +} diff --git a/src/bin/library_profile/scenarios/subagent_storm.rs b/src/bin/library_profile/scenarios/subagent_storm.rs new file mode 100644 index 000000000..76e8b6fb2 --- /dev/null +++ b/src/bin/library_profile/scenarios/subagent_storm.rs @@ -0,0 +1,191 @@ +//! `subagent-storm`: fuzz the *width* of delegation inside ONE core instance. +//! +//! One orchestrator turn fans out to **K** parallel researcher subagents (all +//! in-process tokio tasks, not child processes) via `spawn_parallel_agents`. +//! K comes from `OPENHUMAN_PROFILE_SUBAGENTS` (default 8; tested up to 32), and +//! each researcher carries per-subagent mock latency drawn from the shared +//! `OPENHUMAN_PROFILE_MOCK_LATENCY_MS` / `_JITTER_MS` knobs. +//! +//! ## Measurement shape (and a hard constraint we hit) +//! +//! The intended shape was: prewarm one width-K fan-out, then measure a second on +//! the same warm process so the delta is attributable to the K children rather +//! than cold bootstrap. That is **not achievable** here: the parallel-spawn +//! machinery is effectively one-shot per process. Once a fan-out's run ledger is +//! finalized, a second `spawn_parallel_agents` returns an empty result and the +//! orchestrator just re-calls the tool without re-running the workers. Worse, +//! merely *constructing* an orchestrator/researcher agent beforehand perturbs +//! the fan-out the same way. The only shape that reliably executes all K real +//! researcher subagents is a single fan-out as the process's first agent +//! activity. +//! +//! So this scenario measures exactly that: one cold width-K fan-out. `retained_delta_kib` +//! therefore includes the shared per-process bootstrap (~20–30 MiB every +//! scenario pays once) amortized across K, so a single run's +//! `marginal_rss_kib_per_agent = retained_delta_kib / K` is an **upper bound**, +//! not the true marginal. Read the true marginal by comparing runs: the fixed +//! bootstrap amortizes, so `(retained(K₂) - retained(K₁)) / (K₂ - K₁)` across +//! two widths (e.g. K=8 vs K=32) isolates the genuine per-additional-subagent +//! cost. `peak_delta_kib` additionally captures the K-concurrent peak. +//! +//! Reported fields: `subagents = K`, `marginal_rss_kib_per_agent` (retained/K, +//! upper bound), `checkpoints` (baseline → storm-turn-done), and +//! `turn_latency_ms` (percentiles across the K researcher child executions). +//! The workload asserts all K researcher subagents actually executed. + +use std::sync::Arc; + +use anyhow::Result; +use openhuman_core::core::event_bus::init_global; +use openhuman_core::openhuman::agent::harness::AgentDefinitionRegistry; +use openhuman_core::openhuman::agent::Agent; +use openhuman_core::openhuman::inference::provider::factory::test_provider_override; +use openhuman_core::openhuman::inference::provider::Provider; + +use crate::harness::{fixture, measure, ProfileResult, TurnLatency}; +use crate::mock::{subagent_marker, SubagentMock}; + +const DEFAULT_SUBAGENTS: usize = 8; + +/// The orchestrator's top-level task. The mock ignores the wording and always +/// fans out to K researchers. +const STORM_PROMPT: &str = "Research every subsystem in parallel and merge the findings."; + +/// Positive identity anchor for a researcher *worker* turn — its own system +/// prompt names it. Distinguishes a real worker from the orchestrator turns that +/// also echo every task marker in the fan-out tool call / result. +const RESEARCHER_IDENTITY: &str = "You are the **Researcher** agent"; + +fn env_usize(key: &str, default: usize) -> usize { + std::env::var(key) + .ok() + .and_then(|value| value.parse::().ok()) + .filter(|n| *n > 0) + .unwrap_or(default) +} + +/// Nearest-rank percentile of an already-sorted slice. `p` in `[0, 100]`. +fn percentile(sorted: &[u128], p: u128) -> u128 { + if sorted.is_empty() { + return 0; + } + let rank = ((p * sorted.len() as u128) + 99) / 100; // ceil(p% * n) + let idx = rank.saturating_sub(1).min(sorted.len() as u128 - 1) as usize; + sorted[idx] +} + +fn latency_summary(mut samples: Vec) -> Option { + if samples.is_empty() { + return None; + } + samples.sort_unstable(); + Some(TurnLatency { + p50: percentile(&samples, 50), + p95: percentile(&samples, 95), + p99: percentile(&samples, 99), + max: *samples.last().unwrap(), + }) +} + +pub async fn run() -> Result { + let width = env_usize("OPENHUMAN_PROFILE_SUBAGENTS", DEFAULT_SUBAGENTS); + + let mut fixture = fixture()?; + // `spawn_parallel_agents` rejects a fan-out wider than the orchestrator's + // `max_parallel_tools` (default 4). Raise it to K so the full width actually + // spawns instead of erroring back to a re-spawn loop. + fixture.config.agent.max_parallel_tools = width.max(4); + let _ = init_global(256); + openhuman_core::openhuman::agent::bus::register_agent_handlers(); + let _ = AgentDefinitionRegistry::init_global_builtins(); + + let mock = SubagentMock::with_width(width); + let provider: Arc = mock.clone(); + let _provider = test_provider_override::install(provider); + eprintln!("[library-profile] subagent-storm: width={width} — single cold width-K fan-out"); + + // We drive the `orchestrator` agent directly: it owns `spawn_parallel_agents` + // and allows the `researcher` subagent (the chat-tier subconscious has + // neither, and would reject the fan-out). One orchestrator turn fans out to + // K real researcher subagents via the parallel graph. This fan-out MUST be + // the process's first agent activity — see the module docs for why prewarming + // is not possible here. + let config = fixture.config.clone(); + let mock_for_workload = mock.clone(); + let mut result = measure("subagent-storm", width, None, move |rec| async move { + rec.checkpoint("baseline")?; + let mut agent = Agent::from_config_for_agent(&config, "orchestrator")?; + let reply = agent.run_single(STORM_PROMPT).await?; + rec.checkpoint("storm-turn-done")?; + anyhow::ensure!(!reply.trim().is_empty(), "empty storm-turn response"); + // Every one of the K researcher subagents must have actually executed as + // its own worker turn: for each i there must be a prompt that carries the + // researcher identity anchor AND that researcher's task marker — not + // merely an orchestrator turn echoing every marker in the fan-out call. + let prompts = mock_for_workload.prompts.lock().expect("mock prompt lock"); + for i in 1..=width { + anyhow::ensure!( + prompts + .iter() + .any(|p| p.contains(RESEARCHER_IDENTITY) && p.contains(&subagent_marker(i))), + "researcher subagent {i}/{width} never executed as its own worker turn" + ); + } + Ok(()) + }) + .await?; + + // Marginal per subagent = retained / K. An UPPER BOUND: `retained_delta_kib` + // still carries the one-time per-process bootstrap (see module docs), so the + // true marginal is the cross-width delta `(retained(K₂)-retained(K₁))/(K₂-K₁)`. + let marginal = if width > 0 { + Some(result.retained_delta_kib as f64 / width as f64) + } else { + None + }; + let latencies = mock + .researcher_latencies_ms + .lock() + .expect("mock latency lock") + .clone(); + eprintln!( + "[library-profile] subagent-storm: width={width} retained_delta_kib={} \ + marginal_rss_kib_per_agent={:?} researcher_executions={}", + result.retained_delta_kib, + marginal, + latencies.len() + ); + + result.subagents = Some(width); + result.marginal_rss_kib_per_agent = marginal; + result.turn_latency_ms = latency_summary(latencies); + Ok(result) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn percentiles_nearest_rank() { + let v: Vec = (1..=100).collect(); + assert_eq!(percentile(&v, 50), 50); + assert_eq!(percentile(&v, 95), 95); + assert_eq!(percentile(&v, 99), 99); + assert_eq!(percentile(&v, 100), 100); + assert_eq!(percentile(&[], 50), 0); + } + + #[test] + fn latency_summary_none_when_empty() { + assert!(latency_summary(Vec::new()).is_none()); + let s = latency_summary(vec![10, 20, 30]).unwrap(); + assert_eq!(s.max, 30); + assert_eq!(s.p50, 20); + } + + #[test] + fn env_usize_falls_back_on_zero_or_unset() { + assert_eq!(env_usize("OPENHUMAN_PROFILE_STORM_UNSET_XYZ", 8), 8); + } +} diff --git a/src/bin/library_profile/scenarios/subagents.rs b/src/bin/library_profile/scenarios/subagents.rs new file mode 100644 index 000000000..2f2c52e4e --- /dev/null +++ b/src/bin/library_profile/scenarios/subagents.rs @@ -0,0 +1,56 @@ +//! `subagents`: run one real orchestrator chat turn that spawns two real +//! researcher subagents through the parallel-delegation tool. + +use std::sync::Arc; + +use anyhow::Result; +use openhuman_core::core::event_bus::init_global; +use openhuman_core::openhuman::agent::harness::AgentDefinitionRegistry; +use openhuman_core::openhuman::config::schema::SubconsciousMode; +use openhuman_core::openhuman::inference::provider::factory::test_provider_override; +use openhuman_core::openhuman::inference::provider::Provider; +use openhuman_core::openhuman::subconscious::LongLivedSession; + +use crate::harness::{fixture, measure, ProfileResult}; +use crate::mock::{subagent_marker, SubagentMock}; + +pub async fn run() -> Result { + let fixture = fixture()?; + let _ = init_global(256); + openhuman_core::openhuman::agent::bus::register_agent_handlers(); + let _ = AgentDefinitionRegistry::init_global_builtins(); + let mock = SubagentMock::new(); + let provider: Arc = mock.clone(); + let _provider = test_provider_override::install(provider); + if std::env::var_os("OPENHUMAN_PROFILE_PREWARM_SUBAGENTS").is_some() { + eprintln!("[library-profile] subagents: prewarming one full turn"); + let warmup = LongLivedSession::with_thread( + fixture.config.workspace_dir.clone(), + SubconsciousMode::Aggressive, + "profile:warmup".into(), + ); + let outcome = warmup + .process_promoted("Please research the Phoenix migration.", false) + .await + .map_err(anyhow::Error::msg)?; + anyhow::ensure!(!outcome.response.is_empty(), "empty warmup response"); + mock.prompts.lock().expect("mock prompt lock").clear(); + } + let session = LongLivedSession::with_thread( + fixture.config.workspace_dir.clone(), + SubconsciousMode::Aggressive, + "profile:orchestrator".into(), + ); + measure("subagents", 2, None, |_rec| async { + let outcome = session + .process_promoted("Please research the Phoenix migration.", false) + .await + .map_err(anyhow::Error::msg)?; + anyhow::ensure!(!outcome.response.is_empty(), "empty orchestrator response"); + let prompts = mock.prompts.lock().expect("mock prompt lock"); + anyhow::ensure!(prompts.iter().any(|p| p.contains(&subagent_marker(1)))); + anyhow::ensure!(prompts.iter().any(|p| p.contains(&subagent_marker(2)))); + Ok(()) + }) + .await +} diff --git a/src/bin/library_profile/scenarios/subconscious.rs b/src/bin/library_profile/scenarios/subconscious.rs new file mode 100644 index 000000000..1721be437 --- /dev/null +++ b/src/bin/library_profile/scenarios/subconscious.rs @@ -0,0 +1,56 @@ +//! `subconscious`: one promoted subconscious turn WITHOUT delegation. Same +//! `LongLivedSession` path as `subagents`, but the mock returns a direct text +//! response (no `spawn_parallel_agents` tool call). Complements `subagents`. + +use std::sync::Arc; + +use anyhow::Result; +use openhuman_core::core::event_bus::init_global; +use openhuman_core::openhuman::agent::harness::AgentDefinitionRegistry; +use openhuman_core::openhuman::config::schema::SubconsciousMode; +use openhuman_core::openhuman::inference::provider::factory::test_provider_override; +use openhuman_core::openhuman::inference::provider::Provider; +use openhuman_core::openhuman::subconscious::LongLivedSession; + +use crate::harness::{fixture, measure, ProfileResult}; +use crate::mock::PlainTextMock; + +pub async fn run() -> Result { + let fixture = fixture()?; + let _ = init_global(256); + openhuman_core::openhuman::agent::bus::register_agent_handlers(); + let _ = AgentDefinitionRegistry::init_global_builtins(); + let mock = PlainTextMock::new("Phoenix migration is on track; nothing needs your attention."); + let provider: Arc = mock.clone(); + let _provider = test_provider_override::install(provider); + + if std::env::var_os("OPENHUMAN_PROFILE_PREWARM_SUBAGENTS").is_some() { + eprintln!("[library-profile] subconscious: prewarming one full turn"); + let warmup = LongLivedSession::with_thread( + fixture.config.workspace_dir.clone(), + SubconsciousMode::Aggressive, + "profile:warmup".into(), + ); + let outcome = warmup + .process_promoted("Please review the Phoenix migration.", false) + .await + .map_err(anyhow::Error::msg)?; + anyhow::ensure!(!outcome.response.is_empty(), "empty warmup response"); + mock.prompts.lock().expect("mock prompt lock").clear(); + } + + let session = LongLivedSession::with_thread( + fixture.config.workspace_dir.clone(), + SubconsciousMode::Aggressive, + "profile:subconscious".into(), + ); + measure("subconscious", 1, None, |_rec| async { + let outcome = session + .process_promoted("Please review the Phoenix migration.", false) + .await + .map_err(anyhow::Error::msg)?; + anyhow::ensure!(!outcome.response.is_empty(), "empty subconscious response"); + Ok(()) + }) + .await +} diff --git a/src/bin/library_profile/scenarios/workflow.rs b/src/bin/library_profile/scenarios/workflow.rs new file mode 100644 index 000000000..2b14b459f --- /dev/null +++ b/src/bin/library_profile/scenarios/workflow.rs @@ -0,0 +1,75 @@ +//! `workflow`: run a real flows-domain workflow end to end. A +//! trigger -> transform -> agent graph is created OUTSIDE the measured region +//! (recorded as a checkpoint), then `flows_run` is measured as the workload. +//! The agent node's LLM routes through the plain-text mock provider. + +use std::sync::Arc; + +use anyhow::Result; +use openhuman_core::core::event_bus::init_global; +use openhuman_core::openhuman::agent::harness::AgentDefinitionRegistry; +use openhuman_core::openhuman::flows::ops::{flows_create, flows_run}; +use openhuman_core::openhuman::flows::FlowRunTrigger; +use openhuman_core::openhuman::inference::provider::factory::test_provider_override; +use openhuman_core::openhuman::inference::provider::Provider; +use serde_json::json; + +use crate::harness::{fixture, measure, ProfileResult}; +use crate::mock::PlainTextMock; + +pub async fn run() -> Result { + let fixture = fixture()?; + let _ = init_global(256); + openhuman_core::openhuman::agent::bus::register_agent_handlers(); + let _ = AgentDefinitionRegistry::init_global_builtins(); + let mock = PlainTextMock::new("Phoenix migration status: healthy, ramp on Friday."); + let provider: Arc = mock.clone(); + let _provider = test_provider_override::install(provider); + + let graph = json!({ + "name": "profile-workflow", + "nodes": [ + { "id": "t", "kind": "trigger", "name": "Trigger" }, + { "id": "prep", "kind": "transform", "name": "Prep", + "config": { "set": { "topic": "Phoenix migration" } } }, + { "id": "summarize", "kind": "agent", "name": "Summarize", + "config": { "agent_ref": "researcher", + "prompt": "Summarise the Phoenix migration status in one line." } } + ], + "edges": [ + { "from_node": "t", "to_node": "prep" }, + { "from_node": "prep", "to_node": "summarize" } + ] + }); + + eprintln!("[library-profile] workflow: creating flow (outside measured region)"); + let flow = flows_create(&fixture.config, "profile-workflow".into(), graph, false) + .await + .map_err(anyhow::Error::msg)? + .value; + let flow_id = flow.id.clone(); + + measure("workflow", 1, None, move |_rec| async move { + let outcome = flows_run( + &fixture.config, + &flow_id, + json!({ "topic": "Phoenix migration" }), + FlowRunTrigger::Rpc, + ) + .await + .map_err(anyhow::Error::msg)?; + let output = outcome.value.get("output"); + anyhow::ensure!( + output.is_some() && !output.unwrap().is_null(), + "workflow run produced no output: {}", + outcome.value + ); + anyhow::ensure!( + outcome.value.get("note").is_none(), + "workflow with an actionable agent node unexpectedly reported nothing-to-run: {}", + outcome.value + ); + Ok(()) + }) + .await +} diff --git a/src/openhuman/agent/prompts/render_helpers.rs b/src/openhuman/agent/prompts/render_helpers.rs index c1a9f98b4..48ce89ba2 100644 --- a/src/openhuman/agent/prompts/render_helpers.rs +++ b/src/openhuman/agent/prompts/render_helpers.rs @@ -111,6 +111,16 @@ pub fn render_datetime(ctx: &PromptContext<'_>) -> Result { /// session. The static grounding *rule* that tells the model to read this /// line lives in [`DateTimeSection`] / [`render_datetime`]. pub fn current_datetime_line() -> String { + #[cfg(feature = "rss-bench")] + if std::env::var_os("OPENHUMAN_PROFILE_FORCE_UTC").is_some() { + let now = chrono::Utc::now(); + return format!( + "Current Date & Time: {} UTC (UTC, UTC+00:00), {}", + now.format("%Y-%m-%d %H:%M:%S"), + now.format("%A"), + ); + } + // When the host resolves an IANA zone, stamp local time + that zone. When // it can't (CI, stripped containers), fall back to true UTC — formatting // `Utc::now()` so the time, offset, and zone label all agree rather than diff --git a/src/openhuman/inference/provider/factory.rs b/src/openhuman/inference/provider/factory.rs index b074a831c..7c6abb4cf 100644 --- a/src/openhuman/inference/provider/factory.rs +++ b/src/openhuman/inference/provider/factory.rs @@ -513,9 +513,9 @@ pub(crate) fn resolve_byok_fallback_provider_string(config: &Config) -> Option anyhow::Result<(Arc>, String)> { let test_override_active = { - #[cfg(any(test, feature = "e2e-test-support"))] + #[cfg(any(test, feature = "e2e-test-support", feature = "rss-bench"))] { test_provider_override::current().is_some() } - #[cfg(not(any(test, feature = "e2e-test-support")))] + #[cfg(not(any(test, feature = "e2e-test-support", feature = "rss-bench")))] { false } @@ -1477,11 +1477,11 @@ pub(crate) fn create_turn_chat_model_with_native_tools( native_tool_calling: bool, ) -> anyhow::Result>> { let test_override_active = { - #[cfg(any(test, feature = "e2e-test-support"))] + #[cfg(any(test, feature = "e2e-test-support", feature = "rss-bench"))] { test_provider_override::current().is_some() } - #[cfg(not(any(test, feature = "e2e-test-support")))] + #[cfg(not(any(test, feature = "e2e-test-support", feature = "rss-bench")))] { false } @@ -1559,11 +1559,11 @@ pub(crate) fn create_turn_chat_model_from_string_with_native_tools( native_tool_calling: bool, ) -> anyhow::Result>> { let test_override_active = { - #[cfg(any(test, feature = "e2e-test-support"))] + #[cfg(any(test, feature = "e2e-test-support", feature = "rss-bench"))] { test_provider_override::current().is_some() } - #[cfg(not(any(test, feature = "e2e-test-support")))] + #[cfg(not(any(test, feature = "e2e-test-support", feature = "rss-bench")))] { false } diff --git a/src/openhuman/inference/provider/ops/provider_factory.rs b/src/openhuman/inference/provider/ops/provider_factory.rs index d92881397..a71bcbe0a 100644 --- a/src/openhuman/inference/provider/ops/provider_factory.rs +++ b/src/openhuman/inference/provider/ops/provider_factory.rs @@ -172,8 +172,8 @@ pub fn create_routed_provider_with_options( ) -> anyhow::Result> { // Test-only: a mock provider injected by an e2e test wins over any // config-derived routing (covers the triage remote arm). Gated on - // cfg(test) / the off-by-default `e2e-test-support` feature. - #[cfg(any(test, feature = "e2e-test-support"))] + // cfg(test) / the off-by-default test and profiling features. + #[cfg(any(test, feature = "e2e-test-support", feature = "rss-bench"))] if let Some(p) = super::super::factory::test_provider_override::current() { return Ok(Box::new( super::super::factory::test_provider_override::ProviderHandle(p), diff --git a/src/openhuman/proc_metrics/mod.rs b/src/openhuman/proc_metrics/mod.rs index 44b036531..32550a07b 100644 --- a/src/openhuman/proc_metrics/mod.rs +++ b/src/openhuman/proc_metrics/mod.rs @@ -1,4 +1,4 @@ -//! Process memory sampling from Linux `/proc`. +//! Cross-platform process memory sampling. //! //! Reads the current process's resident-memory breakdown from //! `/proc/self/smaps_rollup` + `/proc/self/status` and aggregates repeated @@ -6,15 +6,19 @@ //! `rss-bench` benchmark harness (#5046), which measures the steady-state RSS //! of an embedded `openhuman_core` agent roster against the 20–30 MiB budget, //! but [`sample_self`] is a general capability: any caller wanting this -//! process's RSS / PSS / private-page / peak-RSS figures on Linux can use it. +//! process's RSS / peak-RSS figures can use it. Linux additionally reports PSS +//! and private-page breakdowns; macOS leaves those Linux-only fields at zero. //! //! The parsers ([`parse_status`], [`parse_smaps_rollup`]) are OS-agnostic and //! take `&str`, so they are unit-tested without a live `/proc`. [`sample_self`] -//! is Linux-only and returns a structured error elsewhere — it never fabricates -//! a reading (a macOS local run fails loudly rather than emitting garbage). +//! supports Linux and macOS and returns a structured error elsewhere — it never +//! fabricates a reading. use serde::{Deserialize, Serialize}; +pub mod tree; +pub use tree::{sample_tree, ChildSample, TreeSample}; + /// Product budget for the embedded roster, in KiB (#5046). Target the agent /// roster should land under. pub const RSS_BUDGET_KIB: u64 = 20 * 1024; @@ -39,6 +43,20 @@ pub struct ProcSample { pub threads: u64, /// On-disk size of the running executable, in bytes. pub binary_size_bytes: u64, + /// Cumulative user-mode CPU time, in milliseconds. Linux: `/proc/self/stat` + /// `utime` (ticks → ms). macOS: `proc_pid_rusage` `ri_user_time` (ns → ms). + /// Defaulted so pre-existing JSON consumers keep deserializing. + #[serde(default)] + pub cpu_user_ms: u64, + /// Cumulative system-mode CPU time, in milliseconds. Linux: `/proc/self/stat` + /// `stime`. macOS: `proc_pid_rusage` `ri_system_time`. + #[serde(default)] + pub cpu_system_ms: u64, + /// Open file-descriptor count. Linux: entries in `/proc/self/fd`. macOS: + /// `proc_pidinfo(PROC_PIDLISTFDS)` buffer size / `proc_fdinfo` size. `None` + /// when the platform lookup is unavailable rather than a misleading zero. + #[serde(default)] + pub open_fds: Option, } /// Fields extracted from `/proc//status`. @@ -100,6 +118,49 @@ pub fn parse_smaps_rollup(contents: &str) -> SmapsRollupFields { fields } +/// User/system CPU jiffies parsed from `/proc//stat`. +#[derive(Debug, Default, Clone, Copy, PartialEq, Eq)] +pub struct StatCpuFields { + pub utime_ticks: u64, + pub stime_ticks: u64, +} + +/// Parse `utime` (field 14) and `stime` (field 15) out of `/proc//stat`. +/// The `comm` field (2) is parenthesised and may itself contain spaces or +/// parens, so we split on the **last** `')'` and index the remaining +/// whitespace-separated fields: after `)`, token 0 is `state`, so `utime` is +/// token 11 and `stime` token 12. Missing/short input yields zero. +pub fn parse_proc_stat_cpu(contents: &str) -> StatCpuFields { + let Some(after) = contents.rsplit_once(')').map(|(_, tail)| tail) else { + return StatCpuFields::default(); + }; + let fields: Vec<&str> = after.split_whitespace().collect(); + // token 0 == state; utime == token 11, stime == token 12. + let utime_ticks = fields.get(11).and_then(|t| t.parse().ok()).unwrap_or(0); + let stime_ticks = fields.get(12).and_then(|t| t.parse().ok()).unwrap_or(0); + StatCpuFields { + utime_ticks, + stime_ticks, + } +} + +/// Convert CPU clock ticks to milliseconds given `CLK_TCK` (ticks per second). +/// A zero or negative tick rate yields zero rather than dividing by zero. +pub fn cpu_ticks_to_ms(ticks: u64, clk_tck: i64) -> u64 { + if clk_tck <= 0 { + return 0; + } + ticks.saturating_mul(1000) / clk_tck as u64 +} + +/// Count entries in `/proc//fd` (excluding `.`/`..`, which `read_dir` +/// already omits). `None` when the directory can't be read. +#[cfg(target_os = "linux")] +fn count_open_fds() -> Option { + let entries = std::fs::read_dir("/proc/self/fd").ok()?; + Some(entries.filter(|e| e.is_ok()).count() as u64) +} + /// Sample this process's resident memory. Linux-only. #[cfg(target_os = "linux")] pub fn sample_self() -> anyhow::Result { @@ -109,6 +170,10 @@ pub fn sample_self() -> anyhow::Result { .context("read /proc/self/smaps_rollup")?; let status = parse_status(&status); let smaps = parse_smaps_rollup(&smaps); + let stat = std::fs::read_to_string("/proc/self/stat").unwrap_or_default(); + let cpu = parse_proc_stat_cpu(&stat); + // SAFETY: `sysconf` is a pure lookup; `_SC_CLK_TCK` is a valid selector. + let clk_tck = unsafe { libc::sysconf(libc::_SC_CLK_TCK) }; let binary_size_bytes = std::env::current_exe() .and_then(std::fs::metadata) .map(|meta| meta.len()) @@ -121,15 +186,103 @@ pub fn sample_self() -> anyhow::Result { vm_hwm_kib: status.vm_hwm_kib, threads: status.threads, binary_size_bytes, + cpu_user_ms: cpu_ticks_to_ms(cpu.utime_ticks, clk_tck), + cpu_system_ms: cpu_ticks_to_ms(cpu.stime_ticks, clk_tck), + open_fds: count_open_fds(), }) } -/// Sample this process's resident memory. Non-Linux stub — fails loudly rather -/// than fabricating a reading. -#[cfg(not(target_os = "linux"))] +/// Sample this process's resident memory on macOS via `proc_pid_rusage`. +/// +/// `ri_resident_size` and `ru_maxrss` are bytes on Darwin. PSS and private-page +/// accounting have no direct macOS equivalent, so those Linux-specific fields +/// remain zero rather than being populated with a misleading substitute. +#[cfg(target_os = "macos")] +pub fn sample_self() -> anyhow::Result { + use std::mem::{size_of, MaybeUninit}; + + let pid = std::process::id() as libc::c_int; + let mut usage = MaybeUninit::::uninit(); + // SAFETY: `usage` points to writable storage of the exact structure size + // required by `RUSAGE_INFO_V2`; the kernel initializes it on success. + let result = unsafe { + libc::proc_pid_rusage( + pid, + libc::RUSAGE_INFO_V2, + usage.as_mut_ptr().cast::(), + ) + }; + if result != 0 { + return Err(std::io::Error::last_os_error()).map_err(Into::into); + } + // SAFETY: `proc_pid_rusage` returned success and initialized `usage`. + let usage = unsafe { usage.assume_init() }; + + let mut peak = MaybeUninit::::uninit(); + // SAFETY: `peak` is valid writable storage for `getrusage`. + let peak_result = unsafe { libc::getrusage(libc::RUSAGE_SELF, peak.as_mut_ptr()) }; + let vm_hwm_kib = if peak_result == 0 { + // SAFETY: `getrusage` returned success and initialized `peak`. + (unsafe { peak.assume_init() }.ru_maxrss as u64) / 1024 + } else { + 0 + }; + + let mut task = MaybeUninit::::uninit(); + // SAFETY: `task` points to writable storage and its size is passed to + // `proc_pidinfo`, which initializes it when the returned byte count matches. + let task_bytes = unsafe { + libc::proc_pidinfo( + pid, + libc::PROC_PIDTASKINFO, + 0, + task.as_mut_ptr().cast(), + size_of::() as libc::c_int, + ) + }; + let threads = if task_bytes == size_of::() as libc::c_int { + // SAFETY: the kernel returned the full structure size. + unsafe { task.assume_init() }.pti_threadnum.max(0) as u64 + } else { + 0 + }; + + // `proc_pidinfo(PROC_PIDLISTFDS, .., NULL, 0)` returns the byte size of the + // fd table; dividing by `proc_fdinfo` size gives the descriptor count. + // SAFETY: passing a null buffer with zero length is the documented + // size-probe form of `proc_pidinfo`. + let fd_bytes = + unsafe { libc::proc_pidinfo(pid, libc::PROC_PIDLISTFDS, 0, std::ptr::null_mut(), 0) }; + let open_fds = if fd_bytes > 0 { + Some(fd_bytes as u64 / size_of::() as u64) + } else { + None + }; + + let binary_size_bytes = std::env::current_exe() + .and_then(std::fs::metadata) + .map(|meta| meta.len()) + .unwrap_or(0); + Ok(ProcSample { + rss_kib: usage.ri_resident_size / 1024, + pss_kib: 0, + private_clean_kib: 0, + private_dirty_kib: 0, + vm_hwm_kib, + threads, + binary_size_bytes, + // `ri_user_time` / `ri_system_time` are nanoseconds on Darwin. + cpu_user_ms: usage.ri_user_time / 1_000_000, + cpu_system_ms: usage.ri_system_time / 1_000_000, + open_fds, + }) +} + +/// Unsupported-platform stub — fails loudly rather than fabricating a reading. +#[cfg(not(any(target_os = "linux", target_os = "macos")))] pub fn sample_self() -> anyhow::Result { anyhow::bail!( - "proc_metrics::sample_self requires Linux /proc/self/smaps_rollup + status (this is a {} build)", + "proc_metrics::sample_self supports Linux and macOS (this is a {} build)", std::env::consts::OS ) } @@ -340,9 +493,41 @@ mod tests { vm_hwm_kib: hwm, threads, binary_size_bytes: 1024, + cpu_user_ms: 0, + cpu_system_ms: 0, + open_fds: None, } } + // A realistic `/proc/self/stat` line whose `comm` field embeds spaces and a + // close-paren, to prove the last-`)` split is robust. + const SAMPLE_STAT: &str = "1234 (weird ) name) R 1 1234 1234 0 -1 4194304 500 0 0 0 \ + 420 137 0 0 20 0 8 0 99999 123456789 512 18446744073709551615"; + + #[test] + fn parse_proc_stat_cpu_extracts_utime_stime() { + let f = parse_proc_stat_cpu(SAMPLE_STAT); + assert_eq!(f.utime_ticks, 420); + assert_eq!(f.stime_ticks, 137); + } + + #[test] + fn parse_proc_stat_cpu_short_input_stays_zero() { + assert_eq!( + parse_proc_stat_cpu("1234 (x) R 1"), + StatCpuFields::default() + ); + assert_eq!(parse_proc_stat_cpu(""), StatCpuFields::default()); + } + + #[test] + fn cpu_ticks_to_ms_converts_and_guards_zero_rate() { + // 420 ticks at 100 Hz == 4200 ms. + assert_eq!(cpu_ticks_to_ms(420, 100), 4200); + assert_eq!(cpu_ticks_to_ms(1000, 0), 0); + assert_eq!(cpu_ticks_to_ms(1000, -1), 0); + } + #[test] fn median_handles_odd_and_even() { assert_eq!(median_u64(&[]), 0); @@ -438,9 +623,22 @@ mod tests { assert_eq!(report.per_agent_increment_kib(), None); } - #[cfg(not(target_os = "linux"))] + #[cfg(not(any(target_os = "linux", target_os = "macos")))] #[test] - fn sample_self_is_linux_only() { + fn sample_self_rejects_unsupported_platform() { assert!(sample_self().is_err()); } + + #[cfg(target_os = "macos")] + #[test] + fn sample_self_reports_macos_resident_memory() { + let sample = sample_self().expect("macOS process metrics"); + assert!(sample.rss_kib > 0); + assert!(sample.vm_hwm_kib >= sample.rss_kib); + assert!(sample.threads > 0); + assert!(sample.binary_size_bytes > 0); + assert_eq!(sample.pss_kib, 0); + // A live process has consumed at least some CPU and holds open fds. + assert!(sample.open_fds.map(|n| n > 0).unwrap_or(false)); + } } diff --git a/src/openhuman/proc_metrics/tree.rs b/src/openhuman/proc_metrics/tree.rs new file mode 100644 index 000000000..6bdff1246 --- /dev/null +++ b/src/openhuman/proc_metrics/tree.rs @@ -0,0 +1,419 @@ +//! Process-*tree* resident memory sampling. +//! +//! Where [`super::sample_self`] measures only the current process, this module +//! measures the process *and all of its descendants* — the interpreter child +//! processes (`node`, `python`, …) that a skill run or shell tool spawns. The +//! true resource cost of "run this skill" includes those children, which never +//! show up in a self-only RSS reading. +//! +//! [`sample_tree`] returns a [`TreeSample`]: this process's own +//! [`ProcSample`](super::ProcSample), a flat list of descendant +//! [`ChildSample`]s (pid + name + RSS), and `tree_rss_kib` (self + every +//! descendant). Per-child RSS lookups that fail (a child that raced away, or a +//! permission error) are skipped with a `[proc_metrics]` stderr note rather +//! than aborting the whole sample. +//! +//! - **Linux** walks `/proc/*/stat` to recover each pid's `ppid`, chains those +//! into a descendant set, and reads RSS from `/proc//status` (`VmRSS`). +//! - **macOS** enumerates descendants via `proc_listchildpids` (recursively), +//! names them via `proc_pidinfo(PROC_PIDTBSDINFO)`, and reads RSS via +//! `proc_pid_rusage` (`ri_resident_size`). +//! +//! The pure graph walk ([`collect_descendants`]) and the Linux `/proc//stat` +//! parser ([`parse_stat_comm_ppid`]) are OS-agnostic and unit-tested without a +//! live `/proc`. + +use std::collections::{HashMap, HashSet}; + +use serde::{Deserialize, Serialize}; + +use super::ProcSample; + +/// One descendant process in a [`TreeSample`]. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct ChildSample { + /// Process id of the descendant. + pub pid: i32, + /// Executable / accounting name (`node`, `python3`, …). May be empty when + /// the platform lookup fails. + pub name: String, + /// Resident set size of this descendant, in KiB. + pub rss_kib: u64, +} + +/// A process-tree resident-memory sample: this process plus every descendant. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct TreeSample { + /// This process's own self sample. + pub self_sample: ProcSample, + /// Every descendant process found at sample time. + pub children: Vec, + /// Self RSS + the RSS of every descendant, in KiB. + pub tree_rss_kib: u64, +} + +impl TreeSample { + /// Number of descendant processes captured. + pub fn child_count(&self) -> usize { + self.children.len() + } + + /// Sum `self.rss` + every child's RSS into `tree_rss_kib`. + fn assemble(self_sample: ProcSample, children: Vec) -> Self { + let tree_rss_kib = + self_sample.rss_kib + children.iter().map(|child| child.rss_kib).sum::(); + Self { + self_sample, + children, + tree_rss_kib, + } + } +} + +/// Collect every transitive descendant of `root` given a `pid -> ppid` map. +/// +/// OS-agnostic and pure so it can be unit-tested without a live process table. +/// Cycles (which a live table should never contain, but a racy snapshot might) +/// are broken by a visited set; `root` itself is never included. +pub fn collect_descendants(root: i32, ppid_of: &HashMap) -> Vec { + let mut children_of: HashMap> = HashMap::new(); + for (&pid, &ppid) in ppid_of { + children_of.entry(ppid).or_default().push(pid); + } + let mut out = Vec::new(); + let mut seen = HashSet::new(); + let mut stack = vec![root]; + while let Some(parent) = stack.pop() { + if let Some(kids) = children_of.get(&parent) { + for &kid in kids { + if kid != root && seen.insert(kid) { + out.push(kid); + stack.push(kid); + } + } + } + } + out.sort_unstable(); + out +} + +/// Parse the `comm` (name) and `ppid` out of a `/proc//stat` line. +/// +/// `stat` is `pid (comm) state ppid …`; `comm` is parenthesised and may itself +/// contain spaces and parens, so we take everything between the first `(` and +/// the **last** `)`. After that `)`, whitespace token 0 is `state` and token 1 +/// is `ppid`. Missing / malformed input yields `None`. +pub fn parse_stat_comm_ppid(contents: &str) -> Option<(String, i32)> { + let open = contents.find('(')?; + let close = contents.rfind(')')?; + if close <= open { + return None; + } + let comm = contents[open + 1..close].to_string(); + let after = &contents[close + 1..]; + let fields: Vec<&str> = after.split_whitespace().collect(); + // token 0 == state, token 1 == ppid. + let ppid = fields.get(1)?.parse::().ok()?; + Some((comm, ppid)) +} + +/// Sample this process and every descendant. Linux implementation. +#[cfg(target_os = "linux")] +pub fn sample_tree() -> anyhow::Result { + use anyhow::Context; + + let self_pid = std::process::id() as i32; + let self_sample = super::sample_self().context("sample self for tree")?; + + // Build the full pid -> ppid map and a pid -> name map from /proc/*/stat. + let mut ppid_of: HashMap = HashMap::new(); + let mut name_of: HashMap = HashMap::new(); + let entries = std::fs::read_dir("/proc").context("read /proc")?; + for entry in entries.flatten() { + let file_name = entry.file_name(); + let Some(name) = file_name.to_str() else { + continue; + }; + let Ok(pid) = name.parse::() else { + continue; // non-pid entry (self, cpuinfo, …) + }; + let stat = match std::fs::read_to_string(format!("/proc/{pid}/stat")) { + Ok(contents) => contents, + Err(_) => continue, // process exited between readdir and read + }; + if let Some((comm, ppid)) = parse_stat_comm_ppid(&stat) { + ppid_of.insert(pid, ppid); + name_of.insert(pid, comm); + } + } + + let descendants = collect_descendants(self_pid, &ppid_of); + let mut children = Vec::with_capacity(descendants.len()); + for pid in descendants { + let status = match std::fs::read_to_string(format!("/proc/{pid}/status")) { + Ok(contents) => contents, + Err(err) => { + eprintln!( + "[proc_metrics] tree: skipping child pid={pid}: status read failed: {err}" + ); + continue; + } + }; + let rss_kib = super::parse_status(&status).vm_rss_kib; + children.push(ChildSample { + pid, + name: name_of.get(&pid).cloned().unwrap_or_default(), + rss_kib, + }); + } + + Ok(TreeSample::assemble(self_sample, children)) +} + +/// Sample this process and every descendant. macOS implementation. +#[cfg(target_os = "macos")] +pub fn sample_tree() -> anyhow::Result { + let self_pid = std::process::id() as i32; + let self_sample = super::sample_self()?; + + let descendants = macos::descendants(self_pid); + let mut children = Vec::with_capacity(descendants.len()); + for pid in descendants { + match macos::child_rss_kib(pid) { + Some(rss_kib) => children.push(ChildSample { + pid, + name: macos::proc_name(pid), + rss_kib, + }), + None => { + eprintln!( + "[proc_metrics] tree: skipping child pid={pid}: proc_pid_rusage unavailable (exited or permission denied)" + ); + } + } + } + + Ok(TreeSample::assemble(self_sample, children)) +} + +/// Unsupported-platform stub — fails loudly rather than fabricating a reading. +#[cfg(not(any(target_os = "linux", target_os = "macos")))] +pub fn sample_tree() -> anyhow::Result { + anyhow::bail!( + "proc_metrics::sample_tree supports Linux and macOS (this is a {} build)", + std::env::consts::OS + ) +} + +#[cfg(target_os = "macos")] +mod macos { + use std::collections::HashSet; + use std::mem::{size_of, MaybeUninit}; + + /// Direct children of `ppid` via `proc_listchildpids`. Empty on any error. + /// + /// Note the Darwin ABI quirk: `proc_listchildpids` returns the **number of + /// pids** written (not a byte count, unlike `proc_listpids`), and fills the + /// buffer with that many `pid_t`. It also doesn't reliably support the NULL + /// size-probe form, so we allocate a real buffer up front and grow it if the + /// kernel filled it completely (the list may have been truncated). + fn child_pids(ppid: i32) -> Vec { + let mut cap = 256usize; + loop { + let mut buf = vec![0 as libc::pid_t; cap]; + let byte_cap = (buf.len() * size_of::()) as libc::c_int; + // SAFETY: `buf` is `cap` writable pid_t slots; `byte_cap` matches its size. + let written = unsafe { + libc::proc_listchildpids(ppid, buf.as_mut_ptr().cast::(), byte_cap) + }; + if written <= 0 { + return Vec::new(); + } + let count = written as usize; + // Buffer filled to the brim ⇒ the list may be truncated; grow + retry. + if count >= cap && cap < 65_536 { + cap *= 4; + continue; + } + buf.truncate(count.min(buf.len())); + return buf.into_iter().filter(|&pid| pid > 0).collect(); + } + } + + /// Every transitive descendant of `root`, walked breadth-first through + /// `proc_listchildpids`. A visited set breaks any cycle a racy snapshot + /// might present. + pub(super) fn descendants(root: i32) -> Vec { + let mut out = Vec::new(); + let mut seen = HashSet::new(); + let mut stack = child_pids(root); + while let Some(pid) = stack.pop() { + if pid == root || !seen.insert(pid) { + continue; + } + out.push(pid); + stack.extend(child_pids(pid)); + } + out.sort_unstable(); + out + } + + /// Resident set size of `pid` in KiB via `proc_pid_rusage`, or `None` when + /// the call fails (process exited, or permission denied). + pub(super) fn child_rss_kib(pid: i32) -> Option { + let mut usage = MaybeUninit::::uninit(); + // SAFETY: `usage` is writable storage of the exact size for + // `RUSAGE_INFO_V2`; the kernel initializes it on success (return 0). + let rc = unsafe { + libc::proc_pid_rusage( + pid, + libc::RUSAGE_INFO_V2, + usage.as_mut_ptr().cast::(), + ) + }; + if rc != 0 { + return None; + } + // SAFETY: `proc_pid_rusage` returned success and initialized `usage`. + let usage = unsafe { usage.assume_init() }; + Some(usage.ri_resident_size / 1024) + } + + /// Executable / accounting name of `pid` via `proc_pidinfo(PROC_PIDTBSDINFO)`. + /// Prefers the longer `pbi_name`, falling back to `pbi_comm`; empty string + /// when the lookup fails. + pub(super) fn proc_name(pid: i32) -> String { + let mut info = MaybeUninit::::uninit(); + let size = size_of::() as libc::c_int; + // SAFETY: `info` is writable storage of the exact structure size passed + // to `proc_pidinfo`, which initializes it when the returned count matches. + let filled = unsafe { + libc::proc_pidinfo( + pid, + libc::PROC_PIDTBSDINFO, + 0, + info.as_mut_ptr().cast::(), + size, + ) + }; + if filled != size { + return String::new(); + } + // SAFETY: the kernel returned the full structure size. + let info = unsafe { info.assume_init() }; + let name = cstr_field(&info.pbi_name); + if name.is_empty() { + cstr_field(&info.pbi_comm) + } else { + name + } + } + + /// Convert a fixed-size, NUL-padded `c_char` array into a `String` up to the + /// first NUL, lossily decoding any non-UTF-8 bytes. + fn cstr_field(raw: &[libc::c_char]) -> String { + let bytes: Vec = raw + .iter() + .take_while(|&&c| c != 0) + .map(|&c| c as u8) + .collect(); + String::from_utf8_lossy(&bytes).into_owned() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn collect_descendants_walks_transitive_chain() { + // 1 -> 2 -> 4, 1 -> 3; 5 is unrelated (parent 99). + let map: HashMap = [(2, 1), (3, 1), (4, 2), (5, 99)].into_iter().collect(); + let mut got = collect_descendants(1, &map); + got.sort_unstable(); + assert_eq!(got, vec![2, 3, 4]); + } + + #[test] + fn collect_descendants_excludes_root_and_unrelated() { + let map: HashMap = [(2, 1), (3, 2)].into_iter().collect(); + assert_eq!(collect_descendants(2, &map), vec![3]); + assert!(collect_descendants(42, &map).is_empty()); + } + + #[test] + fn collect_descendants_survives_a_cycle() { + // Degenerate self-parent + mutual cycle must terminate, not hang. + let map: HashMap = [(2, 1), (1, 2)].into_iter().collect(); + let got = collect_descendants(1, &map); + assert_eq!(got, vec![2]); + } + + #[test] + fn parse_stat_extracts_comm_and_ppid() { + // pid (comm) state ppid pgrp … + let line = "4321 (node) R 4300 4321 4300 0 -1 4194304 100 0 0 0 5 2 0 0 20 0 11 0"; + let (comm, ppid) = parse_stat_comm_ppid(line).unwrap(); + assert_eq!(comm, "node"); + assert_eq!(ppid, 4300); + } + + #[test] + fn parse_stat_handles_comm_with_spaces_and_parens() { + let line = "7 (weird ) proc) S 3 7 3 0 -1 0 0 0 0 0 1 1 0 0 20 0 2 0"; + let (comm, ppid) = parse_stat_comm_ppid(line).unwrap(); + assert_eq!(comm, "weird ) proc"); + assert_eq!(ppid, 3); + } + + #[test] + fn parse_stat_rejects_short_or_malformed() { + assert!(parse_stat_comm_ppid("").is_none()); + assert!(parse_stat_comm_ppid("123 no-parens here").is_none()); + assert!(parse_stat_comm_ppid("1 (x) R").is_none()); + } + + #[test] + fn assemble_sums_self_plus_children() { + let self_sample = ProcSample { + rss_kib: 1000, + pss_kib: 0, + private_clean_kib: 0, + private_dirty_kib: 0, + vm_hwm_kib: 0, + threads: 1, + binary_size_bytes: 0, + cpu_user_ms: 0, + cpu_system_ms: 0, + open_fds: None, + }; + let children = vec![ + ChildSample { + pid: 2, + name: "node".into(), + rss_kib: 400, + }, + ChildSample { + pid: 3, + name: "python3".into(), + rss_kib: 250, + }, + ]; + let tree = TreeSample::assemble(self_sample, children); + assert_eq!(tree.tree_rss_kib, 1650); + assert_eq!(tree.child_count(), 2); + } + + #[cfg(target_os = "macos")] + #[test] + fn sample_tree_reports_self_on_macos() { + // A leaf test process has no children, but tree_rss must equal self RSS + // and the self sample must be populated. + let tree = sample_tree().expect("macOS tree sample"); + assert!(tree.self_sample.rss_kib > 0); + assert_eq!( + tree.tree_rss_kib, + tree.self_sample.rss_kib + tree.children.iter().map(|c| c.rss_kib).sum::() + ); + } +} diff --git a/vendor/tinycortex b/vendor/tinycortex index 9a0603afb..daaaf6ba5 160000 --- a/vendor/tinycortex +++ b/vendor/tinycortex @@ -1 +1 @@ -Subproject commit 9a0603afbebac608eac2ca0fa606caecd31ed1e7 +Subproject commit daaaf6ba5f02635c08deae2b2b2ed7fcc8c06b6a