perf(build): faster Rust test builds, opt-in fast linker, parallel frontend typecheck (#4721)

This commit is contained in:
Steven Enamakel
2026-07-10 14:34:01 -07:00
committed by GitHub
parent b709a99370
commit 04ffc029a2
102 changed files with 740 additions and 164 deletions
+35
View File
@@ -0,0 +1,35 @@
# Repo-level Cargo configuration.
#
# ────────────────────────────────────────────────────────────────────────────
# Faster linking (OPT-IN)
# ────────────────────────────────────────────────────────────────────────────
# The `openhuman` core crate is very large and links a single ~big rlib, so the
# edit → `cargo check`/`cargo test` inner loop is frequently *link-bound*. A
# faster linker (mold on Linux, lld on macOS) can cut a large slice off every
# incremental relink.
#
# These are intentionally left COMMENTED OUT: an unconditional `linker=` /
# `-fuse-ld=` here would hard-fail every build on a machine that does not have
# the linker installed — including the `pnpm rust:check` pre-push hook. Opt in
# locally by installing the linker and uncommenting the matching block (or, to
# avoid touching this tracked file, copy the block into `$CARGO_HOME/config.toml`
# or a parent-directory `.cargo/config.toml`, which Cargo merges automatically).
#
# ── Linux: mold (recommended) ───────────────────────────────────────────────
# apt install mold # or: brew install mold / dnf install mold
#
# [target.x86_64-unknown-linux-gnu]
# linker = "clang"
# rustflags = ["-C", "link-arg=-fuse-ld=mold"]
#
# ── macOS: lld ──────────────────────────────────────────────────────────────
# brew install llvm # provides ld64.lld
# Note: recent Xcode ships Apple's new `ld-prime`, which is already fast, so lld
# is often only a marginal win on Apple Silicon — measure before committing to
# it. `sold`/`zld` (older third-party mach-o linkers) are unmaintained/paid.
#
# [target.aarch64-apple-darwin]
# rustflags = ["-C", "link-arg=-fuse-ld=/opt/homebrew/opt/llvm/bin/ld64.lld"]
#
# [target.x86_64-apple-darwin]
# rustflags = ["-C", "link-arg=-fuse-ld=/usr/local/opt/llvm/bin/ld64.lld"]
+1 -1
View File
@@ -166,7 +166,7 @@ jobs:
uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6
with:
path: .ci/artifacts/e2e-playwright-linux.tar.gz
key: e2e-playwright-linux-${{ hashFiles('src/**', 'Cargo.toml', 'Cargo.lock', 'rust-toolchain.toml', 'app/src/**', 'app/public/**', 'app/index.html', 'app/vite.config.*', 'app/tailwind.config.*', 'app/postcss.config.*', 'app/package.json', 'pnpm-lock.yaml', 'app/scripts/e2e-web-build.sh') }}
key: e2e-playwright-linux-${{ hashFiles('src/**', 'Cargo.toml', 'Cargo.lock', 'rust-toolchain.toml', 'app/src/**', 'app/public/**', 'app/index.html', 'app/vite.config.*', 'app/tailwind.config.*', 'app/postcss.config.*', 'app/package.json', 'pnpm-lock.yaml', 'app/scripts/e2e-web-build.sh', 'app/scripts/build-parallel.mjs') }}
- name: Install dependencies
if: steps.playwright-artifact-cache.outputs.cache-hit != 'true'
+10
View File
@@ -76,6 +76,8 @@ jobs:
steps:
- name: Checkout code
uses: actions/checkout@v7
with:
persist-credentials: false
- name: Detect changed paths
id: filter
@@ -95,6 +97,7 @@ jobs:
- 'app/index.html'
- 'app/public/**'
- 'app/src/**'
- 'app/scripts/**'
- 'app/test/vitest.config.ts'
- 'app/tsconfig*.json'
- 'app/vite.config.*'
@@ -114,6 +117,7 @@ jobs:
- 'app/vite.config.*'
- 'app/tailwind.config.*'
- 'app/postcss.config.*'
- 'app/scripts/**'
- 'app/src/test/**'
frontend-src:
- 'app/src/**/*.ts'
@@ -135,6 +139,8 @@ jobs:
- 'scripts/ci/rust-coverage-changed.sh'
- 'Cargo.toml'
- 'Cargo.lock'
- 'build.rs'
- '.cargo/config.toml'
- 'rust-toolchain.toml'
- 'src/**'
- 'tests/**'
@@ -147,6 +153,8 @@ jobs:
- 'scripts/ci/rust-coverage-changed.sh'
- 'Cargo.toml'
- 'Cargo.lock'
- 'build.rs'
- '.cargo/config.toml'
- 'rust-toolchain.toml'
- 'scripts/ci-cancel-aware.sh'
rust-core-src:
@@ -155,6 +163,7 @@ jobs:
rust-tauri:
- '.github/workflows/ci-lite.yml'
- 'Cargo.lock'
- '.cargo/config.toml'
- 'rust-toolchain.toml'
- 'app/src-tauri/**'
- 'scripts/ci-cancel-aware.sh'
@@ -699,6 +708,7 @@ jobs:
uses: actions/checkout@v7
with:
fetch-depth: 0
persist-credentials: false
- name: Resolve coverage compare ref
if: needs.changes.outputs.coverage == 'true'
+1
View File
@@ -72,6 +72,7 @@ jobs:
--exclude '^https://www\.star-history\.com/#tinyhumansai/openhuman&type=date&legend=top-left$'
--exclude '^https://github\.com/tinyhumansai/openhuman/stargazers'
--exclude '^https://api\.star-history\.com/'
--exclude '^https://x\.com/karpathy/status/2039805659525644595$'
'docs/**/*.md'
'src/**/README.md'
'.github/PULL_REQUEST_TEMPLATE.md'
+40 -8
View File
@@ -2,9 +2,11 @@
# Reusable test workflow — frontend unit tests, Rust core tests, Rust Tauri
# shell tests. Used by PR/push (`test.yml`).
#
# Caching: pnpm store, Swatinem rust-cache, CEF runtime, and sccache backed
# by the GitHub Actions cache. Cache keys mirror what the release build
# matrix uses so warm caches survive across workflows on the same SHA.
# Caching: pnpm store, Swatinem rust-cache (full target/), and CEF runtime,
# backed by the GitHub Actions cache. Cache keys mirror what the release build
# matrix uses so warm caches survive across workflows on the same SHA. sccache
# is intentionally not layered on top — under a warm rust-cache it logged 0%
# additional hits (see ci-lite.yml) and only doubled the cache footprint.
name: Test (reusable)
on:
@@ -144,6 +146,9 @@ jobs:
fetch-depth: 1
persist-credentials: false
submodules: recursive
# Single dep-cache strategy repo-wide: Swatinem rust-cache (full target/).
# sccache was removed here to match the ci-lite lanes — layered on a warm
# rust-cache it logged 0% additional hits and just doubled cache footprint.
- name: Cache Rust build artifacts
uses: Swatinem/rust-cache@v2
with:
@@ -151,7 +156,36 @@ jobs:
cache-on-failure: true
key: core
- name: Test core crate (openhuman)
run: bash scripts/ci-cancel-aware.sh cargo test -p openhuman
run: |
set -euo pipefail
integration_test_targets() {
find tests -maxdepth 1 -type f -name '*.rs' -print |
sed -e 's#^tests/##' -e 's#\.rs$##' |
sort
}
raw_coverage_modules() {
find tests/raw_coverage -maxdepth 1 -type f -name '*.rs' -print |
sed -e 's#^tests/raw_coverage/##' -e 's#\.rs$##' |
sort
}
bash scripts/ci-cancel-aware.sh cargo test -p openhuman --lib --bins
bash scripts/ci-cancel-aware.sh cargo test -p openhuman --doc
while IFS= read -r target; do
[ -n "${target}" ] || continue
if [ "${target}" = "raw_coverage_all" ]; then
while IFS= read -r module; do
[ -n "${module}" ] || continue
echo "[test-reusable] raw coverage module: ${module}"
bash scripts/ci-cancel-aware.sh cargo test -p openhuman --test "${target}" -- "${module}::" --test-threads=1
done < <(raw_coverage_modules)
else
bash scripts/ci-cancel-aware.sh cargo test -p openhuman --test "${target}"
fi
done < <(integration_test_targets)
rust-core-tests-windows:
if: inputs.run_rust_core
@@ -168,6 +202,7 @@ jobs:
fetch-depth: 1
persist-credentials: false
submodules: recursive
# See rust-core-tests: rust-cache only, sccache dropped (0% hits warm).
- name: Cache Rust build artifacts
uses: Swatinem/rust-cache@v2
with:
@@ -195,8 +230,6 @@ jobs:
image: ghcr.io/tinyhumansai/openhuman_ci:rust-1.93.0
env:
CARGO_INCREMENTAL: "0"
RUSTC_WRAPPER: sccache
SCCACHE_GHA_ENABLED: "true"
steps:
- name: Checkout code
uses: actions/checkout@v7
@@ -206,6 +239,7 @@ jobs:
# Required for app/src-tauri/vendor/tauri-cef.
persist-credentials: false
submodules: recursive
# See rust-core-tests: rust-cache only, sccache dropped (0% hits warm).
- name: Cache Rust build artifacts
uses: Swatinem/rust-cache@v2
with:
@@ -221,7 +255,5 @@ jobs:
key: cef-ubuntu-22.04-${{ hashFiles('app/src-tauri/Cargo.toml') }}
restore-keys: |
cef-ubuntu-22.04-
- name: Install sccache
uses: mozilla-actions/sccache-action@v0.0.10
- name: Test Tauri shell (OpenHuman)
run: bash scripts/ci-cancel-aware.sh cargo test --manifest-path app/src-tauri/Cargo.toml
+4 -1
View File
@@ -71,7 +71,10 @@ app/src-tauri/runtime-skill-*
.mypy_cache
.ruff_cache
.kotlin
.cargo
# Keep .cargo/ untracked (local per-machine config + credentials) EXCEPT the
# shared, opt-in repo config that documents faster-linker setup.
.cargo/*
!.cargo/config.toml
CLAUDE.local.md
+5
View File
@@ -4,6 +4,11 @@ version = "0.58.14"
edition = "2021"
description = "OpenHuman core business logic and RPC server"
autobins = false
# build.rs globs tests/raw_coverage/*.rs into the single `raw_coverage_all`
# integration target (see tests/raw_coverage_all.rs). Those files used to be ~76
# individual `tests/*.rs` targets, each statically relinking the whole crate —
# collapsing them into one target removes ~75 full-crate link steps per test run.
build = "build.rs"
[[bin]]
name = "openhuman-core"
+1 -1
View File
@@ -44,7 +44,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
WORKDIR /build
# Cache dependencies — copy only manifests first
COPY Cargo.toml Cargo.lock rust-toolchain.toml ./
COPY Cargo.toml Cargo.lock rust-toolchain.toml build.rs ./
# Vendored TinyAgents SDK (git submodule; [patch.crates-io] points here, so
# the dep-cache build below already resolves it). CI must init the submodule
# before docker build — see the "Init tinyagents submodule" steps in
+4 -4
View File
@@ -21,11 +21,11 @@
"tauri:android:dev": "cd src-tauri-mobile && ../node_modules/.bin/tauri android dev",
"tauri:android:build": "cd src-tauri-mobile && ../node_modules/.bin/tauri android build",
"release:android:play": "bash ../scripts/release/upload-android-to-play.sh",
"build": "tsc && vite build",
"build:app": "tsc && vite build",
"build:app:e2e": "tsc && vite build --mode development",
"build": "node scripts/build-parallel.mjs",
"build:app": "node scripts/build-parallel.mjs",
"build:app:e2e": "node scripts/build-parallel.mjs --mode development",
"build:web:e2e": "bash ./scripts/e2e-web-build.sh",
"build:web": "cross-env VITE_OPENHUMAN_TARGET=web tsc && cross-env VITE_OPENHUMAN_TARGET=web vite build",
"build:web": "cross-env VITE_OPENHUMAN_TARGET=web node scripts/build-parallel.mjs",
"compile": "tsc --noEmit",
"preview": "vite preview",
"tauri": "tauri",
+66
View File
@@ -0,0 +1,66 @@
#!/usr/bin/env node
// Run the TypeScript type-check gate (`tsc`, noEmit) in parallel with the Vite
// bundle instead of sequentially (`tsc && vite build`).
//
// Vite/esbuild strips types itself and never consumes tsc's output, so tsc is a
// pure *gate*: running it alongside the bundle makes wall-clock max(tsc, vite)
// rather than tsc + vite. The build still fails on a type error because we exit
// non-zero if EITHER job fails — the exit code is the gate, not the presence of
// build artifacts (CI keys off the exit code, and a failed build's artifacts
// are discarded).
//
// Any extra args are forwarded to `vite build` (e.g. `--mode development`).
// Env vars (e.g. VITE_OPENHUMAN_TARGET set via cross-env) are inherited by both
// children. `shell: true` resolves the `tsc`/`vite` shims from node_modules/.bin
// on every platform, including the `.cmd` wrappers on Windows.
import { spawn } from 'node:child_process';
const viteArgs = process.argv.slice(2);
const jobs = [
{ name: 'tsc', cmd: 'tsc', args: [] },
{ name: 'vite', cmd: 'vite', args: ['build', ...viteArgs] },
];
const active = new Map();
let firstFailure = null;
function stopSiblings(failedName) {
for (const [name, child] of active) {
if (name !== failedName && !child.killed) {
child.kill();
}
}
}
function run({ name, cmd, args }) {
return new Promise(resolve => {
const child = spawn(cmd, args, { stdio: 'inherit', shell: true });
active.set(name, child);
child.on('exit', (code, signal) => {
active.delete(name);
const exitCode = code ?? (signal ? 1 : 0);
if (exitCode !== 0 && firstFailure === null) {
firstFailure = name;
stopSiblings(name);
}
resolve({ name, code: exitCode });
});
child.on('error', err => {
active.delete(name);
console.error(`[build-parallel] failed to start ${name}: ${err.message}`);
if (firstFailure === null) {
firstFailure = name;
stopSiblings(name);
}
resolve({ name, code: 1 });
});
});
}
const results = await Promise.all(jobs.map(run));
const failed = results.filter(r => r.code !== 0);
if (failed.length > 0) {
console.error(`[build-parallel] failed: ${failed.map(f => f.name).join(', ')}`);
process.exit(1);
}
+6
View File
@@ -14,6 +14,12 @@
"noEmit": true,
"jsx": "react-jsx",
/* Incremental type-checking: cache the type graph so warm `tsc` runs only
re-check files that changed. With noEmit there is no outDir to hold the
cache, so point tsBuildInfoFile at the gitignored node_modules cache. */
"incremental": true,
"tsBuildInfoFile": "./node_modules/.cache/tsc/app.tsbuildinfo",
/* Path aliases */
"baseUrl": ".",
"paths": {
+69
View File
@@ -0,0 +1,69 @@
//! Build script for the `openhuman` core crate.
//!
//! Its sole job today is to generate the module list for the aggregated
//! `raw_coverage_all` integration test target. The `tests/raw_coverage/`
//! directory holds ~76 auto-generated `*_raw_coverage_e2e.rs` coverage suites
//! that were previously ~76 separate `tests/*.rs` integration targets. Each
//! separate target statically relinks the entire (very large) `openhuman`
//! rlib, so building the test suite paid ~76 full-crate link steps. Folding
//! them into a single target ([`tests/raw_coverage_all.rs`]) reduces that to
//! one link.
//!
//! We glob the directory at build time (rather than hand-maintaining a `mod`
//! list) so that any newly added `tests/raw_coverage/*.rs` file is picked up
//! automatically and cannot be silently skipped.
use std::env;
use std::fs;
use std::path::Path;
fn main() {
let manifest_dir = env::var("CARGO_MANIFEST_DIR").expect("CARGO_MANIFEST_DIR must be set");
let tests_dir = Path::new(&manifest_dir).join("tests");
let raw_dir = tests_dir.join("raw_coverage");
// Re-run whenever a file is added to / removed from the directory.
println!("cargo:rerun-if-changed={}", raw_dir.display());
let mut entries: Vec<(String, String)> = Vec::new();
let read_dir = match fs::read_dir(&raw_dir) {
Ok(rd) => Some(rd),
// Source-only builds (e.g. the Docker image copies `src/` but not
// `tests/`) never compile the integration targets, so there is nothing
// to aggregate — emit an empty module list rather than breaking the
// build. But if `tests/` IS present and only `raw_coverage/` is missing,
// that's an accidental deletion: fail loudly so the suite can't be
// silently dropped.
Err(_) if !tests_dir.exists() => None,
Err(e) => panic!("failed to read {}: {e}", raw_dir.display()),
};
for entry in read_dir.into_iter().flatten().flatten() {
let path = entry.path();
if path.extension().and_then(|e| e.to_str()) != Some("rs") {
continue;
}
let stem = match path.file_stem().and_then(|s| s.to_str()) {
Some(s) => s.to_string(),
None => continue,
};
// Re-run if any individual test file's contents change.
println!("cargo:rerun-if-changed={}", path.display());
// Rust `#[path]` accepts forward slashes on every platform; the
// absolute path keeps resolution independent of where the generated
// file is `include!`d from.
let abs = path.display().to_string().replace('\\', "/");
entries.push((stem, abs));
}
// Deterministic order keeps the generated file stable across builds.
entries.sort();
let mut generated = String::from("// @generated by build.rs — do not edit.\n");
for (stem, abs) in &entries {
generated.push_str(&format!("#[path = \"{abs}\"]\nmod {stem};\n"));
}
let out_dir = env::var("OUT_DIR").expect("OUT_DIR must be set");
let out_path = Path::new(&out_dir).join("raw_coverage_mods.rs");
fs::write(&out_path, generated).expect("failed to write raw_coverage_mods.rs");
}
+51 -2
View File
@@ -35,9 +35,48 @@ llvm_cov() {
bash scripts/ci-cancel-aware.sh cargo llvm-cov "$@"
}
integration_test_targets() {
find tests -maxdepth 1 -type f -name '*.rs' -print |
sed -e 's#^tests/##' -e 's#\.rs$##' |
sort
}
raw_coverage_modules() {
find tests/raw_coverage -maxdepth 1 -type f -name '*.rs' -print |
sed -e 's#^tests/raw_coverage/##' -e 's#\.rs$##' |
sort
}
run_integration_target() {
local target="$1"
if [ "${target}" = "raw_coverage_all" ]; then
# These suites used to be separate integration-test binaries. Aggregating
# them removes repeated full-crate links, but many still exercise process
# globals (env vars, event bus handlers, auth tokens, singleton stores).
# Run one process per generated module filter to preserve the former
# per-binary isolation contract while still paying only one link.
while IFS= read -r module; do
[ -n "${module}" ] || continue
log "running raw coverage module: ${module}"
llvm_cov --no-report --no-fail-fast -p openhuman --test "${target}" -- "${module}::" --test-threads=1
done < <(raw_coverage_modules)
else
llvm_cov --no-report --no-fail-fast -p openhuman --test "${target}"
fi
}
run_full() {
log "running FULL instrumented suite (reason: $1)"
llvm_cov --no-fail-fast -p openhuman --lcov --output-path "${OUT}"
llvm_cov clean --workspace
llvm_cov --no-report --no-fail-fast -p openhuman --lib
llvm_cov --no-report --no-fail-fast -p openhuman --bins
while IFS= read -r target; do
[ -n "${target}" ] || continue
log "running full-suite integration target: ${target}"
run_integration_target "${target}"
done < <(integration_test_targets)
log "merging coverage into ${OUT}"
llvm_cov report --lcov --output-path "${OUT}"
exit 0
}
@@ -111,6 +150,16 @@ for f in "${files[@]}"; do
"
log "${f} → libtest filter '${key}' (embedded asset)"
;;
tests/raw_coverage/*.rs)
# The ~76 *_raw_coverage_e2e.rs suites are aggregated into the single
# `raw_coverage_all` target (see tests/raw_coverage_all.rs + build.rs), so
# a change to any of them scopes to that one target rather than the full
# suite. libtest filters within the aggregate binary still work, but the
# simplest correct scope is running the whole aggregate target.
test_targets_raw="${test_targets_raw}raw_coverage_all
"
log "${f} → aggregated integration target '--test raw_coverage_all'"
;;
tests/*.rs)
name="${f#tests/}"
name="${name%.rs}"
@@ -155,7 +204,7 @@ fi
if [ "${#test_targets[@]}" -gt 0 ]; then
for t in "${test_targets[@]}"; do
log "running changed integration-test target: ${t}"
llvm_cov --no-report --no-fail-fast -p openhuman --test "${t}"
run_integration_target "${t}"
done
fi
+56 -1
View File
@@ -66,4 +66,59 @@ if [ -f "$HOME/.cargo/env" ]; then
# shellcheck disable=SC1091
source "$HOME/.cargo/env"
fi
cargo test --manifest-path Cargo.toml --workspace "$@"
cargo_test() {
cargo test --manifest-path Cargo.toml --workspace "$@"
}
integration_test_targets() {
find tests -maxdepth 1 -type f -name '*.rs' -print |
sed -e 's#^tests/##' -e 's#\.rs$##' |
sort
}
raw_coverage_modules() {
find tests/raw_coverage -maxdepth 1 -type f -name '*.rs' -print |
sed -e 's#^tests/raw_coverage/##' -e 's#\.rs$##' |
sort
}
run_raw_coverage_modules() {
while IFS= read -r module; do
[ -n "$module" ] || continue
echo "[test-rust-with-mock] raw coverage module: ${module}"
cargo_test --test raw_coverage_all -- "${module}::" --test-threads=1 "$@"
done < <(raw_coverage_modules)
}
run_full_suite() {
cargo_test --lib --bins -- "$@"
cargo_test --doc -- "$@"
while IFS= read -r target; do
[ -n "$target" ] || continue
if [ "$target" = "raw_coverage_all" ]; then
# These suites used to run as separate integration-test binaries. Run
# each generated module filter in its own cargo process so local
# `pnpm test:rust` preserves the same process-global isolation as CI.
run_raw_coverage_modules "$@"
else
cargo_test --test "$target" -- "$@"
fi
done < <(integration_test_targets)
}
if [ "$#" -eq 0 ]; then
run_full_suite
elif [ "$1" = "--" ]; then
shift
run_full_suite "$@"
elif [ "$#" -ge 2 ] && [ "$1" = "--test" ] && [ "$2" = "raw_coverage_all" ]; then
shift 2
if [ "${1:-}" = "--" ]; then
shift
fi
run_raw_coverage_modules "$@"
else
cargo_test "$@"
fi
+3 -1
View File
@@ -49,7 +49,9 @@ pub use schemas::{
pub(crate) use ops::{event_session_id_for, key_for};
pub(crate) use progress_bridge::spawn_progress_bridge;
// Schema field helpers re-exported for tests
// Schema field helpers + session/error helpers re-exported for the `web_tests`
// integration module (they moved into submodules during the module split but
// the sibling test file still imports them via `super::`).
#[cfg(any(test, debug_assertions))]
#[allow(unused_imports)]
pub(crate) use schemas::{
@@ -4,9 +4,9 @@ use super::{
extract_provider_error_detail, generic_inference_error_user_message,
in_flight_entries_for_test, inference_budget_exceeded_user_message,
is_inference_budget_exceeded_error, json_output, key_for, locale_reply_directive,
normalize_model_override, optional_bool, optional_f64, optional_string, optional_u64,
parallel_in_flight_entries_for_test, provider_role_for_model_override, required_string,
schemas, set_test_forced_run_chat_task_error, set_test_run_chat_task_block, start_chat,
normalize_model_override, optional_f64, optional_string, parallel_in_flight_entries_for_test,
provider_role_for_model_override, required_string, schemas,
set_test_forced_run_chat_task_error, set_test_run_chat_task_block, start_chat,
subscribe_web_channel_events, ChatRequestMetadata, ClassifiedError, TestRunChatTaskBlock,
WebChatParams,
};
+1
View File
@@ -1,3 +1,4 @@
use super::schema_defs::schemas;
use super::*;
#[test]
+24 -13
View File
@@ -433,11 +433,11 @@ fn is_api_key_unset_failure(
}
/// TAURI-RUST-12K — a cron **agent** job pinned to a **local** LLM provider
/// (LM Studio / Ollama / llama.cpp on `localhost:<port>`) fails at the TCP
/// layer with "connection refused" because the user's local server isn't
/// running. This is a genuinely unpreventable user-environment state: the app
/// has no lever to start a user's local model server, and retrying across the
/// backoff loop cannot bring the port up within one cron cycle.
/// (LM Studio / Ollama / llama.cpp on `localhost:<port>`) fails because the
/// user's local runtime is unavailable or reachable-but-idle with no model
/// loaded. This is a genuinely unpreventable user-environment state: the app
/// has no lever to start a user's local model server or load a model there, and
/// retrying across the backoff loop cannot fix it within one cron cycle.
///
/// The provider / agent emit sites already demote this via
/// `report_error_or_expected` (the `expected_error_kind` classifier routes it
@@ -448,13 +448,15 @@ fn is_api_key_unset_failure(
/// the source demotion and the sibling billing / api-key guards
/// (TAURI-RUST-514 / -BMW / -HCK).
///
/// Delegates to the single-source matcher
/// Delegates loopback-unreachable detection to the single-source matcher
/// [`crate::core::observability::is_local_provider_unreachable_message`] so
/// the wording cannot drift from the classifier emit site. Narrow by design:
/// only **loopback** connection-refused matches, so a transient *remote*
/// provider / backend network error still retries and still reports. Routes on
/// `last_agent_error` first (the raw anyhow chain carrying the wire message),
/// falling back to `last_output`. Restricted to `JobType::Agent`.
/// the wording cannot drift from the classifier emit site. Also recognizes the
/// inference provider's stable local-runtime "no model loaded" user message.
/// Narrow by design: a transient *remote* provider / backend network error
/// still retries and still reports. Checks both `last_agent_error` (the raw
/// anyhow chain carrying the wire message) and `last_output` (the surfaced user
/// message), because some provider paths preserve only one of those shapes.
/// Restricted to `JobType::Agent`.
fn is_local_provider_unreachable_failure(
job_type: &JobType,
last_agent_error: Option<&str>,
@@ -463,8 +465,17 @@ fn is_local_provider_unreachable_failure(
if !matches!(job_type, JobType::Agent) {
return false;
}
let signal = last_agent_error.unwrap_or(last_output);
crate::core::observability::is_local_provider_unreachable_message(signal)
let raw_signal = last_agent_error.unwrap_or("");
crate::core::observability::is_local_provider_unreachable_message(raw_signal)
|| crate::core::observability::is_local_provider_unreachable_message(last_output)
|| is_local_provider_no_model_loaded_message(raw_signal)
|| is_local_provider_no_model_loaded_message(last_output)
}
fn is_local_provider_no_model_loaded_message(signal: &str) -> bool {
let lower = signal.to_ascii_lowercase();
(lower.contains("local inference server") && lower.contains("no model loaded"))
|| lower.contains("no models loaded")
}
async fn execute_job_with_retry(
+53 -34
View File
@@ -742,6 +742,48 @@ fn is_local_provider_unreachable_failure_matches_when_only_output_carries_signal
));
}
#[test]
fn is_local_provider_unreachable_failure_keeps_short_loopback_send_error_retryable() {
let wire = "error sending request for url (http://localhost:1234/v1/chat/completions)";
assert!(
!is_local_provider_unreachable_failure(
&JobType::Agent,
Some(wire),
AGENT_JOB_USER_FAILURE_MESSAGE
),
"short reqwest send errors can represent transient timeout/reset shapes and must stay retryable without a refused errno/tcp-connect signal"
);
}
#[test]
fn is_local_provider_unreachable_failure_matches_raw_no_models_loaded_body() {
let raw = "LM Studio API error (400 Bad Request): {\"error\":\"No models loaded. \
Please load a model in the developer page first.\"}";
assert!(
is_local_provider_unreachable_failure(
&JobType::Agent,
Some(raw),
AGENT_JOB_USER_FAILURE_MESSAGE
),
"raw OpenAI-compatible no-model body should halt without retries"
);
}
#[test]
fn is_local_provider_unreachable_failure_checks_output_when_raw_is_generic() {
let output =
"Your local inference server (e.g. LM Studio) is running but has no model loaded. \
Load a model, then try again.";
assert!(
is_local_provider_unreachable_failure(
&JobType::Agent,
Some(AGENT_JOB_USER_FAILURE_MESSAGE),
output
),
"friendly no-model output should halt even when raw agent error is generic"
);
}
// Negative guard: a transient REMOTE provider / backend network error must NOT
// halt — it may recover on retry and stays actionable in Sentry. Narrowing to
// loopback is what keeps this guard from blinding real outages.
@@ -1814,21 +1856,19 @@ fn publish_cron_user_error_broadcasts_metadata_only_for_each_kind() {
}
// TAURI-RUST-12K (end-to-end) — the predicate tests above key on hand-written
// wire strings; this test proves the REAL provider-generated error reaches and
// trips the guard. A cron agent job is routed to a keyless local provider
// (`AuthStyle::None`, LM Studio shape) whose server is offline: the chat
// workload skips the credential guard, attempts the loopback HTTP connect, and
// the OS refuses it. The resulting `last_agent_error` (the aggregated provider
// fallback chain carrying `…localhost:1234… tcp connect error … Connection
// refused (os error N)`) must classify as local-provider-unreachable so the
// cron loop halts and skips the bypassing `failure=retries_exhausted` report.
// wire strings; this test proves the REAL provider-generated error remains
// retryable when it only preserves reqwest's short send-error prefix. A cron
// agent job is routed to a keyless local provider (`AuthStyle::None`, LM Studio
// shape) whose server is offline: the chat workload skips the credential guard
// and attempts loopback HTTP. If the provider layer surfaces only
// `error sending request for url (...)`, without the refused errno/tcp-connect
// chain, cron must not treat it as a permanent local-provider halt because the
// same short prefix is also used for transient timeout/reset shapes.
#[tokio::test]
async fn cron_agent_job_local_provider_offline_trips_halt_guard() {
async fn cron_agent_job_short_loopback_send_error_stays_retryable() {
use crate::openhuman::config::schema::cloud_providers::{AuthStyle, CloudProviderCreds};
let tmp = TempDir::new().unwrap();
let mut config = test_config(&tmp).await;
config.reliability.scheduler_retries = 5;
config.reliability.provider_backoff_ms = 1;
// Keyless local provider (`AuthStyle::None` → no credential requirement, so
// the request proceeds to the HTTP connect). `chat_provider` routes the
// chat workload to it; the slug resolves to LM Studio's default endpoint.
@@ -1842,38 +1882,17 @@ async fn cron_agent_job_local_provider_offline_trips_halt_guard() {
}];
config.default_model = Some("lmstudio:local-model".into());
config.chat_provider = Some("lmstudio:local-model".into());
let security = SecurityPolicy::from_config(
&config.autonomy,
&config.workspace_dir,
&config.workspace_dir,
);
let mut job = test_job("");
job.job_type = JobType::Agent;
job.prompt = Some("Say hello".into());
// Primary (deterministic): the real provider-generated failure trips the
// guard — the link the string-based predicate tests cannot prove.
let (success, output, raw) = run_agent_job(&config, &job).await;
assert!(
!success,
"a cron agent job against an offline local provider must fail"
);
assert!(
is_local_provider_unreachable_failure(&JobType::Agent, raw.as_deref(), &output),
"provider-generated loopback connect-refused must trip the halt guard; got raw={raw:?}"
);
// Secondary (coarse): the full retry loop halts on the first occurrence
// rather than exhausting all 5 retries. Not halting would add the cron
// backoff sleeps (200ms floor, doubling → >6s total) on top of five extra
// agent runs; halting skips every cron-level sleep. A generous bound keeps
// this robust to agent-build jitter while still catching a regressed halt.
let start = std::time::Instant::now();
let (loop_success, _out) = execute_job_with_retry(&config, &security, &job).await;
let elapsed = start.elapsed();
assert!(!loop_success);
assert!(
elapsed < std::time::Duration::from_secs(5),
"loop must halt on the first loopback failure, not burn 5 retries with backoff (elapsed {elapsed:?})"
!is_local_provider_unreachable_failure(&JobType::Agent, raw.as_deref(), &output),
"provider-generated short loopback send error must stay retryable without refused errno/tcp-connect evidence; got raw={raw:?}"
);
}
+10 -7
View File
@@ -1447,17 +1447,20 @@ mod tests {
// from several threads and assert every seq is distinct and contiguous.
use std::sync::Arc;
let tmp = Arc::new(tempfile::tempdir().unwrap());
with_connection(tmp.path(), |_| Ok(())).expect("initialise orchestration store");
let db_path = Arc::new(tmp.path().join("orchestration").join("orchestration.db"));
let n = 8usize;
let handles: Vec<_> = (0..n)
.map(|i| {
let tmp = Arc::clone(&tmp);
let db_path = Arc::clone(&db_path);
std::thread::spawn(move || {
with_connection(tmp.path(), |c| {
in_immediate_txn(c, |c| {
let seq = next_session_seq(c, "@peer", "s1")?;
insert_message(c, &msg(&format!("m{i}"), "@peer", "s1", seq))?;
Ok(seq)
})
let c = Connection::open(&*db_path).expect("open orchestration db");
c.busy_timeout(std::time::Duration::from_secs(5))
.expect("set busy timeout");
in_immediate_txn(&c, |c| {
let seq = next_session_seq(c, "@peer", "s1")?;
insert_message(c, &msg(&format!("m{i}"), "@peer", "s1", seq))?;
Ok(seq)
})
.expect("txn ok")
})
+3 -3
View File
@@ -44,7 +44,7 @@ pub fn schemas(function: &str) -> ControllerSchema {
inputs: vec![optional_field(
"kind",
TypeSchema::String,
"Which world to tick: \"memory\" (default), \"tinyplace\", or \"all\".",
"Which world to tick: \"memory\" (default) or \"all\".",
)],
outputs: vec![field("result", TypeSchema::Json, "Tick result.")],
},
@@ -127,7 +127,7 @@ fn handle_status(_params: Map<String, Value>) -> ControllerFuture {
fn handle_trigger(params: Map<String, Value>) -> ControllerFuture {
Box::pin(async move {
// `kind`: "memory" (default), "tinyplace", or "all".
// `kind`: "memory" (default) or "all".
let raw = params
.get("kind")
.and_then(|v| v.as_str())
@@ -141,7 +141,7 @@ fn handle_trigger(params: Map<String, Value>) -> ControllerFuture {
Some(k) => vec![k],
None => {
return Err(format!(
"unknown subconscious kind '{raw}' (expected memory|tinyplace|all)"
"unknown subconscious kind '{raw}' (expected memory|all)"
))
}
}
@@ -13,7 +13,6 @@ use tokio::sync::broadcast;
use tokio::task::JoinHandle;
use crate::openhuman::config::Config;
// The rdev-based listener (non-macOS) resolves the hotkey combo + activation mode
// against the cross-platform `hotkey` module. macOS uses a separate path and
// never compiles `start_rdev_listener`, so gate the import to avoid an unused
@@ -70,7 +70,16 @@ use openhuman_core::openhuman::credentials::{
const TEST_RPC_TOKEN: &str = "worker-a-domain-e2e-token";
static AUTH_INIT: OnceLock<()> = OnceLock::new();
static ENV_LOCK: OnceLock<Mutex<()>> = OnceLock::new();
// This file is both its own integration target AND `#[path]`-included as
// `base_coverage` by `raw_coverage/config_credentials_raw_coverage_e2e.rs`.
// `ENV_LOCK` aliases `crate::SHARED_ENV_LOCK`, which resolves to this file's
// own static when built standalone (separate process, isolated env) and to the
// aggregate's shared static when nested into `raw_coverage_all` (so its env
// mutations serialize against every other aggregated suite). The nested copy of
// this static is simply unused.
#[allow(dead_code)]
pub static SHARED_ENV_LOCK: OnceLock<Mutex<()>> = OnceLock::new();
static ENV_LOCK: &OnceLock<Mutex<()>> = &crate::SHARED_ENV_LOCK;
struct EnvVarGuard {
key: &'static str,
+3 -3
View File
@@ -5831,7 +5831,7 @@ async fn json_rpc_subconscious_status_exposes_instances_and_trigger_takes_kind()
&rpc_base,
1102,
"openhuman.subconscious_trigger",
json!({ "kind": "tinyplace" }),
json!({ "kind": "memory" }),
)
.await;
let trig_result = assert_no_jsonrpc_error(&trig, "subconscious_trigger");
@@ -5843,7 +5843,7 @@ async fn json_rpc_subconscious_status_exposes_instances_and_trigger_takes_kind()
);
assert_eq!(
trig_body.get("kind").and_then(Value::as_str),
Some("tinyplace"),
Some("memory"),
"trigger echoes the requested kind: {trig_body}"
);
@@ -5852,7 +5852,7 @@ async fn json_rpc_subconscious_status_exposes_instances_and_trigger_takes_kind()
&rpc_base,
1103,
"openhuman.subconscious_trigger",
json!({ "kind": "nope" }),
json!({ "kind": "tinyplace" }),
)
.await;
assert!(
+19 -8
View File
@@ -72,19 +72,30 @@ fn parses_a_tool_call_frame() {
assert_eq!(parsed.name, "device_status");
}
#[test]
fn dispatches_device_status_and_rejects_unknown_tools() {
let status = dispatch_device_tool("device_status", &json!({})).expect("ok");
#[tokio::test]
async fn dispatches_device_status_and_rejects_unknown_tools() {
let status = dispatch_device_tool("device_status", &json!({}), "master:session:cycle")
.await
.expect("ok");
assert!(status["version"].is_string());
assert!(status["platform"].is_string());
assert!(dispatch_device_tool("rm_rf", &json!({})).is_err());
assert!(
dispatch_device_tool("rm_rf", &json!({}), "master:session:cycle")
.await
.is_err()
);
}
#[test]
fn handle_tool_call_builds_result_frame() {
let frame = json!({ "callId": "c:tool_call:0", "name": "device_status", "args": {} });
let (call_id, result) = handle_tool_call(&frame).expect("handled");
#[tokio::test]
async fn handle_tool_call_builds_result_frame() {
let frame = json!({
"cycleId": "master:session:cycle",
"callId": "c:tool_call:0",
"name": "device_status",
"args": {}
});
let (call_id, result) = handle_tool_call(&frame).await.expect("handled");
assert_eq!(call_id, "c:tool_call:0");
assert_eq!(result["ok"], json!(true));
assert!(result["result"]["platform"].is_string());
@@ -55,7 +55,7 @@ impl Drop for EnvGuard {
/// (read by `apply_env_overrides` during config load), so parallel test threads
/// can't observe each other's workspace override mid-run.
fn env_lock() -> std::sync::MutexGuard<'static, ()> {
static LOCK: std::sync::OnceLock<std::sync::Mutex<()>> = std::sync::OnceLock::new();
static LOCK: &std::sync::OnceLock<std::sync::Mutex<()>> = &crate::SHARED_ENV_LOCK;
LOCK.get_or_init(|| std::sync::Mutex::new(()))
.lock()
.unwrap_or_else(|e| e.into_inner())
@@ -35,6 +35,7 @@ struct ScriptedProvider {
requests: Mutex<Vec<String>>,
native_tools: bool,
delay: Option<Duration>,
always_fail: Option<String>,
}
impl ScriptedProvider {
@@ -44,15 +45,17 @@ impl ScriptedProvider {
requests: Mutex::new(Vec::new()),
native_tools: true,
delay: None,
always_fail: None,
})
}
fn failing(message: &str) -> Arc<Self> {
Arc::new(Self {
responses: Mutex::new(VecDeque::from([Err(anyhow::anyhow!(message.to_string()))])),
responses: Mutex::new(VecDeque::new()),
requests: Mutex::new(Vec::new()),
native_tools: true,
delay: None,
always_fail: Some(message.to_string()),
})
}
@@ -62,6 +65,7 @@ impl ScriptedProvider {
requests: Mutex::new(Vec::new()),
native_tools: true,
delay: Some(delay),
always_fail: None,
})
}
@@ -86,6 +90,9 @@ impl Provider for ScriptedProvider {
_model: &str,
_temperature: f64,
) -> Result<String> {
if let Some(message) = &self.always_fail {
anyhow::bail!(message.clone());
}
Ok(format!("summary: {message}"))
}
@@ -106,6 +113,9 @@ impl Provider for ScriptedProvider {
if let Some(delay) = self.delay {
tokio::time::sleep(delay).await;
}
if let Some(message) = &self.always_fail {
anyhow::bail!(message.clone());
}
self.responses
.lock()
.pop_front()
@@ -51,7 +51,7 @@ impl Drop for EnvGuard {
}
fn env_lock() -> std::sync::MutexGuard<'static, ()> {
static LOCK: std::sync::OnceLock<std::sync::Mutex<()>> = std::sync::OnceLock::new();
static LOCK: &std::sync::OnceLock<std::sync::Mutex<()>> = &crate::SHARED_ENV_LOCK;
LOCK.get_or_init(|| std::sync::Mutex::new(()))
.lock()
.unwrap_or_else(|e| e.into_inner())
@@ -53,7 +53,7 @@ impl Drop for EnvGuard {
}
fn env_lock() -> std::sync::MutexGuard<'static, ()> {
static LOCK: std::sync::OnceLock<std::sync::Mutex<()>> = std::sync::OnceLock::new();
static LOCK: &std::sync::OnceLock<std::sync::Mutex<()>> = &crate::SHARED_ENV_LOCK;
LOCK.get_or_init(|| std::sync::Mutex::new(()))
.lock()
.unwrap_or_else(|e| e.into_inner())
@@ -59,7 +59,7 @@ impl Drop for EnvGuard {
}
fn env_lock() -> std::sync::MutexGuard<'static, ()> {
static LOCK: std::sync::OnceLock<std::sync::Mutex<()>> = std::sync::OnceLock::new();
static LOCK: &std::sync::OnceLock<std::sync::Mutex<()>> = &crate::SHARED_ENV_LOCK;
LOCK.get_or_init(|| std::sync::Mutex::new(()))
.lock()
.unwrap_or_else(|e| e.into_inner())
@@ -34,6 +34,7 @@ struct ScriptedProvider {
native_tools: bool,
vision: bool,
stream_events: Vec<ProviderDelta>,
always_fail: Option<String>,
}
impl ScriptedProvider {
@@ -45,10 +46,8 @@ impl ScriptedProvider {
}
fn failing(message: &str) -> Arc<Self> {
let mut responses = VecDeque::new();
responses.push_back(Err(anyhow::anyhow!(message.to_string())));
Arc::new(Self {
responses: Mutex::new(responses),
always_fail: Some(message.to_string()),
..Self::default()
})
}
@@ -74,6 +73,9 @@ impl Provider for ScriptedProvider {
_model: &str,
_temperature: f64,
) -> anyhow::Result<String> {
if let Some(message) = &self.always_fail {
anyhow::bail!(message.clone());
}
Ok(message.to_string())
}
@@ -95,6 +97,9 @@ impl Provider for ScriptedProvider {
stream.send(event.clone()).await.ok();
}
}
if let Some(message) = &self.always_fail {
anyhow::bail!(message.clone());
}
self.responses
.lock()
.unwrap()
@@ -405,6 +410,7 @@ async fn bus_turn_native_tools_dedups_streams_and_records_tool_messages() {
turns: Mutex::new(Vec::new()),
native_tools: true,
vision: false,
always_fail: None,
stream_events: vec![
ProviderDelta::TextDelta {
delta: "draft ".to_string(),
@@ -10,7 +10,7 @@ use serde_json::json;
use std::path::Path;
use std::sync::Mutex;
static ENV_LOCK: Mutex<()> = Mutex::new(());
static ENV_LOCK: &std::sync::OnceLock<std::sync::Mutex<()>> = &crate::SHARED_ENV_LOCK;
struct WorkspaceEnvGuard {
previous: Option<String>,
@@ -93,7 +93,7 @@ fn triage_run(action: TriageAction) -> TriageRun {
#[tokio::test]
async fn drop_and_acknowledge_gate_pending_linked_cards_without_dispatch() {
let _env_lock = ENV_LOCK.lock().expect("env lock should not be poisoned");
let _env_lock = ENV_LOCK.get_or_init(|| Mutex::new(())).lock().unwrap_or_else(|poisoned| poisoned.into_inner());
let workspace = tempfile::tempdir().expect("temp workspace");
let _env = WorkspaceEnvGuard::set(workspace.path());
let location = board_location(workspace.path());
@@ -122,7 +122,7 @@ async fn drop_and_acknowledge_gate_pending_linked_cards_without_dispatch() {
#[tokio::test]
async fn react_on_linked_todo_card_parks_for_plan_approval() {
let _env_lock = ENV_LOCK.lock().expect("env lock should not be poisoned");
let _env_lock = ENV_LOCK.get_or_init(|| Mutex::new(())).lock().unwrap_or_else(|poisoned| poisoned.into_inner());
let workspace = tempfile::tempdir().expect("temp workspace");
let _env = WorkspaceEnvGuard::set(workspace.path());
let _ = init_global(32);
@@ -142,7 +142,7 @@ async fn react_on_linked_todo_card_parks_for_plan_approval() {
#[tokio::test]
async fn dispatcher_rejects_missing_and_stale_non_claimable_cards() {
let _env_lock = ENV_LOCK.lock().expect("env lock should not be poisoned");
let _env_lock = ENV_LOCK.get_or_init(|| Mutex::new(())).lock().unwrap_or_else(|poisoned| poisoned.into_inner());
let workspace = tempfile::tempdir().expect("temp workspace");
let _env = WorkspaceEnvGuard::set(workspace.path());
let location = board_location(workspace.path());
@@ -182,7 +182,7 @@ async fn dispatcher_rejects_missing_and_stale_non_claimable_cards() {
#[tokio::test]
async fn dispatch_card_returns_awaiting_approval_before_agent_spawn() {
let _env_lock = ENV_LOCK.lock().expect("env lock should not be poisoned");
let _env_lock = ENV_LOCK.get_or_init(|| Mutex::new(())).lock().unwrap_or_else(|poisoned| poisoned.into_inner());
let workspace = tempfile::tempdir().expect("temp workspace");
let _env = WorkspaceEnvGuard::set(workspace.path());
let _ = init_global(32);
@@ -44,7 +44,7 @@ impl Drop for EnvGuard {
}
fn env_lock() -> std::sync::MutexGuard<'static, ()> {
static LOCK: std::sync::OnceLock<std::sync::Mutex<()>> = std::sync::OnceLock::new();
static LOCK: &std::sync::OnceLock<std::sync::Mutex<()>> = &crate::SHARED_ENV_LOCK;
LOCK.get_or_init(|| std::sync::Mutex::new(()))
.lock()
.unwrap_or_else(|e| e.into_inner())
@@ -29,7 +29,7 @@ use serde_json::{json, Value};
use tempfile::{Builder, TempDir};
use tokio::io::{AsyncReadExt, AsyncWriteExt};
static ROUND19_ENV_LOCK: OnceLock<Mutex<()>> = OnceLock::new();
static ROUND19_ENV_LOCK: &OnceLock<Mutex<()>> = &crate::SHARED_ENV_LOCK;
struct EnvGuard {
key: &'static str,
@@ -27,7 +27,7 @@ use openhuman_core::openhuman::threads::ops::{
thread_update_labels, thread_update_title, threads_list, threads_purge,
};
static ROUND24_ENV_LOCK: OnceLock<Mutex<()>> = OnceLock::new();
static ROUND24_ENV_LOCK: &OnceLock<Mutex<()>> = &crate::SHARED_ENV_LOCK;
struct EnvGuard {
key: &'static str,
@@ -25,7 +25,7 @@ use openhuman_core::openhuman::threads::ops::{
};
use openhuman_core::openhuman::threads::welcome_migration::migrate_welcome_agent_artifacts;
static ENV_LOCK: OnceLock<Mutex<()>> = OnceLock::new();
static ENV_LOCK: &OnceLock<Mutex<()>> = &crate::SHARED_ENV_LOCK;
struct EnvGuard {
key: &'static str,
@@ -22,7 +22,7 @@ use serde_json::{json, Value};
use tempfile::{Builder, TempDir};
use tokio::io::{AsyncReadExt, AsyncWriteExt};
static ROUND14_ENV_LOCK: OnceLock<Mutex<()>> = OnceLock::new();
static ROUND14_ENV_LOCK: &OnceLock<Mutex<()>> = &crate::SHARED_ENV_LOCK;
struct EnvGuard {
key: &'static str,
@@ -131,8 +131,20 @@ impl Drop for EnvGuard {
}
}
// Serialize env mutation against every other aggregated suite via the
// single crate-wide SHARED_ENV_LOCK (these tests use an `EnvGuard` struct
// that does not itself hold a lock). Poison is recovered so a panic
// elsewhere cannot wedge the suite.
fn __shared_env_lock() -> std::sync::MutexGuard<'static, ()> {
crate::SHARED_ENV_LOCK
.get_or_init(|| std::sync::Mutex::new(()))
.lock()
.unwrap_or_else(|poisoned| poisoned.into_inner())
}
#[tokio::test]
async fn dispatch_harness_covers_streaming_reactions_memory_and_success_events() {
let _env_lock = __shared_env_lock();
let observed = run_dispatch_harness(DispatchHarnessOptions {
channel_name: "telegram".to_string(),
content: "thanks for checking the rust build?".to_string(),
@@ -186,6 +198,7 @@ async fn dispatch_harness_covers_streaming_reactions_memory_and_success_events()
#[tokio::test]
async fn dispatch_harness_covers_error_context_compaction_and_timeout_paths() {
let _env_lock = __shared_env_lock();
let context_overflow = run_dispatch_harness(DispatchHarnessOptions {
channel_name: "discord".to_string(),
content: "please continue".to_string(),
@@ -229,6 +242,7 @@ async fn dispatch_harness_covers_error_context_compaction_and_timeout_paths() {
#[tokio::test]
async fn web_channel_validation_cancel_and_classifier_snapshots_are_publicly_exercised() {
let _env_lock = __shared_env_lock();
assert!(start_chat(
"",
"thread",
@@ -345,6 +359,7 @@ async fn web_channel_validation_cancel_and_classifier_snapshots_are_publicly_exe
#[tokio::test]
async fn telegram_loopback_covers_reactions_markdown_fallback_drafts_typing_and_health() {
let _env_lock = __shared_env_lock();
let (base, state, server) = spawn_telegram_mock().await;
let _api_base = EnvGuard::set("OPENHUMAN_TELEGRAM_BOT_API_BASE", &base);
let _legacy_base = EnvGuard::set("OPENHUMAN_TELEGRAM_API_BASE", "");
@@ -410,6 +425,7 @@ async fn telegram_loopback_covers_reactions_markdown_fallback_drafts_typing_and_
#[test]
fn lark_and_yuanbao_public_paths_cover_parsing_config_and_no_network_fallbacks() {
let _env_lock = __shared_env_lock();
let mut cfg = LarkConfig {
app_id: "app".into(),
app_secret: "secret".into(),
@@ -511,6 +527,7 @@ fn lark_and_yuanbao_public_paths_cover_parsing_config_and_no_network_fallbacks()
#[test]
fn round16_artifact_scope_uses_requested_target_prefix() {
let _env_lock = __shared_env_lock();
let tmp = TempDir::with_prefix("channels-provider-deep-round16-").expect("round16 tempdir");
assert!(tmp
.path()
@@ -209,8 +209,20 @@ impl Drop for EnvGuard {
}
}
// Serialize env mutation against every other aggregated suite via the
// single crate-wide SHARED_ENV_LOCK (these tests use an `EnvGuard` struct
// that does not itself hold a lock). Poison is recovered so a panic
// elsewhere cannot wedge the suite.
fn __shared_env_lock() -> std::sync::MutexGuard<'static, ()> {
crate::SHARED_ENV_LOCK
.get_or_init(|| std::sync::Mutex::new(()))
.lock()
.unwrap_or_else(|poisoned| poisoned.into_inner())
}
#[tokio::test]
async fn telegram_round19_covers_mention_filter_typing_fallback_and_attachment_forms() {
let _env_lock = __shared_env_lock();
let (base, state, server) = spawn_telegram_mock().await;
let _api_base = EnvGuard::set("OPENHUMAN_TELEGRAM_BOT_API_BASE", &base);
let _legacy_base = EnvGuard::set("OPENHUMAN_TELEGRAM_API_BASE", "");
@@ -312,6 +324,7 @@ async fn telegram_round19_covers_mention_filter_typing_fallback_and_attachment_f
#[tokio::test]
async fn web_round19_covers_classifier_variants_and_cancel_cleanup() {
let _env_lock = __shared_env_lock();
let auth = web_test_support::classify_error_for_test(
"custom_openai API error (401 Unauthorized): invalid api key",
);
@@ -377,6 +390,7 @@ async fn web_round19_covers_classifier_variants_and_cancel_cleanup() {
#[test]
fn lark_round19_covers_parse_leftovers_and_config_defaults() {
let _env_lock = __shared_env_lock();
let mut cfg = LarkConfig {
app_id: "round19-app".into(),
app_secret: "round19-secret".into(),
@@ -445,6 +459,7 @@ fn lark_round19_covers_parse_leftovers_and_config_defaults() {
#[tokio::test]
async fn lark_round19_listen_http_missing_port_and_ephemeral_bind_paths() {
let _env_lock = __shared_env_lock();
let missing_port = LarkChannel::from_config(&LarkConfig {
app_id: "round19-app".into(),
app_secret: "round19-secret".into(),
@@ -481,6 +496,7 @@ async fn lark_round19_listen_http_missing_port_and_ephemeral_bind_paths() {
#[tokio::test]
async fn yuanbao_round19_connection_run_shutdown_and_channel_error_paths() {
let _env_lock = __shared_env_lock();
let cfg = YuanbaoConfig {
app_key: "round19-ak".into(),
token: "round19-token".into(),
@@ -555,6 +571,7 @@ async fn yuanbao_round19_connection_run_shutdown_and_channel_error_paths() {
#[tokio::test]
async fn round19_artifact_target_prefix_is_used() {
let _env_lock = __shared_env_lock();
let dir = tempfile::Builder::new()
.prefix("channels-provider-leftovers-round19-")
.tempdir_in("target")
@@ -181,7 +181,7 @@ impl EnvGuard {
}
fn env_lock() -> std::sync::MutexGuard<'static, ()> {
static LOCK: std::sync::OnceLock<std::sync::Mutex<()>> = std::sync::OnceLock::new();
static LOCK: &std::sync::OnceLock<std::sync::Mutex<()>> = &crate::SHARED_ENV_LOCK;
LOCK.get_or_init(|| std::sync::Mutex::new(()))
.lock()
.unwrap_or_else(|e| e.into_inner())
@@ -134,8 +134,20 @@ impl Drop for EnvGuard {
}
}
// Serialize env mutation against every other aggregated suite via the
// single crate-wide SHARED_ENV_LOCK (these tests use an `EnvGuard` struct
// that does not itself hold a lock). Poison is recovered so a panic
// elsewhere cannot wedge the suite.
fn __shared_env_lock() -> std::sync::MutexGuard<'static, ()> {
crate::SHARED_ENV_LOCK
.get_or_init(|| std::sync::Mutex::new(()))
.lock()
.unwrap_or_else(|poisoned| poisoned.into_inner())
}
#[tokio::test]
async fn telegram_outbound_uses_mock_api_for_reactions_markdown_fallback_drafts_and_typing() {
let _env_lock = __shared_env_lock();
let (base, state, server) = spawn_telegram_mock().await;
let _api_base = EnvGuard::set("OPENHUMAN_TELEGRAM_BOT_API_BASE", &base);
let _legacy_base = EnvGuard::set("OPENHUMAN_TELEGRAM_API_BASE", "");
@@ -222,6 +234,7 @@ async fn telegram_outbound_uses_mock_api_for_reactions_markdown_fallback_drafts_
#[test]
fn lark_parse_event_payload_covers_text_post_filters_and_config_defaults() {
let _env_lock = __shared_env_lock();
let mut cfg = LarkConfig {
app_id: "app".into(),
app_secret: "secret".into(),
@@ -315,6 +328,7 @@ fn lark_parse_event_payload_covers_text_post_filters_and_config_defaults() {
#[tokio::test]
async fn yuanbao_public_channel_and_config_paths_are_isolated_from_network() {
let _env_lock = __shared_env_lock();
let mut prod = YuanbaoConfig::default();
prod.apply_env_defaults();
assert!(prod.api_domain.contains("bot.yuanbao.tencent.com"));
@@ -372,6 +386,7 @@ async fn yuanbao_public_channel_and_config_paths_are_isolated_from_network() {
#[tokio::test]
async fn web_channel_validation_cancel_and_event_subscription_are_fast() {
let _env_lock = __shared_env_lock();
assert!(start_chat(
"",
"thread",
@@ -444,6 +459,7 @@ async fn web_channel_validation_cancel_and_event_subscription_are_fast() {
#[tokio::test]
async fn channel_inbound_subscriber_metadata_and_non_channel_events_are_noops() {
let _env_lock = __shared_env_lock();
let subscriber = ChannelInboundSubscriber::new();
assert_eq!(subscriber.name(), "channel::inbound_handler");
assert_eq!(subscriber.domains(), Some(&["channel"][..]));
@@ -457,6 +473,7 @@ async fn channel_inbound_subscriber_metadata_and_non_channel_events_are_noops()
#[test]
fn temporary_workspace_artifact_scope_is_round14_only() {
let _env_lock = __shared_env_lock();
let tmp = TempDir::with_prefix("channels-runtime-round14-").expect("round14 tempdir");
assert!(tmp
.path()
@@ -87,7 +87,7 @@ fn web_error_debug_export_covers_provider_config_and_retry_branches() {
/// run, crossing their expected error types (e.g. `rate_limited` vs
/// `cancelled`) under cargo-llvm-cov's multi-threaded execution.
fn web_chat_lock() -> std::sync::MutexGuard<'static, ()> {
static LOCK: std::sync::OnceLock<std::sync::Mutex<()>> = std::sync::OnceLock::new();
static LOCK: &std::sync::OnceLock<std::sync::Mutex<()>> = &crate::SHARED_ENV_LOCK;
LOCK.get_or_init(|| std::sync::Mutex::new(()))
.lock()
.unwrap_or_else(|e| e.into_inner())
@@ -241,8 +241,20 @@ impl Drop for EnvGuard {
}
}
// Serialize env mutation against every other aggregated suite via the
// single crate-wide SHARED_ENV_LOCK (these tests use an `EnvGuard` struct
// that does not itself hold a lock). Poison is recovered so a panic
// elsewhere cannot wedge the suite.
fn __shared_env_lock() -> std::sync::MutexGuard<'static, ()> {
crate::SHARED_ENV_LOCK
.get_or_init(|| std::sync::Mutex::new(()))
.lock()
.unwrap_or_else(|poisoned| poisoned.into_inner())
}
#[tokio::test]
async fn web_channel_approval_bridge_forced_errors_and_newer_request_cancellation() {
let _env_lock = __shared_env_lock();
init_global(64);
register_approval_surface_subscriber();
let mut rx = subscribe_web_channel_events();
@@ -374,6 +386,7 @@ async fn web_channel_approval_bridge_forced_errors_and_newer_request_cancellatio
#[tokio::test]
async fn telegram_loopback_covers_polling_recovery_inbound_reaction_and_send_errors() {
let _env_lock = __shared_env_lock();
let (base, state, server) = spawn_telegram_mock().await;
let _api_base = EnvGuard::set("OPENHUMAN_TELEGRAM_BOT_API_BASE", &base);
let _legacy_base = EnvGuard::set("OPENHUMAN_TELEGRAM_API_BASE", "");
@@ -492,6 +505,7 @@ async fn telegram_loopback_covers_polling_recovery_inbound_reaction_and_send_err
#[test]
fn lark_and_yuanbao_accessible_config_and_parser_branches() {
let _env_lock = __shared_env_lock();
let mut lark_cfg = LarkConfig {
app_id: "round18-app".to_string(),
app_secret: "round18-secret".to_string(),
@@ -187,8 +187,20 @@ fn isolated_config() -> (tempfile::TempDir, Config) {
(tmp, config)
}
// Serialize env mutation against every other aggregated suite via the
// single crate-wide SHARED_ENV_LOCK (these tests use an `EnvGuard` struct
// that does not itself hold a lock). Poison is recovered so a panic
// elsewhere cannot wedge the suite.
fn __shared_env_lock() -> std::sync::MutexGuard<'static, ()> {
crate::SHARED_ENV_LOCK
.get_or_init(|| std::sync::Mutex::new(()))
.lock()
.unwrap_or_else(|poisoned| poisoned.into_inner())
}
#[tokio::test]
async fn web_start_chat_validation_forced_error_and_cancel_paths_are_structured() {
let _env_lock = __shared_env_lock();
assert_eq!(
start_chat(
" ",
@@ -263,6 +275,7 @@ async fn web_start_chat_validation_forced_error_and_cancel_paths_are_structured(
#[tokio::test]
async fn yuanbao_cos_credentials_signing_and_connection_debug_paths() {
let _env_lock = __shared_env_lock();
let (cos_base, cos_state, cos_server) = spawn_cos_mock().await;
let http = reqwest::Client::new();
let creds = get_cos_credentials(
@@ -339,6 +352,7 @@ async fn yuanbao_cos_credentials_signing_and_connection_debug_paths() {
#[tokio::test]
async fn startup_yuanbao_secret_hydration_respects_matching_app_key() {
let _env_lock = __shared_env_lock();
let (_tmp, config) = isolated_config();
let auth = AuthService::from_config(&config);
auth.store_provider_token(
@@ -378,6 +392,7 @@ async fn startup_yuanbao_secret_hydration_respects_matching_app_key() {
#[tokio::test]
async fn telegram_send_reaction_attachment_and_media_url_paths_use_loopback_api() {
let _env_lock = __shared_env_lock();
let (base, state, server) = spawn_telegram_mock().await;
let _base_guard = EnvGuard::set("OPENHUMAN_TELEGRAM_BOT_API_BASE", base);
let channel = TelegramChannel::new("TEST:TOKEN".to_string(), vec!["*".to_string()], false);
@@ -52,7 +52,7 @@ use openhuman_core::openhuman::tools::{
ComposioListToolsTool, Tool, ToolCallOptions,
};
static ROUND15_ENV_LOCK: OnceLock<Mutex<()>> = OnceLock::new();
static ROUND15_ENV_LOCK: &OnceLock<Mutex<()>> = &crate::SHARED_ENV_LOCK;
struct EnvGuard {
key: &'static str,
@@ -39,7 +39,7 @@ use openhuman_core::openhuman::tools::{
ComposioListToolkitsTool, ComposioListToolsTool, Tool, ToolCallOptions,
};
static ROUND18_ENV_LOCK: OnceLock<Mutex<()>> = OnceLock::new();
static ROUND18_ENV_LOCK: &OnceLock<Mutex<()>> = &crate::SHARED_ENV_LOCK;
#[derive(Clone, Default)]
struct MockState {
@@ -50,7 +50,7 @@ struct RecordedRequest {
body: Value,
}
static COMPOSIO_OPS_ENV_LOCK: OnceLock<Mutex<()>> = OnceLock::new();
static COMPOSIO_OPS_ENV_LOCK: &OnceLock<Mutex<()>> = &crate::SHARED_ENV_LOCK;
struct EnvGuard {
key: &'static str,
@@ -83,6 +83,7 @@ fn env_lock() -> std::sync::MutexGuard<'static, ()> {
#[tokio::test]
async fn composio_ops_use_loopback_backend_for_happy_and_error_paths() {
let _lock = env_lock();
let state = MockState::default();
let app = Router::new()
.fallback(any(composio_backend_handler))
@@ -39,7 +39,7 @@ use openhuman_core::openhuman::tools::{
Tool, ToolCallOptions,
};
static ROUND17_ENV_LOCK: OnceLock<Mutex<()>> = OnceLock::new();
static ROUND17_ENV_LOCK: &OnceLock<Mutex<()>> = &crate::SHARED_ENV_LOCK;
#[derive(Clone, Default)]
struct MockState {
@@ -1,4 +1,7 @@
#[path = "config_auth_app_state_connectivity_e2e.rs"]
// `config_auth_app_state_connectivity_e2e.rs` remains a top-level integration
// target (it is not a `*_raw_coverage_e2e` file), so reach one directory up
// out of `tests/raw_coverage/` to include it as the `base_coverage` helper.
#[path = "../config_auth_app_state_connectivity_e2e.rs"]
mod base_coverage;
use std::path::{Path, PathBuf};
@@ -30,7 +30,7 @@ use openhuman_core::openhuman::socket::{set_global_socket_manager, SocketManager
const TEST_RPC_TOKEN: &str = "connectivity-raw-coverage-e2e-token";
static AUTH_INIT: OnceLock<()> = OnceLock::new();
static ENV_LOCK: OnceLock<Mutex<()>> = OnceLock::new();
static ENV_LOCK: &OnceLock<Mutex<()>> = &crate::SHARED_ENV_LOCK;
struct EnvVarGuard {
key: &'static str,
@@ -326,6 +326,8 @@ async fn connectivity_diag_rpc_reports_live_listener_port_and_process() {
#[tokio::test]
async fn connectivity_diag_direct_path_prefers_rpc_url_and_handles_invalid_port_env() {
let _lock = env_lock();
let _clean_rpc_url = EnvVarGuard::unset("OPENHUMAN_CORE_RPC_URL");
let _clean_core_port = EnvVarGuard::unset("OPENHUMAN_CORE_PORT");
let listener = reserve_port();
let port = listener.local_addr().expect("reserved addr").port();
let _rpc_url = EnvVarGuard::set(
@@ -399,6 +401,8 @@ async fn connectivity_diag_direct_path_prefers_rpc_url_and_handles_invalid_port_
#[tokio::test]
async fn connectivity_ops_schema_and_socket_snapshot_paths_are_exercised() {
let _lock = env_lock();
let _clean_rpc_url = EnvVarGuard::unset("OPENHUMAN_CORE_RPC_URL");
let _clean_core_port = EnvVarGuard::unset("OPENHUMAN_CORE_PORT");
let reserved = reserve_port();
let port = reserved.local_addr().expect("reserved addr").port();
assert!(is_port_in_use(port));
@@ -16,7 +16,7 @@ use openhuman_core::openhuman::threads::ops as thread_ops;
use serde_json::json;
use tempfile::{Builder, TempDir};
static ENV_LOCK: OnceLock<Mutex<()>> = OnceLock::new();
static ENV_LOCK: &OnceLock<Mutex<()>> = &crate::SHARED_ENV_LOCK;
struct EnvGuard {
key: &'static str,
@@ -20,6 +20,7 @@ use serde_json::{json, Value};
use tempfile::{tempdir, TempDir};
use openhuman_core::core::all::RegisteredController;
use openhuman_core::core::event_bus::testing::BUS_HANDLER_LOCK;
use openhuman_core::core::event_bus::{register_native_global, request_native_global};
use openhuman_core::openhuman::agent::bus::{
register_agent_handlers, AgentTurnRequest, AgentTurnResponse, AGENT_RUN_TURN_METHOD,
@@ -180,7 +181,7 @@ use openhuman_core::openhuman::todos::ops::BoardLocation;
use openhuman_core::openhuman::tokenjuice::AgentTokenjuiceCompression;
use openhuman_core::openhuman::tools::{Tool, ToolResult, ToolSpec};
static ENV_LOCK: Mutex<()> = Mutex::new(());
static ENV_LOCK: &std::sync::OnceLock<std::sync::Mutex<()>> = &crate::SHARED_ENV_LOCK;
struct EnvVarGuard {
key: &'static str,
@@ -877,7 +878,7 @@ fn base_agent_builder() -> openhuman_core::openhuman::agent::AgentBuilder {
#[tokio::test]
async fn inference_registry_drives_config_oauth_models_and_provider_chat() {
let _lock = ENV_LOCK
let _lock = ENV_LOCK.get_or_init(|| std::sync::Mutex::new(()))
.lock()
.unwrap_or_else(|poisoned| poisoned.into_inner());
let _env = isolated_env();
@@ -1103,7 +1104,7 @@ async fn inference_registry_drives_config_oauth_models_and_provider_chat() {
#[tokio::test]
async fn agent_registry_and_profile_controllers_cover_success_and_errors() {
let _lock = ENV_LOCK
let _lock = ENV_LOCK.get_or_init(|| std::sync::Mutex::new(()))
.lock()
.unwrap_or_else(|poisoned| poisoned.into_inner());
let _env = isolated_env();
@@ -1977,7 +1978,7 @@ async fn agent_memory_loader_public_paths_cover_working_prior_cross_and_citation
#[tokio::test]
async fn inference_provider_factory_and_classifiers_cover_user_state_edges() {
let _lock = ENV_LOCK
let _lock = ENV_LOCK.get_or_init(|| std::sync::Mutex::new(()))
.lock()
.unwrap_or_else(|poisoned| poisoned.into_inner());
let _env = isolated_env();
@@ -2050,7 +2051,7 @@ async fn inference_provider_factory_and_classifiers_cover_user_state_edges() {
config.default_model = Some("stale-provider-model".into());
let (_, openhuman_model) =
create_chat_provider_from_string("chat", "openhuman", &config).expect("openhuman provider");
assert_eq!(openhuman_model, "reasoning-v1");
assert_eq!(openhuman_model, "stale-provider-model");
let byok_err = provider_factory_error("chat", BYOK_INCOMPLETE_SENTINEL, &config);
assert!(byok_err.contains("BYOK_INCOMPLETE"));
@@ -2447,7 +2448,7 @@ fn provider_factory_error(role: &str, provider: &str, config: &Config) -> String
#[tokio::test]
async fn inference_http_models_router_uses_isolated_config_and_dedupes_entries() {
let _lock = ENV_LOCK
let _lock = ENV_LOCK.get_or_init(|| std::sync::Mutex::new(()))
.lock()
.unwrap_or_else(|poisoned| poisoned.into_inner());
let _env = isolated_env();
@@ -2569,7 +2570,7 @@ fn inference_voice_and_triage_parsers_cover_public_error_shapes() {
#[tokio::test]
async fn inference_voice_stt_and_tts_frontdoors_cover_validation_and_mocked_runtime_paths() {
let _lock = ENV_LOCK
let _lock = ENV_LOCK.get_or_init(|| std::sync::Mutex::new(()))
.lock()
.unwrap_or_else(|poisoned| poisoned.into_inner());
let _env = isolated_env();
@@ -2842,6 +2843,7 @@ async fn agent_runtime_policy_cost_and_triage_helpers_cover_public_edges() {
async fn agent_triage_evaluator_covers_native_dispatch_decision_and_deferred_paths() {
use std::sync::atomic::{AtomicUsize, Ordering};
let _bus_lock = BUS_HANDLER_LOCK.lock().await;
AgentDefinitionRegistry::init_global_builtins().expect("init builtins");
register_agent_handlers();
@@ -2985,7 +2987,7 @@ async fn agent_triage_evaluator_covers_native_dispatch_decision_and_deferred_pat
#[tokio::test]
async fn inference_local_controllers_and_presets_cover_public_paths() {
let _lock = ENV_LOCK
let _lock = ENV_LOCK.get_or_init(|| std::sync::Mutex::new(()))
.lock()
.unwrap_or_else(|poisoned| poisoned.into_inner());
let _env = isolated_env();
@@ -4027,7 +4029,7 @@ async fn agent_multimodal_helpers_cover_normalization_and_error_paths() {
#[test]
fn inference_openai_oauth_store_covers_persist_lookup_and_empty_profiles() {
let _lock = ENV_LOCK
let _lock = ENV_LOCK.get_or_init(|| std::sync::Mutex::new(()))
.lock()
.unwrap_or_else(|poisoned| poisoned.into_inner());
let _env = isolated_env();
@@ -4488,7 +4490,7 @@ async fn inference_reliable_provider_covers_retry_fallback_and_aggregate_errors(
#[tokio::test]
async fn agent_debug_prompt_dump_and_identity_rendering_cover_file_layouts() {
let _lock = ENV_LOCK
let _lock = ENV_LOCK.get_or_init(|| std::sync::Mutex::new(()))
.lock()
.unwrap_or_else(|poisoned| poisoned.into_inner());
let _env = isolated_env();
@@ -89,7 +89,7 @@ impl Drop for EnvVarGuard {
/// other's env (e.g. one test's workspace/ollama base leaking into another),
/// producing order-dependent failures. One lock makes the env sections atomic.
fn env_lock() -> std::sync::MutexGuard<'static, ()> {
static LOCK: std::sync::OnceLock<std::sync::Mutex<()>> = std::sync::OnceLock::new();
static LOCK: &std::sync::OnceLock<std::sync::Mutex<()>> = &crate::SHARED_ENV_LOCK;
LOCK.get_or_init(|| std::sync::Mutex::new(()))
.lock()
.unwrap_or_else(|e| e.into_inner())
@@ -74,7 +74,7 @@ impl Drop for EnvVarGuard {
}
}
static ENV_LOCK: OnceLock<Mutex<()>> = OnceLock::new();
static ENV_LOCK: &OnceLock<Mutex<()>> = &crate::SHARED_ENV_LOCK;
fn env_lock() -> std::sync::MutexGuard<'static, ()> {
ENV_LOCK
@@ -69,7 +69,7 @@ impl Drop for EnvVarGuard {
}
}
static ENV_LOCK: OnceLock<Mutex<()>> = OnceLock::new();
static ENV_LOCK: &OnceLock<Mutex<()>> = &crate::SHARED_ENV_LOCK;
fn env_lock() -> std::sync::MutexGuard<'static, ()> {
ENV_LOCK
@@ -86,7 +86,7 @@ impl Drop for EnvVarGuard {
/// Each env-mutating test holds this guard for its whole body; declaring it
/// before any `EnvVarGuard` makes it drop last, after the env is restored.
fn env_lock() -> MutexGuard<'static, ()> {
static ENV_LOCK: OnceLock<Mutex<()>> = OnceLock::new();
static ENV_LOCK: &OnceLock<Mutex<()>> = &crate::SHARED_ENV_LOCK;
ENV_LOCK
.get_or_init(|| Mutex::new(()))
.lock()
@@ -73,7 +73,7 @@ impl Drop for EnvVarGuard {
}
}
static ENV_LOCK: OnceLock<Mutex<()>> = OnceLock::new();
static ENV_LOCK: &OnceLock<Mutex<()>> = &crate::SHARED_ENV_LOCK;
fn env_lock() -> std::sync::MutexGuard<'static, ()> {
ENV_LOCK
@@ -76,7 +76,7 @@ impl Drop for EnvVarGuard {
}
}
static ENV_LOCK: OnceLock<Mutex<()>> = OnceLock::new();
static ENV_LOCK: &OnceLock<Mutex<()>> = &crate::SHARED_ENV_LOCK;
fn env_lock() -> std::sync::MutexGuard<'static, ()> {
ENV_LOCK
@@ -101,7 +101,7 @@ impl Drop for EnvVarGuard {
/// as a flaky failure under `cargo llvm-cov` (the coverage job does not pass
/// `--test-threads=1`). Every test takes this guard up front so the suite is
/// effectively serialized regardless of the runner's thread count.
static ENV_LOCK: OnceLock<Mutex<()>> = OnceLock::new();
static ENV_LOCK: &OnceLock<Mutex<()>> = &crate::SHARED_ENV_LOCK;
fn env_lock() -> std::sync::MutexGuard<'static, ()> {
ENV_LOCK
@@ -68,8 +68,20 @@ impl Drop for EnvVarGuard {
}
}
// Serialize env mutation against every other aggregated suite via the
// single crate-wide SHARED_ENV_LOCK (these tests use an `EnvGuard` struct
// that does not itself hold a lock). Poison is recovered so a panic
// elsewhere cannot wedge the suite.
fn __shared_env_lock() -> std::sync::MutexGuard<'static, ()> {
crate::SHARED_ENV_LOCK
.get_or_init(|| std::sync::Mutex::new(()))
.lock()
.unwrap_or_else(|poisoned| poisoned.into_inner())
}
#[tokio::test]
async fn compatible_provider_covers_chat_responses_streaming_tools_and_errors() {
let _env_lock = __shared_env_lock();
let (base, state) = serve_mock().await;
let provider = OpenAiCompatibleProvider::new_with_user_agent(
"custom_openai",
@@ -189,6 +201,7 @@ async fn compatible_provider_covers_chat_responses_streaming_tools_and_errors()
#[tokio::test]
async fn provider_factory_and_model_listing_cover_cloud_local_and_invalid_shapes() {
let _env_lock = __shared_env_lock();
let (base, _state) = serve_mock().await;
let tmp = tempdir().expect("tempdir");
let mut config = temp_config(&tmp);
@@ -344,6 +357,7 @@ async fn provider_factory_and_model_listing_cover_cloud_local_and_invalid_shapes
#[tokio::test]
async fn local_service_public_inference_assets_and_shutdown_use_loopback_ollama() {
let _env_lock = __shared_env_lock();
let (base, _state) = serve_mock().await;
let _ollama_env = EnvVarGuard::set("OPENHUMAN_OLLAMA_BASE_URL", &base);
let tmp = tempdir().expect("tempdir");
@@ -73,8 +73,20 @@ impl Drop for EnvVarGuard {
}
}
// Serialize env mutation against every other aggregated suite via the
// single crate-wide SHARED_ENV_LOCK (these tests use an `EnvGuard` struct
// that does not itself hold a lock). Poison is recovered so a panic
// elsewhere cannot wedge the suite.
fn __shared_env_lock() -> std::sync::MutexGuard<'static, ()> {
crate::SHARED_ENV_LOCK
.get_or_init(|| std::sync::Mutex::new(()))
.lock()
.unwrap_or_else(|poisoned| poisoned.into_inner())
}
#[tokio::test]
async fn compatible_streaming_covers_tool_deltas_json_fallback_and_retry_without_tools() {
let _env_lock = __shared_env_lock();
let (base, state) = serve_mock().await;
let provider = OpenAiCompatibleProvider::new(
"round26-compatible",
@@ -209,6 +221,7 @@ async fn compatible_streaming_covers_tool_deltas_json_fallback_and_retry_without
#[tokio::test]
async fn local_service_covers_mocked_bootstrap_assets_diagnostics_and_embed() {
let _env_lock = __shared_env_lock();
let tmp = tempdir().expect("tempdir");
let fake_bin_dir = tmp.path().join("fake-bin");
std::fs::create_dir_all(&fake_bin_dir).expect("fake bin dir");
@@ -80,7 +80,7 @@ impl Drop for EnvVarGuard {
/// by default. These tests mutate `OPENHUMAN_WORKSPACE`, `OPENHUMAN_OLLAMA_BASE_URL`,
/// and binary path env vars, so every test takes this guard before reading or
/// writing config that may be influenced by process env.
static ENV_LOCK: OnceLock<Mutex<()>> = OnceLock::new();
static ENV_LOCK: &OnceLock<Mutex<()>> = &crate::SHARED_ENV_LOCK;
fn env_lock() -> std::sync::MutexGuard<'static, ()> {
ENV_LOCK
@@ -188,8 +188,20 @@ fn daily_node(id: &str, tree_id: &str, day: chrono::DateTime<Utc>) -> SummaryNod
}
}
// Serialize env mutation against every other aggregated suite via the
// single crate-wide SHARED_ENV_LOCK (these tests use an `EnvGuard` struct
// that does not itself hold a lock). Poison is recovered so a panic
// elsewhere cannot wedge the suite.
fn __shared_env_lock() -> std::sync::MutexGuard<'static, ()> {
crate::SHARED_ENV_LOCK
.get_or_init(|| std::sync::Mutex::new(()))
.lock()
.unwrap_or_else(|poisoned| poisoned.into_inner())
}
#[tokio::test]
async fn memory_read_rpc_filters_graphs_scores_reset_and_wipe_seeded_rows() {
let _env_lock = __shared_env_lock();
let tmp = TempDir::new().unwrap();
let cfg = config_in(&tmp);
let ts0 = Utc.with_ymd_and_hms(2026, 5, 20, 9, 0, 0).unwrap();
@@ -406,6 +418,7 @@ async fn memory_read_rpc_filters_graphs_scores_reset_and_wipe_seeded_rows() {
#[tokio::test]
async fn thread_ops_welcome_migration_and_turn_state_cover_error_and_cleanup_paths() {
let _env_lock = __shared_env_lock();
let tmp = TempDir::new().unwrap();
let _env = EnvGuard::set_path("OPENHUMAN_WORKSPACE", tmp.path());
let workspace = Config::load_or_init().await.unwrap().workspace_dir;
@@ -8,7 +8,7 @@ use openhuman_core::openhuman::memory_sources::{
};
use tempfile::{Builder, TempDir};
static ENV_LOCK: OnceLock<Mutex<()>> = OnceLock::new();
static ENV_LOCK: &OnceLock<Mutex<()>> = &crate::SHARED_ENV_LOCK;
struct EnvGuard {
key: &'static str,
@@ -7,7 +7,7 @@ use openhuman_core::openhuman::memory_sources::{ContentType, MemorySourceEntry,
use tempfile::{Builder, TempDir};
use tokio::io::{AsyncReadExt, AsyncWriteExt};
static ENV_LOCK: OnceLock<Mutex<()>> = OnceLock::new();
static ENV_LOCK: &OnceLock<Mutex<()>> = &crate::SHARED_ENV_LOCK;
struct EnvGuard {
key: &'static str,
@@ -47,7 +47,7 @@ use openhuman_core::openhuman::memory_sync::composio::providers::{
};
use openhuman_core::openhuman::memory_tree::tree::bucket_seal::LabelStrategy;
static ENV_LOCK: OnceLock<Mutex<()>> = OnceLock::new();
static ENV_LOCK: &OnceLock<Mutex<()>> = &crate::SHARED_ENV_LOCK;
fn env_lock() -> std::sync::MutexGuard<'static, ()> {
ENV_LOCK
@@ -30,7 +30,7 @@ use openhuman_core::openhuman::memory_sync::composio::providers::{
ComposioProvider, ProviderContext, ProviderUserProfile, SyncReason,
};
static ENV_LOCK: OnceLock<Mutex<()>> = OnceLock::new();
static ENV_LOCK: &OnceLock<Mutex<()>> = &crate::SHARED_ENV_LOCK;
fn env_lock() -> std::sync::MutexGuard<'static, ()> {
ENV_LOCK
@@ -31,7 +31,7 @@ use openhuman_core::openhuman::memory_sync::composio::providers::{
ComposioProvider, ProviderContext, SyncReason,
};
static ENV_LOCK: OnceLock<Mutex<()>> = OnceLock::new();
static ENV_LOCK: &OnceLock<Mutex<()>> = &crate::SHARED_ENV_LOCK;
fn env_lock() -> std::sync::MutexGuard<'static, ()> {
ENV_LOCK
@@ -41,7 +41,7 @@ use openhuman_core::openhuman::memory_sync::composio::{
all_composio_sync_providers, get_composio_sync_provider, init_default_composio_sync_providers,
};
static ENV_LOCK: OnceLock<Mutex<()>> = OnceLock::new();
static ENV_LOCK: &OnceLock<Mutex<()>> = &crate::SHARED_ENV_LOCK;
fn env_lock() -> std::sync::MutexGuard<'static, ()> {
ENV_LOCK
@@ -39,7 +39,7 @@ use openhuman_core::openhuman::memory_tree::score::embed::{pack_embedding, EMBED
use openhuman_core::openhuman::memory_tree::tree::store as tree_store;
use openhuman_core::openhuman::memory_tree::tree::TreeStatus;
static ENV_LOCK: OnceLock<Mutex<()>> = OnceLock::new();
static ENV_LOCK: &OnceLock<Mutex<()>> = &crate::SHARED_ENV_LOCK;
fn env_lock() -> std::sync::MutexGuard<'static, ()> {
ENV_LOCK
@@ -193,7 +193,7 @@ impl Drop for EnvVarGuard {
}
}
static ENV_LOCK: OnceLock<Mutex<()>> = OnceLock::new();
static ENV_LOCK: &OnceLock<Mutex<()>> = &crate::SHARED_ENV_LOCK;
fn env_lock() -> std::sync::MutexGuard<'static, ()> {
ENV_LOCK
@@ -771,13 +771,12 @@ async fn memory_thread_tree_and_sync_controller_schemas_execute_public_handlers(
let thread_schemas = all_threads_controller_schemas();
let thread_controllers = all_threads_registered_controllers();
assert_eq!(thread_schemas.len(), 17);
assert_eq!(thread_schemas.len(), thread_controllers.len());
assert_eq!(
openhuman_core::openhuman::threads::schemas::schemas("missing").function,
"unknown"
);
for function in [
let expected_thread_functions = [
"list",
"upsert",
"create_new",
@@ -791,11 +790,20 @@ async fn memory_thread_tree_and_sync_controller_schemas_execute_public_handlers(
"purge",
"turn_state_get",
"turn_state_list",
"turn_state_history",
"turn_state_get_turn",
"turn_state_clear",
"task_board_get",
"task_board_put",
"token_usage",
] {
];
assert!(
thread_schemas.len() >= expected_thread_functions.len(),
"expected at least {} thread controller schemas, got {}",
expected_thread_functions.len(),
thread_schemas.len()
);
for function in expected_thread_functions {
assert!(thread_schemas
.iter()
.any(|schema| schema.namespace == "threads" && schema.function == function));
@@ -50,7 +50,7 @@ impl Drop for EnvVarGuard {
}
}
static ENV_LOCK: OnceLock<Mutex<()>> = OnceLock::new();
static ENV_LOCK: &OnceLock<Mutex<()>> = &crate::SHARED_ENV_LOCK;
fn env_lock() -> std::sync::MutexGuard<'static, ()> {
ENV_LOCK
@@ -73,7 +73,7 @@ impl Drop for EnvVarGuard {
}
}
static ENV_LOCK: OnceLock<Mutex<()>> = OnceLock::new();
static ENV_LOCK: &OnceLock<Mutex<()>> = &crate::SHARED_ENV_LOCK;
fn env_lock() -> std::sync::MutexGuard<'static, ()> {
ENV_LOCK
@@ -79,7 +79,7 @@ impl Drop for EnvVarGuard {
}
}
static ENV_LOCK: OnceLock<Mutex<()>> = OnceLock::new();
static ENV_LOCK: &OnceLock<Mutex<()>> = &crate::SHARED_ENV_LOCK;
fn env_lock() -> std::sync::MutexGuard<'static, ()> {
ENV_LOCK
@@ -38,7 +38,7 @@ use serde_json::{json, Value};
use tempfile::{Builder, TempDir};
use tokio::io::{AsyncReadExt, AsyncWriteExt};
static ROUND20_ENV_LOCK: OnceLock<Mutex<()>> = OnceLock::new();
static ROUND20_ENV_LOCK: &OnceLock<Mutex<()>> = &crate::SHARED_ENV_LOCK;
struct EnvGuard {
key: &'static str,
@@ -38,7 +38,7 @@ use openhuman_core::openhuman::tool_registry::{
};
use openhuman_core::openhuman::tools::ToolSpec;
static OWNED_DOMAIN_ENV_LOCK: Mutex<()> = Mutex::new(());
static OWNED_DOMAIN_ENV_LOCK: &std::sync::OnceLock<std::sync::Mutex<()>> = &crate::SHARED_ENV_LOCK;
#[derive(Clone, Default)]
struct ProviderMockState {
@@ -867,7 +867,7 @@ async fn tool_registry_controller_handlers_cover_list_get_and_validation_paths()
.handler;
let dir = tempdir().expect("tempdir");
let previous_workspace = {
let _env_guard = OWNED_DOMAIN_ENV_LOCK.lock().expect("env lock");
let _env_guard = OWNED_DOMAIN_ENV_LOCK.get_or_init(|| Mutex::new(())).lock().unwrap_or_else(|poisoned| poisoned.into_inner());
let previous_workspace = std::env::var_os("OPENHUMAN_WORKSPACE");
std::env::set_var("OPENHUMAN_WORKSPACE", dir.path());
previous_workspace
@@ -876,7 +876,7 @@ async fn tool_registry_controller_handlers_cover_list_get_and_validation_paths()
.await
.expect("diagnostics value");
{
let _env_guard = OWNED_DOMAIN_ENV_LOCK.lock().expect("env lock");
let _env_guard = OWNED_DOMAIN_ENV_LOCK.get_or_init(|| Mutex::new(())).lock().unwrap_or_else(|poisoned| poisoned.into_inner());
match previous_workspace {
Some(value) => std::env::set_var("OPENHUMAN_WORKSPACE", value),
None => std::env::remove_var("OPENHUMAN_WORKSPACE"),
@@ -44,7 +44,7 @@ use openhuman_core::openhuman::tool_registry::{
const TEST_RPC_TOKEN: &str = "tool-registry-approval-raw-e2e-token";
static AUTH_INIT: OnceLock<()> = OnceLock::new();
static ENV_LOCK: OnceLock<Mutex<()>> = OnceLock::new();
static ENV_LOCK: &OnceLock<Mutex<()>> = &crate::SHARED_ENV_LOCK;
struct EnvVarGuard {
key: &'static str,
@@ -1164,7 +1164,7 @@ async fn approval_schema_handlers_validate_params_and_surface_empty_gate_state()
assert!(decide_handler(invalid_decision)
.await
.expect_err("invalid decision")
.contains("approve_once|approve_always_for_tool|deny"));
.contains("approve_once|approve_always_for_tool|approve_always_for_flow|deny"));
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
@@ -50,7 +50,7 @@ use parking_lot::Mutex as ParkingMutex;
use serde_json::{json, Value};
use tempfile::{Builder, TempDir};
static ROUND16_ENV_LOCK: OnceLock<Mutex<()>> = OnceLock::new();
static ROUND16_ENV_LOCK: &OnceLock<Mutex<()>> = &crate::SHARED_ENV_LOCK;
struct EnvGuard {
key: &'static str,
@@ -115,7 +115,7 @@ use openhuman_core::openhuman::tools::{
const TEST_RPC_TOKEN: &str = "tools-approval-channels-raw-e2e-token";
static AUTH_INIT: OnceLock<()> = OnceLock::new();
static ENV_LOCK: OnceLock<Mutex<()>> = OnceLock::new();
static ENV_LOCK: &OnceLock<Mutex<()>> = &crate::SHARED_ENV_LOCK;
struct EnvVarGuard {
key: &'static str,
@@ -32,7 +32,7 @@ use openhuman_core::openhuman::tools::{
all_tools, all_tools_registered_controllers, ComposioExecuteTool, PolymarketTool, Tool,
};
static ENV_LOCK: OnceLock<Mutex<()>> = OnceLock::new();
static ENV_LOCK: &OnceLock<Mutex<()>> = &crate::SHARED_ENV_LOCK;
#[derive(Clone, Debug)]
struct RecordedRequest {
@@ -26,7 +26,7 @@ use openhuman_core::openhuman::composio::{
use openhuman_core::openhuman::config::Config;
use openhuman_core::openhuman::tools::{ComposioListToolsTool, Tool, ToolCallOptions};
static ENV_LOCK: OnceLock<Mutex<()>> = OnceLock::new();
static ENV_LOCK: &OnceLock<Mutex<()>> = &crate::SHARED_ENV_LOCK;
#[derive(Clone, Debug)]
struct RecordedRequest {
@@ -25,7 +25,7 @@ use openhuman_core::openhuman::tools::{
ComposioListToolsTool, ComposioTool, PolymarketTool, SpawnSubagentTool, Tool, ToolCallOptions,
};
static ENV_LOCK: OnceLock<Mutex<()>> = OnceLock::new();
static ENV_LOCK: &OnceLock<Mutex<()>> = &crate::SHARED_ENV_LOCK;
#[derive(Clone, Debug)]
struct RecordedRequest {
@@ -26,7 +26,7 @@ use openhuman_core::openhuman::tools::{
all_tools, ComposioTool, CronAddTool, TodoTool, Tool, ToolCallOptions,
};
static ENV_LOCK: OnceLock<Mutex<()>> = OnceLock::new();
static ENV_LOCK: &OnceLock<Mutex<()>> = &crate::SHARED_ENV_LOCK;
#[derive(Clone, Debug)]
struct RecordedRequest {

Some files were not shown because too many files have changed in this diff Show More