mirror of
https://github.com/tinyhumansai/openhuman.git
synced 2026-07-27 21:08:00 +00:00
fix(core): clean up startup timeout task (#2407)
Co-authored-by: Steven Enamakel <enamakel@tinyhumans.ai>
This commit is contained in:
co-authored by
Steven Enamakel
parent
27f8d7776e
commit
b47b71ea3f
@@ -32,7 +32,9 @@ use tokio_util::sync::CancellationToken;
|
||||
|
||||
use crate::process_kill::{kill_pid_force, kill_pid_term};
|
||||
|
||||
const EMBEDDED_CORE_READY_WAIT_ATTEMPTS: u16 = 200;
|
||||
const CORE_READY_POLL_MS: u64 = 100;
|
||||
const CORE_READY_ATTEMPTS: usize = 200;
|
||||
const CORE_READY_TIMEOUT_MS: u64 = CORE_READY_POLL_MS * CORE_READY_ATTEMPTS as u64;
|
||||
|
||||
/// Generate a 256-bit cryptographically-random bearer token as a hex string.
|
||||
///
|
||||
@@ -288,9 +290,11 @@ impl CoreProcessHandle {
|
||||
// (issue: core_process tests intermittently failing with
|
||||
// "core process did not become ready"), especially under
|
||||
// cargo-llvm-cov instrumentation where the binary runs ~2x
|
||||
// slower. Normal runs still exit the loop as soon as the ready
|
||||
// signal arrives and the listener is open.
|
||||
for _ in 0..EMBEDDED_CORE_READY_WAIT_ATTEMPTS {
|
||||
// slower. 20s is still well under any user-visible startup
|
||||
// expectation: in normal runs the ready signal arrives in well
|
||||
// under 1s and the loop exits immediately; the headroom only
|
||||
// matters on heavily loaded instrumented CI workers.
|
||||
for _ in 0..CORE_READY_ATTEMPTS {
|
||||
if !received_ready {
|
||||
match ready_rx.try_recv() {
|
||||
Ok(ready_signal) => {
|
||||
@@ -341,16 +345,56 @@ impl CoreProcessHandle {
|
||||
};
|
||||
}
|
||||
}
|
||||
tokio::time::sleep(Duration::from_millis(100)).await;
|
||||
tokio::time::sleep(Duration::from_millis(CORE_READY_POLL_MS)).await;
|
||||
}
|
||||
|
||||
if retry_after_takeover {
|
||||
continue;
|
||||
}
|
||||
return Err("core process did not become ready".to_string());
|
||||
|
||||
// One last non-sleeping check avoids declaring a timeout when the
|
||||
// ready signal arrived during the final poll sleep.
|
||||
if !received_ready {
|
||||
if let Ok(ready_signal) = ready_rx.try_recv() {
|
||||
self.apply_embedded_ready_signal(ready_signal);
|
||||
received_ready = true;
|
||||
}
|
||||
}
|
||||
if received_ready && self.is_rpc_port_open().await {
|
||||
log::info!("[core] core rpc became ready at {}", self.rpc_url());
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
let port_open = self.is_rpc_port_open().await;
|
||||
return Err(self
|
||||
.cleanup_startup_timeout(received_ready, port_open)
|
||||
.await);
|
||||
}
|
||||
|
||||
Err("core process did not become ready".to_string())
|
||||
let port_open = self.is_rpc_port_open().await;
|
||||
Err(self.cleanup_startup_timeout(false, port_open).await)
|
||||
}
|
||||
|
||||
async fn cleanup_startup_timeout(&self, received_ready: bool, port_open: bool) -> String {
|
||||
let task_state = {
|
||||
let guard = self.task.lock().await;
|
||||
match guard.as_ref() {
|
||||
None => "missing",
|
||||
Some(task) if task.is_finished() => "finished",
|
||||
Some(_) => "running",
|
||||
}
|
||||
};
|
||||
log::error!(
|
||||
"[core] startup timed out after {CORE_READY_TIMEOUT_MS}ms \
|
||||
(ready_signal={received_ready}, port_open={port_open}, task_state={task_state}); \
|
||||
aborting embedded startup task before retry"
|
||||
);
|
||||
self.cancel_shutdown_token(" after startup timeout").await;
|
||||
self.abort_task(" after startup timeout").await;
|
||||
format!(
|
||||
"core process did not become ready within {CORE_READY_TIMEOUT_MS}ms \
|
||||
(ready_signal={received_ready}, port_open={port_open}, task_state={task_state})"
|
||||
)
|
||||
}
|
||||
|
||||
fn apply_embedded_ready_signal(
|
||||
|
||||
@@ -375,3 +375,47 @@ fn send_terminate_signal_cancels_shutdown_token() {
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn startup_timeout_cleanup_aborts_task_and_clears_slot() {
|
||||
let rt = tokio::runtime::Runtime::new().expect("runtime");
|
||||
rt.block_on(async {
|
||||
let handle = CoreProcessHandle::new(19006);
|
||||
let task = tokio::spawn(async {
|
||||
tokio::time::sleep(tokio::time::Duration::from_secs(60)).await;
|
||||
Ok::<(), anyhow::Error>(())
|
||||
});
|
||||
|
||||
{
|
||||
let mut guard = handle.task.lock().await;
|
||||
*guard = Some(task);
|
||||
}
|
||||
|
||||
let message = handle.cleanup_startup_timeout(false, false).await;
|
||||
|
||||
assert!(
|
||||
message.contains("core process did not become ready within"),
|
||||
"timeout message should include the readiness budget: {message}"
|
||||
);
|
||||
assert!(
|
||||
message.contains("ready_signal=false"),
|
||||
"timeout message should include ready signal state: {message}"
|
||||
);
|
||||
assert!(
|
||||
message.contains("port_open=false"),
|
||||
"timeout message should include final port state: {message}"
|
||||
);
|
||||
assert!(
|
||||
message.contains("task_state=running"),
|
||||
"timeout message should include task state: {message}"
|
||||
);
|
||||
assert!(
|
||||
handle.task.lock().await.is_none(),
|
||||
"cleanup must clear the managed task slot so retry can spawn fresh"
|
||||
);
|
||||
assert!(
|
||||
handle.shutdown_token_is_cancelled().await,
|
||||
"cleanup must cancel the startup token before aborting"
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user