feat(core): add managed runtime_python installer (#1976)

This commit is contained in:
Steven Enamakel
2026-05-16 18:36:02 -07:00
committed by GitHub
parent f0b5fdb9d2
commit 95b7d084be
17 changed files with 1490 additions and 0 deletions
+18
View File
@@ -190,6 +190,24 @@ SKILLS_LOCAL_DIR=
# Set to false to disable persisting `working.user.*` docs from skill sync payloads.
OPENHUMAN_SKILLS_WORKING_MEMORY_ENABLED=true
# ---------------------------------------------------------------------------
# Python runtime
# ---------------------------------------------------------------------------
# [optional] Default: true. Controls whether core may resolve Python for
# Python-backed integrations such as stdio MCP servers.
# OPENHUMAN_RUNTIME_PYTHON_ENABLED=true
# [optional] Minimum acceptable interpreter version (default: 3.12.0).
# OPENHUMAN_RUNTIME_PYTHON_MINIMUM_VERSION=3.12.0
# [optional] Pin a specific python-build-standalone release tag. Blank means
# query the latest release at install time.
# OPENHUMAN_RUNTIME_PYTHON_MANAGED_RELEASE_TAG=
# [optional] Reuse a host interpreter before the managed runtime. Default: false.
# OPENHUMAN_RUNTIME_PYTHON_PREFER_SYSTEM=false
# [optional] Preferred executable name or absolute path.
# OPENHUMAN_RUNTIME_PYTHON_PREFERRED_COMMAND=python3.12
# [optional] Reserved cache directory for future managed CPython installs.
# OPENHUMAN_RUNTIME_PYTHON_CACHE_DIR=
# ---------------------------------------------------------------------------
# Error Reporting (Sentry)
# ---------------------------------------------------------------------------
+11
View File
@@ -607,6 +607,17 @@ const CAPABILITIES: &[Capability] = &[
status: CapabilityStatus::Beta,
privacy: MODEL_DOWNLOAD,
},
Capability {
id: "local_ai.python_runtime_installer",
name: "Managed Python Runtime",
domain: "runtime_python",
category: CapabilityCategory::LocalAI,
description:
"Download and reuse an OpenHuman-managed CPython runtime for Python-backed local integrations such as MCP servers, with a system-Python override reserved for development.",
how_to: "Configured by the core `runtime_python` module; future UI surfaces can expose install state and overrides.",
status: CapabilityStatus::Beta,
privacy: MODEL_DOWNLOAD,
},
Capability {
id: "team.create",
name: "Create Teams",
+29
View File
@@ -1148,6 +1148,35 @@ impl Config {
}
}
// Python runtime overrides
if let Some(flag) = env.get("OPENHUMAN_RUNTIME_PYTHON_ENABLED") {
if let Some(enabled) = parse_env_bool("OPENHUMAN_RUNTIME_PYTHON_ENABLED", &flag) {
self.runtime_python.enabled = enabled;
}
}
if let Some(version) = env.get("OPENHUMAN_RUNTIME_PYTHON_MINIMUM_VERSION") {
let trimmed = version.trim();
if !trimmed.is_empty() {
self.runtime_python.minimum_version = trimmed.to_string();
}
}
if let Some(dir) = env.get("OPENHUMAN_RUNTIME_PYTHON_CACHE_DIR") {
self.runtime_python.cache_dir = dir.trim().to_string();
}
if let Some(tag) = env.get("OPENHUMAN_RUNTIME_PYTHON_MANAGED_RELEASE_TAG") {
self.runtime_python.managed_release_tag = tag.trim().to_string();
}
if let Some(flag) = env.get("OPENHUMAN_RUNTIME_PYTHON_PREFER_SYSTEM") {
if let Some(prefer_system) =
parse_env_bool("OPENHUMAN_RUNTIME_PYTHON_PREFER_SYSTEM", &flag)
{
self.runtime_python.prefer_system = prefer_system;
}
}
if let Some(command) = env.get("OPENHUMAN_RUNTIME_PYTHON_PREFERRED_COMMAND") {
self.runtime_python.preferred_command = command.trim().to_string();
}
// Prefer the namespaced name. `OPENHUMAN_SENTRY_DSN` is the legacy
// unprefixed name kept as a fallback so existing CI vars and local
// `.env` files keep working until the GH org-level variable can be
+47
View File
@@ -612,6 +612,53 @@ fn env_overlay_node_flags_respect_bool_parser() {
assert_eq!(cfg.node.version, original_version);
}
#[test]
fn env_overlay_runtime_python_flags_respect_bool_parser() {
let mut cfg = Config::default();
let original_version = cfg.runtime_python.minimum_version.clone();
cfg.apply_env_overlay_with(
&HashMapEnv::new()
.with("OPENHUMAN_RUNTIME_PYTHON_ENABLED", "yes")
.with("OPENHUMAN_RUNTIME_PYTHON_PREFER_SYSTEM", "off")
.with("OPENHUMAN_RUNTIME_PYTHON_CACHE_DIR", "/tmp/oh-python")
.with("OPENHUMAN_RUNTIME_PYTHON_MANAGED_RELEASE_TAG", "20260510")
.with("OPENHUMAN_RUNTIME_PYTHON_PREFERRED_COMMAND", "python3.12"),
);
assert!(cfg.runtime_python.enabled);
assert!(!cfg.runtime_python.prefer_system);
assert_eq!(cfg.runtime_python.cache_dir, "/tmp/oh-python");
assert_eq!(cfg.runtime_python.managed_release_tag, "20260510");
assert_eq!(cfg.runtime_python.preferred_command, "python3.12");
assert_eq!(
cfg.runtime_python.minimum_version, original_version,
"untouched keys stay at defaults"
);
cfg.apply_env_overlay_with(
&HashMapEnv::new().with("OPENHUMAN_RUNTIME_PYTHON_ENABLED", "perhaps"),
);
assert!(cfg.runtime_python.enabled);
cfg.apply_env_overlay_with(
&HashMapEnv::new().with("OPENHUMAN_RUNTIME_PYTHON_MINIMUM_VERSION", " "),
);
assert_eq!(cfg.runtime_python.minimum_version, original_version);
cfg.runtime_python.cache_dir = "/tmp/seed".into();
cfg.runtime_python.managed_release_tag = "20260510".into();
cfg.runtime_python.preferred_command = "python3.12".into();
cfg.apply_env_overlay_with(
&HashMapEnv::new()
.with("OPENHUMAN_RUNTIME_PYTHON_CACHE_DIR", " ")
.with("OPENHUMAN_RUNTIME_PYTHON_MANAGED_RELEASE_TAG", " ")
.with("OPENHUMAN_RUNTIME_PYTHON_PREFERRED_COMMAND", " "),
);
assert_eq!(cfg.runtime_python.cache_dir, "");
assert_eq!(cfg.runtime_python.managed_release_tag, "");
assert_eq!(cfg.runtime_python.preferred_command, "");
}
#[test]
fn env_overlay_sentry_dsn_trims_and_ignores_blank() {
let mut cfg = Config::default();
+2
View File
@@ -30,6 +30,7 @@ mod observability;
mod proxy;
mod routes;
mod runtime;
mod runtime_python;
mod scheduler_gate;
mod storage_memory;
mod tools;
@@ -61,6 +62,7 @@ pub use proxy::{
};
pub use routes::{EmbeddingRouteConfig, ModelRouteConfig};
pub use runtime::{DockerRuntimeConfig, ReliabilityConfig, RuntimeConfig, SchedulerConfig};
pub use runtime_python::RuntimePythonConfig;
pub use scheduler_gate::{SchedulerGateConfig, SchedulerGateMode};
pub use storage_memory::{
LlmBackend, MemoryConfig, MemoryTreeConfig, StorageConfig, StorageProviderConfig,
@@ -0,0 +1,67 @@
//! Python runtime configuration.
//!
//! Controls how the core resolves a Python 3.12+ interpreter for features
//! that need to launch Python subprocesses such as MCP servers.
//!
//! Product direction: `runtime_python` should eventually own a managed
//! CPython distribution so OpenHuman does not depend on host Python being
//! installed correctly. The system-interpreter probe is a compatibility and
//! developer override path, not the desired long-term contract.
use schemars::JsonSchema;
use serde::{Deserialize, Serialize};
#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)]
#[serde(default)]
pub struct RuntimePythonConfig {
/// Master switch. When `false`, the runtime refuses to resolve a Python
/// interpreter and callers must skip Python-backed features entirely.
#[serde(default = "default_enabled")]
pub enabled: bool,
/// Minimum accepted Python version. Interpreters older than this are
/// rejected even when present on `PATH`.
#[serde(default = "default_minimum_version")]
pub minimum_version: String,
/// Absolute path to a directory reserved for future managed Python
/// installs. Empty string means "use the runtime default cache dir".
#[serde(default)]
pub cache_dir: String,
/// Optional upstream release tag for managed standalone Python builds.
/// Empty string means "query the latest release at install time".
#[serde(default)]
pub managed_release_tag: String,
/// When `true`, probe the host `PATH` for a compatible interpreter before
/// attempting any managed-runtime flow. Useful for development; the
/// intended shipped path is a managed interpreter owned by OpenHuman.
#[serde(default = "default_prefer_system")]
pub prefer_system: bool,
/// Optional preferred executable name or absolute path. Examples:
/// `python3.12`, `/opt/homebrew/bin/python3.12`.
#[serde(default)]
pub preferred_command: String,
}
fn default_enabled() -> bool {
true
}
fn default_minimum_version() -> String {
"3.12.0".to_string()
}
fn default_prefer_system() -> bool {
false
}
impl Default for RuntimePythonConfig {
fn default() -> Self {
Self {
enabled: default_enabled(),
minimum_version: default_minimum_version(),
cache_dir: String::new(),
managed_release_tag: String::new(),
prefer_system: default_prefer_system(),
preferred_command: String::new(),
}
}
}
+6
View File
@@ -229,6 +229,11 @@ pub struct Config {
#[serde(default)]
pub node: NodeConfig,
/// Python managed runtime configuration (Python-backed MCP servers and
/// other Python subprocess integrations).
#[serde(default)]
pub runtime_python: RuntimePythonConfig,
#[serde(default)]
pub voice_server: VoiceServerConfig,
@@ -442,6 +447,7 @@ impl Default for Config {
learning_provider: None,
subconscious_provider: None,
node: NodeConfig::default(),
runtime_python: RuntimePythonConfig::default(),
voice_server: VoiceServerConfig::default(),
integrations: IntegrationsConfig::default(),
learning: LearningConfig::default(),
+1
View File
@@ -58,6 +58,7 @@ pub mod redirect_links;
pub mod referral;
pub mod routing;
pub mod runtime_node;
pub mod runtime_python;
pub mod scheduler_gate;
pub mod screen_intelligence;
pub mod security;
+277
View File
@@ -0,0 +1,277 @@
//! Python bootstrap orchestrator.
//!
//! Resolves a managed standalone CPython distribution by default, with an
//! optional system-Python override for development.
use anyhow::{bail, Context, Result};
use reqwest::Client;
use std::fs::OpenOptions;
use std::path::{Path, PathBuf};
use std::sync::Arc;
use tokio::sync::Mutex;
use super::downloader::{download_distribution, select_distribution};
use super::extractor::{atomic_install, extract_distribution};
use super::resolver::{detect_system_python, SystemPython};
use crate::openhuman::config::schema::RuntimePythonConfig;
/// Origin of the resolved interpreter.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum PythonSource {
/// Reused a compatible Python already available on the host.
System,
/// Reserved for a future managed CPython distribution.
Managed,
}
/// Fully-resolved Python interpreter.
#[derive(Debug, Clone)]
pub struct ResolvedPython {
/// Absolute path to the Python executable.
pub python_bin: std::path::PathBuf,
/// Normalized interpreter version, e.g. `3.12.4`.
pub version: String,
/// Where the interpreter came from.
pub source: PythonSource,
}
/// Serialised bootstrap entrypoint for Python runtime resolution.
pub struct PythonBootstrap {
config: RuntimePythonConfig,
client: Client,
cached: Arc<Mutex<Option<ResolvedPython>>>,
}
impl PythonBootstrap {
pub fn new(config: RuntimePythonConfig) -> Self {
Self::new_with_client(config, Client::new())
}
pub(crate) fn new_with_client(config: RuntimePythonConfig, client: Client) -> Self {
Self {
config,
client,
cached: Arc::new(Mutex::new(None)),
}
}
/// Peek at the memoized interpreter without triggering a probe.
pub fn try_cached(&self) -> Option<ResolvedPython> {
self.cached.try_lock().ok().and_then(|g| g.clone())
}
/// Resolve a Python 3.12+ interpreter. The first successful result is
/// memoized for subsequent callers.
pub async fn resolve(&self) -> Result<ResolvedPython> {
let mut guard = self.cached.lock().await;
if let Some(existing) = guard.as_ref() {
tracing::debug!(
version = %existing.version,
source = ?existing.source,
"[runtime_python::bootstrap] returning cached ResolvedPython"
);
return Ok(existing.clone());
}
if !self.config.enabled {
bail!(
"runtime_python is disabled (set runtime_python.enabled = true to use Python-backed integrations)"
);
}
if self.config.prefer_system {
if let Some(system) = detect_system_python(
&self.config.minimum_version,
empty_to_none(&self.config.preferred_command),
) {
let resolved = resolve_from_system(system);
*guard = Some(resolved.clone());
return Ok(resolved);
}
}
let managed = self
.install_managed_from_api(super::downloader::RELEASES_API)
.await?;
*guard = Some(managed.clone());
Ok(managed)
}
/// Build a preconfigured child-process launcher for stdio-oriented Python
/// workloads such as MCP servers.
pub async fn spawn_stdio(
&self,
spec: &crate::openhuman::runtime_python::process::PythonLaunchSpec,
) -> Result<tokio::process::Child> {
let resolved = self.resolve().await?;
crate::openhuman::runtime_python::process::spawn_stdio_process(&resolved, spec)
}
}
impl PythonBootstrap {
async fn install_managed(&self) -> Result<ResolvedPython> {
self.install_managed_from_api(super::downloader::RELEASES_API)
.await
}
async fn install_managed_from_api(&self, releases_api_base: &str) -> Result<ResolvedPython> {
let cache_root = self.cache_root();
tokio::fs::create_dir_all(&cache_root)
.await
.with_context(|| format!("creating python runtime cache {}", cache_root.display()))?;
let release = super::downloader::fetch_release_metadata_from_base(
&self.client,
releases_api_base,
empty_to_none(&self.config.managed_release_tag),
)
.await?;
let dist = select_distribution(&release, &self.config.minimum_version)?;
let install_dir = cache_root.join(dist.install_dir_name());
let _install_lock = acquire_install_lock(&install_dir).await?;
if let Some(existing) = probe_managed_install(&install_dir) {
tracing::info!(
install_dir = %install_dir.display(),
version = %existing.version,
"[runtime_python::bootstrap] reusing existing managed python install"
);
return Ok(existing);
}
tracing::info!(
asset = %dist.asset_name,
release = %release.tag_name,
install_dir = %install_dir.display(),
"[runtime_python::bootstrap] installing managed python"
);
let archive_path = cache_root.join(&dist.asset_name);
download_distribution(&self.client, &dist, &archive_path).await?;
let scratch = cache_root.join(format!(
".stage-{}-{}",
std::process::id(),
uuid::Uuid::new_v4()
));
let _ = tokio::fs::remove_dir_all(&scratch).await;
let top_level = extract_distribution(&archive_path, &scratch).await?;
atomic_install(&top_level, &install_dir).await?;
let _ = tokio::fs::remove_dir_all(&scratch).await;
let _ = tokio::fs::remove_file(&archive_path).await;
probe_managed_install(&install_dir).with_context(|| {
format!(
"managed python install completed but no interpreter was found under {}",
install_dir.display()
)
})
}
fn cache_root(&self) -> PathBuf {
let configured = self.config.cache_dir.trim();
if !configured.is_empty() {
return PathBuf::from(configured);
}
if let Some(user_cache) = dirs::cache_dir() {
return user_cache.join("openhuman").join("runtime-python");
}
PathBuf::from(".openhuman").join("runtime-python")
}
}
fn resolve_from_system(system: SystemPython) -> ResolvedPython {
tracing::info!(
path = %system.path.display(),
version = %system.version,
"[runtime_python::bootstrap] reusing compatible system python"
);
ResolvedPython {
python_bin: system.path,
version: system.version,
source: PythonSource::System,
}
}
fn empty_to_none(value: &str) -> Option<&str> {
let trimmed = value.trim();
if trimmed.is_empty() {
None
} else {
Some(trimmed)
}
}
fn probe_managed_install(install_dir: &Path) -> Option<ResolvedPython> {
let python_bin = find_python_binary(install_dir)?;
let version = super::resolver::probe_python_version_public(&python_bin)?;
let version_info = super::resolver::parse_python_version(&version)?;
Some(ResolvedPython {
python_bin,
version: version_info.display(),
source: PythonSource::Managed,
})
}
fn find_python_binary(install_dir: &Path) -> Option<PathBuf> {
let candidates = [
install_dir.join("bin").join("python3.12"),
install_dir.join("bin").join("python3"),
install_dir.join("bin").join("python"),
install_dir.join("python.exe"),
install_dir.join("python3.12.exe"),
install_dir.join("python3.exe"),
];
for candidate in candidates {
if candidate.is_file() {
return Some(candidate);
}
}
for entry in walkdir::WalkDir::new(install_dir).into_iter().flatten() {
let path = entry.path();
if !path.is_file() {
continue;
}
let Some(name) = path.file_name().and_then(|n| n.to_str()) else {
continue;
};
if matches!(
name,
"python" | "python3" | "python3.12" | "python.exe" | "python3.exe" | "python3.12.exe"
) {
return Some(path.to_path_buf());
}
}
None
}
async fn acquire_install_lock(install_dir: &Path) -> Result<std::fs::File> {
let lock_path = install_dir.with_extension("lock");
if let Some(parent) = lock_path.parent() {
tokio::fs::create_dir_all(parent)
.await
.with_context(|| format!("creating lock parent {}", parent.display()))?;
}
let lock_path_for_task = lock_path.clone();
tokio::task::spawn_blocking(move || -> Result<std::fs::File> {
use fs2::FileExt;
let file = OpenOptions::new()
.create(true)
.read(true)
.write(true)
.open(&lock_path_for_task)
.with_context(|| format!("opening install lock {}", lock_path_for_task.display()))?;
file.lock_exclusive()
.with_context(|| format!("locking install target {}", lock_path_for_task.display()))?;
Ok(file)
})
.await
.context("join failure while acquiring runtime_python install lock")?
}
#[cfg(test)]
#[path = "bootstrap_tests.rs"]
mod tests;
@@ -0,0 +1,169 @@
use super::*;
use axum::extract::State;
use axum::http::{HeaderValue, StatusCode};
use axum::response::IntoResponse;
use axum::routing::get;
use axum::{Json, Router};
use serde_json::json;
use sha2::{Digest, Sha256};
use std::sync::Arc;
use tokio::net::TcpListener;
#[cfg(unix)]
#[tokio::test]
async fn install_managed_from_mock_astral_release_downloads_and_resolves_executable() {
use std::os::unix::fs::PermissionsExt;
let archive_bytes = build_test_python_archive().expect("archive bytes");
let archive_sha = hex::encode(Sha256::digest(&archive_bytes));
let asset_name = test_asset_name();
let listener = TcpListener::bind("127.0.0.1:0").await.expect("bind");
let addr = listener.local_addr().expect("local addr");
let app_state = Arc::new(MockAstralState {
asset_name: asset_name.to_string(),
archive_bytes,
archive_sha,
artifact_url: format!("http://{addr}/artifacts/{asset_name}"),
});
let app = Router::new()
.route("/releases/latest", get(mock_release_latest))
.route("/artifacts/{asset}", get(mock_artifact))
.with_state(app_state.clone());
let server = tokio::spawn(async move { axum::serve(listener, app).await.unwrap() });
let tmp = tempfile::tempdir().expect("tempdir");
let mut cfg = RuntimePythonConfig::default();
cfg.cache_dir = tmp.path().join("runtime-python").display().to_string();
cfg.managed_release_tag = String::new();
cfg.prefer_system = false;
let bootstrap = PythonBootstrap::new_with_client(cfg, reqwest::Client::new());
let api_base = format!("http://{addr}/releases");
let resolved = bootstrap
.install_managed_from_api(&api_base)
.await
.expect("managed install should succeed");
assert_eq!(resolved.source, PythonSource::Managed);
assert!(resolved.python_bin.is_file(), "python binary should exist");
let mode = std::fs::metadata(&resolved.python_bin)
.expect("metadata")
.permissions()
.mode();
assert_ne!(mode & 0o111, 0, "python binary must be executable");
let version = crate::openhuman::runtime_python::resolver::probe_python_version_public(
&resolved.python_bin,
)
.expect("version probe");
assert_eq!(version.trim(), "Python 3.12.13");
server.abort();
}
#[derive(Clone)]
struct MockAstralState {
asset_name: String,
archive_bytes: Vec<u8>,
archive_sha: String,
artifact_url: String,
}
async fn mock_release_latest(State(state): State<Arc<MockAstralState>>) -> impl IntoResponse {
Json(json!({
"tag_name": "20260510",
"assets": [
{
"name": state.asset_name,
"browser_download_url": state.artifact_url,
"digest": format!("sha256:{}", state.archive_sha),
}
]
}))
}
async fn mock_artifact(
State(state): State<Arc<MockAstralState>>,
axum::extract::Path(asset): axum::extract::Path<String>,
) -> impl IntoResponse {
if asset != state.asset_name {
return (StatusCode::NOT_FOUND, Vec::new()).into_response();
}
(
[(
axum::http::header::CONTENT_TYPE,
HeaderValue::from_static("application/gzip"),
)],
state.archive_bytes.clone(),
)
.into_response()
}
#[cfg(unix)]
fn build_test_python_archive() -> anyhow::Result<Vec<u8>> {
use flate2::write::GzEncoder;
use flate2::Compression;
use tar::{Builder, Header};
let mut tar_bytes = Vec::new();
{
let encoder = GzEncoder::new(&mut tar_bytes, Compression::default());
let mut builder = Builder::new(encoder);
let root = test_asset_name().trim_end_matches(".tar.gz");
let bin_dir = format!("{root}/bin");
let python_path = format!("{bin_dir}/python3.12");
let mut root_header = Header::new_gnu();
root_header.set_entry_type(tar::EntryType::Directory);
root_header.set_mode(0o755);
root_header.set_size(0);
root_header.set_cksum();
builder.append_data(&mut root_header, root, std::io::empty())?;
let mut bin_header = Header::new_gnu();
bin_header.set_entry_type(tar::EntryType::Directory);
bin_header.set_mode(0o755);
bin_header.set_size(0);
bin_header.set_cksum();
builder.append_data(&mut bin_header, &bin_dir, std::io::empty())?;
let script = b"#!/bin/sh\necho 'Python 3.12.13'\n";
let mut python_header = Header::new_gnu();
python_header.set_entry_type(tar::EntryType::Regular);
python_header.set_mode(0o755);
python_header.set_size(script.len() as u64);
python_header.set_cksum();
builder.append_data(&mut python_header, &python_path, &script[..])?;
builder.into_inner()?.finish()?;
}
Ok(tar_bytes)
}
#[cfg(all(target_os = "macos", target_arch = "aarch64"))]
fn test_asset_name() -> &'static str {
"cpython-3.12.13+20260510-aarch64-apple-darwin-install_only.tar.gz"
}
#[cfg(all(target_os = "macos", target_arch = "x86_64"))]
fn test_asset_name() -> &'static str {
"cpython-3.12.13+20260510-x86_64-apple-darwin-install_only.tar.gz"
}
#[cfg(all(target_os = "linux", target_arch = "x86_64"))]
fn test_asset_name() -> &'static str {
"cpython-3.12.13+20260510-x86_64-unknown-linux-gnu-install_only.tar.gz"
}
#[cfg(all(target_os = "linux", target_arch = "aarch64"))]
fn test_asset_name() -> &'static str {
"cpython-3.12.13+20260510-aarch64-unknown-linux-gnu-install_only.tar.gz"
}
+257
View File
@@ -0,0 +1,257 @@
//! Managed standalone Python distribution downloader.
//!
//! Pulls release metadata from `astral-sh/python-build-standalone`, selects a
//! host-compatible `install_only` archive satisfying the configured minimum
//! Python version, downloads it, and verifies the published SHA-256 digest.
use anyhow::{anyhow, bail, Context, Result};
use reqwest::Client;
use serde::Deserialize;
use sha2::{Digest, Sha256};
use std::path::Path;
use tokio::fs::File;
use tokio::io::AsyncWriteExt;
use super::resolver::{parse_python_version, PythonVersion};
pub(crate) const RELEASES_API: &str =
"https://api.github.com/repos/astral-sh/python-build-standalone/releases";
#[derive(Debug, Clone, Deserialize)]
pub struct GithubRelease {
pub tag_name: String,
pub assets: Vec<GithubAsset>,
}
#[derive(Debug, Clone, Deserialize)]
pub struct GithubAsset {
pub name: String,
pub browser_download_url: String,
pub digest: Option<String>,
}
#[derive(Debug, Clone)]
pub struct PythonDistribution {
pub release_tag: String,
pub asset_name: String,
pub url: String,
pub version: PythonVersion,
pub expected_sha256: Option<String>,
}
impl PythonDistribution {
pub fn install_dir_name(&self) -> String {
self.asset_name.trim_end_matches(".tar.gz").to_string()
}
}
pub async fn fetch_release_metadata(
client: &Client,
release_tag: Option<&str>,
) -> Result<GithubRelease> {
fetch_release_metadata_from_base(client, RELEASES_API, release_tag).await
}
pub(crate) async fn fetch_release_metadata_from_base(
client: &Client,
releases_api_base: &str,
release_tag: Option<&str>,
) -> Result<GithubRelease> {
let url = if let Some(tag) = release_tag {
format!("{releases_api_base}/tags/{tag}")
} else {
format!("{releases_api_base}/latest")
};
tracing::debug!(url = %url, "[runtime_python::downloader] fetching release metadata");
client
.get(&url)
.header(reqwest::header::USER_AGENT, "openhuman-core/runtime_python")
.send()
.await
.with_context(|| format!("GET {url}"))?
.error_for_status()
.with_context(|| format!("non-success status on {url}"))?
.json::<GithubRelease>()
.await
.with_context(|| format!("decoding release metadata from {url}"))
}
pub fn select_distribution(
release: &GithubRelease,
minimum_version: &str,
) -> Result<PythonDistribution> {
let Some(minimum) = parse_python_version(minimum_version) else {
bail!("invalid runtime_python.minimum_version `{minimum_version}`");
};
let target_suffix = host_asset_suffix()?;
let mut candidates = release
.assets
.iter()
.filter_map(|asset| parse_distribution_asset(asset, &release.tag_name))
.filter(|dist| asset_matches_target(&dist.asset_name, target_suffix))
.filter(|dist| dist.version >= minimum)
.collect::<Vec<_>>();
if candidates.is_empty() {
bail!(
"no managed python-build-standalone asset found for host suffix `{target_suffix}` with version >= {} in release {}",
minimum.display(),
release.tag_name
);
}
candidates.sort_by(|a, b| {
b.version
.cmp(&a.version)
.then_with(|| a.asset_name.cmp(&b.asset_name))
});
if let Some(preferred) = candidates
.iter()
.find(|dist| dist.asset_name.contains("install_only_stripped"))
.cloned()
{
return Ok(preferred);
}
candidates
.into_iter()
.next()
.ok_or_else(|| anyhow!("internal error selecting managed python asset"))
}
fn parse_distribution_asset(asset: &GithubAsset, release_tag: &str) -> Option<PythonDistribution> {
let name = asset.name.as_str();
if !name.starts_with("cpython-") || !name.ends_with(".tar.gz") || !name.contains("install_only")
{
return None;
}
let rest = name.strip_prefix("cpython-")?;
let version_str = rest.split('+').next()?;
let version = parse_python_version(version_str)?;
let expected_sha256 = asset
.digest
.as_deref()
.and_then(|digest| digest.strip_prefix("sha256:"))
.map(str::to_string);
Some(PythonDistribution {
release_tag: release_tag.to_string(),
asset_name: asset.name.clone(),
url: asset.browser_download_url.clone(),
version,
expected_sha256,
})
}
fn host_asset_suffix() -> Result<&'static str> {
let os = std::env::consts::OS;
let arch = std::env::consts::ARCH;
match (os, arch) {
("macos", "aarch64") => Ok("aarch64-apple-darwin-install_only.tar.gz"),
("macos", "x86_64") => Ok("x86_64-apple-darwin-install_only.tar.gz"),
("linux", "aarch64") => Ok("aarch64-unknown-linux-gnu-install_only.tar.gz"),
("linux", "x86_64") => Ok("x86_64-unknown-linux-gnu-install_only.tar.gz"),
("windows", "aarch64") => Ok("aarch64-pc-windows-msvc-install_only.tar.gz"),
("windows", "x86_64") => Ok("x86_64-pc-windows-msvc-install_only.tar.gz"),
_ => Err(anyhow!(
"no managed standalone Python distribution for host {os}/{arch}"
)),
}
}
fn asset_matches_target(asset_name: &str, target_suffix: &str) -> bool {
asset_name.ends_with(target_suffix)
|| asset_name.ends_with(
&target_suffix.replace("-install_only.tar.gz", "-install_only_stripped.tar.gz"),
)
}
pub async fn download_distribution(
client: &Client,
dist: &PythonDistribution,
target_path: &Path,
) -> Result<()> {
tracing::info!(
url = %dist.url,
target = %target_path.display(),
"[runtime_python::downloader] starting download"
);
if let Some(parent) = target_path.parent() {
tokio::fs::create_dir_all(parent)
.await
.with_context(|| format!("creating cache dir {}", parent.display()))?;
}
let mut response = client
.get(&dist.url)
.header(reqwest::header::USER_AGENT, "openhuman-core/runtime_python")
.send()
.await
.with_context(|| format!("GET {}", dist.url))?
.error_for_status()
.with_context(|| format!("non-success status on {}", dist.url))?;
let mut file = File::create(target_path)
.await
.with_context(|| format!("creating {}", target_path.display()))?;
let mut hasher = Sha256::new();
let stream_result: Result<()> = async {
while let Some(chunk) = response
.chunk()
.await
.with_context(|| format!("streaming {}", dist.url))?
{
hasher.update(&chunk);
file.write_all(&chunk)
.await
.with_context(|| format!("writing chunk to {}", target_path.display()))?;
}
file.flush()
.await
.with_context(|| format!("flushing {}", target_path.display()))?;
Ok(())
}
.await;
drop(file);
if let Err(err) = stream_result {
let _ = tokio::fs::remove_file(target_path).await;
return Err(err);
}
if let Some(expected) = dist.expected_sha256.as_deref() {
let actual_hex = hex::encode(hasher.finalize());
if actual_hex != expected {
let _ = tokio::fs::remove_file(target_path).await;
bail!(
"SHA-256 mismatch for {} (expected {expected}, got {actual_hex})",
dist.asset_name
);
}
} else {
tracing::warn!(
asset = %dist.asset_name,
"[runtime_python::downloader] release metadata did not include a digest; skipping SHA-256 verification"
);
}
tracing::info!(
target = %target_path.display(),
asset = %dist.asset_name,
"[runtime_python::downloader] download complete"
);
Ok(())
}
#[cfg(test)]
#[path = "downloader_tests.rs"]
mod tests;
@@ -0,0 +1,24 @@
use super::*;
#[test]
fn parses_asset_into_distribution() {
let asset = GithubAsset {
name: "cpython-3.12.13+20260510-x86_64-apple-darwin-install_only.tar.gz".to_string(),
browser_download_url: "https://example.invalid/python.tar.gz".to_string(),
digest: Some("sha256:abc123".to_string()),
};
let dist = parse_distribution_asset(&asset, "20260510").expect("dist");
assert_eq!(dist.release_tag, "20260510");
assert_eq!(dist.version.display(), "3.12.13");
assert_eq!(dist.expected_sha256.as_deref(), Some("abc123"));
}
#[test]
fn ignores_non_install_only_assets() {
let asset = GithubAsset {
name: "cpython-3.12.13+20260510-x86_64-apple-darwin-full.tar.zst".to_string(),
browser_download_url: "https://example.invalid/python.tar.zst".to_string(),
digest: None,
};
assert!(parse_distribution_asset(&asset, "20260510").is_none());
}
+113
View File
@@ -0,0 +1,113 @@
//! Archive extraction for managed standalone Python distributions.
use anyhow::{anyhow, Context, Result};
use std::fs::{self, File};
use std::path::{Path, PathBuf};
pub async fn extract_distribution(archive: &Path, extract_root: &Path) -> Result<PathBuf> {
let archive = archive.to_path_buf();
let extract_root = extract_root.to_path_buf();
tracing::info!(
archive = %archive.display(),
extract_root = %extract_root.display(),
"[runtime_python::extractor] starting extraction"
);
tokio::task::spawn_blocking(move || -> Result<PathBuf> {
fs::create_dir_all(&extract_root)
.with_context(|| format!("creating extract root {}", extract_root.display()))?;
let file = File::open(&archive)
.with_context(|| format!("opening archive {}", archive.display()))?;
let decoder = flate2::read::GzDecoder::new(file);
let mut tar = tar::Archive::new(decoder);
tar.set_preserve_permissions(true);
tar.set_overwrite(true);
tar.unpack(&extract_root)
.with_context(|| format!("unpacking tar.gz into {}", extract_root.display()))?;
find_single_top_level(&extract_root)
})
.await
.context("spawn_blocking join failure during extraction")?
}
fn find_single_top_level(extract_root: &Path) -> Result<PathBuf> {
let mut entries = fs::read_dir(extract_root)
.with_context(|| format!("listing {}", extract_root.display()))?
.collect::<Result<Vec<_>, _>>()
.with_context(|| format!("reading entries of {}", extract_root.display()))?;
entries.sort_by_key(|e| e.file_name());
let mut dirs = entries
.into_iter()
.filter(|e| e.file_type().map(|t| t.is_dir()).unwrap_or(false))
.map(|e| e.path())
.collect::<Vec<_>>();
match dirs.len() {
1 => Ok(dirs.pop().expect("single dir")),
0 => Err(anyhow!(
"expected one top-level folder under {}, found none",
extract_root.display()
)),
n => Err(anyhow!(
"expected one top-level folder under {}, found {n}",
extract_root.display()
)),
}
}
pub async fn atomic_install(staged: &Path, final_dest: &Path) -> Result<PathBuf> {
let staged = staged.to_path_buf();
let final_dest = final_dest.to_path_buf();
tokio::task::spawn_blocking(move || -> Result<PathBuf> {
if let Some(parent) = final_dest.parent() {
fs::create_dir_all(parent)
.with_context(|| format!("creating parent {}", parent.display()))?;
}
let backup = if final_dest.exists() {
let candidate = final_dest.with_extension(format!("old-{}", std::process::id()));
fs::rename(&final_dest, &candidate).with_context(|| {
format!(
"moving existing install {} aside to {}",
final_dest.display(),
candidate.display()
)
})?;
Some(candidate)
} else {
None
};
if let Err(err) = fs::rename(&staged, &final_dest).with_context(|| {
format!(
"renaming staged {} -> {}",
staged.display(),
final_dest.display()
)
}) {
if let Some(backup_path) = backup.as_ref() {
if let Err(restore_err) = fs::rename(backup_path, &final_dest) {
return Err(anyhow!(
"{err}; rollback from {} to {} also failed: {restore_err}",
backup_path.display(),
final_dest.display()
));
}
}
return Err(err);
}
if let Some(backup_path) = backup {
let _ = fs::remove_dir_all(backup_path);
}
Ok(final_dest)
})
.await
.context("spawn_blocking join failure during atomic install")?
}
+18
View File
@@ -0,0 +1,18 @@
//! Managed Python runtime for Python-backed integrations.
//!
//! The immediate use case is stdio MCP servers implemented in Python. This
//! module owns interpreter discovery and process-launch primitives so callers
//! do not need to care whether Python came from the host or a future managed
//! distribution.
pub mod bootstrap;
pub mod downloader;
pub mod extractor;
pub mod process;
pub mod resolver;
pub use bootstrap::{PythonBootstrap, PythonSource, ResolvedPython};
pub use downloader::{fetch_release_metadata, select_distribution, PythonDistribution};
pub use extractor::{atomic_install, extract_distribution};
pub use process::PythonLaunchSpec;
pub use resolver::{detect_system_python, parse_python_version, PythonVersion, SystemPython};
+78
View File
@@ -0,0 +1,78 @@
//! Python child-process launch helpers.
//!
//! Uses unbuffered stdio (`-u`) by default so line-oriented protocols such as
//! MCP do not stall behind Python's output buffering.
use anyhow::{Context, Result};
use std::collections::BTreeMap;
use std::path::PathBuf;
use std::process::Stdio;
use super::bootstrap::ResolvedPython;
/// Launch spec for a Python stdio subprocess.
#[derive(Debug, Clone)]
pub struct PythonLaunchSpec {
/// Absolute or caller-resolved path to the Python script.
pub script_path: PathBuf,
/// Positional arguments forwarded after the script path.
pub args: Vec<String>,
/// Optional working directory for the child process.
pub cwd: Option<PathBuf>,
/// Extra environment variables to set on the child process.
pub env: BTreeMap<String, String>,
/// Whether to pass `-u` for unbuffered stdio. Defaults to `true`.
pub unbuffered: bool,
}
impl PythonLaunchSpec {
pub fn new(script_path: PathBuf) -> Self {
Self {
script_path,
args: Vec::new(),
cwd: None,
env: BTreeMap::new(),
unbuffered: true,
}
}
}
pub fn spawn_stdio_process(
resolved: &ResolvedPython,
spec: &PythonLaunchSpec,
) -> Result<tokio::process::Child> {
let mut cmd = tokio::process::Command::new(&resolved.python_bin);
if spec.unbuffered {
cmd.arg("-u");
}
cmd.arg(&spec.script_path);
cmd.args(&spec.args);
if let Some(cwd) = spec.cwd.as_ref() {
cmd.current_dir(cwd);
}
for (key, value) in &spec.env {
cmd.env(key, value);
}
cmd.stdin(Stdio::piped())
.stdout(Stdio::piped())
.stderr(Stdio::piped())
.kill_on_drop(true);
let child = cmd.spawn().with_context(|| {
format!(
"failed to spawn python process `{}` for script {}",
resolved.python_bin.display(),
spec.script_path.display()
)
})?;
tracing::info!(
python_bin = %resolved.python_bin.display(),
script = %spec.script_path.display(),
arg_count = spec.args.len(),
cwd = spec.cwd.as_ref().map(|p| p.display().to_string()),
"[runtime_python::process] spawned stdio python child"
);
Ok(child)
}
+277
View File
@@ -0,0 +1,277 @@
//! System Python resolver.
//!
//! Walks the configured command candidates / `PATH`, probes `--version`, and
//! returns a [`SystemPython`] when the interpreter satisfies the configured
//! minimum version floor.
use std::ffi::OsString;
use std::path::{Path, PathBuf};
use std::process::{Command, Stdio};
use std::time::Duration;
/// Parsed Python semantic version.
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
pub struct PythonVersion {
pub major: u32,
pub minor: u32,
pub patch: u32,
}
impl PythonVersion {
pub fn display(self) -> String {
format!("{}.{}.{}", self.major, self.minor, self.patch)
}
}
/// A usable Python interpreter discovered on the host.
#[derive(Debug, Clone)]
pub struct SystemPython {
/// Absolute path to the executable.
pub path: PathBuf,
/// Parsed semantic version.
pub version_info: PythonVersion,
/// Normalized `major.minor.patch` string.
pub version: String,
}
/// Parse a version line like `Python 3.12.4` or `3.12.4`.
pub fn parse_python_version(raw: &str) -> Option<PythonVersion> {
let trimmed = raw.trim();
let stripped = trimmed.strip_prefix("Python ").unwrap_or(trimmed);
let mut parts = stripped.split('.');
let major = parts.next()?.parse::<u32>().ok()?;
let minor = parts.next()?.parse::<u32>().ok()?;
let patch = parts
.next()
.and_then(|segment| {
let digits = segment
.chars()
.take_while(|c| c.is_ascii_digit())
.collect::<String>();
if digits.is_empty() {
None
} else {
digits.parse::<u32>().ok()
}
})
.unwrap_or(0);
Some(PythonVersion {
major,
minor,
patch,
})
}
/// Probe the host for a Python interpreter satisfying `minimum_version`.
///
/// Candidate order:
/// 1. `preferred_command` when supplied
/// 2. `python3.12`
/// 3. `python3`
/// 4. `python`
pub fn detect_system_python(
minimum_version: &str,
preferred_command: Option<&str>,
) -> Option<SystemPython> {
detect_system_python_in_path(
minimum_version,
preferred_command,
std::env::var_os("PATH").as_ref(),
)
}
fn detect_system_python_in_path(
minimum_version: &str,
preferred_command: Option<&str>,
path_var: Option<&OsString>,
) -> Option<SystemPython> {
let Some(minimum) = parse_python_version(minimum_version) else {
tracing::warn!(
minimum_version,
"[runtime_python::resolver] invalid minimum_version, skipping system-python probe"
);
return None;
};
for candidate in candidate_commands(preferred_command, minimum) {
let Some(path) = resolve_candidate(&candidate, path_var) else {
tracing::debug!(candidate, "[runtime_python::resolver] candidate not found");
continue;
};
tracing::debug!(
candidate,
path = %path.display(),
minimum_version = %minimum.display(),
"[runtime_python::resolver] probing python candidate"
);
let Some(raw_version) = probe_python_version(&path) else {
tracing::warn!(
candidate,
path = %path.display(),
"[runtime_python::resolver] `python --version` failed; skipping candidate"
);
continue;
};
let Some(version_info) = parse_python_version(&raw_version) else {
tracing::warn!(
candidate,
path = %path.display(),
raw_version = %raw_version,
"[runtime_python::resolver] could not parse python version output"
);
continue;
};
if version_info < minimum {
tracing::info!(
candidate,
path = %path.display(),
found = %version_info.display(),
minimum = %minimum.display(),
"[runtime_python::resolver] python candidate below minimum version"
);
continue;
}
let normalized = version_info.display();
tracing::info!(
candidate,
path = %path.display(),
version = %normalized,
"[runtime_python::resolver] reusing compatible system python"
);
return Some(SystemPython {
path,
version_info,
version: normalized,
});
}
None
}
fn candidate_commands(preferred_command: Option<&str>, minimum: PythonVersion) -> Vec<String> {
let mut candidates = Vec::new();
if let Some(preferred) = preferred_command.map(str::trim).filter(|s| !s.is_empty()) {
candidates.push(preferred.to_string());
}
let minimum_specific = format!("python{}.{}", minimum.major, minimum.minor);
for fallback in [minimum_specific.as_str(), "python3.12", "python3", "python"] {
if !candidates.iter().any(|existing| existing == fallback) {
candidates.push(fallback.to_string());
}
}
candidates
}
fn resolve_candidate(candidate: &str, path_var: Option<&OsString>) -> Option<PathBuf> {
let path = Path::new(candidate);
if path.components().count() > 1 || path.is_absolute() {
return is_executable_candidate(path).then(|| path.to_path_buf());
}
let path_var = path_var?;
for dir in std::env::split_paths(path_var) {
let base = dir.join(candidate);
if is_executable_candidate(&base) {
return Some(base);
}
#[cfg(windows)]
{
let exe = dir.join(format!("{candidate}.exe"));
if is_executable_candidate(&exe) {
return Some(exe);
}
}
}
None
}
#[cfg(unix)]
fn is_executable_candidate(path: &Path) -> bool {
use std::os::unix::fs::PermissionsExt;
std::fs::metadata(path)
.map(|meta| meta.is_file() && (meta.permissions().mode() & 0o111 != 0))
.unwrap_or(false)
}
#[cfg(not(unix))]
fn is_executable_candidate(path: &Path) -> bool {
path.is_file()
}
fn probe_python_version(path: &Path) -> Option<String> {
use std::io::Read;
use wait_timeout::ChildExt;
let mut cmd = Command::new(path);
cmd.arg("--version")
.stdin(Stdio::null())
.stdout(Stdio::piped())
.stderr(Stdio::piped());
#[cfg(windows)]
{
use std::os::windows::process::CommandExt;
cmd.creation_flags(0x0800_0000); // CREATE_NO_WINDOW
}
let mut child = cmd.spawn().ok()?;
let timeout = Duration::from_secs(5);
let status = match child.wait_timeout(timeout).ok()? {
Some(status) => status,
None => {
tracing::warn!(
path = %path.display(),
timeout_secs = 5,
"[runtime_python::resolver] `<bin> --version` timed out; killing process"
);
let _ = child.kill();
let _ = child.wait();
return None;
}
};
if !status.success() {
let mut stderr_buf = String::new();
if let Some(mut s) = child.stderr.take() {
let _ = s.read_to_string(&mut stderr_buf);
}
tracing::debug!(
path = %path.display(),
status = ?status,
stderr = %stderr_buf,
"[runtime_python::resolver] `<bin> --version` exited non-zero"
);
return None;
}
let mut stdout_buf = String::new();
if let Some(mut s) = child.stdout.take() {
let _ = s.read_to_string(&mut stdout_buf);
}
let mut stderr_buf = String::new();
if let Some(mut s) = child.stderr.take() {
let _ = s.read_to_string(&mut stderr_buf);
}
let combined = if stdout_buf.trim().is_empty() {
stderr_buf.trim().to_string()
} else {
stdout_buf.trim().to_string()
};
if combined.is_empty() {
None
} else {
Some(combined)
}
}
pub(crate) fn probe_python_version_public(path: &Path) -> Option<String> {
probe_python_version(path)
}
#[cfg(test)]
#[path = "resolver_tests.rs"]
mod tests;
@@ -0,0 +1,96 @@
use super::*;
#[test]
fn parses_standard_python_version() {
assert_eq!(
parse_python_version("Python 3.12.4"),
Some(PythonVersion {
major: 3,
minor: 12,
patch: 4
})
);
}
#[test]
fn parses_without_python_prefix() {
assert_eq!(
parse_python_version("3.12.0"),
Some(PythonVersion {
major: 3,
minor: 12,
patch: 0
})
);
}
#[test]
fn parses_patchless_version_as_zero() {
assert_eq!(
parse_python_version("Python 3.12"),
Some(PythonVersion {
major: 3,
minor: 12,
patch: 0
})
);
}
#[test]
fn rejects_invalid_versions() {
assert_eq!(parse_python_version("Python three.twelve"), None);
assert_eq!(parse_python_version(""), None);
}
#[cfg(unix)]
#[test]
fn detects_preferred_python_from_custom_path() {
use std::fs;
use std::os::unix::fs::PermissionsExt;
let dir = tempfile::tempdir().expect("tempdir");
let script = dir.path().join("python3.12");
fs::write(&script, "#!/bin/sh\necho 'Python 3.12.7'\n").expect("write script");
let mut perms = fs::metadata(&script).expect("metadata").permissions();
perms.set_mode(0o755);
fs::set_permissions(&script, perms).expect("chmod");
let path_var = OsString::from(dir.path().display().to_string());
let found = detect_system_python_in_path("3.12.0", Some("python3.12"), Some(&path_var))
.expect("python should resolve");
assert_eq!(found.version, "3.12.7");
assert_eq!(found.path, script);
}
#[cfg(unix)]
#[test]
fn rejects_python_below_minimum() {
use std::fs;
use std::os::unix::fs::PermissionsExt;
let dir = tempfile::tempdir().expect("tempdir");
let script = dir.path().join("python3");
fs::write(&script, "#!/bin/sh\necho 'Python 3.11.9'\n").expect("write script");
let mut perms = fs::metadata(&script).expect("metadata").permissions();
perms.set_mode(0o755);
fs::set_permissions(&script, perms).expect("chmod");
let path_var = OsString::from(dir.path().display().to_string());
let found = detect_system_python_in_path("3.12.0", None, Some(&path_var));
assert!(found.is_none(), "3.11 must be rejected");
}
#[test]
fn candidate_commands_include_minimum_specific_binary() {
let candidates = candidate_commands(
None,
PythonVersion {
major: 3,
minor: 13,
patch: 0,
},
);
assert_eq!(candidates[0], "python3.13");
assert!(candidates.iter().any(|candidate| candidate == "python3"));
}