fix(flows): warn (not block) on inference-provider readiness while authoring, fail runs cleanly (B45) (#5212)

This commit is contained in:
Cyrus Gray
2026-07-27 21:15:15 +05:30
committed by GitHub
parent 44783cefc9
commit fc2279f776
6 changed files with 1557 additions and 4 deletions
@@ -222,6 +222,39 @@ Typical setup arc: user asks for a Slack step → `composio_list_connections`
shows Slack isn't linked → `composio_connect { toolkit: "slack" }` → once
connected, `list_flow_connections` → build the `tool_call` node with the real
`connection_ref` + a `search_tool_catalog` slug → dry-run → propose.
## Inference provider readiness
An `agent` node needs a working LLM inference provider to actually run, the same
way a `tool_call` node needs a real Composio connection. This is a separate,
independent concern from app connections above. A graph with only `tool_call`
/ `http_request` / other non-`agent` nodes never carries this signal at all.
This is advisory, not a blocker. Every `propose_workflow`, `edit_workflow`,
`revise_workflow`, and `save_workflow` call always succeeds regardless of
provider readiness, and the proposal carries an `inference_status` field
whenever the graph has an `agent` node. Always build and propose the graph.
When `inference_status` is not `"ready"`, propose normally and, alongside the
proposal, tell the user in plain language that the workflow is built correctly
but needs their AI provider connected before it will run:
- **`signed_out`** ("you are signed out" / no active session): tell the user
the workflow is ready to go, they just need to sign in to OpenHuman before
running it.
- **`provider_not_configured`** (the backend reports something like "API key
not configured for provider"): tell the user the workflow is ready to go,
they just need to configure their provider API key in Settings > Providers
before running it.
- **`error`**: a more specific construction problem (for example an
incomplete custom or BYOK provider setup). Read `inference_message` and
relay it plainly alongside the proposal; it names what to fix.
You cannot configure a provider or sign the user in yourself. Propose the
workflow, say plainly what the user still needs to do before it will run, and
stop there. Do not refuse to propose over this. Do not swap the `agent` node
for a code or transform node to work around it. Do not loop trying to resolve
it yourself. Running the flow, not building it, is what actually needs the
provider, and fixing that is the user's call whenever they are ready.
3. **Build the graph** (see the model below).
4. **Self-check with `dry_run_workflow`** on the draft — catch missing edges,
wrong ports, unreachable nodes. Fix and re-run.
+537
View File
@@ -520,6 +520,22 @@ pub(crate) async fn run_builder_gates(config: &Config, graph: &WorkflowGraph) ->
if !agent_ref_errors.is_empty() {
return agent_ref_errors;
}
// NOTE (B45 design correction, judge finding on live run 104aab90):
// provider-connectivity (issue B45 — signed out, or a managed-backend
// account with no provider API key configured) is deliberately NOT a
// hard author gate here. It used to reject `propose_workflow` /
// `edit_workflow` outright, which meant a graph whose only problem was
// "not runnable yet" could never even be SHOWN to the user — the copilot
// detected the problem, could not propose past it, and trailed off with
// no proposal at all. `evaluate_inference_readiness` still runs (see
// `build_builder_proposal` below) and surfaces `inference_status` /
// `inference_message` as an ADVISORY warning on the proposal payload, so
// authoring always succeeds and the UI can render a "connect your
// provider" nudge alongside the built workflow. The hard rejection moved
// to run time instead — see `validate_inference_readiness`'s use in
// `run_flow_body`, which fails a real run cleanly before the engine
// executes rather than blocking the author from ever seeing the graph.
//
// Async, live connection list: a tool_call whose `connection_ref` names the
// wrong toolkit for its slug, or a connection id the user doesn't actually
// have (WS3 — the transcript bug where a TIKTOK connection id was wired onto
@@ -722,6 +738,19 @@ pub(crate) async fn build_builder_proposal(
// toolkits this graph needs and whether they're connected, so it can render
// "Connect <toolkit>" CTAs instead of a bare gate error later.
let required_connections = compute_required_connections(config, graph).await;
// B45 (design correction): the LLM-provider-connectivity evaluation is
// ADVISORY here, never a rejection — `run_builder_gates` above no longer
// includes it (that used to hard-block `propose_workflow`/`edit_workflow`
// on a graph the copilot couldn't then show the user at all — judge
// finding on live run 104aab90). So `evaluation.status` here can
// legitimately be `"ready"`, `"signed_out"`, `"provider_not_configured"`,
// or `"error"` — the UI renders a "Connect a provider" / "Sign in" CTA
// for the non-ready cases, alongside the toolkit-connection CTAs above.
// The graph is proposed regardless of this value. Computed via the same
// shared, cached evaluator the run-time preflight (`validate_inference_readiness`
// in `run_flow_body`) consumes, so a run right after this proposal reads
// the cached result instead of re-probing the network.
let inference_readiness = evaluate_inference_readiness(config, graph).await;
let graph_value = serde_json::to_value(graph).map_err(|e| e.to_string())?;
tracing::info!(
@@ -748,6 +777,15 @@ pub(crate) async fn build_builder_proposal(
"warnings": warnings,
"required_connections": required_connections,
});
// Only present when the graph has at least one applicable `agent` node;
// a tool_call-only graph omits both fields entirely rather than claiming
// a meaningless "ready".
if let Some(evaluation) = inference_readiness {
payload["inference_status"] = json!(evaluation.status);
if let Some(message) = evaluation.message {
payload["inference_message"] = json!(message);
}
}
if let Some(instruction) = instruction {
payload["instruction"] = json!(instruction);
}
@@ -1808,6 +1846,457 @@ pub(crate) async fn validate_agent_refs(config: &Config, graph: &WorkflowGraph)
errors
}
// ─────────────────────────────────────────────────────────────────────────────
// Inference-readiness check: provider-connectivity (issue B45)
// ─────────────────────────────────────────────────────────────────────────────
//
// An `agent` node's completion (`OpenHumanLlm::complete` in
// `tinyflows/caps.rs`) resolves a chat model exactly like every other
// inference caller in this host — but no check previously inspected that
// resolution at all. `compute_required_connections` only walks `tool_call`
// Composio nodes; an `agent` node's own hard dependency, a working LLM
// provider, went completely unchecked. The confirmed failure: a signed-in
// user whose managed-backend account has no provider API key configured gets
// an HTTP 400 `{"success":false,"error":"API key not configured for
// provider","errorCode":"BAD_REQUEST"}` — but only mid-run, wrapped several
// layers deep as `capability error: graph error: capability error: model
// error: ...`.
//
// **Design correction (judge finding on live run 104aab90 — see git log for
// the full writeup):** this was originally wired in as a HARD author gate
// (`run_builder_gates`), rejecting `propose_workflow`/`edit_workflow`
// outright. In practice that meant a graph whose only problem was "the user
// hasn't configured a provider yet" could never be proposed at all — the
// copilot detected `provider_not_configured`, tried to propose anyway, was
// blocked, and trailed off with no workflow shown to the user. The correct
// placement is:
//
// - **Author time (`build_builder_proposal`)** — ADVISORY ONLY. Authoring
// always succeeds; `evaluate_inference_readiness`'s result rides along on
// the proposal payload as `inference_status`/`inference_message` so the UI
// can render a "connect your provider" nudge next to the built workflow.
// - **Run time (`run_flow_body`)** — HARD gate. A real run (never
// `dry_run_workflow`, which is a sandbox) checks readiness before invoking
// the tinyflows engine and fails the run row cleanly with an actionable
// message if the graph's agent node(s) can't currently reach a provider —
// see `validate_inference_readiness`'s call site in `run_flow_body`.
//
// Two layers, cheapest and most decisive first:
//
// - **Layer 1 (sync)** — the desktop session itself: signed out
// (`scheduler_gate::is_signed_out`), or no valid `app-session` JWT
// (`inference::provider::factory::verify_session_active`, the exact check
// every custom-provider construction already gates on).
// - **Layer 2 (async, cached)** — one cheap real probe per DISTINCT resolved
// role (`inference::provider::probe_inference_readiness`) to catch the
// "signed in but no provider API key configured for this account" class of
// failure that Layer 1 cannot see. A graph can mix agent nodes pinned to
// different models (e.g. one `hint:reasoning`, one plain `chat`) that route
// to different provider configs — each distinct role is probed once, not
// once per node, and every probe's result caches BOTH a successful and a
// definitively-negative result for a short TTL — a propose → edit → save →
// run authoring/run burst hits the network at most once per role per TTL
// window, whichever way the probe comes back. This is safe to cache
// negative because `probe_inference_readiness` (and, beneath it,
// `OpenHumanBackendModel::probe_readiness`) already fails OPEN (`Ok(())`)
// on anything transient — a timeout, a transport error, a 5xx — so an
// `Err` reaching this cache is always the definitive, config-level "not
// ready" signal, never a flake that a naive cache would freeze in place.
//
// [`evaluate_inference_readiness`] is the single evaluation both
// [`validate_inference_readiness`] (the hard gate) and
// [`build_builder_proposal`]'s `inference_status` payload field consume, so
// the gate and the UI-facing status can never disagree.
/// Cache TTL for the Layer-2 managed-backend/role probe.
const INFERENCE_PROBE_CACHE_TTL: std::time::Duration = std::time::Duration::from_secs(60);
/// Cache key: (workload role, session identity). `config.config_path` stands
/// in for "session identity" — within one desktop process there is exactly
/// one active config/session, so this is stable in production, while
/// distinct `Config`s (as every test builds its own `tempfile` workspace)
/// naturally get distinct cache entries instead of bleeding a cached result
/// from one test/session into an unrelated one. Keying on `role` alone would
/// NOT be enough: two different sessions (or two tests) can both resolve the
/// literal role `"summarization"` to entirely different, unrelated outcomes.
type InferenceProbeCacheKey = (String, std::path::PathBuf);
/// A cached probe outcome: when it was taken, and the definitive result.
type InferenceProbeCacheEntry = (std::time::Instant, Result<(), String>);
/// The probe cache map, factored out to keep the `static` type readable
/// (clippy::type-complexity).
type InferenceProbeCacheMap =
std::collections::HashMap<InferenceProbeCacheKey, InferenceProbeCacheEntry>;
/// Process-global cache of Layer-2 probe outcomes, keyed by
/// [`InferenceProbeCacheKey`]. Both `Ok` and `Err` entries are served from
/// cache within [`INFERENCE_PROBE_CACHE_TTL`] (design correction, B45 —
/// previously only `Ok` was cached, so a signed-in-but-unconfigured account
/// re-hit the network on every one of `edit_workflow` / `validate_workflow` /
/// `propose_workflow` / a run's own preflight in a single authoring turn — up
/// to 4 network round trips observed in one live judge-flagged turn). A
/// cached `Err` is still only ever the definitive class (see the module doc
/// above on fail-open) — a fixed provider becomes visible again at most
/// `INFERENCE_PROBE_CACHE_TTL` later, or immediately on sign-out/back-in via
/// [`invalidate_inference_probe_cache_if_signed_out`].
static INFERENCE_PROBE_CACHE: LazyLock<std::sync::Mutex<InferenceProbeCacheMap>> =
LazyLock::new(|| std::sync::Mutex::new(std::collections::HashMap::new()));
/// Invalidate every cached Layer-2 probe result. Checked defensively on every
/// call so a signed-out session (whether the initial one or a later
/// account-switch) can never serve a stale cached "ready" — the moment
/// `is_signed_out` flips true the next successful probe starts a fresh TTL
/// window. Clears the whole cache rather than just the current key: a
/// sign-out is a session-wide event, not scoped to one role.
fn invalidate_inference_probe_cache_if_signed_out() {
if crate::openhuman::scheduler_gate::is_signed_out() {
INFERENCE_PROBE_CACHE
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.clear();
}
}
async fn cached_probe_inference_readiness(role: &str, config: &Config) -> Result<(), String> {
invalidate_inference_probe_cache_if_signed_out();
let key: InferenceProbeCacheKey = (role.to_string(), config.config_path.clone());
if let Some((checked_at, result)) = INFERENCE_PROBE_CACHE
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.get(&key)
.cloned()
{
if checked_at.elapsed() < INFERENCE_PROBE_CACHE_TTL {
tracing::debug!(
target: "flows",
role,
cached_ready = result.is_ok(),
"[flows] inference-readiness: reusing cached probe result"
);
return result;
}
}
let result =
crate::openhuman::inference::provider::probe_inference_readiness(role, config).await;
INFERENCE_PROBE_CACHE
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.insert(key, (std::time::Instant::now(), result.clone()));
result
}
/// The workload role an `agent` node's completion effectively runs on —
/// mirrors the exact mapping `OpenHumanLlm::complete` (`tinyflows/caps.rs`)
/// applies, so this probe checks the same route the node will actually
/// dispatch to at run time. Precedence (findings A+B on this gate):
///
/// 1. Node `config.model` — a managed tier or `hint:*` alias, translated via
/// [`role_for_model_tier`](crate::openhuman::inference::provider::role_for_model_tier).
/// 2. A static (non-`=`) `agent_ref` whose custom
/// [`AgentRegistryEntry`](crate::openhuman::agent_registry::AgentRegistryEntry)
/// itself pins a `model` (e.g. `hint:reasoning`) — resolved the same way
/// [`OpenHumanAgentRunner::run_via_harness`](crate::openhuman::tinyflows::caps::OpenHumanAgentRunner)
/// does via `resolve_node_model(&request, entry_model)`, using the same
/// sync, config-only accessor
/// ([`find_custom_in_config`](crate::openhuman::agent_registry::find_custom_in_config))
/// it calls.
/// 3. Otherwise, caps.rs's own default role (`"summarization"`, its fallback
/// absent a `role` field on the completion request).
///
/// A static `agent_ref` that instead resolves to a shipped/TOML harness
/// `AgentDefinition` (`AgentRoute::Harness`) can *also* pin a model via
/// `ModelSpec::Exact`/`ModelSpec::Hint` — but `ModelSpec::Inherit` (the
/// default) resolves against the *parent* agent's live model at spawn time,
/// which this static, pre-run gate has no parent turn to read. Resolving only
/// the Exact/Hint cases here — while silently mis-defaulting every
/// `Inherit`-using definition — would be a half-correct, fragile lookup, so
/// this case falls back to the default role rather than guess.
/// TODO(B45): resolve agent_ref-pinned model for harness `AgentDefinition`s
/// once a parent-model-free resolution path exists.
fn agent_node_role(config: &Config, node: &tinyflows::model::Node) -> &'static str {
let pinned_model = node
.config
.get("model")
.and_then(Value::as_str)
.map(str::trim)
.filter(|s| !s.is_empty());
if let Some(model) = pinned_model {
return crate::openhuman::inference::provider::role_for_model_tier(model);
}
let static_agent_ref = node
.config
.get("agent_ref")
.and_then(Value::as_str)
.map(str::trim)
.filter(|s| !s.is_empty() && !s.starts_with('='));
if let Some(agent_ref) = static_agent_ref {
if let Some(entry_model) =
crate::openhuman::agent_registry::find_custom_in_config(config, agent_ref)
.and_then(|entry| entry.model)
{
let entry_model = entry_model.trim();
if !entry_model.is_empty() {
return crate::openhuman::inference::provider::role_for_model_tier(entry_model);
}
}
}
"summarization"
}
/// Classifies an inference-readiness failure message into the fixed wire
/// vocabulary `build_builder_proposal`'s `inference_status` payload and this
/// gate's prose both use (`"signed_out" | "provider_not_configured" |
/// "error"`).
///
/// Defensive ordering: a message that still smells like a dead session (an
/// unlikely race between this gate's own signed-out check and the async
/// probe) is classified `signed_out` before the more specific
/// `provider_not_configured` pattern; anything else falls back to the generic
/// `error` bucket (a BYOK-incomplete config, an unknown provider slug, a
/// local-only privacy-mode block, …) rather than mislabeling it as a
/// provider-key problem.
fn classify_inference_error_message(message: &str) -> &'static str {
let lower = message.to_ascii_lowercase();
if lower.contains("session_expired") || lower.contains("sign in") {
"signed_out"
} else if lower.contains("api key not configured") {
"provider_not_configured"
} else {
"error"
}
}
/// Outcome of [`evaluate_inference_readiness`] for a graph that has at least
/// one applicable `agent` node.
struct InferenceReadinessEvaluation {
/// One of `"ready"`, `"signed_out"`, `"provider_not_configured"`, `"error"`
/// — the fixed vocabulary shared with the proposal payload.
status: &'static str,
/// User-actionable prose; `None` only when `status == "ready"`.
message: Option<String>,
/// The offending node id, when applicable (absent for `"ready"`).
node_id: Option<String>,
}
/// Evaluate the B45 provider-connectivity gate for `graph`.
///
/// Returns `None` when the graph has no `agent` node at all — a tool_call-only
/// graph never pays this check's cost. A dynamic `=`-derived `agent_ref` node
/// is still in scope (finding C): its concrete route is not knowable
/// statically, so its exact per-model role can't be resolved, but the node
/// still means "this graph runs inference" — it stays in scope for Layer 1
/// (signed-out/session) and gets a default-role Layer 2 probe. Only the
/// per-model role resolution is skipped for such a node, never the whole
/// check.
///
/// Every DISTINCT role across the graph's applicable `agent` nodes is probed
/// (findings A+B): Layer 1 (signed-out/session) runs once for the whole
/// graph — every agent node shares one backend session — then Layer 2 runs
/// once per distinct role (via [`cached_probe_inference_readiness`], so a
/// role already probed elsewhere in this process within the TTL is served
/// from cache). `status`/`message` report `provider_not_configured`/`error`
/// if ANY role's probe fails, naming every offending node and role.
async fn evaluate_inference_readiness(
config: &Config,
graph: &WorkflowGraph,
) -> Option<InferenceReadinessEvaluation> {
let agent_nodes: Vec<&tinyflows::model::Node> = graph
.nodes
.iter()
.filter(|node| node.kind == NodeKind::Agent)
.collect();
let first_node = *agent_nodes.first()?;
// Layer 1: signed-out is the cheapest, most decisive check. Session-wide
// — checked once for the whole graph, not per node/role.
if crate::openhuman::scheduler_gate::is_signed_out() {
tracing::debug!(
target: "flows",
node = %first_node.id,
"[flows] inference-readiness: signed out — rejecting"
);
return Some(InferenceReadinessEvaluation {
status: "signed_out",
message: Some(
"Inference unavailable: you are signed out. Sign in to OpenHuman to run agent \
nodes."
.to_string(),
),
node_id: Some(first_node.id.clone()),
});
}
// Skipped under `#[cfg(test)]`, matching every other call site of this
// exact check (`factory.rs`'s `unresolved_chat_model_error` and friends):
// unit-test configs use a fresh `tempfile::tempdir()` workspace with no
// stored `app-session` JWT by design, so this would otherwise reject
// every agent-node graph built by the hundreds of existing flows tests
// that have nothing to do with session state. Layer 2 below still fails
// OPEN on a construction failure caused by a genuinely missing session
// (see `OpenHumanBackendModel::probe_readiness`'s own doc), so production
// behavior for a real signed-out desktop user is unchanged — only the
// (redundant, in that case) early rejection here is test-only skipped.
#[cfg(not(test))]
if let Err(e) = crate::openhuman::inference::provider::factory::verify_session_active(config) {
tracing::debug!(
target: "flows",
node = %first_node.id,
error = %e,
"[flows] inference-readiness: no active backend session — rejecting"
);
return Some(InferenceReadinessEvaluation {
status: "signed_out",
message: Some(format!(
"Inference unavailable: {e} Sign in to OpenHuman to run agent nodes."
)),
node_id: Some(first_node.id.clone()),
});
}
// Layer 2: each node's effective role, grouped so every DISTINCT role is
// probed exactly once (a graph with several agent nodes pinning the same
// role must not pay the network/cache-lookup cost twice). `BTreeMap` for
// deterministic iteration/message ordering (test-friendly, and stable
// prose across runs).
let mut nodes_by_role: std::collections::BTreeMap<&'static str, Vec<String>> =
std::collections::BTreeMap::new();
for node in &agent_nodes {
let role = agent_node_role(config, node);
nodes_by_role.entry(role).or_default().push(node.id.clone());
}
let mut failures: Vec<(&'static str, String, Vec<String>)> = Vec::new();
for (role, node_ids) in &nodes_by_role {
tracing::debug!(
target: "flows",
nodes = ?node_ids,
role,
"[flows] inference-readiness: probing managed-backend/role readiness"
);
if let Err(msg) = cached_probe_inference_readiness(role, config).await {
tracing::warn!(
target: "flows",
nodes = ?node_ids,
role,
"[flows] inference-readiness: probe rejected — {msg}"
);
failures.push((role, msg, node_ids.clone()));
}
}
if failures.is_empty() {
return Some(InferenceReadinessEvaluation {
status: "ready",
message: None,
node_id: None,
});
}
// Defensive ordering matches `classify_inference_error_message`'s own doc:
// `signed_out` (unlikely to reach Layer 2, given the Layer 1 check above,
// but a race is not impossible) outranks `provider_not_configured`, which
// outranks the generic `error` bucket.
let statuses: Vec<&'static str> = failures
.iter()
.map(|(_, msg, _)| classify_inference_error_message(msg))
.collect();
let status = if statuses.contains(&"signed_out") {
"signed_out"
} else if statuses.contains(&"provider_not_configured") {
"provider_not_configured"
} else {
"error"
};
// Single failing role naming a single node: keep the original flat
// message shape (no node-list preamble) so the existing single-node
// contract/tests read exactly as before. Anything broader (several
// failing roles, or one role shared by several nodes) names every
// offending node/role explicitly, since a flat message can no longer
// unambiguously point at "the" offending node.
if let [(_role, msg, node_ids)] = failures.as_slice() {
if let [node_id] = node_ids.as_slice() {
let message = if status == "provider_not_configured" {
format!(
"This flow's agent step needs a working AI provider, but the provider \
returned: '{msg}'. Configure your provider API key in OpenHuman Settings > \
Providers, then try again."
)
} else {
format!("This flow's agent step needs a working AI provider: {msg}")
};
return Some(InferenceReadinessEvaluation {
status,
message: Some(message),
node_id: Some(node_id.clone()),
});
}
}
let message = failures
.iter()
.map(|(role, msg, node_ids)| {
let nodes = node_ids
.iter()
.map(|id| format!("'{id}'"))
.collect::<Vec<_>>()
.join(", ");
let role_status = classify_inference_error_message(msg);
if role_status == "provider_not_configured" {
format!(
"Node(s) {nodes} (role `{role}`): the provider returned: '{msg}'. Configure \
your provider API key in OpenHuman Settings > Providers, then try again."
)
} else {
format!("Node(s) {nodes} (role `{role}`): {msg}")
}
})
.collect::<Vec<_>>()
.join("\n\n");
Some(InferenceReadinessEvaluation {
status,
message: Some(format!(
"This flow has {} agent step(s) that need a working AI provider:\n\n{message}",
failures.len()
)),
node_id: None,
})
}
/// The B45 provider-connectivity check as a gate-shaped `Vec<String>`: empty
/// when the graph's `agent` node(s) (if any) can currently reach a working
/// LLM provider, otherwise the offending node's error, naming it.
///
/// **No longer wired into `run_builder_gates`** (design correction — see the
/// module doc above): authoring is never blocked by this. Its one production
/// caller is `run_flow_body`'s run-time preflight, which fails a real run
/// cleanly before the tinyflows engine executes rather than hard-blocking the
/// author from proposing/saving the graph in the first place. See the module
/// doc above for the two-layer evaluation design.
pub(crate) async fn validate_inference_readiness(
config: &Config,
graph: &WorkflowGraph,
) -> Vec<String> {
let Some(evaluation) = evaluate_inference_readiness(config, graph).await else {
return Vec::new();
};
if evaluation.status == "ready" {
return Vec::new();
}
let message = evaluation
.message
.unwrap_or_else(|| "This flow's agent step needs a working AI provider.".to_string());
match evaluation.node_id {
Some(node_id) => vec![format!("Node '{node_id}': {message}")],
None => vec![message],
}
}
// ─────────────────────────────────────────────────────────────────────────────
// Tool-contract enforcement gate (systemic tool-contract fix, Part 2)
// ─────────────────────────────────────────────────────────────────────────────
@@ -4136,6 +4625,54 @@ async fn run_flow_body(
let config: &Config = config_arc.as_ref();
let flow_id: &str = flow_id.as_str();
// B45 run-time preflight (design correction — see the "Inference-readiness
// check" module doc above): an `agent` node needs a working LLM provider
// to run at all, but that is no longer enforced as an author-time gate —
// `propose_workflow`/`edit_workflow`/`save_workflow` always succeed now,
// so a graph can reach here whose agent node(s) cannot currently complete.
// Catch that HERE, before the tinyflows engine (and any upstream
// fetch/prep nodes) does real work for nothing, and finalize the run row
// as `failed` with a clear, actionable message instead of the opaque,
// several-layers-deep "capability error: graph error: capability error:
// model error: ... API key not configured for provider" a mid-run failure
// surfaces as. Reuses `validate_inference_readiness` — backed by the same
// cached evaluation `build_builder_proposal`'s advisory `inference_status`
// warns on — so a run right after a proposal/edit reads the cached
// negative (`INFERENCE_PROBE_CACHE`) instead of re-probing the network.
// Returns an empty `Vec` (no-op here) for a tool_call-only graph, and is
// never consulted by `dry_run_workflow` (sandbox runs are exempt by
// design — that tool doesn't route through `run_flow_body` at all).
let inference_errors = validate_inference_readiness(config, &flow.graph).await;
if !inference_errors.is_empty() {
let detail = inference_errors.join(" ");
let msg = format!("This flow's AI step needs a working AI provider to run. {detail}");
tracing::warn!(
target: "flows",
flow_id,
"[flows] run_flow_body: inference-readiness preflight failed — finalizing run as \
failed without invoking the engine: {msg}"
);
if let Err(rec_err) = store::record_run(config, flow_id, "failed") {
tracing::warn!(
target: "flows",
flow_id,
error = %rec_err,
"[flows] run_flow_body: failed to record failed run (inference preflight)"
);
}
let observed = current_persisted_steps(config, &thread_id);
finish_flow_run_row(
config,
&thread_id,
flow_id,
"failed",
&observed,
&[],
Some(&msg),
);
return Err(msg);
}
// Recompile to execute — the entry point already compile-checked to fail
// fast before the running row existed. A failure *now* (after the row was
// inserted) must finalize the row as failed, never orphan it.
+553
View File
@@ -3453,6 +3453,559 @@ async fn agent_ref_unknown_is_rejected() {
);
}
// ── validate_inference_readiness (provider-connectivity author gate, B45) ──
//
// An `agent` node needs a working LLM inference provider the same way a
// `tool_call` node needs a real Composio connection — but no author-time gate
// previously checked it at all, so a signed-in user with no provider API key
// configured on the managed backend only found out mid-run. These tests never
// touch the network AND never install the process-global
// `test_provider_override` seam (which would race any other test in this
// binary that also installs it): the "construction succeeds" case points the
// role at a local runtime (`ollama:...`), which `resolves_to_managed_backend`
// correctly identifies as non-managed, so `probe_inference_readiness` never
// reaches for the network; the construction-error case is engineered to fail
// purely on a config lookup (`resolve_cloud_slug`'s "no cloud provider
// configured for slug" branch), before any HTTP client is built.
fn seed_app_session_for_gate_test(tmp: &TempDir) {
use crate::openhuman::credentials::{
AuthService, APP_SESSION_PROVIDER, DEFAULT_AUTH_PROFILE_NAME,
};
// `verify_session_active` reads from `config.config_path.parent()`, which
// `test_config` sets to `tmp.path()` itself (distinct from
// `tmp.path()/workspace`) — seed the session there.
AuthService::new(tmp.path(), false)
.store_provider_token(
APP_SESSION_PROVIDER,
DEFAULT_AUTH_PROFILE_NAME,
"test.session.jwt",
std::collections::HashMap::new(),
true,
)
.expect("seed app-session token");
}
#[tokio::test]
async fn inference_gate_skips_when_no_agent_nodes() {
// A tool_call-only graph never has an inference dependency to check — the
// gate must short-circuit to empty without touching sign-in state or the
// network at all.
let tmp = TempDir::new().unwrap();
let config = test_config(&tmp);
let g = graph(json!({
"nodes": [
{ "id": "t", "kind": "trigger", "name": "Manual" },
{ "id": "post", "kind": "tool_call", "name": "Post",
"config": { "slug": "SLACK_SEND_MESSAGE", "args": { "channel": "#general" } } }
],
"edges": [ { "from_node": "t", "to_node": "post" } ]
}));
let errors = validate_inference_readiness(&config, &g).await;
assert!(errors.is_empty(), "{errors:?}");
}
// B45 design correction (judge finding on live run 104aab90): the gate used
// to hard-reject `run_builder_gates` when signed out, which blocked
// `propose_workflow`/`edit_workflow` from ever showing the user the graph at
// all. Authoring must now succeed unconditionally; readiness only ever
// surfaces as an advisory `inference_status` on the proposal. These two tests
// replace the old `inference_gate_rejects_when_signed_out`, which asserted
// the opposite (a hard reject) of the now-correct contract.
#[tokio::test]
async fn run_builder_gates_does_not_reject_when_signed_out() {
// Authoring is never blocked by inference readiness (design correction,
// B45): a signed-out session must NOT appear among `run_builder_gates`'
// errors for an otherwise-valid agent-node graph.
let _signed_out = crate::openhuman::scheduler_gate::SignedOutTestGuard::set(true);
let tmp = TempDir::new().unwrap();
let config = test_config(&tmp);
let g = graph(json!({
"nodes": [
{ "id": "t", "kind": "trigger", "name": "Manual" },
{ "id": "a", "kind": "agent", "name": "Plan", "config": { "prompt": "outline it" } }
],
"edges": [ { "from_node": "t", "to_node": "a" } ]
}));
let errors = run_builder_gates(&config, &g).await;
assert!(
errors.is_empty(),
"authoring must not be blocked by a signed-out session: {errors:?}"
);
// `SignedOutTestGuard` restores the prior flag on drop at the end of this
// scope — no other test observes this override.
}
#[tokio::test]
async fn proposal_surfaces_signed_out_inference_status() {
// The proposal still WARNS about the signed-out state (advisory, never a
// rejection) so the UI can render a "sign in" nudge alongside the built
// workflow.
let _signed_out = crate::openhuman::scheduler_gate::SignedOutTestGuard::set(true);
let tmp = TempDir::new().unwrap();
let config = test_config(&tmp);
let g = graph(json!({
"nodes": [
{ "id": "t", "kind": "trigger", "name": "Manual" },
{ "id": "a", "kind": "agent", "name": "Plan", "config": { "prompt": "outline it" } }
],
"edges": [ { "from_node": "t", "to_node": "a" } ]
}));
let payload = build_builder_proposal(
&config,
"propose_workflow",
"agent-flow",
&g,
false,
false,
None,
None,
None,
)
.await
.expect("a signed-out session must NOT block proposing the graph");
assert_eq!(payload["inference_status"], json!("signed_out"));
let message = payload["inference_message"]
.as_str()
.expect("a non-ready status must carry inference_message");
assert!(
message.to_ascii_lowercase().contains("signed out"),
"message must tell the user they are signed out: {message}"
);
// `SignedOutTestGuard` restores the prior flag on drop at the end of this
// scope — no other test observes this override.
}
#[tokio::test]
async fn inference_gate_passes_when_model_constructs() {
// Layer 2 (async probe), happy path: the resolved role ("summarization" —
// the default for a plain agent node) points at a local runtime
// (`ollama:...`), which `probe_inference_readiness` never probes over the
// network at all — `resolves_to_managed_backend` is false for a local
// provider, so construction succeeding is the whole check (no HTTP, no
// process-global test seam, so this can never race another test that
// installs `test_provider_override`).
let tmp = TempDir::new().unwrap();
let mut config = test_config(&tmp);
config.memory_provider = Some("ollama:llama3".to_string());
let g = graph(json!({
"nodes": [
{ "id": "t", "kind": "trigger", "name": "Manual" },
{ "id": "a", "kind": "agent", "name": "Plan", "config": { "prompt": "outline it" } }
],
"edges": [ { "from_node": "t", "to_node": "a" } ]
}));
let errors = validate_inference_readiness(&config, &g).await;
assert!(errors.is_empty(), "{errors:?}");
}
#[tokio::test]
async fn inference_gate_surfaces_construction_error() {
// Layer 2 (async probe), construction-failure path: the resolved role
// ("summarization" — the default for a plain agent node with no pinned
// `config.model`) points at a cloud slug that isn't in `cloud_providers`
// at all, so `create_chat_model_with_model_id_inner` fails on a pure
// config lookup — no test override installed, no network involved — and
// the gate must surface that failure, naming the offending node.
let tmp = TempDir::new().unwrap();
let mut config = test_config(&tmp);
seed_app_session_for_gate_test(&tmp);
config.memory_provider = Some("no_such_slug:some-model".to_string());
let g = graph(json!({
"nodes": [
{ "id": "t", "kind": "trigger", "name": "Manual" },
{ "id": "a", "kind": "agent", "name": "Plan", "config": { "prompt": "outline it" } }
],
"edges": [ { "from_node": "t", "to_node": "a" } ]
}));
let errors = validate_inference_readiness(&config, &g).await;
assert!(!errors.is_empty(), "a construction failure must reject");
assert!(
errors.iter().any(|e| e.contains("Node 'a'")),
"error must name the offending node 'a': {errors:?}"
);
assert!(
errors
.iter()
.any(|e| e.contains("no_such_slug") || e.contains("no cloud provider configured")),
"error must surface the construction failure detail: {errors:?}"
);
}
// ── multi-role agent-node graphs (findings A+B, P1) ─────────────────────────
//
// Previously `evaluate_inference_readiness` collected every applicable
// `agent` node but derived the Layer-2 probe role from ONLY the graph's
// first node — a second (or later) node pinned to a different `config.model`
// (and therefore routed to a different, possibly broken, provider) was never
// probed at all. These tests wire each role to its own pure-config-lookup
// failure (no network, no test-provider-override seam) so a bug that skips a
// role would show up as a falsely-empty `errors` list.
#[test]
fn agent_node_role_prefers_custom_registry_entry_model_pin_over_default() {
// Finding A/B: a node with no per-node `config.model` but a STATIC
// (non-`=`) `agent_ref` naming a custom registry entry that itself pins a
// model (e.g. `hint:reasoning`) must resolve to THAT role — the same
// precedence `OpenHumanAgentRunner::run_via_harness` applies via
// `resolve_node_model(&request, entry_model)`, reusing the same sync,
// config-only accessor (`find_custom_in_config`) it calls.
use crate::openhuman::agent_registry::types::{AgentRegistryEntry, AgentRegistrySource};
let tmp = TempDir::new().unwrap();
let mut config = test_config(&tmp);
config.agent_registry.entries.push(AgentRegistryEntry {
id: "researcher_custom".to_string(),
name: "Researcher".to_string(),
description: "does research".to_string(),
source: AgentRegistrySource::Custom,
enabled: true,
model: Some("hint:reasoning".to_string()),
system_prompt: None,
tool_allowlist: Vec::new(),
tool_denylist: Vec::new(),
subagents: Default::default(),
tags: Vec::new(),
metadata: Value::Null,
});
let g = graph(json!({
"nodes": [
{ "id": "t", "kind": "trigger", "name": "Manual" },
{ "id": "a", "kind": "agent", "name": "Research",
"config": { "agent_ref": "researcher_custom", "prompt": "go" } }
],
"edges": [ { "from_node": "t", "to_node": "a" } ]
}));
let node = g.nodes.iter().find(|n| n.id == "a").expect("node 'a'");
assert_eq!(
agent_node_role(&config, node),
"reasoning",
"the custom registry entry's `hint:reasoning` pin must win over the default role"
);
}
#[tokio::test]
async fn inference_gate_probes_every_distinct_agent_node_role() {
// A graph with TWO `agent` nodes, each pinned (via `config.model`) to a
// DIFFERENT role — `chat` and `reasoning` — each wired to its own broken
// provider slug for that specific role's config knob
// (`chat_provider`/`reasoning_provider`). If the gate only probed the
// first node's role (the pre-fix bug), the second node's broken
// `reasoning` provider would never be checked and this graph would
// incorrectly pass. Both failures must be named.
let tmp = TempDir::new().unwrap();
let mut config = test_config(&tmp);
seed_app_session_for_gate_test(&tmp);
config.chat_provider = Some("no_such_chat_slug:some-model".to_string());
config.reasoning_provider = Some("no_such_reasoning_slug:some-model".to_string());
let g = graph(json!({
"nodes": [
{ "id": "t", "kind": "trigger", "name": "Manual" },
{ "id": "a", "kind": "agent", "name": "Chat step",
"config": { "prompt": "chat", "model": "chat-v1" } },
{ "id": "b", "kind": "agent", "name": "Reasoning step",
"config": { "prompt": "reason", "model": "reasoning-v1" } }
],
"edges": [
{ "from_node": "t", "to_node": "a" },
{ "from_node": "a", "to_node": "b" }
]
}));
let errors = validate_inference_readiness(&config, &g).await;
assert!(
!errors.is_empty(),
"both roles are broken, the gate must reject"
);
let combined = errors.join("\n");
assert!(
combined.contains("'a'") && combined.contains("no_such_chat_slug"),
"the `chat` role's failure (node 'a') must be named: {combined}"
);
assert!(
combined.contains("'b'") && combined.contains("no_such_reasoning_slug"),
"the `reasoning` role's failure (node 'b') must be named — this is the exact \
regression the pre-fix \"probe only the first node's role\" bug would have hidden: \
{combined}"
);
}
// ── dynamic agent_ref still gets the Layer-1 check (finding C, P2) ─────────
#[tokio::test]
async fn inference_gate_reports_signed_out_for_dynamic_agent_ref_only_graph() {
// Finding C: a graph whose only `agent` node has a DYNAMIC (`=`-derived)
// `agent_ref` still means "this graph runs inference" at run time — it
// must stay in scope for Layer 1 (signed-out/session), even though its
// exact per-model role can't be resolved statically. Previously the
// dynamic-ref filter excluded such nodes entirely, so a graph made up
// only of them returned `None` (no readiness signal at all) and a
// signed-out session went completely unreported.
let _signed_out = crate::openhuman::scheduler_gate::SignedOutTestGuard::set(true);
let tmp = TempDir::new().unwrap();
let config = test_config(&tmp);
let g = graph(json!({
"nodes": [
{ "id": "t", "kind": "trigger", "name": "Manual" },
{ "id": "a", "kind": "agent", "name": "Dynamic",
"config": { "agent_ref": "=nodes.t.item.agent_choice", "prompt": "go" } }
],
"edges": [ { "from_node": "t", "to_node": "a" } ]
}));
let errors = validate_inference_readiness(&config, &g).await;
assert!(
!errors.is_empty(),
"a signed-out session must still be reported even though the only agent node's \
agent_ref is dynamic: {errors:?}"
);
assert!(
errors
.iter()
.any(|e| e.to_ascii_lowercase().contains("signed out")),
"{errors:?}"
);
// `SignedOutTestGuard` restores the prior flag on drop at the end of this
// scope — no other test observes this override.
}
#[tokio::test]
async fn proposal_includes_inference_status_for_agent_graph() {
// `build_builder_proposal`'s payload carries the same inference-readiness
// evaluation, ADVISORY only (B45 design correction), so the UI can render
// provider-connectivity state alongside the built workflow. This pins the
// happy-path shape: a `"ready"` graph carries no `inference_message`. A
// local (`ollama:...`) provider construction is the pass path, matching
// `inference_gate_passes_when_model_constructs` — no network, no
// process-global test seam.
let tmp = TempDir::new().unwrap();
let mut config = test_config(&tmp);
config.memory_provider = Some("ollama:llama3".to_string());
let g = graph(json!({
"nodes": [
{ "id": "t", "kind": "trigger", "name": "Manual" },
{ "id": "a", "kind": "agent", "name": "Plan", "config": { "prompt": "outline it" } }
],
"edges": [ { "from_node": "t", "to_node": "a" } ]
}));
let payload = build_builder_proposal(
&config,
"propose_workflow",
"agent-flow",
&g,
false,
false,
None,
None,
None,
)
.await
.expect("proposal must succeed for a well-formed agent graph");
assert_eq!(payload["inference_status"], json!("ready"));
assert!(
payload.get("inference_message").is_none(),
"a ready status must omit inference_message: {payload:?}"
);
}
#[tokio::test]
async fn proposal_omits_inference_status_for_tool_call_only_graph() {
// A graph with no `agent` node has nothing for this check to evaluate —
// the field must be absent entirely, never a meaningless "ready".
let tmp = TempDir::new().unwrap();
let config = test_config(&tmp);
let g = graph(json!({
"nodes": [
{ "id": "t", "kind": "trigger", "name": "Manual" },
{ "id": "post", "kind": "tool_call", "name": "Post",
"config": { "slug": "oh:noop" } }
],
"edges": [ { "from_node": "t", "to_node": "post" } ]
}));
let payload = build_builder_proposal(
&config,
"propose_workflow",
"tool-flow",
&g,
false,
false,
None,
None,
None,
)
.await
.expect("proposal must succeed for a tool_call-only graph");
assert!(
payload.get("inference_status").is_none(),
"a graph with no agent node must omit inference_status: {payload:?}"
);
}
/// B45 run-time preflight (design correction, judge finding on live run
/// 104aab90): since authoring no longer hard-blocks on inference readiness, a
/// flow whose `agent` node cannot currently reach a working LLM provider can
/// be created and then RUN. `run_flow_body` must catch that BEFORE invoking
/// the tinyflows engine, finalizing the run row as `failed` with a clear,
/// actionable message rather than letting the engine attempt (and fail) real
/// work, or surface the opaque several-layers-deep "capability error: graph
/// error: capability error: model error: ... API key not configured for
/// provider" a mid-run failure produces.
///
/// Uses the signed-out seam (`SignedOutTestGuard`) rather than a mock
/// provider-not-configured backend response: both are classified `Err` by
/// `evaluate_inference_readiness` and reach the same preflight code path in
/// `run_flow_body`, and signed-out needs no network/mock server at all
/// (matching the existing gate tests' no-network convention). The
/// provider_not_configured class is covered end-to-end by
/// `probe_readiness_surfaces_api_key_not_configured` (construction) and the
/// negative-cache test below (through `cached_probe_inference_readiness`).
#[tokio::test]
async fn flows_run_fails_cleanly_without_invoking_engine_when_inference_not_ready() {
let _signed_out = crate::openhuman::scheduler_gate::SignedOutTestGuard::set(true);
let tmp = TempDir::new().unwrap();
let config = test_config(&tmp);
let g = json!({
"name": "needs-a-provider",
"nodes": [
{ "id": "t", "kind": "trigger", "name": "Manual" },
{ "id": "a", "kind": "agent", "name": "Plan", "config": { "prompt": "outline it" } }
],
"edges": [ { "from_node": "t", "to_node": "a" } ]
});
let created = flows_create(&config, "needs-a-provider".to_string(), g, false)
.await
.expect("creating (authoring) an agent-node flow must succeed even when signed out");
let err = flows_run(&config, &created.value.id, json!({}), FlowRunTrigger::Rpc)
.await
.expect_err("a run whose agent node cannot reach a provider must fail cleanly");
assert!(
err.to_ascii_lowercase().contains("ai provider"),
"error must explain the AI-provider problem: {err}"
);
assert!(
err.to_ascii_lowercase().contains("signed out"),
"error must surface the specific reason (signed out): {err}"
);
// The run row settled `failed` with that same message, and the engine
// never ran (no persisted steps) — this is the "no pointless work" half
// of the contract, not just "the RPC call returned an error".
let runs = flows_list_runs(&config, &created.value.id, 1)
.await
.unwrap()
.value;
let run = runs.first().expect("a run row must exist");
assert_eq!(run.status, "failed");
assert!(
run.steps.is_empty(),
"the engine must never have executed a step: {:?}",
run.steps
);
let run_error = run
.error
.as_deref()
.expect("a failed run must carry an error message");
assert!(
run_error.to_ascii_lowercase().contains("ai provider"),
"the persisted run error must explain the AI-provider problem: {run_error}"
);
// `SignedOutTestGuard` restores the prior flag on drop at the end of this
// scope — no other test observes this override.
}
/// The negative-probe cache (design correction, item 3): a definitive
/// `provider_not_configured` result must be served from cache within the TTL
/// exactly like a `"ready"` result, so an edit -> validate -> propose -> run
/// authoring/run burst hits the mock backend once, not once per call (the judge's
/// live run observed 4 network round trips in a single ~80s turn before this
/// fix). Uses a real local axum server (no real network) that counts requests
/// so a cache hit is provable, not just plausible.
#[tokio::test]
async fn cached_probe_inference_readiness_caches_a_negative_result() {
use std::sync::atomic::{AtomicUsize, Ordering};
let tmp = TempDir::new().unwrap();
let mut config = test_config(&tmp);
seed_app_session_for_gate_test(&tmp);
let hit_count = std::sync::Arc::new(AtomicUsize::new(0));
let counter = hit_count.clone();
let listener = tokio::net::TcpListener::bind("127.0.0.1:0")
.await
.expect("bind");
let addr = listener.local_addr().expect("local_addr");
let app = axum::Router::new().route(
"/openai/v1/chat/completions",
axum::routing::post(move || {
let counter = counter.clone();
async move {
counter.fetch_add(1, Ordering::SeqCst);
use axum::response::IntoResponse;
(
axum::http::StatusCode::BAD_REQUEST,
axum::Json(json!({
"success": false,
"error": "API key not configured for provider",
"errorCode": "BAD_REQUEST"
})),
)
.into_response()
}
}),
);
tokio::spawn(async move {
axum::serve(listener, app).await.expect("serve");
});
config.api_url = Some(format!("http://{addr}"));
// First call: a real (mock) network round trip, definitively rejected.
let first = cached_probe_inference_readiness("summarization", &config).await;
let err = first.expect_err("a confirmed provider-not-configured 400 must reject");
assert!(
err.to_ascii_lowercase()
.contains("api key not configured for provider"),
"error must surface the backend's own message: {err}"
);
assert_eq!(
hit_count.load(Ordering::SeqCst),
1,
"the first call must hit the (mock) network exactly once"
);
// Second call, same (role, config_path) key, well within the TTL: must be
// served from cache — the mock server's hit count must NOT increase.
let second = cached_probe_inference_readiness("summarization", &config).await;
assert!(
second.is_err(),
"the cached negative result must still be an Err"
);
assert_eq!(
hit_count.load(Ordering::SeqCst),
1,
"a repeat probe within the TTL must be served from cache, not hit the network again"
);
}
// ── validate_tool_contracts (systemic tool-contract fix, Part 2) ───────────
//
// The live-catalog cache is process-global (`LIVE_CATALOG_CACHE`) — every
+67 -1
View File
@@ -755,6 +755,66 @@ fn resolves_to_managed_backend(role: &str, config: &Config) -> bool {
resolved.trim() == PROVIDER_OPENHUMAN
}
/// Probe whether `role` can actually complete an inference call right now
/// (issue B45 — the flows provider-connectivity author gate).
///
/// Two-stage check, mirroring the two ways a `role` can be un-runnable:
///
/// 1. **Construction** — [`create_chat_model_with_model_id_inner`] must
/// succeed. This is the existing Layer 1 check (BYOK-incomplete config,
/// unknown provider slug, local-only privacy-mode block, …) reused
/// verbatim so this probe never re-implements it.
/// 2. **Managed-backend readiness** — when `role` resolves to the managed
/// OpenHuman backend, [`OpenHumanBackendModel::probe_readiness`] makes one
/// cheap real completion attempt to catch the "account has no provider API
/// key configured" class of failure that construction alone cannot see
/// (construction only builds the client; it never calls the backend).
/// BYOK/local models have no such hidden failure mode — their construction
/// step already validates what it can, so they return `Ok(())` here
/// unconditionally.
///
/// Respects the [`test_provider_override`] test seam: when a mock model is
/// installed, construction returns it immediately and this function returns
/// `Ok(())` without ever touching the network or resolving `role` again —
/// `resolves_to_managed_backend` is a pure config read that would otherwise
/// still call this "managed" in a test with a bare default `Config`.
pub async fn probe_inference_readiness(role: &str, config: &Config) -> Result<(), String> {
#[cfg(any(test, feature = "e2e-test-support", feature = "rss-bench"))]
if test_provider_override::current().is_some() {
log::debug!(
"[flows][inference-probe] role={role} test model override active — skipping probe"
);
return Ok(());
}
log::debug!("[flows][inference-probe] role={role} verifying model construction");
if let Err(e) = create_chat_model_with_model_id_inner(role, config) {
log::debug!("[flows][inference-probe] role={role} construction failed: {e}");
return Err(e.to_string());
}
if !resolves_to_managed_backend(role, config) {
log::debug!(
"[flows][inference-probe] role={role} resolves to a non-managed provider — \
construction succeeded, nothing further to probe"
);
return Ok(());
}
log::debug!(
"[flows][inference-probe] role={role} resolves to the managed OpenHuman backend — \
probing readiness"
);
let (managed_model, model_id) =
resolve_managed_backend(role, config).map_err(|e| e.to_string())?;
let result = managed_model.probe_readiness().await;
log::debug!(
"[flows][inference-probe] role={role} model={model_id} probe result: {}",
if result.is_ok() { "ready" } else { "not ready" }
);
result
}
/// Build an `Arc<dyn ChatModel>` from an explicit provider string and config.
///
/// The explicit-string counterpart of [`create_chat_model`].
@@ -1781,7 +1841,13 @@ pub(crate) fn create_local_chat_model_from_string(
/// entirely. This function ensures that custom providers (Ollama,
/// `<slug>:<model>`) are only reachable when the workspace holds a valid
/// `app-session` JWT.
fn verify_session_active(config: &Config) -> anyhow::Result<()> {
///
/// `pub(crate)`: also reused directly by the flows provider-connectivity
/// author gate (issue B45, `openhuman::flows::ops::evaluate_inference_readiness`)
/// as its Layer 1 sync session check, so the author-time gate and this
/// construction-time chokepoint can never diverge on what "session active"
/// means.
pub(crate) fn verify_session_active(config: &Config) -> anyhow::Result<()> {
// AgentBox marketplace containers run headless with no desktop
// `app-session` JWT — the deployment is operator-controlled and ships its
// own GMI MaaS credentials via `GMI_*` env vars. The session gate exists to
+2 -2
View File
@@ -39,8 +39,8 @@ pub use error_code::{
pub(crate) use factory::is_raw_passthrough_model;
pub use factory::{
create_chat_model, create_chat_model_from_string, create_chat_model_from_string_with_model_id,
create_chat_model_with_model_id, provider_for_role, role_for_model_tier,
BYOK_INCOMPLETE_SENTINEL,
create_chat_model_with_model_id, probe_inference_readiness, provider_for_role,
role_for_model_tier, BYOK_INCOMPLETE_SENTINEL,
};
pub use openhuman_backend_model::{OpenHumanBackendModel, PROVIDER_LABEL};
pub use ops::*;
@@ -26,9 +26,11 @@
use async_trait::async_trait;
use serde_json::Value;
use std::path::PathBuf;
use std::time::Duration;
use tinyagents::harness::message::Message;
use tinyagents::harness::model::{
ChatModel, Modalities, ModelProfile, ModelRequest, ModelResponse, ModelStream,
ChatModel, Modalities, ModelProfile, ModelRequest, ModelResponse, ModelStream, ProviderError,
};
use tinyagents::harness::providers::openai::OpenAiModel;
use tinyagents::{Result as TaResult, TinyAgentsError};
@@ -144,6 +146,137 @@ impl OpenHumanBackendModel {
.with_native_tool_calling(self.native_tool_calling),
)
}
/// Probe whether the managed backend account actually has a working
/// inference provider configured, cheaply and without inflating usage
/// (issue B45 — flows provider-connectivity author gate).
///
/// [`build_wire_model`](Self::build_wire_model) only resolves the session
/// JWT and builds the request client — it says nothing about whether the
/// account has a provider API key configured server-side. That only
/// surfaces on a real completion attempt, as an HTTP 400
/// `{"success":false,"error":"API key not configured for provider","errorCode":"BAD_REQUEST"}`.
/// Previously the first time a flows author found this out was mid-run,
/// deep inside a tinyflows `agent` node. This probe moves that discovery
/// to author time by issuing one minimal completion (`"ping"`,
/// `max_tokens: 1`) and classifying the result.
///
/// Fails OPEN on everything except a definitive client-configuration
/// error: a 5-second timeout, a transport failure, a 5xx, or any other
/// non-matching provider error all return `Ok(())` so a flaky backend or
/// slow network never blocks authoring. Only a backend-confirmed "no
/// provider configured for this account" response returns `Err` —
/// carrying the backend's own error string so the author sees exactly
/// what run time would have shown them.
pub async fn probe_readiness(&self) -> Result<(), String> {
log::debug!(
"[flows][inference-probe] entering probe_readiness model={}",
self.default_model
);
let model = match self.build_wire_model() {
Ok(model) => model,
Err(e) => {
// The flows readiness gate's Layer 1 (sign-in / session
// checks) is responsible for catching a genuinely
// absent/expired session before this ever runs — a
// construction failure reaching here is a race, not a
// provider-configuration problem, so fail open rather than
// duplicate or contradict that gate's message.
log::debug!(
"[flows][inference-probe] wire model construction failed, failing open: {e}"
);
return Ok(());
}
};
let request = ModelRequest::new(vec![Message::user("ping")]).with_max_tokens(1);
let outcome =
match tokio::time::timeout(Duration::from_secs(5), model.invoke(&(), request)).await {
Ok(result) => result,
Err(_) => {
log::debug!(
"[flows][inference-probe] model={} timed out after 5s, failing open",
self.default_model
);
return Ok(());
}
};
match outcome {
Ok(_) => {
log::debug!(
"[flows][inference-probe] model={} probe completion succeeded — provider ready",
self.default_model
);
Ok(())
}
Err(TinyAgentsError::Provider(err)) => {
if is_provider_not_configured_error(&err) {
log::warn!(
"[flows][inference-probe] model={} backend reports no provider configured: {}",
self.default_model,
err.message
);
Err(err.message.clone())
} else if err.status.is_some_and(|status| status >= 500) {
log::debug!(
"[flows][inference-probe] model={} backend {:?}, failing open: {}",
self.default_model,
err.status,
err.message
);
Ok(())
} else {
// Any other structured provider failure (401, 429, a
// malformed request, …) is not the definitive "provider
// not configured" signal this probe exists to catch —
// fail open rather than risk a false-positive
// author-time block.
log::debug!(
"[flows][inference-probe] model={} non-definitive provider error {:?}, \
failing open: {}",
self.default_model,
err.status,
err.message
);
Ok(())
}
}
Err(e) => {
log::debug!(
"[flows][inference-probe] model={} transport/model error, failing open: {e}",
self.default_model
);
Ok(())
}
}
}
}
/// Whether `err` is the definitive "no inference provider configured for this
/// account" signal the managed backend returns as an HTTP 400 with body
/// `{"success":false,"error":"API key not configured for provider","errorCode":"BAD_REQUEST"}`.
///
/// Deliberately narrow: matches ONLY a 400 whose message contains the specific
/// `"api key not configured for provider"` phrasing, or (as a `BAD_REQUEST`-
/// coded tolerance for message wording drift) the narrower `"not configured
/// for provider"` substring — never a bare `"not configured"`, which an
/// unrelated 400 (a malformed request naming some other unconfigured field,
/// a validation error, …) could also contain. Every other 4xx/5xx/transport
/// failure fails open (see [`OpenHumanBackendModel::probe_readiness`]'s doc).
fn is_provider_not_configured_error(err: &ProviderError) -> bool {
if err.status != Some(400) {
return false;
}
let message = err.message.to_ascii_lowercase();
let code_is_bad_request = err
.code
.as_deref()
.is_some_and(|c| c.eq_ignore_ascii_case("BAD_REQUEST"));
message.contains("api key not configured for provider")
|| (code_is_bad_request && message.contains("not configured for provider"))
}
fn resolve_model(model: &str) -> String {
@@ -450,4 +583,235 @@ mod tests {
assert_eq!(usage.charged_amount_usd, 0.0);
assert_eq!(usage.cached_input_tokens, 3, "crate cached count preserved");
}
// ── probe_readiness (B45 — flows provider-connectivity author gate) ────
#[test]
fn is_provider_not_configured_error_matches_exact_backend_shape() {
let err = ProviderError {
provider: "OpenHuman".to_string(),
model: None,
status: Some(400),
code: Some("BAD_REQUEST".to_string()),
message: "API key not configured for provider".to_string(),
retryable: false,
raw: None,
};
assert!(is_provider_not_configured_error(&err));
}
#[test]
fn is_provider_not_configured_error_rejects_other_400s() {
// A 400 that isn't the "no provider configured" class (e.g. a bad
// request shape) must NOT be classified as provider-not-configured —
// only the exact backend-confirmed signal should ever reject.
let err = ProviderError {
provider: "OpenHuman".to_string(),
model: None,
status: Some(400),
code: Some("BAD_REQUEST".to_string()),
message: "invalid request: messages must not be empty".to_string(),
retryable: false,
raw: None,
};
assert!(!is_provider_not_configured_error(&err));
}
#[test]
fn is_provider_not_configured_error_tolerates_not_configured_for_provider_wording_drift() {
// The `code_is_bad_request` branch still matches the narrower
// "not configured for provider" substring even when it isn't
// introduced by the exact "api key" prefix — tolerance for backend
// message wording drift, not a broadening to any "not configured".
let err = ProviderError {
provider: "OpenHuman".to_string(),
model: None,
status: Some(400),
code: Some("BAD_REQUEST".to_string()),
message: "credentials not configured for provider 'anthropic'".to_string(),
retryable: false,
raw: None,
};
assert!(is_provider_not_configured_error(&err));
}
#[test]
fn is_provider_not_configured_error_rejects_generic_not_configured_400() {
// Tightened contract (finding D): a 400 `BAD_REQUEST` whose message
// contains only the generic word "not configured" — but not the
// specific "not configured for provider" phrasing — must fail OPEN,
// not be misclassified as the provider-key signal. Otherwise an
// unrelated backend validation error ("model X not configured", "this
// feature is not configured for your account", …) would falsely
// reject a run/proposal as a provider problem.
let err = ProviderError {
provider: "OpenHuman".to_string(),
model: None,
status: Some(400),
code: Some("BAD_REQUEST".to_string()),
message: "webhook target not configured".to_string(),
retryable: false,
raw: None,
};
assert!(!is_provider_not_configured_error(&err));
}
#[test]
fn is_provider_not_configured_error_rejects_non_400_status() {
let err = ProviderError {
provider: "OpenHuman".to_string(),
model: None,
status: Some(401),
code: None,
message: "API key not configured for provider".to_string(),
retryable: false,
raw: None,
};
assert!(!is_provider_not_configured_error(&err));
}
fn seed_app_session(dir: &std::path::Path) {
use crate::openhuman::credentials::{
AuthService, APP_SESSION_PROVIDER, DEFAULT_AUTH_PROFILE_NAME,
};
AuthService::new(dir, false)
.store_provider_token(
APP_SESSION_PROVIDER,
DEFAULT_AUTH_PROFILE_NAME,
"test.session.jwt",
std::collections::HashMap::new(),
true,
)
.expect("seed app-session token");
}
fn backend_pointed_at(addr: &str, dir: &std::path::Path) -> OpenHumanBackendModel {
OpenHumanBackendModel::new(
Some(&format!("http://{addr}")),
&ProviderRuntimeOptions {
openhuman_dir: Some(dir.to_path_buf()),
secrets_encrypt: false,
..ProviderRuntimeOptions::default()
},
"reasoning-v1",
)
}
#[derive(Clone)]
struct StaticChatResponse {
status: axum::http::StatusCode,
body: Value,
}
async fn static_chat_handler(
axum::extract::State(s): axum::extract::State<StaticChatResponse>,
) -> axum::response::Response {
use axum::response::IntoResponse;
(s.status, axum::Json(s.body)).into_response()
}
async fn spawn_static_chat_server(status: axum::http::StatusCode, body: Value) -> String {
let listener = tokio::net::TcpListener::bind("127.0.0.1:0")
.await
.expect("bind");
let addr = listener.local_addr().expect("local_addr");
let app = axum::Router::new()
.route(
"/openai/v1/chat/completions",
axum::routing::post(static_chat_handler),
)
.with_state(StaticChatResponse { status, body });
tokio::spawn(async move {
axum::serve(listener, app).await.expect("serve");
});
addr.to_string()
}
async fn slow_chat_handler() -> axum::response::Response {
use axum::response::IntoResponse;
// Longer than the probe's 5s timeout — the probe must return before
// this ever resolves.
tokio::time::sleep(Duration::from_secs(8)).await;
(
axum::http::StatusCode::OK,
axum::Json(serde_json::json!({
"choices": [{ "message": { "role": "assistant", "content": "pong" } }]
})),
)
.into_response()
}
async fn spawn_slow_chat_server() -> String {
let listener = tokio::net::TcpListener::bind("127.0.0.1:0")
.await
.expect("bind");
let addr = listener.local_addr().expect("local_addr");
let app = axum::Router::new().route(
"/openai/v1/chat/completions",
axum::routing::post(slow_chat_handler),
);
tokio::spawn(async move {
axum::serve(listener, app).await.expect("serve");
});
addr.to_string()
}
#[tokio::test]
async fn probe_readiness_surfaces_api_key_not_configured() {
let tmp = tempfile::TempDir::new().unwrap();
seed_app_session(tmp.path());
let addr = spawn_static_chat_server(
axum::http::StatusCode::BAD_REQUEST,
serde_json::json!({
"success": false,
"error": "API key not configured for provider",
"errorCode": "BAD_REQUEST"
}),
)
.await;
let backend = backend_pointed_at(&addr, tmp.path());
let err = backend
.probe_readiness()
.await
.expect_err("a confirmed provider-not-configured 400 must reject");
assert!(
err.to_ascii_lowercase()
.contains("api key not configured for provider"),
"error must surface the backend's own message: {err}"
);
}
#[tokio::test]
async fn probe_readiness_fails_open_on_timeout_or_5xx() {
// 5xx sub-case: a transient backend failure must never block authoring.
let tmp = tempfile::TempDir::new().unwrap();
seed_app_session(tmp.path());
let addr = spawn_static_chat_server(
axum::http::StatusCode::SERVICE_UNAVAILABLE,
serde_json::json!({ "error": "temporarily unavailable" }),
)
.await;
let backend = backend_pointed_at(&addr, tmp.path());
backend
.probe_readiness()
.await
.expect("a transient 5xx must fail open (Ok)");
// Timeout sub-case: a hung backend must fail open once the 5s probe
// timeout fires, without waiting for the slow handler to respond.
let tmp2 = tempfile::TempDir::new().unwrap();
seed_app_session(tmp2.path());
let addr2 = spawn_slow_chat_server().await;
let backend2 = backend_pointed_at(&addr2, tmp2.path());
let started = std::time::Instant::now();
backend2
.probe_readiness()
.await
.expect("a hung backend must fail open (Ok) once the 5s timeout fires");
assert!(
started.elapsed() < Duration::from_secs(7),
"probe must return around the 5s timeout, not wait for the slow handler"
);
}
}