From b489ead52e552197fb541aea429fa760c0950a9d Mon Sep 17 00:00:00 2001 From: Cyrus Gray <144336577+graycyrus@users.noreply.github.com> Date: Fri, 17 Jul 2026 17:23:18 +0530 Subject: [PATCH] =?UTF-8?q?fix(flows):=20builder=20chat=20hygiene=20?= =?UTF-8?q?=E2=80=94=20run-tool=20honesty=20(Bld=20#4)=20+=20reply=20hygie?= =?UTF-8?q?ne=20(#5007)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../agents/workflow_builder/builder_prompt.rs | 152 +++++++++++++++++- .../flows/agents/workflow_builder/prompt.md | 37 ++++- 2 files changed, 178 insertions(+), 11 deletions(-) diff --git a/src/openhuman/flows/agents/workflow_builder/builder_prompt.rs b/src/openhuman/flows/agents/workflow_builder/builder_prompt.rs index 432104917..b6a2d0f3b 100644 --- a/src/openhuman/flows/agents/workflow_builder/builder_prompt.rs +++ b/src/openhuman/flows/agents/workflow_builder/builder_prompt.rs @@ -97,8 +97,10 @@ const DIRECTIVE_PROPOSE: &str = const DIRECTIVE_REVISE: &str = "Revise this tinyflows automation and return the revised proposal. Do not save \ unless I explicitly ask you to (when I do, use save_workflow on the saved flow id), and never enable or \ - disable anything. You may run_flow the SAVED flow to test it, but ONLY if I ask and only after you \ - confirm with me first."; + disable anything. If I ask you to run/test the SAVED flow, follow the run_flow capability rule from \ + your standing instructions: only run_flow it if that tool is on your belt and only after you confirm \ + with me first; if it isn't on your belt, point me to the Run control in the Workflows UI instead of \ + offering."; const DIRECTIVE_BUILD_PROPOSE_ONLY: &str = "Build this tinyflows automation END-TO-END and return the workflow \ proposal. The flow already exists (created blank just now) — design the graph and verify it with \ @@ -135,8 +137,9 @@ pub fn render_prompt(req: &BuilderRequest) -> String { if let Some(flow_id) = req.flow_id.as_deref().filter(|s| !s.is_empty()) { lines.push(String::new()); lines.push(format!( - "This workflow is saved with flow id `{flow_id}` — if I ask you to run/test it, you \ - may run_flow that id, but confirm with me first." + "This workflow is saved with flow id `{flow_id}` — if I ask you to run/test it, follow \ + the run_flow capability rule: only run_flow that id if the tool is on your belt and \ + I've confirmed first; otherwise point me to the Run control in the Workflows UI." )); } lines.push(String::new()); @@ -248,6 +251,46 @@ mod tests { assert!(p.contains("add a Slack step")); } + #[test] + fn revise_run_guidance_is_capability_conditional() { + // Regression: the revise-turn directive (and its per-turn flow_id + // note) used to unconditionally assert "you may run_flow" — + // contradicting the standing prompt's capability check (Bld §4), + // which hides run_flow/resume_flow_run/cancel_flow_run on the + // flows_build path (`FLOWS_BUILD_HIDDEN_TOOLS`). Because the + // per-turn brief is appended AFTER the standing prompt, an + // unconditional per-turn assertion would override the standing + // prompt's capability check and reproduce the offer-then-refuse bug + // the standing-prompt fix was meant to close. Both the mode-level + // directive and the flow_id-specific note must defer to the + // capability rule instead of asserting the tool is available. + let mut r = req(BuildMode::Revise); + r.flow_id = Some("flow_77".into()); + let p = render_prompt(&r); + + assert!( + p.contains("run_flow capability rule"), + "revise directive must defer to the run_flow capability rule rather than \ + assert the tool is available" + ); + assert!( + p.contains("Run control in the Workflows UI"), + "revise directive must point to the Workflows UI Run control as the \ + off-the-belt fallback" + ); + + for banned in [ + "You may run_flow the SAVED flow to test it, but ONLY if I ask", + "may run_flow that id, but confirm with me first.", + ] { + assert!( + !p.contains(banned), + "revise directive must not carry the stale unconditional run_flow \ + phrasing `{banned}`" + ); + } + } + #[test] fn build_is_propose_only_and_injects_flow_id_as_context() { // Regression for #4596: the instant-create build turn must NOT @@ -359,6 +402,81 @@ mod tests { created flows are always born disabled (issue #6)" ); + // Positive (Bld §4): run guidance is capability-conditional. `run_flow` + // (and resume/cancel) are hidden on the `flows_build` path, so the + // prompt must NOT unconditionally claim the builder can run a flow — + // it must first check whether the tool is on its belt and, when it is + // not, point the user to the Workflows UI Run control instead of + // offering-then-refusing (the confusing "want me to run it?" → "I + // don't have access" behavior). + assert!( + STANDING_PROMPT.contains("only if the tool is on your belt") + && STANDING_PROMPT.contains("never offer to run the flow") + && STANDING_PROMPT.contains("Workflows UI"), + "standing prompt must make run_flow capability-conditional: never offer to run \ + when the tool is off the belt, and point the user to the Workflows UI Run \ + control instead (Bld §4 offer-then-refuse)" + ); + + // Negative: the pre-fix heading ("ask first!") asserted run_flow was + // simply a confirm-before-use tool, with no capability check at all — + // it must not reappear (that's the exact offer-then-refuse regression + // Bld §4 closed). + assert!( + !STANDING_PROMPT.contains("`run_flow` (ask first!)"), + "standing prompt must not regress to the pre-Bld-§4 unconditional \ + \"ask first!\" run_flow heading" + ); + + // Positive: the run_flow section must explicitly gate the real-run + // instructions behind the capability check, not just mention the + // check somewhere else in the doc — bind the assertion to the two + // halves of the actual contract (off-belt fallback, on-belt usage). + assert!( + STANDING_PROMPT + .contains("If you do **not** have a `run_flow` tool, never offer to run the flow"), + "standing prompt must state the off-belt fallback as a direct consequence \ + of the capability check, not a generic nearby mention" + ); + assert!( + STANDING_PROMPT + .contains("If you **do** have `run_flow`: once the user has **saved** a flow"), + "standing prompt must gate the on-belt run_flow usage behind the same \ + capability check" + ); + + // Positive (CodeRabbit follow-up on Bld §4): `resume_flow_run` / + // `cancel_flow_run` get the identical capability-conditional + // treatment as `run_flow` — both are hidden alongside it on the + // `flows_build` path (`FLOWS_BUILD_HIDDEN_TOOLS`), so a fix that only + // gated `run_flow` while leaving these two unconditional would + // reopen the same offer-then-refuse bug one hop later. + assert!( + STANDING_PROMPT + .contains("those tools are on your belt** — `resume_flow_run` (approval-gated) or"), + "standing prompt must gate resume_flow_run/cancel_flow_run behind the \ + same on-your-belt capability check as run_flow" + ); + assert!( + STANDING_PROMPT.contains("(if they're not available, point the"), + "standing prompt must state the resume/cancel off-belt fallback condition" + ); + assert!( + STANDING_PROMPT + .contains("user to the runs list in the Workflows UI instead of offering)."), + "standing prompt must point resume/cancel's off-belt fallback to the \ + Workflows UI runs list, matching run_flow's UI fallback pattern" + ); + + // Negative: the pre-fix wording offered resume/cancel unconditionally + // right after `edit_workflow`, with no capability check in between — + // must not reappear. + assert!( + !STANDING_PROMPT.contains("patch with `edit_workflow`; `resume_flow_run`"), + "standing prompt must not regress to the pre-fix unconditional \ + resume_flow_run/cancel_flow_run offer" + ); + // Positive: self-DM resolution — the prompt must teach the builder to // wire "DM me" onto the connection's own `platform_user_id`, not a // public channel (the #general/#team-product fallback bug). @@ -464,6 +582,32 @@ mod tests { } } + /// The standing prompt must teach reply hygiene: no deliberation + /// narration, no draft-then-restate, lead with substance. Without these + /// the reasoning-tier model narrates its chain of thought in the visible + /// reply ("let me think… actually wait… let me reconsider") and restates + /// its questions twice in the same message. (The harness already keeps + /// real reasoning blocks out of the visible text — this is the model + /// choosing to narrate in its output, so a prompt rule is the fix.) + #[test] + fn standing_prompt_teaches_reply_hygiene() { + const STANDING_PROMPT: &str = include_str!("prompt.md"); + + for rule in [ + "finished reply", + "No deliberation narration", + "No draft-then-restate", + "Lead with substance", + ] { + assert!( + STANDING_PROMPT.contains(rule), + "standing prompt must teach the reply-hygiene rule `{rule}` — the \ + reply is the finished answer, not a thinking scratchpad (no \ + deliberation narration, no draft-then-restate)" + ); + } + } + #[test] fn repair_includes_run_id_error_and_failing_nodes() { let mut r = req(BuildMode::Repair); diff --git a/src/openhuman/flows/agents/workflow_builder/prompt.md b/src/openhuman/flows/agents/workflow_builder/prompt.md index 79076686f..ff8a6fc30 100644 --- a/src/openhuman/flows/agents/workflow_builder/prompt.md +++ b/src/openhuman/flows/agents/workflow_builder/prompt.md @@ -71,12 +71,20 @@ that already exists (creating one is `create_workflow`'s job, not auto-disable the flow if the graph's trigger just transitioned from manual to automatic on an already-enabled flow; say so if it happens. -## Testing a saved flow: `run_flow` (ask first!) +## Testing a saved flow: `run_flow` (only if the tool is on your belt) -Once the user has **saved** a flow, you can `run_flow { flow_id }` to test it -end-to-end. Unlike `dry_run_workflow`, this is a **real run** — real effects can -fire (the flow's own approval gate still pauses outbound-action nodes, but treat -it as real). Rules: +**First check whether `run_flow` is in your available tools — on some surfaces it +is not.** If you do **not** have a `run_flow` tool, never offer to run the flow +yourself and never say you'll run it: instead tell the user they can run it +themselves from the **Run** control on the flow in the Workflows UI (or by +triggering it however it's configured). The one thing to avoid is offering to run +it and then saying you can't — if you can't run it, don't offer; point to the Run +control up front. + +If you **do** have `run_flow`: once the user has **saved** a flow, you can +`run_flow { flow_id }` to test it end-to-end. Unlike `dry_run_workflow`, this is a +**real run** — real effects can fire (the flow's own approval gate still pauses +outbound-action nodes, but treat it as real). Rules: 1. **Only a saved flow.** `run_flow` needs a `flow_id`; if the graph isn't saved yet, save it first (`save_workflow` when you have the flow id, @@ -173,8 +181,10 @@ You have a machine-readable belt; use it instead of relying on memory: already connected — prefer those; the proposal's `required_connections` enumerates what still needs linking. - **Debug a run:** `list_flow_runs { flow_id }` → find a failing run; - `get_flow_run` → diagnose it; patch with `edit_workflow`; `resume_flow_run` - (approval-gated) or `cancel_flow_run` to progress/stop a run. `get_flow_history` + `get_flow_run` → diagnose it; patch with `edit_workflow`; and — **only if + those tools are on your belt** — `resume_flow_run` (approval-gated) or + `cancel_flow_run` to progress/stop a run (if they're not available, point the + user to the runs list in the Workflows UI instead of offering). `get_flow_history` → prior graph snapshots. - **Persist (only when the user explicitly asks):** `create_workflow` makes a NEW flow (always born disabled); `duplicate_flow` clones one (disabled) for @@ -601,6 +611,19 @@ needs zero questions is still the happy path. Don't let "ask when truly unsure" turn into "ask about everything": most requests carry enough signal to build immediately. +### Reply hygiene + +Every message you send is the **finished reply**, not a thinking scratchpad. + +- **No deliberation narration.** Never write "let me think", "actually wait", + "let me reconsider", "actually, I have several questions", "hold on", or any + stream-of-consciousness preamble. Decide what to say, then say it. +- **No draft-then-restate.** State your questions or your answer exactly once. + Never write a set of questions and then rewrite the same questions "more + concisely" in the same message. +- **Lead with substance.** Open with the answer, the proposal summary, or the + clarifying question — never with a narration of your own reasoning process. + ### The ask-vs-just-build rule Once `get_tool_contract` hands you a node's `required_args`, sort each one