From bcc95373209c2f7c129fe5a0e1df101de7de78da Mon Sep 17 00:00:00 2001 From: Steven Enamakel <31011319+senamakel@users.noreply.github.com> Date: Mon, 6 Jul 2026 00:37:01 -0700 Subject: [PATCH] =?UTF-8?q?feat(flows):=20prompt=20bar=20=E2=86=92=20insta?= =?UTF-8?q?nt=20canvas=20with=20backend-driven=20copilot=20builder=20(#457?= =?UTF-8?q?8)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/src-tauri/Cargo.lock | 1 + .../flows/SuggestedWorkflows.test.tsx | 5 +- .../components/flows/SuggestedWorkflows.tsx | 3 +- .../flows/WorkflowCopilotPanel.test.tsx | 102 +++- .../components/flows/WorkflowCopilotPanel.tsx | 122 ++++- .../flows/WorkflowPromptBar.test.tsx | 88 ++- .../components/flows/WorkflowPromptBar.tsx | 67 ++- .../flows/canvas/AgentNodeInspector.tsx | 176 ++++++ .../__tests__/AgentNodeInspector.test.tsx | 78 +++ .../__tests__/nodeConfigForms.test.tsx | 34 +- .../canvas/nodeConfig/nodeConfigForms.tsx | 13 +- app/src/hooks/useWorkflowBuilderChat.test.ts | 137 ++++- app/src/hooks/useWorkflowBuilderChat.ts | 149 +++-- app/src/lib/flows/newFlow.test.ts | 27 +- app/src/lib/flows/newFlow.ts | 17 + app/src/lib/flows/templates/index.ts | 2 + .../flows/templates/opus-sonnet-brief.json | 62 +++ .../lib/flows/workflowBuilderPrompt.test.ts | 52 -- app/src/lib/flows/workflowBuilderPrompt.ts | 121 ----- app/src/lib/i18n/ar.ts | 7 + app/src/lib/i18n/bn.ts | 9 + app/src/lib/i18n/de.ts | 8 + app/src/lib/i18n/en.ts | 8 + app/src/lib/i18n/es.ts | 9 + app/src/lib/i18n/fr.ts | 8 + app/src/lib/i18n/hi.ts | 9 + app/src/lib/i18n/id.ts | 8 + app/src/lib/i18n/it.ts | 8 + app/src/lib/i18n/ko.ts | 8 + app/src/lib/i18n/pl.ts | 8 + app/src/lib/i18n/pt.ts | 8 + app/src/lib/i18n/ru.ts | 9 + app/src/lib/i18n/zh-CN.ts | 7 + app/src/pages/FlowCanvasPage.tsx | 38 +- .../pages/__tests__/FlowCanvasPage.test.tsx | 67 ++- app/src/services/api/flowsApi.test.ts | 71 +++ app/src/services/api/flowsApi.ts | 144 ++++- gitbooks/SUMMARY.md | 1 + .../architecture/flows-on-tinyagents.md | 347 ++++++++++++ gitbooks/features/workflows.md | 4 +- scripts/live-flows-demo.sh | 43 ++ src/openhuman/agent_registry/agents/loader.rs | 35 +- src/openhuman/agent_registry/agents/mod.rs | 2 - .../agents/workflow_builder/mod.rs | 1 - .../agents/flow_discovery/agent.toml | 0 .../agents/flow_discovery/mod.rs | 0 .../agents/flow_discovery/prompt.md | 0 .../agents/flow_discovery/prompt.rs | 0 src/openhuman/flows/agents/mod.rs | 18 + .../agents/workflow_builder/agent.toml | 24 +- .../agents/workflow_builder/builder_prompt.rs | 305 +++++++++++ .../flows/agents/workflow_builder/mod.rs | 2 + .../agents/workflow_builder/prompt.md | 46 +- .../agents/workflow_builder/prompt.rs | 0 src/openhuman/flows/builder_tools.rs | 257 ++++++++- src/openhuman/flows/builder_tools_tests.rs | 139 +++++ src/openhuman/flows/mod.rs | 1 + src/openhuman/flows/ops.rs | 349 +++++++++++- src/openhuman/flows/ops_tests.rs | 40 ++ src/openhuman/flows/schemas.rs | 210 ++++++- src/openhuman/inference/provider/factory.rs | 46 ++ .../inference/provider/factory_tests.rs | 33 ++ src/openhuman/inference/provider/mod.rs | 4 +- src/openhuman/mcp_server/resources.rs | 4 +- src/openhuman/tinyflows/caps.rs | 512 +++++++++++++++++- src/openhuman/tinyflows/tests.rs | 146 +++++ src/openhuman/tools/ops.rs | 10 + tests/json_rpc_e2e.rs | 507 +++++++++++++++++ tests/live_flows_demo_e2e.rs | 401 ++++++++++++++ vendor/tinyflows | 2 +- 70 files changed, 4742 insertions(+), 437 deletions(-) create mode 100644 app/src/components/flows/canvas/AgentNodeInspector.tsx create mode 100644 app/src/components/flows/canvas/__tests__/AgentNodeInspector.test.tsx create mode 100644 app/src/lib/flows/templates/opus-sonnet-brief.json delete mode 100644 app/src/lib/flows/workflowBuilderPrompt.test.ts delete mode 100644 app/src/lib/flows/workflowBuilderPrompt.ts create mode 100644 gitbooks/developing/architecture/flows-on-tinyagents.md create mode 100755 scripts/live-flows-demo.sh delete mode 100644 src/openhuman/agent_registry/agents/workflow_builder/mod.rs rename src/openhuman/{agent_registry => flows}/agents/flow_discovery/agent.toml (100%) rename src/openhuman/{agent_registry => flows}/agents/flow_discovery/mod.rs (100%) rename src/openhuman/{agent_registry => flows}/agents/flow_discovery/prompt.md (100%) rename src/openhuman/{agent_registry => flows}/agents/flow_discovery/prompt.rs (100%) create mode 100644 src/openhuman/flows/agents/mod.rs rename src/openhuman/{agent_registry => flows}/agents/workflow_builder/agent.toml (55%) create mode 100644 src/openhuman/flows/agents/workflow_builder/builder_prompt.rs create mode 100644 src/openhuman/flows/agents/workflow_builder/mod.rs rename src/openhuman/{agent_registry => flows}/agents/workflow_builder/prompt.md (85%) rename src/openhuman/{agent_registry => flows}/agents/workflow_builder/prompt.rs (100%) create mode 100644 tests/live_flows_demo_e2e.rs diff --git a/app/src-tauri/Cargo.lock b/app/src-tauri/Cargo.lock index f6d13faf5..915c7f27b 100644 --- a/app/src-tauri/Cargo.lock +++ b/app/src-tauri/Cargo.lock @@ -5584,6 +5584,7 @@ dependencies = [ "bitcoin", "block2 0.6.2", "bs58", + "bytes", "chacha20poly1305", "chrono", "chrono-tz", diff --git a/app/src/components/flows/SuggestedWorkflows.test.tsx b/app/src/components/flows/SuggestedWorkflows.test.tsx index 6efa246de..948935c4e 100644 --- a/app/src/components/flows/SuggestedWorkflows.test.tsx +++ b/app/src/components/flows/SuggestedWorkflows.test.tsx @@ -121,7 +121,10 @@ describe('SuggestedWorkflows', () => { await waitFor(() => expect(hookState.send).toHaveBeenCalledTimes(1)); const arg = hookState.send.mock.calls[0][0]; expect(arg.displayText).toBe('Auto-file receipts'); - expect(arg.prompt).toContain('Build a workflow that files receipts.'); + // The build_prompt is now the instruction of a structured create request; + // the server renders the agent brief from it. + expect(arg.request.mode).toBe('create'); + expect(arg.request.instruction).toBe('Build a workflow that files receipts.'); }); it('marks the suggestion built when the inline proposal is saved', async () => { diff --git a/app/src/components/flows/SuggestedWorkflows.tsx b/app/src/components/flows/SuggestedWorkflows.tsx index bf52035a7..d1e06682f 100644 --- a/app/src/components/flows/SuggestedWorkflows.tsx +++ b/app/src/components/flows/SuggestedWorkflows.tsx @@ -22,7 +22,6 @@ import createDebug from 'debug'; import { useCallback, useEffect, useState } from 'react'; import { useWorkflowBuilderChat } from '../../hooks/useWorkflowBuilderChat'; -import { buildCreatePrompt } from '../../lib/flows/workflowBuilderPrompt'; import { useT } from '../../lib/i18n/I18nContext'; import { discoverWorkflows, @@ -155,7 +154,7 @@ export default function SuggestedWorkflows() { setBuildingId(suggestion.id); await send({ displayText: suggestion.title, - prompt: buildCreatePrompt(suggestion.build_prompt), + request: { mode: 'create', instruction: suggestion.build_prompt }, }); }, [sending, send] diff --git a/app/src/components/flows/WorkflowCopilotPanel.test.tsx b/app/src/components/flows/WorkflowCopilotPanel.test.tsx index b1d01608e..6318e6fc4 100644 --- a/app/src/components/flows/WorkflowCopilotPanel.test.tsx +++ b/app/src/components/flows/WorkflowCopilotPanel.test.tsx @@ -2,7 +2,7 @@ import { fireEvent, render, screen } from '@testing-library/react'; import { beforeEach, describe, expect, it, vi } from 'vitest'; import type { WorkflowGraph, WorkflowNode } from '../../lib/flows/types'; -import type { WorkflowProposal } from '../../store/chatRuntimeSlice'; +import type { ToolTimelineEntry, WorkflowProposal } from '../../store/chatRuntimeSlice'; import WorkflowCopilotPanel from './WorkflowCopilotPanel'; vi.mock('../../lib/i18n/I18nContext', () => ({ useT: () => ({ t: (key: string) => key }) })); @@ -11,6 +11,8 @@ const hookState = vi.hoisted(() => ({ sending: false, proposal: null as WorkflowProposal | null, messages: [] as Array<{ id: string; content: string; sender: 'user' | 'agent' }>, + toolTimeline: [] as ToolTimelineEntry[], + liveResponse: '', error: null as string | null, send: vi.fn(), clearProposal: vi.fn(), @@ -40,6 +42,8 @@ describe('WorkflowCopilotPanel', () => { hookState.sending = false; hookState.proposal = null; hookState.messages = []; + hookState.toolTimeline = []; + hookState.liveResponse = ''; hookState.error = null; hookState.send = vi.fn().mockResolvedValue(undefined); hookState.clearProposal = vi.fn(); @@ -65,7 +69,11 @@ describe('WorkflowCopilotPanel', () => { expect(hookState.send).toHaveBeenCalledTimes(1); const arg = hookState.send.mock.calls[0][0]; expect(arg.displayText).toBe('add a Slack notification on failure'); - expect(arg.prompt).toContain(JSON.stringify(baseGraph)); + // The brief is rendered server-side now; the panel sends a structured + // revise request carrying the current graph as context. + expect(arg.request.mode).toBe('revise'); + expect(arg.request.instruction).toBe('add a Slack notification on failure'); + expect(arg.request.graph).toEqual(baseGraph); }); it('renders the conversation transcript (user + agent turns)', () => { @@ -90,6 +98,49 @@ describe('WorkflowCopilotPanel', () => { expect(screen.queryByTestId('workflow-copilot-empty')).not.toBeInTheDocument(); }); + it('renders the shared tool timeline + streaming reply during a builder turn', () => { + hookState.sending = true; + hookState.toolTimeline = [ + { id: 'call-1', name: 'propose_workflow', round: 0, status: 'running' } as ToolTimelineEntry, + ]; + hookState.liveResponse = 'Drafting your workflow…'; + render( + + ); + // The shared ToolTimelineBlock renders (not the bespoke transcript), and the + // one-shot "thinking" placeholder is suppressed once activity is streaming. + expect(screen.getByTestId('workflow-copilot-timeline')).toBeInTheDocument(); + expect(screen.queryByTestId('workflow-copilot-thinking')).not.toBeInTheDocument(); + }); + + it('shows the live reply as a bubble before the first tool call streams', () => { + hookState.sending = true; + hookState.toolTimeline = []; + hookState.liveResponse = 'Thinking about your Slack digest…'; + render( + + ); + expect(screen.getByTestId('workflow-copilot-streaming')).toHaveTextContent( + 'Thinking about your Slack digest…' + ); + // No tool timeline yet, and the plain "thinking" line is replaced by the + // streamed text. + expect(screen.queryByTestId('workflow-copilot-timeline')).not.toBeInTheDocument(); + expect(screen.queryByTestId('workflow-copilot-thinking')).not.toBeInTheDocument(); + }); + it('surfaces a new proposal to the host and shows the added/removed diff', () => { const onProposal = vi.fn(); // proposed drops "b" and adds "c" vs. base [a, b]. @@ -158,7 +209,50 @@ describe('WorkflowCopilotPanel', () => { ); expect(hookState.send).toHaveBeenCalledTimes(1); const arg = hookState.send.mock.calls[0][0]; - expect(arg.prompt).toContain('run-7'); - expect(arg.prompt).toContain('get_flow_run'); + expect(arg.request.mode).toBe('repair'); + expect(arg.request.runId).toBe('run-7'); + expect(arg.request.error).toBe('boom'); + expect(arg.request.graph).toEqual(baseGraph); + }); + + it('auto-sends a build turn once when opened with a prompt-bar build seed', () => { + const { rerender } = render( + + ); + expect(hookState.send).toHaveBeenCalledTimes(1); + const arg = hookState.send.mock.calls[0][0]; + // The user's description reads as their own first turn in the transcript, + // while the real prompt injects the blank graph + flow id and asks for the + // full build → dry-run → save arc onto the already-created flow. + expect(arg.displayText).toBe('digest my Slack every morning'); + // The user's description reads as their own first turn; the structured + // build request carries the blank graph + flow id so the server's brief + // asks for the full build → dry-run → save arc onto the created flow. + expect(arg.request.mode).toBe('build'); + expect(arg.request.instruction).toBe('digest my Slack every morning'); + expect(arg.request.graph).toEqual(baseGraph); + expect(arg.request.flowId).toBe('flow-1'); + + // A re-render (e.g. a graph edit) must not re-fire the seed turn. + rerender( + + ); + expect(hookState.send).toHaveBeenCalledTimes(1); }); }); diff --git a/app/src/components/flows/WorkflowCopilotPanel.tsx b/app/src/components/flows/WorkflowCopilotPanel.tsx index 1053d9003..38611b874 100644 --- a/app/src/components/flows/WorkflowCopilotPanel.tsx +++ b/app/src/components/flows/WorkflowCopilotPanel.tsx @@ -8,10 +8,13 @@ * transcript, surfaces each proposal's node-level diff, and hands Accept/Reject * up to the host, which applies it to the local draft overlay. * - * Chat UI parity: the composer is the same {@link ChatComposer} the main chat - * windows use (mic/attachments off here), and turns render as bubbles via the - * shared {@link BubbleMarkdown}, so the copilot reads like a real chat rather - * than a one-shot form. + * Chat UI parity: the copilot reuses the SHARED chat surface end-to-end — the + * same {@link ChatComposer} the main chat windows use (mic/attachments off + * here), turns render as bubbles via the shared {@link BubbleMarkdown}, and the + * builder turn's live tool activity + streaming reply render through the shared + * {@link ToolTimelineBlock} (fed from the runtime's `toolTimelineByThread` / + * `streamingAssistantByThread`, streamed here by Phase B). So the copilot reads + * like a real chat rather than a one-shot form. * * Invariant: the copilot only PROPOSES. Accept applies to the UNSAVED local * draft (no `flows_update`); persistence stays behind the canvas's own Save. @@ -21,17 +24,28 @@ import { useCallback, useEffect, useRef, useState } from 'react'; import { useWorkflowBuilderChat } from '../../hooks/useWorkflowBuilderChat'; import { diffGraphs } from '../../lib/flows/graphDiff'; import type { WorkflowGraph } from '../../lib/flows/types'; -import { - buildRepairPrompt, - buildRevisePrompt, - type RepairPromptContext, -} from '../../lib/flows/workflowBuilderPrompt'; import { useT } from '../../lib/i18n/I18nContext'; import { BubbleMarkdown } from '../../pages/conversations/components/AgentMessageBubble'; +import { ToolTimelineBlock } from '../../pages/conversations/components/ToolTimelineBlock'; import type { WorkflowProposal } from '../../store/chatRuntimeSlice'; import ChatComposer from '../chat/ChatComposer'; import Button from '../ui/Button'; +/** + * Context for a repair turn opened from a failed run's inspector ("Fix with + * agent"). Maps directly onto a `repair`-mode builder request. + */ +export interface RepairPromptContext { + /** The failed run id (== thread_id) so the agent can `get_flow_run` it. */ + runId: string; + /** The run-level error message, if any. */ + error?: string | null; + /** Node ids that failed / are implicated, if known. */ + failingNodeIds?: string[]; + /** The flow's current graph, injected so the fix builds on the real draft. */ + graph: WorkflowGraph; +} + interface Props { /** The current draft graph, injected as context for each revise turn. */ graph: WorkflowGraph; @@ -57,6 +71,12 @@ interface Props { * repair turn once on mount so the copilot opens already diagnosing. */ repairSeed?: RepairPromptContext | null; + /** + * Optional build seed (from the Flows prompt bar's instant-create path) — + * auto-sends the user's workflow description once on mount so the copilot + * opens already building it against the just-created blank flow. + */ + buildSeed?: { description: string } | null; /** * The workflow's persisted copilot thread id (from the per-flow cache), so * reopening the panel resumes the same conversation instead of starting fresh. @@ -74,12 +94,22 @@ export default function WorkflowCopilotPanel({ onReject, onClose, repairSeed = null, + buildSeed = null, seedThreadId = null, onThreadIdChange, }: Props) { const { t } = useT(); - const { threadId, sending, proposal, messages, error, send, clearProposal } = - useWorkflowBuilderChat(seedThreadId); + const { + threadId, + sending, + proposal, + messages, + toolTimeline, + liveResponse, + error, + send, + clearProposal, + } = useWorkflowBuilderChat(seedThreadId); const [text, setText] = useState(''); // Report the (lazily-created) thread id up so the host persists it per flow — @@ -110,22 +140,53 @@ export default function WorkflowCopilotPanel({ repairSentRef.current = true; void send({ displayText: t('flows.copilot.repairDisplay'), - prompt: buildRepairPrompt(repairSeed), + request: { + mode: 'repair', + instruction: '', + graph: repairSeed.graph, + runId: repairSeed.runId, + error: repairSeed.error ?? null, + failingNodeIds: repairSeed.failingNodeIds ?? [], + }, }); }, [repairSeed, send, t]); - // Keep the transcript pinned to the newest message / thinking indicator. + // Auto-send the build turn once when opened from the prompt bar's + // instant-create path: the user's description becomes the first user turn on + // this thread, and the prompt asks for the full build → dry-run → save arc + // against the just-created flow (its proposal still lands as the usual + // Accept/Reject diff preview). Falls back to a propose-only revise turn if + // the flow id is somehow missing (a draft canvas has nothing to save onto). + const buildSentRef = useRef(false); + useEffect(() => { + if (!buildSeed || buildSentRef.current) return; + buildSentRef.current = true; + void send({ + displayText: buildSeed.description, + request: flowId + ? { mode: 'build', instruction: buildSeed.description, graph, flowId } + : { mode: 'revise', instruction: buildSeed.description, graph, flowId }, + }); + // `graph`/`flowId` are read once for the seed turn — later edits must not + // re-fire it (guarded by the ref regardless). + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [buildSeed, send]); + + // Keep the transcript pinned to the newest message / streamed activity. // `scrollTo` is optional-chained: jsdom (tests) doesn't implement it. useEffect(() => { scrollRef.current?.scrollTo?.({ top: scrollRef.current.scrollHeight }); - }, [messages, sending, proposal]); + }, [messages, sending, proposal, toolTimeline, liveResponse]); const submit = useCallback( async (raw?: string) => { const trimmed = (raw ?? text).trim(); if (!trimmed || sending) return; setText(''); - await send({ displayText: trimmed, prompt: buildRevisePrompt(trimmed, graph, flowId) }); + await send({ + displayText: trimmed, + request: { mode: 'revise', instruction: trimmed, graph, flowId }, + }); }, [text, sending, send, graph, flowId] ); @@ -157,7 +218,10 @@ export default function WorkflowCopilotPanel({ }, [onReject, clearProposal]); const diff = proposal ? diffGraphs(graph, proposal.graph as WorkflowGraph) : null; - const isEmpty = messages.length === 0 && !proposal && !sending && !error; + const hasTimeline = toolTimeline.length > 0; + const hasLiveText = liveResponse.trim().length > 0; + const isEmpty = + messages.length === 0 && !proposal && !sending && !error && !hasTimeline && !hasLiveText; return (