diff --git a/Cargo.lock b/Cargo.lock index c03087ad..fe4af0c2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3792,7 +3792,7 @@ dependencies = [ [[package]] name = "openfang-api" -version = "0.3.34" +version = "0.3.35" dependencies = [ "async-trait", "axum", @@ -3829,7 +3829,7 @@ dependencies = [ [[package]] name = "openfang-channels" -version = "0.3.34" +version = "0.3.35" dependencies = [ "async-trait", "axum", @@ -3861,7 +3861,7 @@ dependencies = [ [[package]] name = "openfang-cli" -version = "0.3.34" +version = "0.3.35" dependencies = [ "clap", "clap_complete", @@ -3888,7 +3888,7 @@ dependencies = [ [[package]] name = "openfang-desktop" -version = "0.3.34" +version = "0.3.35" dependencies = [ "axum", "open", @@ -3914,7 +3914,7 @@ dependencies = [ [[package]] name = "openfang-extensions" -version = "0.3.34" +version = "0.3.35" dependencies = [ "aes-gcm", "argon2", @@ -3942,7 +3942,7 @@ dependencies = [ [[package]] name = "openfang-hands" -version = "0.3.34" +version = "0.3.35" dependencies = [ "chrono", "dashmap", @@ -3959,7 +3959,7 @@ dependencies = [ [[package]] name = "openfang-kernel" -version = "0.3.34" +version = "0.3.35" dependencies = [ "async-trait", "chrono", @@ -3996,7 +3996,7 @@ dependencies = [ [[package]] name = "openfang-memory" -version = "0.3.34" +version = "0.3.35" dependencies = [ "async-trait", "chrono", @@ -4015,7 +4015,7 @@ dependencies = [ [[package]] name = "openfang-migrate" -version = "0.3.34" +version = "0.3.35" dependencies = [ "chrono", "dirs 6.0.0", @@ -4034,7 +4034,7 @@ dependencies = [ [[package]] name = "openfang-runtime" -version = "0.3.34" +version = "0.3.35" dependencies = [ "anyhow", "async-trait", @@ -4068,7 +4068,7 @@ dependencies = [ [[package]] name = "openfang-skills" -version = "0.3.34" +version = "0.3.35" dependencies = [ "chrono", "hex", @@ -4091,7 +4091,7 @@ dependencies = [ [[package]] name = "openfang-types" -version = "0.3.34" +version = "0.3.35" dependencies = [ "async-trait", "chrono", @@ -4110,7 +4110,7 @@ dependencies = [ [[package]] name = "openfang-wire" -version = "0.3.34" +version = "0.3.35" dependencies = [ "async-trait", "chrono", @@ -8772,7 +8772,7 @@ checksum = "b9cc00251562a284751c9973bace760d86c0276c471b4be569fe6b068ee97a56" [[package]] name = "xtask" -version = "0.3.34" +version = "0.3.35" [[package]] name = "yoke" diff --git a/Cargo.toml b/Cargo.toml index 7802c64f..ec3fae55 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -18,7 +18,7 @@ members = [ ] [workspace.package] -version = "0.3.35" +version = "0.3.36" edition = "2021" license = "Apache-2.0 OR MIT" repository = "https://github.com/RightNow-AI/openfang" diff --git a/crates/openfang-api/src/channel_bridge.rs b/crates/openfang-api/src/channel_bridge.rs index 9abfc743..11693b00 100644 --- a/crates/openfang-api/src/channel_bridge.rs +++ b/crates/openfang-api/src/channel_bridge.rs @@ -73,6 +73,33 @@ impl ChannelBridgeHandle for KernelBridgeAdapter { Ok(result.response) } + async fn send_message_with_blocks( + &self, + agent_id: AgentId, + blocks: Vec, + ) -> Result { + // Extract text for the message parameter (used for memory recall / logging) + let text: String = blocks + .iter() + .filter_map(|b| match b { + openfang_types::message::ContentBlock::Text { text } => Some(text.as_str()), + _ => None, + }) + .collect::>() + .join("\n"); + let text = if text.is_empty() { + "[Image]".to_string() + } else { + text + }; + let result = self + .kernel + .send_message_with_blocks(agent_id, &text, blocks) + .await + .map_err(|e| format!("{e}"))?; + Ok(result.response) + } + async fn find_agent_by_name(&self, name: &str) -> Result, String> { Ok(self.kernel.registry.find_by_name(name).map(|e| e.id)) } diff --git a/crates/openfang-api/src/middleware.rs b/crates/openfang-api/src/middleware.rs index 6cac9464..c12ee21f 100644 --- a/crates/openfang-api/src/middleware.rs +++ b/crates/openfang-api/src/middleware.rs @@ -45,9 +45,10 @@ pub async fn request_logging(request: Request, next: Next) -> Response`. If the key is empty, only whitelisted -/// public endpoints are accessible — all others return 401. +/// When `api_key` is non-empty (after trimming), requests to non-public +/// endpoints must include `Authorization: Bearer `. +/// If the key is empty or whitespace-only, auth is disabled entirely +/// (public/local development mode). pub async fn auth( axum::extract::State(api_key): axum::extract::State, request: Request, @@ -119,9 +120,10 @@ pub async fn auth( return next.run(request).await; } - // If no API key configured, skip auth entirely. - // Users who don't set api_key accept that all endpoints are open. - // To secure the dashboard, set api_key in config.toml. + // If no API key configured (empty, whitespace-only, or missing), skip auth + // entirely. Users who don't set api_key accept that all endpoints are open. + // To secure the dashboard, set a non-empty api_key in config.toml. + let api_key = api_key.trim(); if api_key.is_empty() { return next.run(request).await; } diff --git a/crates/openfang-api/src/server.rs b/crates/openfang-api/src/server.rs index 87e78fa8..061003a9 100644 --- a/crates/openfang-api/src/server.rs +++ b/crates/openfang-api/src/server.rs @@ -55,7 +55,7 @@ pub async fn build_router( // CORS: allow localhost origins by default. If API key is set, the API // is protected anyway. For development, permissive CORS is convenient. - let cors = if state.kernel.config.api_key.is_empty() { + let cors = if state.kernel.config.api_key.trim().is_empty() { // No auth → restrict CORS to localhost origins (include both 127.0.0.1 and localhost) let port = listen_addr.port(); let mut origins: Vec = vec![ @@ -103,7 +103,8 @@ pub async fn build_router( .allow_headers(tower_http::cors::Any) }; - let api_key = state.kernel.config.api_key.clone(); + // Trim whitespace so `api_key = ""` or `api_key = " "` both disable auth. + let api_key = state.kernel.config.api_key.trim().to_string(); let gcra_limiter = rate_limiter::create_rate_limiter(); let app = Router::new() diff --git a/crates/openfang-api/src/ws.rs b/crates/openfang-api/src/ws.rs index ebdd8183..108eaf93 100644 --- a/crates/openfang-api/src/ws.rs +++ b/crates/openfang-api/src/ws.rs @@ -146,7 +146,9 @@ pub async fn agent_ws( uri: axum::http::Uri, ) -> impl IntoResponse { // SECURITY: Authenticate WebSocket upgrades (bypasses middleware). - let api_key = &state.kernel.config.api_key; + // Trim whitespace so empty/whitespace-only api_key disables auth. + let api_key_raw = &state.kernel.config.api_key; + let api_key = api_key_raw.trim(); if !api_key.is_empty() { // SECURITY: Use constant-time comparison to prevent timing attacks on API key let ct_eq = |token: &str, key: &str| -> bool { diff --git a/crates/openfang-api/static/index_body.html b/crates/openfang-api/static/index_body.html index a09ad358..dfb6dcef 100644 --- a/crates/openfang-api/static/index_body.html +++ b/crates/openfang-api/static/index_body.html @@ -1468,7 +1468,7 @@
- +
@@ -1476,7 +1476,7 @@
- @@ -1499,7 +1499,7 @@
- +
Top port = true, bottom port = false
@@ -1510,11 +1510,11 @@
- +
- +
@@ -1524,7 +1524,7 @@
- +
@@ -1534,7 +1534,7 @@
- diff --git a/crates/openfang-api/static/js/pages/settings.js b/crates/openfang-api/static/js/pages/settings.js index 54861a6f..2df7f318 100644 --- a/crates/openfang-api/static/js/pages/settings.js +++ b/crates/openfang-api/static/js/pages/settings.js @@ -352,7 +352,10 @@ function settingsPage() { providerAuthText(p) { if (p.auth_status === 'configured') return 'Configured'; - if (p.auth_status === 'not_set' || p.auth_status === 'missing') return 'Not Set'; + if (p.auth_status === 'not_set' || p.auth_status === 'missing') { + if (p.id === 'claude-code') return 'Not Installed'; + return 'Not Set'; + } return 'No Key Needed'; }, diff --git a/crates/openfang-api/static/js/pages/workflow-builder.js b/crates/openfang-api/static/js/pages/workflow-builder.js index eb33c48b..45f89cd3 100644 --- a/crates/openfang-api/static/js/pages/workflow-builder.js +++ b/crates/openfang-api/static/js/pages/workflow-builder.js @@ -38,6 +38,11 @@ function workflowBuilder() { ], _renderScheduled: false, + _lastClickNodeId: null, + _lastClickTime: 0, + _didDrag: false, + _didConnect: false, + _didPan: false, async init() { var self = this; @@ -334,8 +339,23 @@ function workflowBuilder() { onNodeMouseDown: function(node, e) { e.stopPropagation(); + // Detect double-click manually — the native dblclick event never fires + // because scheduleRender() destroys and recreates all SVG elements between + // the first and second click, so the browser loses the DOM target for dblclick. + var now = Date.now(); + if (this._lastClickNodeId === node.id && (now - this._lastClickTime) < 350) { + // Double-click detected — open editor instead of starting drag + this._lastClickNodeId = null; + this._lastClickTime = 0; + this.editNode(node); + return; + } + this._lastClickNodeId = node.id; + this._lastClickTime = now; + this.selectedNode = node; this.selectedConnection = null; + this._didDrag = false; this.dragging = node.id; var rect = this._getCanvasRect(); this.dragOffset = { @@ -350,6 +370,7 @@ function workflowBuilder() { this.selectedConnection = null; this.showNodeEditor = false; // Start canvas pan + this._didPan = false; this.canvasDragging = true; this.canvasDragStart = { x: e.clientX - this.canvasOffset.x * this.zoom, y: e.clientY - this.canvasOffset.y * this.zoom }; }, @@ -357,6 +378,7 @@ function workflowBuilder() { onCanvasMouseMove: function(e) { var rect = this._getCanvasRect(); if (this.dragging) { + this._didDrag = true; var node = this.getNode(this.dragging); if (node) { node.x = Math.max(0, (e.clientX - rect.left) / this.zoom - this.canvasOffset.x - this.dragOffset.x); @@ -364,12 +386,14 @@ function workflowBuilder() { } this.scheduleRender(); } else if (this.connecting) { + this._didConnect = true; this.connectPreview = { x: (e.clientX - rect.left) / this.zoom - this.canvasOffset.x, y: (e.clientY - rect.top) / this.zoom - this.canvasOffset.y }; this.scheduleRender(); } else if (this.canvasDragging) { + this._didPan = true; this.canvasOffset = { x: (e.clientX - this.canvasDragStart.x) / this.zoom, y: (e.clientY - this.canvasDragStart.y) / this.zoom @@ -378,11 +402,19 @@ function workflowBuilder() { }, onCanvasMouseUp: function() { + // Only re-render if something actually moved. Rendering on every mouseup + // destroys SVG elements between clicks, which prevents dblclick detection. + var needsRender = this._didDrag || this._didConnect || this._didPan; this.dragging = null; this.connecting = null; this.connectPreview = null; this.canvasDragging = false; - this.scheduleRender(); + this._didDrag = false; + this._didConnect = false; + this._didPan = false; + if (needsRender) { + this.scheduleRender(); + } }, onCanvasWheel: function(e) { @@ -427,6 +459,12 @@ function workflowBuilder() { editNode: function(node) { this.selectedNode = node; this.showNodeEditor = true; + this.scheduleRender(); + }, + + // Called from editor panel inputs to reflect changes on the canvas SVG + applyNodeEdit: function() { + this.scheduleRender(); }, // ── TOML Generation ────────────────────────────────── diff --git a/crates/openfang-channels/src/bridge.rs b/crates/openfang-channels/src/bridge.rs index 1daedb94..e687de46 100644 --- a/crates/openfang-channels/src/bridge.rs +++ b/crates/openfang-channels/src/bridge.rs @@ -8,6 +8,7 @@ use crate::router::AgentRouter; use crate::types::{ChannelAdapter, ChannelContent, ChannelMessage, ChannelUser}; use async_trait::async_trait; use dashmap::DashMap; +use openfang_types::message::ContentBlock; use futures::StreamExt; use openfang_types::agent::AgentId; use openfang_types::config::{ChannelOverrides, DmPolicy, GroupPolicy, OutputFormat}; @@ -25,6 +26,26 @@ pub trait ChannelBridgeHandle: Send + Sync { /// Send a message to an agent and get the text response. async fn send_message(&self, agent_id: AgentId, message: &str) -> Result; + /// Send a message with structured content blocks (text + images) to an agent. + /// + /// Default implementation extracts text from blocks and falls back to `send_message()`. + async fn send_message_with_blocks( + &self, + agent_id: AgentId, + blocks: Vec, + ) -> Result { + // Default: extract text from blocks and send as plain text + let text: String = blocks + .iter() + .filter_map(|b| match b { + ContentBlock::Text { text } => Some(text.as_str()), + _ => None, + }) + .collect::>() + .join("\n"); + self.send_message(agent_id, &text).await + } + /// Find an agent by name, returning its ID. async fn find_agent_by_name(&self, name: &str) -> Result, String>; @@ -446,19 +467,43 @@ async fn dispatch_message( } } - let text = match &message.content { - ChannelContent::Text(t) => t.clone(), - ChannelContent::Command { name, args } => { - let result = handle_command(name, args, handle, router, &message.sender).await; - send_response(adapter, &message.sender, result, thread_id, output_format).await; + // Handle commands first (early return) + if let ChannelContent::Command { ref name, ref args } = message.content { + let result = handle_command(name, args, handle, router, &message.sender).await; + send_response(adapter, &message.sender, result, thread_id, output_format).await; + return; + } + + // For images: download, base64 encode, and send as multimodal content blocks + if let ChannelContent::Image { ref url, ref caption } = message.content { + let blocks = download_image_to_blocks(url, caption.as_deref()).await; + if blocks.iter().any(|b| matches!(b, ContentBlock::Image { .. })) { + // We have actual image data — send as structured blocks for vision + dispatch_with_blocks( + blocks, + message, + handle, + router, + adapter, + ct_str, + thread_id, + output_format, + ) + .await; return; } + // Image download failed — fall through to text description below + } + + let text = match &message.content { + ChannelContent::Text(t) => t.clone(), + ChannelContent::Command { .. } => unreachable!(), // handled above ChannelContent::Image { ref url, ref caption } => { - let desc = match caption { + // Fallback when image download failed + match caption { Some(c) => format!("[User sent a photo: {url}]\nCaption: {c}"), None => format!("[User sent a photo: {url}]"), - }; - desc + } } ChannelContent::File { ref url, ref filename } => { format!("[User sent a file ({filename}): {url}]") @@ -685,6 +730,187 @@ async fn dispatch_message( } } +/// Download an image from a URL and build content blocks for multimodal LLM input. +/// +/// Returns a `Vec` containing an image block (base64-encoded) and +/// optionally a text block for the caption. If the download fails, returns a +/// text-only block describing the failure. +async fn download_image_to_blocks(url: &str, caption: Option<&str>) -> Vec { + use base64::Engine; + + // 5 MB limit to prevent memory abuse from oversized images + const MAX_IMAGE_BYTES: usize = 5 * 1024 * 1024; + + let client = reqwest::Client::new(); + let resp = match client.get(url).send().await { + Ok(r) => r, + Err(e) => { + warn!("Failed to download image from channel: {e}"); + return vec![ContentBlock::Text { + text: format!("[Image download failed: {e}]"), + }]; + } + }; + + // Detect media type from Content-Type header, fall back to URL extension + let content_type = resp + .headers() + .get("content-type") + .and_then(|v| v.to_str().ok()) + .map(|ct| ct.split(';').next().unwrap_or(ct).trim().to_string()); + + let media_type = content_type.unwrap_or_else(|| { + if url.contains(".png") { + "image/png".to_string() + } else if url.contains(".gif") { + "image/gif".to_string() + } else if url.contains(".webp") { + "image/webp".to_string() + } else { + "image/jpeg".to_string() + } + }); + + let bytes = match resp.bytes().await { + Ok(b) => b, + Err(e) => { + warn!("Failed to read image bytes: {e}"); + return vec![ContentBlock::Text { + text: format!("[Image read failed: {e}]"), + }]; + } + }; + + if bytes.len() > MAX_IMAGE_BYTES { + warn!( + "Image too large ({} bytes), skipping vision — sending as text", + bytes.len() + ); + let desc = match caption { + Some(c) => format!("[Image too large for vision ({} KB)]\nCaption: {c}", bytes.len() / 1024), + None => format!("[Image too large for vision ({} KB)]", bytes.len() / 1024), + }; + return vec![ContentBlock::Text { text: desc }]; + } + + let data = base64::engine::general_purpose::STANDARD.encode(&bytes); + + let mut blocks = Vec::new(); + + // Caption as text block first (gives the LLM context about the image) + if let Some(cap) = caption { + if !cap.is_empty() { + blocks.push(ContentBlock::Text { + text: cap.to_string(), + }); + } + } + + blocks.push(ContentBlock::Image { media_type, data }); + + blocks +} + +/// Dispatch a multimodal message (content blocks) to an agent, handling routing +/// and RBAC the same way as the text path. +#[allow(clippy::too_many_arguments)] +async fn dispatch_with_blocks( + blocks: Vec, + message: &ChannelMessage, + handle: &Arc, + router: &Arc, + adapter: &dyn ChannelAdapter, + ct_str: &str, + thread_id: Option<&str>, + output_format: OutputFormat, +) { + // Route to agent (same logic as text path) + let agent_id = router.resolve( + &message.channel, + &message.sender.platform_id, + message.sender.openfang_user.as_deref(), + ); + + let agent_id = match agent_id { + Some(id) => id, + None => { + let fallback = handle.find_agent_by_name("assistant").await.ok().flatten(); + let fallback = match fallback { + Some(id) => Some(id), + None => handle + .list_agents() + .await + .ok() + .and_then(|agents| agents.first().map(|(id, _)| *id)), + }; + match fallback { + Some(id) => { + router.set_user_default(message.sender.platform_id.clone(), id); + id + } + None => { + send_response( + adapter, + &message.sender, + "No agents available. Start the dashboard at http://127.0.0.1:4200 to create one.".to_string(), + thread_id, + output_format, + ).await; + return; + } + } + } + }; + + // RBAC check + if let Err(denied) = handle + .authorize_channel_user(ct_str, &message.sender.platform_id, "chat") + .await + { + send_response( + adapter, + &message.sender, + format!("Access denied: {denied}"), + thread_id, + output_format, + ) + .await; + return; + } + + let _ = adapter.send_typing(&message.sender).await; + + match handle.send_message_with_blocks(agent_id, blocks).await { + Ok(response) => { + send_response(adapter, &message.sender, response, thread_id, output_format).await; + handle + .record_delivery(agent_id, ct_str, &message.sender.platform_id, true, None) + .await; + } + Err(e) => { + warn!("Agent error for {agent_id}: {e}"); + let err_msg = format!("Agent error: {e}"); + send_response( + adapter, + &message.sender, + err_msg.clone(), + thread_id, + output_format, + ) + .await; + handle + .record_delivery( + agent_id, + ct_str, + &message.sender.platform_id, + false, + Some(&err_msg), + ) + .await; + } + } +} + /// Handle a bot command (returns the response text). async fn handle_command( name: &str, @@ -1124,4 +1350,52 @@ mod tests { "irc" ); } + + #[tokio::test] + async fn test_send_message_with_blocks_default_fallback() { + // The default implementation of send_message_with_blocks extracts text + // from blocks and calls send_message + let agent_id = AgentId::new(); + let handle: Arc = Arc::new(MockHandle { + agents: Mutex::new(vec![(agent_id, "vision-agent".to_string())]), + }); + + let blocks = vec![ + ContentBlock::Text { + text: "What is in this photo?".to_string(), + }, + ContentBlock::Image { + media_type: "image/jpeg".to_string(), + data: "base64data".to_string(), + }, + ]; + + // Default impl should extract text and call send_message + let result = handle + .send_message_with_blocks(agent_id, blocks) + .await + .unwrap(); + assert_eq!(result, "Echo: What is in this photo?"); + } + + #[tokio::test] + async fn test_send_message_with_blocks_image_only() { + // When there's no text block, the default should still work + let agent_id = AgentId::new(); + let handle: Arc = Arc::new(MockHandle { + agents: Mutex::new(vec![(agent_id, "vision-agent".to_string())]), + }); + + let blocks = vec![ContentBlock::Image { + media_type: "image/png".to_string(), + data: "base64data".to_string(), + }]; + + // Default impl sends empty text when no text blocks + let result = handle + .send_message_with_blocks(agent_id, blocks) + .await + .unwrap(); + assert_eq!(result, "Echo: "); + } } diff --git a/crates/openfang-channels/src/telegram.rs b/crates/openfang-channels/src/telegram.rs index 1469d1af..6620f711 100644 --- a/crates/openfang-channels/src/telegram.rs +++ b/crates/openfang-channels/src/telegram.rs @@ -848,4 +848,103 @@ mod tests { let msg = parse_telegram_update(&update, &[], "fake:token", &client).await.unwrap(); assert!(matches!(msg.content, ChannelContent::Location { .. })); } + + #[tokio::test] + async fn test_parse_telegram_photo_fallback() { + // When getFile fails (fake token), photo messages should fall back to + // a text description rather than being silently dropped. + let update = serde_json::json!({ + "update_id": 300, + "message": { + "message_id": 60, + "from": { "id": 123, "first_name": "Alice" }, + "chat": { "id": 123, "type": "private" }, + "date": 1700000000, + "photo": [ + { "file_id": "small_id", "file_unique_id": "a", "width": 90, "height": 90, "file_size": 1234 }, + { "file_id": "large_id", "file_unique_id": "b", "width": 800, "height": 600, "file_size": 45678 } + ], + "caption": "Check this out" + } + }); + + let client = test_client(); + let msg = parse_telegram_update(&update, &[], "fake:token", &client).await.unwrap(); + // With a fake token, getFile will fail, so we get a text fallback + match &msg.content { + ChannelContent::Text(t) => { + assert!(t.contains("Photo received")); + assert!(t.contains("Check this out")); + } + ChannelContent::Image { caption, .. } => { + // If somehow the HTTP call succeeded (unlikely with fake token), + // verify caption was extracted + assert_eq!(caption.as_deref(), Some("Check this out")); + } + other => panic!("Expected Text or Image fallback for photo, got {other:?}"), + } + } + + #[tokio::test] + async fn test_parse_telegram_document_fallback() { + let update = serde_json::json!({ + "update_id": 301, + "message": { + "message_id": 61, + "from": { "id": 123, "first_name": "Alice" }, + "chat": { "id": 123, "type": "private" }, + "date": 1700000000, + "document": { + "file_id": "doc_id", + "file_unique_id": "c", + "file_name": "report.pdf", + "file_size": 102400 + } + } + }); + + let client = test_client(); + let msg = parse_telegram_update(&update, &[], "fake:token", &client).await.unwrap(); + match &msg.content { + ChannelContent::Text(t) => { + assert!(t.contains("Document received")); + assert!(t.contains("report.pdf")); + } + ChannelContent::File { filename, .. } => { + assert_eq!(filename, "report.pdf"); + } + other => panic!("Expected Text or File for document, got {other:?}"), + } + } + + #[tokio::test] + async fn test_parse_telegram_voice_fallback() { + let update = serde_json::json!({ + "update_id": 302, + "message": { + "message_id": 62, + "from": { "id": 123, "first_name": "Alice" }, + "chat": { "id": 123, "type": "private" }, + "date": 1700000000, + "voice": { + "file_id": "voice_id", + "file_unique_id": "d", + "duration": 15 + } + } + }); + + let client = test_client(); + let msg = parse_telegram_update(&update, &[], "fake:token", &client).await.unwrap(); + match &msg.content { + ChannelContent::Text(t) => { + assert!(t.contains("Voice message")); + assert!(t.contains("15s")); + } + ChannelContent::Voice { duration_seconds, .. } => { + assert_eq!(*duration_seconds, 15); + } + other => panic!("Expected Text or Voice for voice message, got {other:?}"), + } + } } diff --git a/crates/openfang-cli/src/main.rs b/crates/openfang-cli/src/main.rs index 004d3d38..6c6e53fa 100644 --- a/crates/openfang-cli/src/main.rs +++ b/crates/openfang-cli/src/main.rs @@ -1073,11 +1073,23 @@ pub(crate) fn find_daemon() -> Option { } /// Build an HTTP client for daemon calls. +/// +/// When api_key is configured in config.toml, the client automatically +/// includes a `Authorization: Bearer ` header on every request. +/// When api_key is empty or missing, no auth header is sent. pub(crate) fn daemon_client() -> reqwest::blocking::Client { - reqwest::blocking::Client::builder() - .timeout(std::time::Duration::from_secs(120)) - .build() - .expect("Failed to build HTTP client") + let mut builder = reqwest::blocking::Client::builder() + .timeout(std::time::Duration::from_secs(120)); + + if let Some(key) = read_api_key() { + let mut headers = reqwest::header::HeaderMap::new(); + if let Ok(val) = reqwest::header::HeaderValue::from_str(&format!("Bearer {key}")) { + headers.insert(reqwest::header::AUTHORIZATION, val); + } + builder = builder.default_headers(headers); + } + + builder.build().expect("Failed to build HTTP client") } /// Helper: send a request to the daemon and parse the JSON body. @@ -1456,11 +1468,14 @@ fn cmd_start(config: Option) { } /// Read the api_key from ~/.openfang/config.toml (if any). +/// +/// Returns `None` when the key is missing, empty, or whitespace-only — +/// meaning the daemon is running in public (unauthenticated) mode. fn read_api_key() -> Option { let config_path = cli_openfang_home().join("config.toml"); let text = std::fs::read_to_string(config_path).ok()?; let table: toml::Value = text.parse().ok()?; - let key = table.get("api_key")?.as_str()?; + let key = table.get("api_key")?.as_str()?.trim(); if key.is_empty() { None } else { @@ -1472,11 +1487,7 @@ fn cmd_stop() { match find_daemon() { Some(base) => { let client = daemon_client(); - let mut req = client.post(format!("{base}/api/shutdown")); - if let Some(key) = read_api_key() { - req = req.bearer_auth(key); - } - match req.send() { + match client.post(format!("{base}/api/shutdown")).send() { Ok(r) if r.status().is_success() => { // Wait for daemon to actually stop (up to 5 seconds) for _ in 0..10 { diff --git a/crates/openfang-hands/bundled/browser/HAND.toml b/crates/openfang-hands/bundled/browser/HAND.toml index e8d3417e..ce8cdc6c 100644 --- a/crates/openfang-hands/bundled/browser/HAND.toml +++ b/crates/openfang-hands/bundled/browser/HAND.toml @@ -13,6 +13,23 @@ tools = [ "file_write", "file_read", ] +[[requires]] +key = "python3" +label = "Python 3 must be installed" +requirement_type = "binary" +check_value = "python3" +description = "Python 3 is required for installing and running the Playwright browser automation library. Python 3.8 or newer is recommended." + +[requires.install] +macos = "brew install python3" +windows = "winget install Python.Python.3.12" +linux_apt = "sudo apt install python3" +linux_dnf = "sudo dnf install python3" +linux_pacman = "sudo pacman -S python" +pip = "python3 --version" +manual_url = "https://www.python.org/downloads/" +estimated_time = "1-3 min" + [[requires]] key = "chromium" label = "Chromium or Google Chrome must be installed" diff --git a/crates/openfang-hands/src/bundled.rs b/crates/openfang-hands/src/bundled.rs index c54c7937..2b9df9e5 100644 --- a/crates/openfang-hands/src/bundled.rs +++ b/crates/openfang-hands/src/bundled.rs @@ -187,8 +187,8 @@ mod tests { assert_eq!(def.name, "Browser Hand"); assert_eq!(def.category, crate::HandCategory::Productivity); assert!(def.skill_content.is_some()); - assert!(!def.requires.is_empty()); // requires chromium - assert_eq!(def.requires.len(), 1); + assert!(!def.requires.is_empty()); // requires python3 + chromium + assert_eq!(def.requires.len(), 2); assert!(def.tools.contains(&"browser_navigate".to_string())); assert!(def.tools.contains(&"browser_click".to_string())); assert!(def.tools.contains(&"browser_type".to_string())); diff --git a/crates/openfang-hands/src/registry.rs b/crates/openfang-hands/src/registry.rs index 18a92d3e..8b841c4c 100644 --- a/crates/openfang-hands/src/registry.rs +++ b/crates/openfang-hands/src/registry.rs @@ -62,6 +62,7 @@ impl HandRegistry { serde_json::json!({ "hand_id": e.hand_id, "config": e.config, + "agent_id": e.agent_id, }) }) .collect(); @@ -73,8 +74,12 @@ impl HandRegistry { } /// Load persisted hand state and re-activate hands. - /// Returns list of (hand_id, config) that should be activated. - pub fn load_state(path: &std::path::Path) -> Vec<(String, HashMap)> { + /// Returns list of (hand_id, config, old_agent_id) that should be activated. + /// The `old_agent_id` is the agent UUID from before the restart, used to + /// reassign cron jobs to the newly spawned agent (issue #402). + pub fn load_state( + path: &std::path::Path, + ) -> Vec<(String, HashMap, Option)> { let data = match std::fs::read_to_string(path) { Ok(d) => d, Err(_) => return Vec::new(), @@ -92,7 +97,10 @@ impl HandRegistry { let hand_id = e["hand_id"].as_str()?.to_string(); let config: HashMap = serde_json::from_value(e["config"].clone()).unwrap_or_default(); - Some((hand_id, config)) + let old_agent_id: Option = e + .get("agent_id") + .and_then(|v| serde_json::from_value(v.clone()).ok()); + Some((hand_id, config, old_agent_id)) }) .collect() } @@ -356,14 +364,16 @@ impl Default for HandRegistry { fn check_requirement(req: &HandRequirement) -> bool { match req.requirement_type { RequirementType::Binary => { + // Special handling for python3: must actually run the command and verify + // the output contains "Python 3", because Windows ships a python3.exe + // Store shim that exists on PATH but doesn't actually work. + if req.check_value == "python3" { + return check_python3_available(); + } // Check if binary exists on PATH. - // For python3, also try "python" (Windows ships python not python3). if which_binary(&req.check_value) { return true; } - if req.check_value == "python3" { - return which_binary("python"); - } if req.check_value == "chromium" { // Try common Chromium/Chrome binary names across platforms return which_binary("chromium-browser") @@ -383,6 +393,44 @@ fn check_requirement(req: &HandRequirement) -> bool { } } +/// Check if Python 3 is actually available by running the command and checking +/// the version output. This avoids false negatives from Windows Store shims +/// (python3.exe that just opens the Microsoft Store) and false positives from +/// Python 2 installations where `python` exists but is Python 2. +fn check_python3_available() -> bool { + // Try "python3 --version" first (Linux/macOS, some Windows installs) + if run_returns_python3("python3") { + return true; + } + // Try "python --version" (Windows commonly uses this, Docker containers too) + if run_returns_python3("python") { + return true; + } + false +} + +/// Run `{cmd} --version` and return true if the output contains "Python 3". +fn run_returns_python3(cmd: &str) -> bool { + match std::process::Command::new(cmd) + .arg("--version") + .stdout(std::process::Stdio::piped()) + .stderr(std::process::Stdio::piped()) + .stdin(std::process::Stdio::null()) + .output() + { + Ok(output) => { + if !output.status.success() { + return false; + } + // Python --version may print to stdout or stderr depending on version + let stdout = String::from_utf8_lossy(&output.stdout); + let stderr = String::from_utf8_lossy(&output.stderr); + stdout.contains("Python 3") || stderr.contains("Python 3") + } + Err(_) => false, + } +} + /// Check if a binary is on PATH (cross-platform). fn which_binary(name: &str) -> bool { let path_var = std::env::var("PATH").unwrap_or_default(); diff --git a/crates/openfang-kernel/src/kernel.rs b/crates/openfang-kernel/src/kernel.rs index 40efa62b..f835dbd6 100644 --- a/crates/openfang-kernel/src/kernel.rs +++ b/crates/openfang-kernel/src/kernel.rs @@ -1365,12 +1365,47 @@ impl OpenFangKernel { .await } + /// Send a multimodal message (text + images) to an agent and get a response. + /// + /// Used by channel bridges when a user sends a photo — the image is downloaded, + /// base64 encoded, and passed as `ContentBlock::Image` alongside any caption text. + pub async fn send_message_with_blocks( + &self, + agent_id: AgentId, + message: &str, + blocks: Vec, + ) -> KernelResult { + let handle: Option> = self + .self_handle + .get() + .and_then(|w| w.upgrade()) + .map(|arc| arc as Arc); + self.send_message_with_handle_and_blocks(agent_id, message, handle, Some(blocks)) + .await + } + /// Send a message with an optional kernel handle for inter-agent tools. pub async fn send_message_with_handle( &self, agent_id: AgentId, message: &str, kernel_handle: Option>, + ) -> KernelResult { + self.send_message_with_handle_and_blocks(agent_id, message, kernel_handle, None) + .await + } + + /// Send a message with optional content blocks and an optional kernel handle. + /// + /// When `content_blocks` is `Some`, the LLM agent loop receives structured + /// multimodal content (text + images) instead of just a text string. This + /// enables vision models to process images sent from channels like Telegram. + pub async fn send_message_with_handle_and_blocks( + &self, + agent_id: AgentId, + message: &str, + kernel_handle: Option>, + content_blocks: Option>, ) -> KernelResult { // Enforce quota before running the agent loop self.scheduler @@ -1389,7 +1424,7 @@ impl OpenFangKernel { self.execute_python_agent(&entry, agent_id, message).await } else { // Default: LLM agent loop (builtin:chat or any unrecognized module) - self.execute_llm_agent(&entry, agent_id, message, kernel_handle) + self.execute_llm_agent(&entry, agent_id, message, kernel_handle, content_blocks) .await }; @@ -1777,6 +1812,7 @@ impl OpenFangKernel { Some(&kernel_clone.hooks), ctx_window, Some(&kernel_clone.process_manager), + None, // content_blocks (streaming path uses text only for now) ) .await; @@ -1996,6 +2032,7 @@ impl OpenFangKernel { agent_id: AgentId, message: &str, kernel_handle: Option>, + content_blocks: Option>, ) -> KernelResult { // Check metering quota before starting self.metering @@ -2257,6 +2294,7 @@ impl OpenFangKernel { Some(&self.hooks), ctx_window, Some(&self.process_manager), + content_blocks, ) .await .map_err(KernelError::OpenFang)?; @@ -3391,9 +3429,34 @@ impl OpenFangKernel { let saved_hands = openfang_hands::registry::HandRegistry::load_state(&state_path); if !saved_hands.is_empty() { info!("Restoring {} persisted hand(s)", saved_hands.len()); - for (hand_id, config) in saved_hands { + for (hand_id, config, old_agent_id) in saved_hands { match self.activate_hand(&hand_id, config) { - Ok(inst) => info!(hand = %hand_id, instance = %inst.instance_id, "Hand restored"), + Ok(inst) => { + info!(hand = %hand_id, instance = %inst.instance_id, "Hand restored"); + // Reassign cron jobs from the pre-restart agent ID to the + // newly spawned agent so scheduled tasks survive daemon + // restarts (issue #402). activate_hand only handles + // reassignment when an existing agent is found in the live + // registry, which is empty on a fresh boot. + if let (Some(old_id), Some(new_id)) = (old_agent_id, inst.agent_id) { + if old_id != new_id { + let migrated = + self.cron_scheduler.reassign_agent_jobs(old_id, new_id); + if migrated > 0 { + info!( + hand = %hand_id, + old_agent = %old_id, + new_agent = %new_id, + migrated, + "Reassigned cron jobs after restart" + ); + if let Err(e) = self.cron_scheduler.persist() { + warn!("Failed to persist cron jobs after hand restore: {e}"); + } + } + } + } + } Err(e) => warn!(hand = %hand_id, error = %e, "Failed to restore hand"), } } @@ -4386,6 +4449,10 @@ impl OpenFangKernel { _ => all_builtins, }; + // Track tool names added by skills/MCP — these already passed their own + // allowlist filters and should bypass the per-agent capability check. + let mut extension_tool_names = std::collections::HashSet::new(); + // Add skill-provided tools (filtered by agent's skill allowlist) let skill_tools = { let registry = self @@ -4399,6 +4466,7 @@ impl OpenFangKernel { } }; for skill_tool in skill_tools { + extension_tool_names.insert(skill_tool.name.clone()); all_tools.push(ToolDefinition { name: skill_tool.name.clone(), description: skill_tool.description.clone(), @@ -4409,6 +4477,9 @@ impl OpenFangKernel { // Add MCP tools (filtered by agent's MCP server allowlist) if let Ok(mcp_tools) = self.mcp_tools.lock() { if mcp_allowlist.is_empty() { + for t in mcp_tools.iter() { + extension_tool_names.insert(t.name.clone()); + } all_tools.extend(mcp_tools.iter().cloned()); } else { // Normalize allowlist names for matching @@ -4416,16 +4487,19 @@ impl OpenFangKernel { .iter() .map(|s| openfang_runtime::mcp::normalize_name(s)) .collect(); - all_tools.extend( - mcp_tools - .iter() - .filter(|t| { - openfang_runtime::mcp::extract_mcp_server(&t.name) - .map(|s| normalized.iter().any(|n| n == s)) - .unwrap_or(false) - }) - .cloned(), - ); + let filtered: Vec<_> = mcp_tools + .iter() + .filter(|t| { + openfang_runtime::mcp::extract_mcp_server(&t.name) + .map(|s| normalized.iter().any(|n| n == s)) + .unwrap_or(false) + }) + .cloned() + .collect(); + for t in &filtered { + extension_tool_names.insert(t.name.clone()); + } + all_tools.extend(filtered); } } @@ -4461,14 +4535,17 @@ impl OpenFangKernel { return all_tools; } - // Filter to tools the agent has capability for + // Filter to tools the agent has capability for. + // MCP and skill tools bypass this check — they already passed their + // own allowlist filters above (mcp_servers / skills on the manifest). all_tools .into_iter() .filter(|tool| { - caps.iter().any(|c| match c { - Capability::ToolInvoke(name) => name == &tool.name || name == "*", - _ => false, - }) + extension_tool_names.contains(&tool.name) + || caps.iter().any(|c| match c { + Capability::ToolInvoke(name) => name == &tool.name || name == "*", + _ => false, + }) }) .collect() } diff --git a/crates/openfang-runtime/src/agent_loop.rs b/crates/openfang-runtime/src/agent_loop.rs index 4a84c040..abd8ca1a 100644 --- a/crates/openfang-runtime/src/agent_loop.rs +++ b/crates/openfang-runtime/src/agent_loop.rs @@ -133,6 +133,7 @@ pub async fn run_agent_loop( hooks: Option<&crate::hooks::HookRegistry>, context_window_tokens: Option, process_manager: Option<&crate::process_manager::ProcessManager>, + user_content_blocks: Option>, ) -> OpenFangResult { info!(agent = %manifest.name, "Starting agent loop"); @@ -217,8 +218,14 @@ pub async fn run_agent_loop( system_prompt.push_str(&crate::prompt_builder::build_memory_section(&mem_pairs)); } - // Add the user message to session history - session.messages.push(Message::user(user_message)); + // Add the user message to session history. + // When content blocks are provided (e.g. text + image from a channel), + // use multimodal message format so the LLM receives the image for vision. + if let Some(blocks) = user_content_blocks { + session.messages.push(Message::user_with_blocks(blocks)); + } else { + session.messages.push(Message::user(user_message)); + } // Build the messages for the LLM, filtering system messages // System prompt goes into the separate `system` field @@ -259,6 +266,10 @@ pub async fn run_agent_loop( "Trimming old messages to prevent context overflow" ); messages.drain(..trim_count); + // Re-validate after trimming: the drain may have split a ToolUse/ToolResult + // pair across the cut boundary, leaving orphaned blocks that cause the LLM + // to return empty responses (input_tokens=0). + messages = crate::session_repair::validate_and_repair(&messages); } // Use autonomous config max_iterations if set, else default @@ -389,14 +400,30 @@ pub async fn run_agent_loop( }); } - // One-shot retry: if the very first LLM call returns empty text - // with no tool use, try once more before accepting the empty result. - // This catches transient LLM hiccups (overload, empty stream, etc.). - if text.trim().is_empty() && iteration == 0 && response.tool_calls.is_empty() { - warn!(agent = %manifest.name, "Empty response on first call, retrying once"); - messages.push(Message::assistant("[no response]".to_string())); - messages.push(Message::user("Please provide your response.".to_string())); - continue; + // One-shot retry: if the LLM returns empty text with no tool use, + // try once more before accepting the empty result. + // Triggers on first call OR when input_tokens=0 (silently failed request). + if text.trim().is_empty() && response.tool_calls.is_empty() { + let is_silent_failure = response.usage.input_tokens == 0 + && response.usage.output_tokens == 0; + if iteration == 0 || is_silent_failure { + warn!( + agent = %manifest.name, + iteration, + input_tokens = response.usage.input_tokens, + output_tokens = response.usage.output_tokens, + silent_failure = is_silent_failure, + "Empty response, retrying once" + ); + // Re-validate messages before retry — the history may have + // broken tool_use/tool_result pairs that caused the failure. + if is_silent_failure { + messages = crate::session_repair::validate_and_repair(&messages); + } + messages.push(Message::assistant("[no response]".to_string())); + messages.push(Message::user("Please provide your response.".to_string())); + continue; + } } // Guard against empty response — covers both iteration 0 and post-tool cycles @@ -1055,6 +1082,7 @@ pub async fn run_agent_loop_streaming( hooks: Option<&crate::hooks::HookRegistry>, context_window_tokens: Option, process_manager: Option<&crate::process_manager::ProcessManager>, + user_content_blocks: Option>, ) -> OpenFangResult { info!(agent = %manifest.name, "Starting streaming agent loop"); @@ -1139,8 +1167,14 @@ pub async fn run_agent_loop_streaming( system_prompt.push_str(&crate::prompt_builder::build_memory_section(&mem_pairs)); } - // Add the user message to session history - session.messages.push(Message::user(user_message)); + // Add the user message to session history. + // When content blocks are provided (e.g. text + image from a channel), + // use multimodal message format so the LLM receives the image for vision. + if let Some(blocks) = user_content_blocks { + session.messages.push(Message::user_with_blocks(blocks)); + } else { + session.messages.push(Message::user(user_message)); + } let llm_messages: Vec = session .messages @@ -1177,6 +1211,10 @@ pub async fn run_agent_loop_streaming( "Trimming old messages to prevent context overflow (streaming)" ); messages.drain(..trim_count); + // Re-validate after trimming: the drain may have split a ToolUse/ToolResult + // pair across the cut boundary, leaving orphaned blocks that cause the LLM + // to return empty responses (input_tokens=0). + messages = crate::session_repair::validate_and_repair(&messages); } // Use autonomous config max_iterations if set, else default @@ -1321,13 +1359,30 @@ pub async fn run_agent_loop_streaming( }); } - // One-shot retry: if the very first LLM call returns empty text - // with no tool use, try once more before accepting the empty result. - if text.trim().is_empty() && iteration == 0 && response.tool_calls.is_empty() { - warn!(agent = %manifest.name, "Empty response on first call (streaming), retrying once"); - messages.push(Message::assistant("[no response]".to_string())); - messages.push(Message::user("Please provide your response.".to_string())); - continue; + // One-shot retry: if the LLM returns empty text with no tool use, + // try once more before accepting the empty result. + // Triggers on first call OR when input_tokens=0 (silently failed request). + if text.trim().is_empty() && response.tool_calls.is_empty() { + let is_silent_failure = response.usage.input_tokens == 0 + && response.usage.output_tokens == 0; + if iteration == 0 || is_silent_failure { + warn!( + agent = %manifest.name, + iteration, + input_tokens = response.usage.input_tokens, + output_tokens = response.usage.output_tokens, + silent_failure = is_silent_failure, + "Empty response (streaming), retrying once" + ); + // Re-validate messages before retry — the history may have + // broken tool_use/tool_result pairs that caused the failure. + if is_silent_failure { + messages = crate::session_repair::validate_and_repair(&messages); + } + messages.push(Message::assistant("[no response]".to_string())); + messages.push(Message::user("Please provide your response.".to_string())); + continue; + } } // Guard against empty response — covers both iteration 0 and post-tool cycles @@ -2218,6 +2273,7 @@ mod tests { None, // hooks None, // context_window_tokens None, // process_manager + None, // user_content_blocks ) .await .expect("Loop should complete without error"); @@ -2270,6 +2326,7 @@ mod tests { None, // hooks None, // context_window_tokens None, // process_manager + None, // user_content_blocks ) .await .expect("Loop should complete without error"); @@ -2322,6 +2379,7 @@ mod tests { None, // hooks None, // context_window_tokens None, // process_manager + None, // user_content_blocks ) .await .expect("Loop should complete without error"); @@ -2367,6 +2425,7 @@ mod tests { None, // hooks None, // context_window_tokens None, // process_manager + None, // user_content_blocks ) .await .expect("Streaming loop should complete without error"); @@ -2489,6 +2548,7 @@ mod tests { None, None, // context_window_tokens None, // process_manager + None, // user_content_blocks ) .await .expect("Loop should recover via retry"); @@ -2535,6 +2595,7 @@ mod tests { None, None, // context_window_tokens None, // process_manager + None, // user_content_blocks ) .await .expect("Loop should complete with fallback"); @@ -2589,6 +2650,7 @@ mod tests { None, // hooks None, // context_window_tokens None, // process_manager + None, // user_content_blocks ) .await .expect("Streaming loop should complete without error"); @@ -3039,6 +3101,7 @@ mod tests { None, // hooks None, // context_window_tokens None, // process_manager + None, // user_content_blocks ) .await .expect("Agent loop should complete"); @@ -3105,6 +3168,7 @@ mod tests { None, None, None, + None, // user_content_blocks ) .await .expect("Normal loop should complete"); @@ -3167,6 +3231,7 @@ mod tests { None, // hooks None, // context_window_tokens None, // process_manager + None, // user_content_blocks ) .await .expect("Streaming loop should complete"); diff --git a/crates/openfang-runtime/src/drivers/openai.rs b/crates/openfang-runtime/src/drivers/openai.rs index 77beed68..b3cd30cd 100644 --- a/crates/openfang-runtime/src/drivers/openai.rs +++ b/crates/openfang-runtime/src/drivers/openai.rs @@ -946,17 +946,31 @@ impl LlmDriver for OpenAIDriver { } // Log stream summary for diagnostics - debug!( - chunks = chunk_count, - sse_lines = sse_line_count, - text_len = text_content.len(), - tool_count = tool_accum.len(), - finish = ?finish_reason, - input_tokens = usage.input_tokens, - output_tokens = usage.output_tokens, - buffer_remaining = buffer.len(), - "SSE stream completed" - ); + let is_empty_stream = text_content.is_empty() + && tool_accum.is_empty() + && usage.input_tokens == 0 + && usage.output_tokens == 0; + if is_empty_stream { + warn!( + chunks = chunk_count, + sse_lines = sse_line_count, + finish = ?finish_reason, + buffer_remaining = buffer.len(), + "SSE stream returned empty: 0 content, 0 tokens — likely a silently failed request" + ); + } else { + debug!( + chunks = chunk_count, + sse_lines = sse_line_count, + text_len = text_content.len(), + tool_count = tool_accum.len(), + finish = ?finish_reason, + input_tokens = usage.input_tokens, + output_tokens = usage.output_tokens, + buffer_remaining = buffer.len(), + "SSE stream completed" + ); + } // Build the final response let mut content = Vec::new(); diff --git a/crates/openfang-runtime/src/model_catalog.rs b/crates/openfang-runtime/src/model_catalog.rs index 67dd66fd..573eed2e 100644 --- a/crates/openfang-runtime/src/model_catalog.rs +++ b/crates/openfang-runtime/src/model_catalog.rs @@ -56,6 +56,18 @@ impl ModelCatalog { /// Only checks presence — never reads or stores the actual secret. pub fn detect_auth(&mut self) { for provider in &mut self.providers { + // Claude Code is special: no API key needed, but we probe for CLI + // installation so the dashboard shows "Configured" vs "Not Installed". + if provider.id == "claude-code" { + provider.auth_status = + if crate::drivers::claude_code::claude_code_available() { + AuthStatus::Configured + } else { + AuthStatus::Missing + }; + continue; + } + if !provider.key_required { provider.auth_status = AuthStatus::NotRequired; continue; @@ -71,7 +83,7 @@ impl ModelCatalog { std::env::var("OPENAI_API_KEY").is_ok() || read_codex_credential().is_some() } - "claude-code" => crate::drivers::claude_code::claude_code_available(), + // claude-code is handled above (before key_required check) _ => false, }; diff --git a/crates/openfang-types/src/message.rs b/crates/openfang-types/src/message.rs index a4a6c9b6..0a47f592 100644 --- a/crates/openfang-types/src/message.rs +++ b/crates/openfang-types/src/message.rs @@ -173,6 +173,14 @@ impl Message { } } + /// Create a user message with structured content blocks (e.g. text + images). + pub fn user_with_blocks(blocks: Vec) -> Self { + Self { + role: Role::User, + content: MessageContent::Blocks(blocks), + } + } + /// Create an assistant message. pub fn assistant(content: impl Into) -> Self { Self { @@ -291,4 +299,27 @@ mod tests { let block: ContentBlock = serde_json::from_value(json).unwrap(); assert!(matches!(block, ContentBlock::Unknown)); } + + #[test] + fn test_user_with_blocks() { + let blocks = vec![ + ContentBlock::Text { + text: "What is in this image?".to_string(), + }, + ContentBlock::Image { + media_type: "image/jpeg".to_string(), + data: "base64data".to_string(), + }, + ]; + let msg = Message::user_with_blocks(blocks); + assert_eq!(msg.role, Role::User); + match msg.content { + MessageContent::Blocks(ref b) => { + assert_eq!(b.len(), 2); + assert!(matches!(&b[0], ContentBlock::Text { text } if text == "What is in this image?")); + assert!(matches!(&b[1], ContentBlock::Image { media_type, .. } if media_type == "image/jpeg")); + } + _ => panic!("Expected blocks content"), + } + } }