diff --git a/app/test/e2e/mock-server.ts b/app/test/e2e/mock-server.ts index d504476f6..b4debf625 100644 --- a/app/test/e2e/mock-server.ts +++ b/app/test/e2e/mock-server.ts @@ -7,6 +7,7 @@ */ export { clearRequestLog, + emitMockAgentAudioStream, getMockBehavior, getRequestLog, resetMockBehavior, diff --git a/app/test/e2e/specs/audio-toolkit-flow.spec.ts b/app/test/e2e/specs/audio-toolkit-flow.spec.ts new file mode 100644 index 000000000..2bd38a467 --- /dev/null +++ b/app/test/e2e/specs/audio-toolkit-flow.spec.ts @@ -0,0 +1,82 @@ +import { waitForApp } from '../helpers/app-helpers'; +import { callOpenhumanRpc } from '../helpers/core-rpc'; +import { resetApp } from '../helpers/reset-app'; +import { startMockServer, stopMockServer } from '../mock-server'; + +const USER_ID = 'e2e-audio-toolkit'; + +describe('Audio toolkit flow', () => { + before(async () => { + await startMockServer(); + await waitForApp(); + await resetApp(USER_ID); + }); + + after(async () => { + await stopMockServer(); + }); + + it('generates an mp3 artifact and captures the email attachment in the workspace', async () => { + const response = await callOpenhumanRpc<{ + result: { + audio: { output_path: string; file_name: string; bytes_written: number; format: string }; + email: { mode: string; capture_path?: string | null; attachment_name: string }; + }; + }>('openhuman.audio_toolkit_generate_and_email_podcast', { + text: 'This is the weekly AI podcast briefing for the team.', + title: 'Weekly briefing', + to: 'listener@example.com', + subject: 'Your weekly audio briefing', + body: 'Attached is the latest audio briefing.', + format: 'mp3', + }); + + expect(response.ok).toBe(true); + const result = (response.result?.result ?? response.result) as + | { + audio: { output_path: string; file_name: string; bytes_written: number; format: string }; + email: { mode: string; capture_path?: string | null; attachment_name: string }; + } + | undefined; + expect(result?.audio.format).toBe('mp3'); + expect(result?.audio.bytes_written).toBeGreaterThan(0); + expect(result?.email.mode).toBe('capture'); + expect(result?.email.capture_path).toBeTruthy(); + + const workspaceFiles = await callOpenhumanRpc<{ + result: { entries: Array<{ rel_path: string; size: number; is_dir: boolean }> }; + }>('openhuman.test_support_list_workspace_files', { rel_root: 'artifacts', max_depth: 4 }); + expect(workspaceFiles.ok).toBe(true); + const entries = + ( + (workspaceFiles.result?.result ?? workspaceFiles.result) as + | { entries?: Array<{ rel_path: string; size: number; is_dir: boolean }> } + | undefined + )?.entries ?? []; + const audioArtifact = entries.find( + e => + e.rel_path === result?.audio.output_path || + !!result?.audio.output_path?.endsWith(`/${e.rel_path}`) || + e.rel_path.endsWith(`/${result?.audio.output_path ?? ''}`) + ); + expect(audioArtifact?.size ?? 0).toBeGreaterThan(0); + const capturedEmail = entries.find( + e => + e.rel_path === result?.email.capture_path || + !!result?.email.capture_path?.endsWith(`/${e.rel_path}`) || + e.rel_path.endsWith(`/${result?.email.capture_path ?? ''}`) + ); + expect(capturedEmail?.size ?? 0).toBeGreaterThan(0); + + const emailRead = await callOpenhumanRpc<{ result: { content_utf8: string } }>( + 'openhuman.test_support_read_workspace_file', + { rel_path: result?.email.capture_path, max_bytes: 131072 } + ); + expect(emailRead.ok).toBe(true); + const wire = + ((emailRead.result?.result ?? emailRead.result) as { content_utf8?: string } | undefined) + ?.content_utf8 ?? ''; + expect(wire).toContain('Subject: Your weekly audio briefing'); + expect(wire).toContain(result?.email.attachment_name ?? 'weekly-briefing.mp3'); + }); +}); diff --git a/scripts/mock-api-core.mjs b/scripts/mock-api-core.mjs index 900621b1e..87e591fbc 100644 --- a/scripts/mock-api-core.mjs +++ b/scripts/mock-api-core.mjs @@ -8,6 +8,7 @@ export { DEFAULT_PORT, clearRequestLog, + emitMockAgentAudioStream, getMockBehavior, getMockServerPort, getRequestLog, diff --git a/scripts/mock-api/index.mjs b/scripts/mock-api/index.mjs index f9b80b0d5..f226cba30 100644 --- a/scripts/mock-api/index.mjs +++ b/scripts/mock-api/index.mjs @@ -27,4 +27,8 @@ export { setMockBehavior, setMockBehaviors, } from "./state.mjs"; -export { disconnectMockSockets, emitMockSocketEvent } from "./socket.mjs"; +export { + disconnectMockSockets, + emitMockAgentAudioStream, + emitMockSocketEvent, +} from "./socket.mjs"; diff --git a/scripts/mock-api/routes/__tests__/audio.test.mjs b/scripts/mock-api/routes/__tests__/audio.test.mjs new file mode 100644 index 000000000..99a7b4276 --- /dev/null +++ b/scripts/mock-api/routes/__tests__/audio.test.mjs @@ -0,0 +1,87 @@ +import assert from "node:assert/strict"; +import test from "node:test"; + +import { resetMockBehavior, setMockBehavior } from "../../state.mjs"; +import { handleAudio } from "../audio.mjs"; + +function createRes() { + return { + statusCode: 0, + headers: {}, + body: "", + writeHead(status, headers = {}) { + this.statusCode = status; + this.headers = headers; + }, + setHeader(name, value) { + this.headers[name] = value; + }, + end(chunk = "") { + this.body += String(chunk); + }, + }; +} + +test.beforeEach(() => { + resetMockBehavior(); +}); + +test("mock audio speech route returns audio + visemes", async () => { + const res = createRes(); + const handled = await handleAudio({ + method: "POST", + url: "/openai/v1/audio/speech", + parsedBody: { + text: "hello world", + with_visemes: true, + }, + res, + }); + + assert.equal(handled, true); + assert.equal(res.statusCode, 200); + const payload = JSON.parse(res.body); + assert.equal(payload.audio_mime, "audio/mpeg"); + assert.ok( + typeof payload.audio_base64 === "string" && payload.audio_base64.length > 0, + ); + assert.ok(Array.isArray(payload.visemes) && payload.visemes.length > 0); + assert.ok(Array.isArray(payload.alignment) && payload.alignment.length > 0); +}); + +test("mock audio speech route honors behavior overrides", async () => { + setMockBehavior("audioSpeechMime", "audio/wav"); + setMockBehavior( + "audioSpeechBase64", + Buffer.from("WAVMOCK", "utf8").toString("base64"), + ); + + const res = createRes(); + await handleAudio({ + method: "POST", + url: "/openai/v1/audio/speech", + parsedBody: { text: "override" }, + res, + }); + + const payload = JSON.parse(res.body); + assert.equal(payload.audio_mime, "audio/wav"); + assert.equal( + Buffer.from(payload.audio_base64, "base64").toString("utf8"), + "WAVMOCK", + ); +}); + +test("mock audio transcription route returns deterministic text", async () => { + setMockBehavior("audioTranscriptionText", "Podcast transcription."); + const res = createRes(); + const handled = await handleAudio({ + method: "POST", + url: "/openai/v1/audio/transcriptions", + parsedBody: {}, + res, + }); + + assert.equal(handled, true); + assert.equal(JSON.parse(res.body).text, "Podcast transcription."); +}); diff --git a/scripts/mock-api/routes/audio.mjs b/scripts/mock-api/routes/audio.mjs new file mode 100644 index 000000000..7bcaca48b --- /dev/null +++ b/scripts/mock-api/routes/audio.mjs @@ -0,0 +1,72 @@ +import { json } from "../http.mjs"; +import { behavior, parseBehaviorJson } from "../state.mjs"; + +const DEFAULT_AUDIO_BYTES = Buffer.from("ID3MOCKAUDIO", "utf8"); + +function defaultVisemes() { + return [ + { viseme: "sil", start_ms: 0, end_ms: 40 }, + { viseme: "aa", start_ms: 40, end_ms: 240 }, + ]; +} + +function defaultAlignment(text) { + const chars = String(text || "ok") + .slice(0, 8) + .split(""); + return chars.map((char, index) => ({ + char, + start_ms: index * 80, + end_ms: index * 80 + 80, + })); +} + +export async function handleAudio(ctx) { + const { method, url, parsedBody, res } = ctx; + + if (method === "POST" && /^\/openai\/v1\/audio\/speech\/?$/.test(url)) { + const text = String(parsedBody?.text || ""); + const mockBehavior = behavior(); + const visemes = parseBehaviorJson("audioSpeechVisemes", defaultVisemes()); + const alignment = parseBehaviorJson( + "audioSpeechAlignment", + defaultAlignment(text), + ); + const audioBytes = mockBehavior.audioSpeechBase64 + ? Buffer.from(String(mockBehavior.audioSpeechBase64), "base64") + : DEFAULT_AUDIO_BYTES; + json(res, 200, { + audio_base64: audioBytes.toString("base64"), + audio_mime: mockBehavior.audioSpeechMime || "audio/mpeg", + visemes: + parsedBody?.with_visemes === true || parsedBody?.with_alignment === true + ? visemes + : [], + alignment: + parsedBody?.with_alignment === true || parsedBody?.with_visemes === true + ? alignment + : undefined, + voice_id: + parsedBody?.voice_id || mockBehavior.audioSpeechVoiceId || "mock-voice", + model_id: + parsedBody?.model_id || + mockBehavior.audioSpeechModelId || + "mock-tts-v1", + }); + return true; + } + + if ( + method === "POST" && + /^\/openai\/v1\/audio\/transcriptions\/?$/.test(url) + ) { + json(res, 200, { + text: + behavior().audioTranscriptionText || + "Mock transcription from the E2E server.", + }); + return true; + } + + return false; +} diff --git a/scripts/mock-api/server.mjs b/scripts/mock-api/server.mjs index db23d5d83..48342fd64 100644 --- a/scripts/mock-api/server.mjs +++ b/scripts/mock-api/server.mjs @@ -10,6 +10,7 @@ import { tryParseJson, } from "./http.mjs"; import { handleAuth } from "./routes/auth.mjs"; +import { handleAudio } from "./routes/audio.mjs"; import { handleConversations } from "./routes/conversations.mjs"; import { handleCron } from "./routes/cron.mjs"; import { handleIntegrations } from "./routes/integrations.mjs"; @@ -42,6 +43,7 @@ const ROUTE_HANDLERS = [ handleUser, handleInvites, handlePayments, + handleAudio, // LLM completions must run before the catch-all stub in // `handleIntegrations` so keyword-driven test scripts can override // the default "Hello from e2e mock agent" reply. diff --git a/scripts/mock-api/socket.mjs b/scripts/mock-api/socket.mjs index 2a177e560..76b101348 100644 --- a/scripts/mock-api/socket.mjs +++ b/scripts/mock-api/socket.mjs @@ -1,5 +1,6 @@ export { disconnectMockSockets, + emitMockAgentAudioStream, emitMockSocketEvent, handleSocketRequest, handleWebSocketUpgrade, diff --git a/scripts/mock-api/socket.transport.test.mjs b/scripts/mock-api/socket.transport.test.mjs index e29b57dc0..2141b2ed7 100644 --- a/scripts/mock-api/socket.transport.test.mjs +++ b/scripts/mock-api/socket.transport.test.mjs @@ -5,6 +5,7 @@ import test from "node:test"; import { clearSocketEventLog, disconnectMockSockets, + emitMockAgentAudioStream, emitMockSocketEvent, listSocketSessions, resetMockBehavior, @@ -101,6 +102,45 @@ test("supports polling-only clients", async () => { } }); +test("streams mock agent audio events through a real socket.io client", async () => { + const started = await startMockServer(18575, { retryIfInUse: true }); + const baseUrl = `http://127.0.0.1:${started.port}`; + const socket = createSocket(baseUrl, { + auth: { token: "mock-jwt-token" }, + transports: ["polling", "websocket"], + }); + + try { + await onceSocket(socket, "ready"); + const startPromise = onceSocket(socket, "agent:audio:start"); + const chunkPromise = onceSocket(socket, "agent:audio:chunk"); + const endPromise = onceSocket(socket, "agent:audio:end"); + + const delivered = emitMockAgentAudioStream({ + sessionId: "session-audio-1", + text: "listen now", + voiceId: "voice-1", + chunks: ["SUQz", "TU9DSw=="], + chunkDelayMs: 5, + }); + assert.equal(delivered, 1); + + const startedPayload = await startPromise; + assert.equal(startedPayload.sessionId, "session-audio-1"); + assert.equal(startedPayload.voiceId, "voice-1"); + assert.equal(startedPayload.contentType, "audio/mpeg"); + + const chunkPayload = await chunkPromise; + assert.equal(chunkPayload.chunk, "SUQz"); + + const endedPayload = await endPromise; + assert.equal(endedPayload.sessionId, "session-audio-1"); + assert.equal(endedPayload.ttsCharCount, "listen now".length); + } finally { + socket.disconnect(); + } +}); + test("keeps polling session alive when websocket probe closes before upgrade", () => { const session = registerSocketSession({ sid: "probe-fallback-sid", diff --git a/scripts/mock-api/socket/core.mjs b/scripts/mock-api/socket/core.mjs index 36ac15d99..f029207e2 100644 --- a/scripts/mock-api/socket/core.mjs +++ b/scripts/mock-api/socket/core.mjs @@ -127,6 +127,13 @@ function scheduleMockSocketActions(session, actions = []) { : (action.data ?? null), targetSid: session.sid, }); + return; + } + if (action?.agentAudioStream) { + emitMockAgentAudioStream({ + targetSid: session.sid, + ...action.agentAudioStream, + }); } }, delayMs); } @@ -407,6 +414,75 @@ export function emitMockSocketEvent({ return matchingSessions.length; } +export function emitMockAgentAudioStream({ + sessionId = "mock-agent-session", + requestId, + text = "", + voiceId = "", + contentType = "audio/mpeg", + chunks, + chunkDelayMs = 0, + targetSid, + targetUserId, + excludeSid, +}) { + const resolvedRequestId = requestId || `mock-audio-${createMockId("req")}`; + const normalizedChunks = + Array.isArray(chunks) && chunks.length > 0 + ? chunks + : [Buffer.from("ID3MOCKAUDIO", "utf8").toString("base64")]; + + let delivered = emitMockSocketEvent({ + event: "agent:audio:start", + data: { + sessionId, + requestId: resolvedRequestId, + contentType, + voiceId, + text, + }, + targetSid, + targetUserId, + excludeSid, + }); + + normalizedChunks.forEach((chunk, index) => { + delivered = Math.max( + delivered, + emitMockSocketEvent({ + event: "agent:audio:chunk", + data: { + sessionId, + requestId: resolvedRequestId, + chunk, + }, + targetSid, + targetUserId, + excludeSid, + delayMs: chunkDelayMs * (index + 1), + }), + ); + }); + + delivered = Math.max( + delivered, + emitMockSocketEvent({ + event: "agent:audio:end", + data: { + sessionId, + requestId: resolvedRequestId, + ttsCharCount: String(text || "").length, + }, + targetSid, + targetUserId, + excludeSid, + delayMs: chunkDelayMs * (normalizedChunks.length + 1), + }), + ); + + return delivered; +} + export function disconnectMockSockets({ targetSid, targetUserId } = {}) { let disconnected = 0; for (const sessionInfo of listSocketSessions()) { diff --git a/scripts/mock-api/socket/index.mjs b/scripts/mock-api/socket/index.mjs index 8d8dc985d..90ba65bf3 100644 --- a/scripts/mock-api/socket/index.mjs +++ b/scripts/mock-api/socket/index.mjs @@ -1,5 +1,6 @@ export { disconnectMockSockets, + emitMockAgentAudioStream, emitMockSocketEvent, handleSocketRequest, handleWebSocketUpgrade, diff --git a/src/core/all.rs b/src/core/all.rs index 2b67247dc..fa0795ac1 100644 --- a/src/core/all.rs +++ b/src/core/all.rs @@ -108,6 +108,8 @@ fn build_registered_controllers() -> Vec { controllers.extend(crate::openhuman::about_app::all_about_app_registered_controllers()); // Core application shell state controllers.extend(crate::openhuman::app_state::all_app_state_registered_controllers()); + // Audio generation + podcast-style email delivery + controllers.extend(crate::openhuman::audio_toolkit::all_audio_toolkit_registered_controllers()); // Composio integration controllers controllers.extend(crate::openhuman::composio::all_composio_registered_controllers()); // Scheduled job management @@ -251,6 +253,7 @@ fn build_declared_controller_schemas() -> Vec { let mut schemas = Vec::new(); schemas.extend(crate::openhuman::about_app::all_about_app_controller_schemas()); schemas.extend(crate::openhuman::app_state::all_app_state_controller_schemas()); + schemas.extend(crate::openhuman::audio_toolkit::all_audio_toolkit_controller_schemas()); schemas.extend(crate::openhuman::composio::all_composio_controller_schemas()); schemas.extend(crate::openhuman::cron::all_cron_controller_schemas()); schemas.extend(crate::openhuman::webview_apis::all_webview_apis_controller_schemas()); diff --git a/src/openhuman/audio_toolkit/mod.rs b/src/openhuman/audio_toolkit/mod.rs new file mode 100644 index 000000000..dab8f8202 --- /dev/null +++ b/src/openhuman/audio_toolkit/mod.rs @@ -0,0 +1,12 @@ +mod ops; +mod schemas; +mod types; + +pub use ops::{ + email_podcast, generate_and_email_podcast, generate_podcast, resolve_email_capture_dir, +}; +pub use schemas::{all_audio_toolkit_controller_schemas, all_audio_toolkit_registered_controllers}; +pub use types::{ + AudioEmailDeliveryResult, AudioFormat, AudioGenerateRequest, AudioGeneratedArtifact, + AudioToolkitGenerateAndEmailResult, EmailPodcastRequest, +}; diff --git a/src/openhuman/audio_toolkit/ops.rs b/src/openhuman/audio_toolkit/ops.rs new file mode 100644 index 000000000..22ea57ec1 --- /dev/null +++ b/src/openhuman/audio_toolkit/ops.rs @@ -0,0 +1,515 @@ +use std::path::{Path, PathBuf}; + +use base64::{engine::general_purpose::STANDARD as BASE64, Engine}; +use lettre::message::{header::ContentType, Attachment, Mailbox, MultiPart, SinglePart}; +use lettre::Message; + +use crate::openhuman::channels::email_channel::EmailChannel; +use crate::openhuman::config::Config; +use crate::openhuman::voice::{create_tts_provider, DEFAULT_PIPER_VOICE}; +use crate::rpc::RpcOutcome; + +use super::types::{ + AudioEmailDeliveryResult, AudioFormat, AudioGenerateRequest, AudioGeneratedArtifact, + AudioToolkitGenerateAndEmailResult, EmailPodcastRequest, +}; + +const LOG_PREFIX: &str = "[audio_toolkit]"; +const DEFAULT_OUTPUT_DIR: &str = "artifacts/audio"; +const DEFAULT_CAPTURE_DIR: &str = "artifacts/email-capture"; +const EMAIL_CAPTURE_ENV: &str = "OPENHUMAN_EMAIL_CAPTURE_DIR"; + +pub async fn generate_podcast( + config: &Config, + request: AudioGenerateRequest, +) -> Result, String> { + let trimmed = request.text.trim(); + if trimmed.is_empty() { + return Err("text is required".to_string()); + } + + let provider = effective_provider_name(config, request.provider.as_deref()); + let format = resolve_format(&provider, request.format)?; + let voice = effective_voice(&provider, request.voice.as_deref()); + let output_rel_path = resolve_output_path( + config.workspace_dir.as_path(), + request.output_path.as_deref(), + request.title.as_deref(), + format, + )?; + + log::debug!( + "{LOG_PREFIX} generate provider={} format={:?} output_path={}", + provider, + format, + output_rel_path.display() + ); + + let provider_impl = create_tts_provider(&provider, voice.as_deref().unwrap_or(""), config) + .map_err(|e| e.to_string())?; + let outcome = provider_impl + .synthesize(config, trimmed, voice.as_deref()) + .await?; + + let bytes = decode_audio_payload(&outcome.value.audio_base64)?; + enforce_audio_format(format, &outcome.value.audio_mime)?; + let resolved = config.workspace_dir.join(&output_rel_path); + if let Some(parent) = resolved.parent() { + tokio::fs::create_dir_all(parent) + .await + .map_err(|e| format!("failed to create {}: {e}", parent.display()))?; + } + tokio::fs::write(&resolved, &bytes) + .await + .map_err(|e| format!("failed to write {}: {e}", resolved.display()))?; + + let result = AudioGeneratedArtifact { + output_path: output_rel_path.to_string_lossy().to_string(), + file_name: resolved + .file_name() + .map(|v| v.to_string_lossy().to_string()) + .unwrap_or_else(|| format!("podcast.{}", format.extension())), + provider, + voice, + format, + audio_mime: outcome.value.audio_mime, + bytes_written: bytes.len(), + chars_synthesized: trimmed.chars().count(), + }; + + Ok(RpcOutcome::single_log( + result, + "audio podcast synthesized to workspace file", + )) +} + +pub async fn email_podcast( + config: &Config, + request: EmailPodcastRequest, +) -> Result, String> { + let to = request.to.trim(); + let subject = request.subject.trim(); + let body = request.body.trim(); + let audio_path = request.audio_path.trim(); + + if to.is_empty() { + return Err("to is required".to_string()); + } + if subject.is_empty() { + return Err("subject is required".to_string()); + } + if body.is_empty() { + return Err("body is required".to_string()); + } + if audio_path.is_empty() { + return Err("audio_path is required".to_string()); + } + + let rel_audio_path = PathBuf::from(audio_path); + if rel_audio_path.is_absolute() { + return Err("audio_path must be workspace-relative".to_string()); + } + if rel_audio_path + .components() + .any(|c| matches!(c, std::path::Component::ParentDir)) + { + return Err("audio_path must not contain parent-directory traversal".to_string()); + } + + let resolved_audio_path = config.workspace_dir.join(&rel_audio_path); + let audio_bytes = tokio::fs::read(&resolved_audio_path) + .await + .map_err(|e| format!("failed to read {}: {e}", resolved_audio_path.display()))?; + let attachment_name = request + .attachment_name + .as_deref() + .map(str::trim) + .filter(|s| !s.is_empty()) + .map(str::to_string) + .or_else(|| { + resolved_audio_path + .file_name() + .map(|v| v.to_string_lossy().to_string()) + }) + .unwrap_or_else(|| "podcast.mp3".to_string()); + let content_type = content_type_for_attachment(&attachment_name)?; + + let email = build_email_message( + config, + to, + subject, + body, + &attachment_name, + content_type, + audio_bytes, + )?; + + if let Some(capture_dir) = resolve_email_capture_dir(config) { + let capture_rel = + capture_email_message(config.workspace_dir.as_path(), &capture_dir, &email).await?; + let result = AudioEmailDeliveryResult { + to: to.to_string(), + subject: subject.to_string(), + attachment_name, + mode: "capture".to_string(), + capture_path: Some(capture_rel), + }; + return Ok(RpcOutcome::single_log( + result, + "audio email captured to workspace file", + )); + } + + let email_cfg = config + .channels_config + .email + .clone() + .ok_or_else(|| "email channel is not configured".to_string())?; + let channel = EmailChannel::new(email_cfg); + channel + .send_message(email) + .map_err(|e| format!("failed to send email attachment: {e}"))?; + + Ok(RpcOutcome::single_log( + AudioEmailDeliveryResult { + to: to.to_string(), + subject: subject.to_string(), + attachment_name, + mode: "smtp".to_string(), + capture_path: None, + }, + "audio email delivered over SMTP", + )) +} + +pub async fn generate_and_email_podcast( + config: &Config, + generate_request: AudioGenerateRequest, + email_request: EmailPodcastRequest, +) -> Result, String> { + let generated = generate_podcast(config, generate_request).await?; + let email_request = EmailPodcastRequest { + audio_path: generated.value.output_path.clone(), + ..email_request + }; + let emailed = email_podcast(config, email_request).await?; + Ok(RpcOutcome::single_log( + AudioToolkitGenerateAndEmailResult { + audio: generated.value, + email: emailed.value, + }, + "audio podcast generated and emailed", + )) +} + +pub fn resolve_email_capture_dir(config: &Config) -> Option { + if let Ok(raw) = std::env::var(EMAIL_CAPTURE_ENV) { + let trimmed = raw.trim(); + if !trimmed.is_empty() { + return Some(PathBuf::from(trimmed)); + } + } + #[cfg(feature = "e2e-test-support")] + { + return Some(config.workspace_dir.join(DEFAULT_CAPTURE_DIR)); + } + #[cfg(not(feature = "e2e-test-support"))] + { + let _ = config; + None + } +} + +fn effective_provider_name(config: &Config, requested: Option<&str>) -> String { + requested + .map(str::trim) + .filter(|s| !s.is_empty()) + .map(str::to_string) + .unwrap_or_else(|| { + let configured = config.local_ai.tts_provider.trim(); + if configured.is_empty() { + "cloud".to_string() + } else { + configured.to_string() + } + }) +} + +fn effective_voice(provider: &str, requested: Option<&str>) -> Option { + let explicit = requested + .map(str::trim) + .filter(|s| !s.is_empty()) + .map(str::to_string); + if explicit.is_some() { + return explicit; + } + if provider == "piper" { + return Some(DEFAULT_PIPER_VOICE.to_string()); + } + None +} + +fn resolve_format(provider: &str, requested: Option) -> Result { + let resolved = requested.unwrap_or_else(|| { + if provider == "piper" { + AudioFormat::Wav + } else { + AudioFormat::Mp3 + } + }); + match (provider, resolved) { + ("piper", AudioFormat::Mp3) => { + Err("provider `piper` only supports wav output; use format=`wav` or provider=`cloud`".to_string()) + } + ("cloud", AudioFormat::Wav) => { + Err("provider `cloud` currently returns mp3 output only; use format=`mp3` or provider=`piper`".to_string()) + } + _ => Ok(resolved), + } +} + +fn decode_audio_payload(audio_base64: &str) -> Result, String> { + BASE64 + .decode(audio_base64.trim()) + .map_err(|e| format!("invalid audio_base64 payload: {e}")) +} + +fn enforce_audio_format(format: AudioFormat, mime: &str) -> Result<(), String> { + let normalized = mime.trim().to_ascii_lowercase(); + match format { + AudioFormat::Mp3 if normalized == "audio/mpeg" => Ok(()), + AudioFormat::Wav if normalized == "audio/wav" => Ok(()), + _ => Err(format!( + "provider returned mime `{mime}` but requested format was `{}`", + format.extension() + )), + } +} + +fn resolve_output_path( + workspace_dir: &Path, + requested: Option<&str>, + title: Option<&str>, + format: AudioFormat, +) -> Result { + let rel_path = match requested.map(str::trim).filter(|s| !s.is_empty()) { + Some(path) => PathBuf::from(path), + None => { + let slug = slugify_title(title.unwrap_or("podcast")); + PathBuf::from(DEFAULT_OUTPUT_DIR).join(format!( + "{}-{}.{}", + chrono::Utc::now().format("%Y%m%d-%H%M%S"), + slug, + format.extension() + )) + } + }; + if rel_path.is_absolute() { + return Err("output_path must be workspace-relative".to_string()); + } + if rel_path + .components() + .any(|c| matches!(c, std::path::Component::ParentDir)) + { + return Err("output_path must not contain parent-directory traversal".to_string()); + } + let resolved = workspace_dir.join(&rel_path); + if !resolved.starts_with(workspace_dir) { + return Err("output_path escapes the workspace".to_string()); + } + Ok(rel_path) +} + +fn slugify_title(title: &str) -> String { + let mut out = String::new(); + let mut last_was_dash = false; + for ch in title.chars() { + if ch.is_ascii_alphanumeric() { + out.push(ch.to_ascii_lowercase()); + last_was_dash = false; + } else if !last_was_dash && !out.is_empty() { + out.push('-'); + last_was_dash = true; + } + } + let trimmed = out.trim_matches('-'); + if trimmed.is_empty() { + "podcast".to_string() + } else { + trimmed.to_string() + } +} + +fn content_type_for_attachment(file_name: &str) -> Result { + let mime = if file_name.to_ascii_lowercase().ends_with(".wav") { + "audio/wav" + } else { + "audio/mpeg" + }; + mime.parse() + .map_err(|e| format!("invalid content type for attachment: {e}")) +} + +fn build_email_message( + config: &Config, + to: &str, + subject: &str, + body: &str, + attachment_name: &str, + content_type: ContentType, + attachment_bytes: Vec, +) -> Result { + let from_address = config + .channels_config + .email + .as_ref() + .map(|cfg| cfg.from_address.trim()) + .filter(|s| !s.is_empty()) + .unwrap_or("openhuman@localhost.test"); + let from: Mailbox = from_address + .parse() + .map_err(|e| format!("invalid from address `{from_address}`: {e}"))?; + let to_mailbox: Mailbox = to + .parse() + .map_err(|e| format!("invalid to address `{to}`: {e}"))?; + let attachment = + Attachment::new(attachment_name.to_string()).body(attachment_bytes, content_type); + Message::builder() + .from(from) + .to(to_mailbox) + .subject(subject) + .multipart( + MultiPart::mixed() + .singlepart(SinglePart::plain(body.to_string())) + .singlepart(attachment), + ) + .map_err(|e| format!("failed to build email message: {e}")) +} + +async fn capture_email_message( + workspace_dir: &Path, + capture_dir: &Path, + email: &Message, +) -> Result { + let resolved_dir = if capture_dir.is_absolute() { + capture_dir.to_path_buf() + } else { + workspace_dir.join(capture_dir) + }; + tokio::fs::create_dir_all(&resolved_dir) + .await + .map_err(|e| format!("failed to create {}: {e}", resolved_dir.display()))?; + let file_name = format!( + "podcast-email-{}-{}.eml", + chrono::Utc::now().format("%Y%m%d-%H%M%S"), + uuid::Uuid::new_v4() + ); + let resolved_path = resolved_dir.join(&file_name); + tokio::fs::write(&resolved_path, email.formatted()) + .await + .map_err(|e| format!("failed to write {}: {e}", resolved_path.display()))?; + let relative = resolved_path + .strip_prefix(workspace_dir) + .map(|v| v.to_string_lossy().to_string()) + .unwrap_or_else(|_| resolved_path.to_string_lossy().to_string()); + Ok(relative) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn test_config() -> Config { + let mut config = Config::default(); + config.workspace_dir = std::env::temp_dir().join("openhuman-audio-toolkit-tests"); + config.local_ai.tts_provider = "cloud".to_string(); + config + } + + #[test] + fn resolve_format_defaults_to_mp3_for_cloud() { + assert_eq!(resolve_format("cloud", None).unwrap(), AudioFormat::Mp3); + } + + #[test] + fn resolve_format_defaults_to_wav_for_piper() { + assert_eq!(resolve_format("piper", None).unwrap(), AudioFormat::Wav); + } + + #[test] + fn resolve_format_rejects_mp3_for_piper() { + let err = resolve_format("piper", Some(AudioFormat::Mp3)).unwrap_err(); + assert!(err.contains("only supports wav")); + } + + #[test] + fn slugify_title_collapses_noise() { + assert_eq!( + slugify_title(" Weekly update: Q2 / AI! "), + "weekly-update-q2-ai" + ); + assert_eq!(slugify_title("###"), "podcast"); + } + + #[test] + fn resolve_output_path_rejects_parent_dir() { + let err = resolve_output_path( + Path::new("/tmp/workspace"), + Some("../escape.mp3"), + None, + AudioFormat::Mp3, + ) + .unwrap_err(); + assert!(err.contains("parent-directory traversal")); + } + + #[test] + fn build_email_message_includes_attachment_name() { + let config = test_config(); + let message = build_email_message( + &config, + "listener@example.com", + "Podcast", + "Attached.", + "briefing.mp3", + "audio/mpeg".parse().unwrap(), + vec![1, 2, 3], + ) + .unwrap(); + let wire = String::from_utf8_lossy(&message.formatted()).to_string(); + assert!(wire.contains("Subject: Podcast")); + assert!(wire.contains("filename=\"briefing.mp3\"")); + assert!(wire.contains("Content-Type: audio/mpeg")); + } + + #[test] + fn resolve_email_capture_dir_uses_workspace_when_e2e_feature_enabled() { + let config = test_config(); + let capture = resolve_email_capture_dir(&config); + #[cfg(feature = "e2e-test-support")] + assert!(capture.unwrap().ends_with(DEFAULT_CAPTURE_DIR)); + #[cfg(not(feature = "e2e-test-support"))] + assert!(capture.is_none()); + } + + #[test] + fn effective_voice_defaults_for_piper_only() { + assert_eq!( + effective_voice("piper", None).as_deref(), + Some(DEFAULT_PIPER_VOICE) + ); + assert!(effective_voice("cloud", None).is_none()); + } + + #[test] + fn enforce_audio_format_requires_matching_mime() { + assert!(enforce_audio_format(AudioFormat::Mp3, "audio/mpeg").is_ok()); + assert!(enforce_audio_format(AudioFormat::Mp3, "audio/wav").is_err()); + } + + #[test] + fn decode_audio_payload_rejects_bad_base64() { + let err = decode_audio_payload("not-base64").unwrap_err(); + assert!(err.contains("invalid audio_base64")); + } +} diff --git a/src/openhuman/audio_toolkit/schemas.rs b/src/openhuman/audio_toolkit/schemas.rs new file mode 100644 index 000000000..1f6baf4d0 --- /dev/null +++ b/src/openhuman/audio_toolkit/schemas.rs @@ -0,0 +1,203 @@ +use serde::Deserialize; +use serde_json::{Map, Value}; + +use crate::core::all::{ControllerFuture, RegisteredController}; +use crate::core::{ControllerSchema, FieldSchema, TypeSchema}; +use crate::openhuman::audio_toolkit::types::{ + AudioFormat, AudioGenerateRequest, EmailPodcastRequest, +}; +use crate::openhuman::config::rpc as config_rpc; +use crate::rpc::RpcOutcome; + +#[derive(Debug, Deserialize)] +struct GenerateAndEmailParams { + text: String, + to: String, + subject: String, + body: String, + #[serde(default)] + title: Option, + #[serde(default)] + output_path: Option, + #[serde(default)] + provider: Option, + #[serde(default)] + voice: Option, + #[serde(default)] + format: Option, + #[serde(default)] + attachment_name: Option, +} + +pub fn all_audio_toolkit_controller_schemas() -> Vec { + vec![ + audio_toolkit_schemas("generate_podcast"), + audio_toolkit_schemas("email_podcast"), + audio_toolkit_schemas("generate_and_email_podcast"), + ] +} + +pub fn all_audio_toolkit_registered_controllers() -> Vec { + vec![ + RegisteredController { + schema: audio_toolkit_schemas("generate_podcast"), + handler: handle_generate_podcast, + }, + RegisteredController { + schema: audio_toolkit_schemas("email_podcast"), + handler: handle_email_podcast, + }, + RegisteredController { + schema: audio_toolkit_schemas("generate_and_email_podcast"), + handler: handle_generate_and_email_podcast, + }, + ] +} + +pub fn audio_toolkit_schemas(function: &str) -> ControllerSchema { + match function { + "generate_podcast" => ControllerSchema { + namespace: "audio_toolkit", + function: "generate_podcast", + description: "Synthesize text into a workspace audio file for listen-later / podcast-style delivery.", + inputs: vec![ + required_string("text", "Text to synthesize."), + optional_string("title", "Optional title used to derive the default file name."), + optional_string("output_path", "Optional workspace-relative output path."), + optional_string("provider", "Optional TTS provider override (`cloud` or `piper`)."), + optional_string("voice", "Optional provider-specific voice id."), + FieldSchema { + name: "format", + ty: TypeSchema::Option(Box::new(TypeSchema::String)), + comment: "Optional output format (`mp3` or `wav`).", + required: false, + }, + ], + outputs: vec![json_output("audio", "Generated audio artifact metadata.")], + }, + "email_podcast" => ControllerSchema { + namespace: "audio_toolkit", + function: "email_podcast", + description: "Email a previously generated workspace audio file as an attachment.", + inputs: vec![ + required_string("to", "Destination email address."), + required_string("subject", "Email subject line."), + required_string("body", "Email body text."), + required_string("audio_path", "Workspace-relative path to the audio attachment."), + optional_string("attachment_name", "Optional attachment file name override."), + ], + outputs: vec![json_output("email", "Email delivery metadata.")], + }, + "generate_and_email_podcast" => ControllerSchema { + namespace: "audio_toolkit", + function: "generate_and_email_podcast", + description: "Generate an audio file from text and immediately email it as a podcast-style attachment.", + inputs: vec![ + required_string("text", "Text to synthesize."), + required_string("to", "Destination email address."), + required_string("subject", "Email subject line."), + required_string("body", "Email body text."), + optional_string("title", "Optional title used for the generated file name."), + optional_string("output_path", "Optional workspace-relative output path."), + optional_string("provider", "Optional TTS provider override (`cloud` or `piper`)."), + optional_string("voice", "Optional provider-specific voice id."), + FieldSchema { + name: "format", + ty: TypeSchema::Option(Box::new(TypeSchema::String)), + comment: "Optional output format (`mp3` or `wav`).", + required: false, + }, + optional_string("attachment_name", "Optional attachment file name override."), + ], + outputs: vec![json_output("result", "Combined audio generation and email-delivery result.")], + }, + _ => ControllerSchema { + namespace: "audio_toolkit", + function: "unknown", + description: "Unknown audio toolkit controller.", + inputs: vec![], + outputs: vec![FieldSchema { + name: "error", + ty: TypeSchema::String, + comment: "Lookup error details.", + required: true, + }], + }, + } +} + +fn handle_generate_podcast(params: Map) -> ControllerFuture { + Box::pin(async move { + let config = config_rpc::load_config_with_timeout().await?; + let request: AudioGenerateRequest = + serde_json::from_value(Value::Object(params)).map_err(|e| e.to_string())?; + to_json(crate::openhuman::audio_toolkit::generate_podcast(&config, request).await?) + }) +} + +fn handle_email_podcast(params: Map) -> ControllerFuture { + Box::pin(async move { + let config = config_rpc::load_config_with_timeout().await?; + let request: EmailPodcastRequest = + serde_json::from_value(Value::Object(params)).map_err(|e| e.to_string())?; + to_json(crate::openhuman::audio_toolkit::email_podcast(&config, request).await?) + }) +} + +fn handle_generate_and_email_podcast(params: Map) -> ControllerFuture { + Box::pin(async move { + let config = config_rpc::load_config_with_timeout().await?; + let request: GenerateAndEmailParams = + serde_json::from_value(Value::Object(params)).map_err(|e| e.to_string())?; + let generated = AudioGenerateRequest { + text: request.text, + title: request.title, + output_path: request.output_path, + provider: request.provider, + voice: request.voice, + format: request.format, + }; + let email = EmailPodcastRequest { + to: request.to, + subject: request.subject, + body: request.body, + audio_path: String::new(), + attachment_name: request.attachment_name, + }; + to_json( + crate::openhuman::audio_toolkit::generate_and_email_podcast(&config, generated, email) + .await?, + ) + }) +} + +fn to_json(outcome: RpcOutcome) -> Result { + serde_json::to_value(outcome.value).map_err(|e| format!("serialize error: {e}")) +} + +fn required_string(name: &'static str, comment: &'static str) -> FieldSchema { + FieldSchema { + name, + ty: TypeSchema::String, + comment, + required: true, + } +} + +fn optional_string(name: &'static str, comment: &'static str) -> FieldSchema { + FieldSchema { + name, + ty: TypeSchema::Option(Box::new(TypeSchema::String)), + comment, + required: false, + } +} + +fn json_output(name: &'static str, comment: &'static str) -> FieldSchema { + FieldSchema { + name, + ty: TypeSchema::Json, + comment, + required: true, + } +} diff --git a/src/openhuman/audio_toolkit/types.rs b/src/openhuman/audio_toolkit/types.rs new file mode 100644 index 000000000..3648c97cc --- /dev/null +++ b/src/openhuman/audio_toolkit/types.rs @@ -0,0 +1,77 @@ +use serde::{Deserialize, Serialize}; + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum AudioFormat { + Mp3, + Wav, +} + +impl AudioFormat { + pub fn extension(self) -> &'static str { + match self { + Self::Mp3 => "mp3", + Self::Wav => "wav", + } + } + + pub fn mime(self) -> &'static str { + match self { + Self::Mp3 => "audio/mpeg", + Self::Wav => "audio/wav", + } + } +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct AudioGenerateRequest { + pub text: String, + #[serde(default)] + pub title: Option, + #[serde(default)] + pub output_path: Option, + #[serde(default)] + pub provider: Option, + #[serde(default)] + pub voice: Option, + #[serde(default)] + pub format: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct EmailPodcastRequest { + pub to: String, + pub subject: String, + pub body: String, + pub audio_path: String, + #[serde(default)] + pub attachment_name: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct AudioGeneratedArtifact { + pub output_path: String, + pub file_name: String, + pub provider: String, + pub voice: Option, + pub format: AudioFormat, + pub audio_mime: String, + pub bytes_written: usize, + pub chars_synthesized: usize, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct AudioEmailDeliveryResult { + pub to: String, + pub subject: String, + pub attachment_name: String, + pub mode: String, + #[serde(default)] + pub capture_path: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct AudioToolkitGenerateAndEmailResult { + pub audio: AudioGeneratedArtifact, + pub email: AudioEmailDeliveryResult, +} diff --git a/src/openhuman/channels/providers/email_channel.rs b/src/openhuman/channels/providers/email_channel.rs index b66f5f46c..9e8a80d98 100644 --- a/src/openhuman/channels/providers/email_channel.rs +++ b/src/openhuman/channels/providers/email_channel.rs @@ -14,7 +14,7 @@ use async_imap::types::Fetch; use async_imap::Session; use async_trait::async_trait; use futures::TryStreamExt; -use lettre::message::SinglePart; +use lettre::message::{header::ContentType, Attachment, MultiPart, SinglePart}; use lettre::transport::smtp::authentication::Credentials; use lettre::{Message, SmtpTransport, Transport}; use mail_parser::{MessageParser, MimeHeaders}; @@ -477,6 +477,50 @@ impl EmailChannel { }; Ok(transport) } + + pub fn send_message(&self, email: Message) -> Result<()> { + let transport = self.create_smtp_transport()?; + transport.send(&email)?; + info!("Email sent"); + Ok(()) + } + + pub fn build_plain_message( + &self, + recipient: &str, + subject: &str, + body: &str, + ) -> Result { + Message::builder() + .from(self.config.from_address.parse()?) + .to(recipient.parse()?) + .subject(subject) + .singlepart(SinglePart::plain(body.to_string())) + .map_err(Into::into) + } + + pub fn build_message_with_attachment( + &self, + recipient: &str, + subject: &str, + body: &str, + attachment_name: &str, + content_type: ContentType, + attachment_bytes: Vec, + ) -> Result { + let attachment = + Attachment::new(attachment_name.to_string()).body(attachment_bytes, content_type); + Message::builder() + .from(self.config.from_address.parse()?) + .to(recipient.parse()?) + .subject(subject) + .multipart( + MultiPart::mixed() + .singlepart(SinglePart::plain(body.to_string())) + .singlepart(attachment), + ) + .map_err(Into::into) + } } /// Internal struct for parsed email data @@ -515,14 +559,8 @@ impl Channel for EmailChannel { ("OpenHuman Message", message.content.as_str()) }; - let email = Message::builder() - .from(self.config.from_address.parse()?) - .to(message.recipient.parse()?) - .subject(subject) - .singlepart(SinglePart::plain(body.to_string()))?; - - let transport = self.create_smtp_transport()?; - transport.send(&email)?; + let email = self.build_plain_message(message.recipient.as_str(), subject, body)?; + self.send_message(email)?; info!("Email sent to {}", message.recipient); Ok(()) } diff --git a/src/openhuman/channels/providers/email_channel_tests.rs b/src/openhuman/channels/providers/email_channel_tests.rs index b8e6d16cd..dd6772486 100644 --- a/src/openhuman/channels/providers/email_channel_tests.rs +++ b/src/openhuman/channels/providers/email_channel_tests.rs @@ -112,6 +112,42 @@ fn email_channel_name() { assert_eq!(channel.name(), "email"); } +#[test] +fn build_plain_message_uses_subject_and_body() { + let channel = EmailChannel::new(EmailConfig { + from_address: "bot@example.com".to_string(), + ..Default::default() + }); + let message = channel + .build_plain_message("listener@example.com", "Podcast", "Hello there") + .expect("plain message"); + let wire = String::from_utf8_lossy(&message.formatted()).to_string(); + assert!(wire.contains("Subject: Podcast")); + assert!(wire.contains("Hello there")); +} + +#[test] +fn build_message_with_attachment_adds_audio_part() { + let channel = EmailChannel::new(EmailConfig { + from_address: "bot@example.com".to_string(), + ..Default::default() + }); + let message = channel + .build_message_with_attachment( + "listener@example.com", + "Weekly briefing", + "Attached.", + "briefing.mp3", + "audio/mpeg".parse().expect("content type"), + vec![1, 2, 3, 4], + ) + .expect("attachment message"); + let wire = String::from_utf8_lossy(&message.formatted()).to_string(); + assert!(wire.contains("Subject: Weekly briefing")); + assert!(wire.contains("filename=\"briefing.mp3\"")); + assert!(wire.contains("Content-Type: audio/mpeg")); +} + // is_sender_allowed tests #[test] diff --git a/src/openhuman/mod.rs b/src/openhuman/mod.rs index 266a3ffb7..fd21c64b5 100644 --- a/src/openhuman/mod.rs +++ b/src/openhuman/mod.rs @@ -19,6 +19,7 @@ pub mod accessibility; pub mod agent; pub mod app_state; pub mod approval; +pub mod audio_toolkit; pub mod autocomplete; pub mod billing; pub mod channels; diff --git a/src/openhuman/tools/impl/audio/mod.rs b/src/openhuman/tools/impl/audio/mod.rs new file mode 100644 index 000000000..3d196dde2 --- /dev/null +++ b/src/openhuman/tools/impl/audio/mod.rs @@ -0,0 +1,5 @@ +mod podcast; + +pub use podcast::{ + AudioEmailPodcastTool, AudioGenerateAndEmailPodcastTool, AudioGeneratePodcastTool, +}; diff --git a/src/openhuman/tools/impl/audio/podcast.rs b/src/openhuman/tools/impl/audio/podcast.rs new file mode 100644 index 000000000..cbb24327b --- /dev/null +++ b/src/openhuman/tools/impl/audio/podcast.rs @@ -0,0 +1,228 @@ +use std::sync::Arc; + +use async_trait::async_trait; +use serde_json::json; + +use crate::openhuman::audio_toolkit::{ + email_podcast, generate_and_email_podcast, generate_podcast, AudioGenerateRequest, + EmailPodcastRequest, +}; +use crate::openhuman::config::Config; +use crate::openhuman::security::{SecurityPolicy, ToolOperation}; +use crate::openhuman::tools::traits::{PermissionLevel, Tool, ToolResult}; + +pub struct AudioGeneratePodcastTool { + config: Arc, + security: Arc, +} + +pub struct AudioEmailPodcastTool { + config: Arc, + security: Arc, +} + +pub struct AudioGenerateAndEmailPodcastTool { + config: Arc, + security: Arc, +} + +impl AudioGeneratePodcastTool { + pub fn new(config: Arc, security: Arc) -> Self { + Self { config, security } + } +} + +impl AudioEmailPodcastTool { + pub fn new(config: Arc, security: Arc) -> Self { + Self { config, security } + } +} + +impl AudioGenerateAndEmailPodcastTool { + pub fn new(config: Arc, security: Arc) -> Self { + Self { config, security } + } +} + +#[async_trait] +impl Tool for AudioGeneratePodcastTool { + fn name(&self) -> &str { + "audio_generate_podcast" + } + + fn description(&self) -> &str { + "Generate an audio file from text and save it into the workspace for listen-later or podcast-style delivery." + } + + fn parameters_schema(&self) -> serde_json::Value { + json!({ + "type": "object", + "required": ["text"], + "properties": { + "text": { "type": "string", "description": "Text to synthesize into audio." }, + "title": { "type": "string", "description": "Optional title used in the default file name." }, + "output_path": { "type": "string", "description": "Optional workspace-relative path to write the audio file to." }, + "provider": { "type": "string", "description": "Optional TTS provider override (`cloud` or `piper`)." }, + "voice": { "type": "string", "description": "Optional voice id for the chosen provider." }, + "format": { "type": "string", "enum": ["mp3", "wav"], "description": "Desired audio format." } + } + }) + } + + fn permission_level(&self) -> PermissionLevel { + PermissionLevel::Execute + } + + async fn execute(&self, args: serde_json::Value) -> anyhow::Result { + self.security + .enforce_tool_operation(ToolOperation::Act, self.name()) + .map_err(anyhow::Error::msg)?; + let request: AudioGenerateRequest = serde_json::from_value(args)?; + let outcome = generate_podcast(&self.config, request) + .await + .map_err(anyhow::Error::msg)?; + Ok(ToolResult::success(serde_json::to_string_pretty( + &outcome.value, + )?)) + } +} + +#[async_trait] +impl Tool for AudioEmailPodcastTool { + fn name(&self) -> &str { + "audio_email_podcast" + } + + fn description(&self) -> &str { + "Email a workspace audio file as an attachment so the recipient can listen to it like a podcast episode." + } + + fn parameters_schema(&self) -> serde_json::Value { + json!({ + "type": "object", + "required": ["to", "subject", "body", "audio_path"], + "properties": { + "to": { "type": "string", "description": "Recipient email address." }, + "subject": { "type": "string", "description": "Email subject line." }, + "body": { "type": "string", "description": "Email body text." }, + "audio_path": { "type": "string", "description": "Workspace-relative path to the generated audio file." }, + "attachment_name": { "type": "string", "description": "Optional attachment file name override." } + } + }) + } + + fn permission_level(&self) -> PermissionLevel { + PermissionLevel::Execute + } + + async fn execute(&self, args: serde_json::Value) -> anyhow::Result { + self.security + .enforce_tool_operation(ToolOperation::Act, self.name()) + .map_err(anyhow::Error::msg)?; + let request: EmailPodcastRequest = serde_json::from_value(args)?; + let outcome = email_podcast(&self.config, request) + .await + .map_err(anyhow::Error::msg)?; + Ok(ToolResult::success(serde_json::to_string_pretty( + &outcome.value, + )?)) + } +} + +#[async_trait] +impl Tool for AudioGenerateAndEmailPodcastTool { + fn name(&self) -> &str { + "audio_generate_and_email_podcast" + } + + fn description(&self) -> &str { + "Generate an audio file from text and immediately email it as a podcast-style attachment." + } + + fn parameters_schema(&self) -> serde_json::Value { + json!({ + "type": "object", + "required": ["text", "to", "subject", "body"], + "properties": { + "text": { "type": "string", "description": "Text to synthesize into audio." }, + "to": { "type": "string", "description": "Recipient email address." }, + "subject": { "type": "string", "description": "Email subject line." }, + "body": { "type": "string", "description": "Email body text." }, + "title": { "type": "string", "description": "Optional title used in the default file name." }, + "output_path": { "type": "string", "description": "Optional workspace-relative output path." }, + "provider": { "type": "string", "description": "Optional TTS provider override (`cloud` or `piper`)." }, + "voice": { "type": "string", "description": "Optional voice id for the chosen provider." }, + "format": { "type": "string", "enum": ["mp3", "wav"], "description": "Desired audio format." }, + "attachment_name": { "type": "string", "description": "Optional attachment file name override." } + } + }) + } + + fn permission_level(&self) -> PermissionLevel { + PermissionLevel::Execute + } + + async fn execute(&self, args: serde_json::Value) -> anyhow::Result { + self.security + .enforce_tool_operation(ToolOperation::Act, self.name()) + .map_err(anyhow::Error::msg)?; + + let text = required_string(&args, "text")?; + let to = required_string(&args, "to")?; + let subject = required_string(&args, "subject")?; + let body = required_string(&args, "body")?; + + let generated = AudioGenerateRequest { + text, + title: optional_string(&args, "title"), + output_path: optional_string(&args, "output_path"), + provider: optional_string(&args, "provider"), + voice: optional_string(&args, "voice"), + format: optional_format(&args, "format")?, + }; + let email = EmailPodcastRequest { + to, + subject, + body, + audio_path: String::new(), + attachment_name: optional_string(&args, "attachment_name"), + }; + let outcome = generate_and_email_podcast(&self.config, generated, email) + .await + .map_err(anyhow::Error::msg)?; + Ok(ToolResult::success(serde_json::to_string_pretty( + &outcome.value, + )?)) + } +} + +fn required_string(args: &serde_json::Value, key: &str) -> anyhow::Result { + args.get(key) + .and_then(|v| v.as_str()) + .map(str::trim) + .filter(|s| !s.is_empty()) + .map(str::to_string) + .ok_or_else(|| anyhow::anyhow!("missing required `{key}`")) +} + +fn optional_string(args: &serde_json::Value, key: &str) -> Option { + args.get(key) + .and_then(|v| v.as_str()) + .map(str::trim) + .filter(|s| !s.is_empty()) + .map(str::to_string) +} + +fn optional_format( + args: &serde_json::Value, + key: &str, +) -> anyhow::Result> { + let Some(raw) = args.get(key).and_then(|v| v.as_str()) else { + return Ok(None); + }; + match raw.trim() { + "mp3" => Ok(Some(crate::openhuman::audio_toolkit::AudioFormat::Mp3)), + "wav" => Ok(Some(crate::openhuman::audio_toolkit::AudioFormat::Wav)), + other => Err(anyhow::anyhow!("invalid format `{other}`")), + } +} diff --git a/src/openhuman/tools/impl/mod.rs b/src/openhuman/tools/impl/mod.rs index 75621b857..668b090e8 100644 --- a/src/openhuman/tools/impl/mod.rs +++ b/src/openhuman/tools/impl/mod.rs @@ -1,4 +1,5 @@ pub mod agent; +pub mod audio; pub mod browser; pub mod computer; pub mod cron; @@ -9,6 +10,7 @@ pub mod system; pub mod whatsapp_data; pub use agent::*; +pub use audio::*; pub use browser::*; pub use computer::*; pub use cron::*; diff --git a/src/openhuman/tools/ops.rs b/src/openhuman/tools/ops.rs index c82437511..bef42bd6c 100644 --- a/src/openhuman/tools/ops.rs +++ b/src/openhuman/tools/ops.rs @@ -155,6 +155,15 @@ pub fn all_tools_with_runtime( security.clone(), workspace_dir.to_path_buf(), )), + Box::new(AudioGeneratePodcastTool::new( + config.clone(), + security.clone(), + )), + Box::new(AudioEmailPodcastTool::new(config.clone(), security.clone())), + Box::new(AudioGenerateAndEmailPodcastTool::new( + config.clone(), + security.clone(), + )), Box::new(GmailUnsubscribeTool), ];