From f05ba5e42f7a4df5528158be6cf6685bb48bb2df Mon Sep 17 00:00:00 2001 From: jaberjaber23 Date: Tue, 12 May 2026 15:38:31 +0300 Subject: [PATCH] tts image urls --- crates/openfang-kernel/src/kernel.rs | 8 +- crates/openfang-runtime/src/image_gen.rs | 54 ++++++- .../src/media_understanding.rs | 7 + crates/openfang-runtime/src/tts.rs | 139 +++++++++++++++++- crates/openfang-types/src/media.rs | 105 +++++++++++++ 5 files changed, 307 insertions(+), 6 deletions(-) diff --git a/crates/openfang-kernel/src/kernel.rs b/crates/openfang-kernel/src/kernel.rs index f08d900a..ec600cd7 100644 --- a/crates/openfang-kernel/src/kernel.rs +++ b/crates/openfang-kernel/src/kernel.rs @@ -1056,7 +1056,13 @@ impl OpenFangKernel { // Initialize media understanding engine let media_engine = openfang_runtime::media_understanding::MediaEngine::new(config.media.clone()); - let tts_engine = openfang_runtime::tts::TtsEngine::new(config.tts.clone()); + // Closes #1051: thread MediaConfig URL overrides into the TTS engine + // so local OpenAI/ElevenLabs-compatible services can be targeted. + let tts_engine = openfang_runtime::tts::TtsEngine::new(config.tts.clone()) + .with_base_urls( + config.media.tts_openai_base_url.clone(), + config.media.tts_elevenlabs_base_url.clone(), + ); let mut pairing = crate::pairing::PairingManager::new(config.pairing.clone()); // Load paired devices from database and set up persistence callback diff --git a/crates/openfang-runtime/src/image_gen.rs b/crates/openfang-runtime/src/image_gen.rs index a3b00f95..ce564b14 100644 --- a/crates/openfang-runtime/src/image_gen.rs +++ b/crates/openfang-runtime/src/image_gen.rs @@ -7,7 +7,15 @@ use tracing::warn; /// Generate images via OpenAI's image generation API. /// /// Requires OPENAI_API_KEY to be set. -pub async fn generate_image(request: &ImageGenRequest) -> Result { +/// +/// `base_url_override` (sourced from `MediaConfig.image_gen_base_url`) lets +/// callers redirect the request to a local OpenAI-compatible image service +/// (e.g. Lemonade/Flux, LM Studio). When `None`, the hardcoded +/// `https://api.openai.com/v1/images/generations` endpoint is used. Closes #1051. +pub async fn generate_image( + request: &ImageGenRequest, + base_url_override: Option<&str>, +) -> Result { // Validate request request.validate()?; @@ -30,9 +38,19 @@ pub async fn generate_image(request: &ImageGenRequest) -> Result) -> String { + base.map(|b| format!("{}/v1/images/generations", b.trim_end_matches('/'))) + .unwrap_or_else(|| "https://api.openai.com/v1/images/generations".to_string()) + } + + // Default: hardcoded URL preserved (backward compatibility). + assert_eq!(build(None), "https://api.openai.com/v1/images/generations"); + + // Override applied. + assert_eq!( + build(Some("http://127.0.0.1:7000")), + "http://127.0.0.1:7000/v1/images/generations" + ); + + // Trailing slash on the user-supplied base is stripped. + assert_eq!( + build(Some("http://127.0.0.1:7000/")), + "http://127.0.0.1:7000/v1/images/generations" + ); + assert_eq!( + build(Some("https://images.example.com/")), + "https://images.example.com/v1/images/generations" + ); + } + #[test] fn test_save_images_creates_dir() { let dir = tempfile::tempdir().unwrap(); diff --git a/crates/openfang-runtime/src/media_understanding.rs b/crates/openfang-runtime/src/media_understanding.rs index 08d73b35..b8ad7e63 100644 --- a/crates/openfang-runtime/src/media_understanding.rs +++ b/crates/openfang-runtime/src/media_understanding.rs @@ -24,6 +24,13 @@ impl MediaEngine { } } + /// Read-only access to the media configuration. Used by callers that + /// need the URL overrides (e.g. image_gen_base_url for #1051) without + /// taking ownership of the engine. + pub fn config(&self) -> &MediaConfig { + &self.config + } + /// Describe an image using a vision-capable LLM. /// Auto-cascade: Anthropic -> OpenAI -> Gemini (based on API key availability). pub async fn describe_image( diff --git a/crates/openfang-runtime/src/tts.rs b/crates/openfang-runtime/src/tts.rs index 3895435a..5d4272fb 100644 --- a/crates/openfang-runtime/src/tts.rs +++ b/crates/openfang-runtime/src/tts.rs @@ -19,11 +19,38 @@ pub struct TtsResult { /// Text-to-speech engine. pub struct TtsEngine { config: TtsConfig, + /// Optional override for OpenAI TTS base URL. When set, the engine POSTs + /// to `/v1/audio/speech` instead of the hardcoded + /// `https://api.openai.com/v1/audio/speech`. Sourced from + /// `MediaConfig.tts_openai_base_url`. Closes #1051. + openai_base_url: Option, + /// Optional override for ElevenLabs TTS base URL. When set, the engine + /// POSTs to `/v1/text-to-speech/{voice_id}` instead + /// of the hardcoded `https://api.elevenlabs.io/...`. Sourced from + /// `MediaConfig.tts_elevenlabs_base_url`. Closes #1051. + elevenlabs_base_url: Option, } impl TtsEngine { pub fn new(config: TtsConfig) -> Self { - Self { config } + Self { + config, + openai_base_url: None, + elevenlabs_base_url: None, + } + } + + /// Attach optional base-URL overrides from `MediaConfig`. Use this to + /// route TTS calls at a local OpenAI-compatible service (e.g. + /// Lemonade/Kokoro, LM Studio) or an ElevenLabs proxy. Closes #1051. + pub fn with_base_urls( + mut self, + openai_base_url: Option, + elevenlabs_base_url: Option, + ) -> Self { + self.openai_base_url = openai_base_url; + self.elevenlabs_base_url = elevenlabs_base_url; + self } /// Detect which TTS provider is available based on environment variables. @@ -100,9 +127,21 @@ impl TtsEngine { "speed": self.config.openai.speed, }); + // `tts_openai_base_url` (config.media.tts_openai_base_url) overrides + // the hardcoded provider URL when set, allowing the same OpenAI-compat + // JSON wire format to be sent to a local TTS service (Lemonade/Kokoro, + // LM Studio, etc.) instead of the cloud provider. The Authorization + // header is still built from `OPENAI_API_KEY`; local services typically + // accept any non-empty bearer token. Closes #1051. + let url = self + .openai_base_url + .as_deref() + .map(|base| format!("{}/v1/audio/speech", base.trim_end_matches('/'))) + .unwrap_or_else(|| "https://api.openai.com/v1/audio/speech".to_string()); + let client = reqwest::Client::new(); let response = client - .post("https://api.openai.com/v1/audio/speech") + .post(&url) .header("Authorization", format!("Bearer {}", api_key)) .header("Content-Type", "application/json") .json(&body) @@ -161,7 +200,17 @@ impl TtsEngine { std::env::var("ELEVENLABS_API_KEY").map_err(|_| "ELEVENLABS_API_KEY not set")?; let voice_id = voice_override.unwrap_or(&self.config.elevenlabs.voice_id); - let url = format!("https://api.elevenlabs.io/v1/text-to-speech/{}", voice_id); + // `tts_elevenlabs_base_url` (config.media.tts_elevenlabs_base_url) + // overrides the hardcoded provider URL when set, allowing the same + // ElevenLabs JSON wire format to be routed through a proxy or + // self-hosted ElevenLabs-compatible gateway. The `xi-api-key` header + // still comes from `ELEVENLABS_API_KEY`. Closes #1051. + let base = self + .elevenlabs_base_url + .as_deref() + .map(|b| b.trim_end_matches('/').to_string()) + .unwrap_or_else(|| "https://api.elevenlabs.io".to_string()); + let url = format!("{}/v1/text-to-speech/{}", base, voice_id); let body = serde_json::json!({ "text": text, @@ -306,4 +355,88 @@ mod tests { fn test_max_audio_constant() { assert_eq!(MAX_AUDIO_RESPONSE_BYTES, 10 * 1024 * 1024); } + + #[test] + fn test_with_base_urls_sets_overrides() { + let engine = TtsEngine::new(default_config()).with_base_urls( + Some("http://127.0.0.1:8000".to_string()), + Some("http://127.0.0.1:9000".to_string()), + ); + assert_eq!( + engine.openai_base_url.as_deref(), + Some("http://127.0.0.1:8000") + ); + assert_eq!( + engine.elevenlabs_base_url.as_deref(), + Some("http://127.0.0.1:9000") + ); + } + + /// Closes #1051: when the OpenAI TTS base URL is overridden, the URL + /// building logic must append `/v1/audio/speech` and strip any trailing + /// slash. When unset, the hardcoded provider URL is used. + #[test] + fn test_tts_openai_base_url_override_logic() { + // Helper mirroring the URL construction in `synthesize_openai`. + fn build(base: Option<&str>) -> String { + base.map(|b| format!("{}/v1/audio/speech", b.trim_end_matches('/'))) + .unwrap_or_else(|| "https://api.openai.com/v1/audio/speech".to_string()) + } + + // Default: hardcoded URL preserved (backward compatibility). + assert_eq!(build(None), "https://api.openai.com/v1/audio/speech"); + + // Override applied. + assert_eq!( + build(Some("http://127.0.0.1:8000")), + "http://127.0.0.1:8000/v1/audio/speech" + ); + + // Trailing slash on the user-supplied base is stripped. + assert_eq!( + build(Some("http://127.0.0.1:8000/")), + "http://127.0.0.1:8000/v1/audio/speech" + ); + assert_eq!( + build(Some("https://tts.example.com/")), + "https://tts.example.com/v1/audio/speech" + ); + } + + /// Closes #1051: when the ElevenLabs TTS base URL is overridden, the URL + /// building logic must append `/v1/text-to-speech/{voice_id}` and strip + /// any trailing slash. When unset, the hardcoded provider URL is used. + #[test] + fn test_tts_elevenlabs_base_url_override_logic() { + fn build(base: Option<&str>, voice_id: &str) -> String { + let b = base + .map(|b| b.trim_end_matches('/').to_string()) + .unwrap_or_else(|| "https://api.elevenlabs.io".to_string()); + format!("{}/v1/text-to-speech/{}", b, voice_id) + } + + let voice = "21m00Tcm4TlvDq8ikWAM"; + + // Default: hardcoded URL preserved. + assert_eq!( + build(None, voice), + format!("https://api.elevenlabs.io/v1/text-to-speech/{voice}") + ); + + // Override applied. + assert_eq!( + build(Some("http://127.0.0.1:9000"), voice), + format!("http://127.0.0.1:9000/v1/text-to-speech/{voice}") + ); + + // Trailing slash stripped. + assert_eq!( + build(Some("http://127.0.0.1:9000/"), voice), + format!("http://127.0.0.1:9000/v1/text-to-speech/{voice}") + ); + assert_eq!( + build(Some("https://eleven.example.com/"), voice), + format!("https://eleven.example.com/v1/text-to-speech/{voice}") + ); + } } diff --git a/crates/openfang-types/src/media.rs b/crates/openfang-types/src/media.rs index deb94620..c8f767c0 100644 --- a/crates/openfang-types/src/media.rs +++ b/crates/openfang-types/src/media.rs @@ -96,6 +96,44 @@ pub struct MediaConfig { /// # works for most local OpenAI-compat servers). /// ``` pub audio_base_url: Option, + + /// Optional override for the OpenAI TTS endpoint base URL. + /// + /// When set, replaces `https://api.openai.com` with + /// `/v1/audio/speech`. Use this to point at a + /// local OpenAI-compatible TTS service (Lemonade/Kokoro, LM Studio, + /// etc.) while keeping the same JSON wire format. The Authorization + /// header is still built from `OPENAI_API_KEY` (local services + /// usually accept any non-empty bearer token). + /// + /// Closes . + #[serde(default)] + pub tts_openai_base_url: Option, + + /// Optional override for the ElevenLabs TTS endpoint base URL. + /// + /// When set, replaces `https://api.elevenlabs.io` with + /// `/v1/text-to-speech/{voice_id}`. Use this + /// to route through a proxy or self-hosted ElevenLabs-compatible + /// gateway. The `xi-api-key` header still comes from + /// `ELEVENLABS_API_KEY`. + /// + /// Closes . + #[serde(default)] + pub tts_elevenlabs_base_url: Option, + + /// Optional override for the OpenAI image generation endpoint base URL. + /// + /// When set, replaces `https://api.openai.com` with + /// `/v1/images/generations`. Use this to point at + /// a local OpenAI-compatible image generation service + /// (Lemonade/Flux, LM Studio, etc.) while keeping the same JSON wire + /// format. The Authorization header is still built from + /// `OPENAI_API_KEY`. + /// + /// Closes . + #[serde(default)] + pub image_gen_base_url: Option, } impl Default for MediaConfig { @@ -108,6 +146,9 @@ impl Default for MediaConfig { image_provider: None, audio_provider: None, audio_base_url: None, + tts_openai_base_url: None, + tts_elevenlabs_base_url: None, + image_gen_base_url: None, } } } @@ -382,6 +423,70 @@ mod tests { assert_eq!(config.max_concurrency, 2); assert!(config.image_provider.is_none()); assert!(config.audio_base_url.is_none()); + assert!(config.tts_openai_base_url.is_none()); + assert!(config.tts_elevenlabs_base_url.is_none()); + assert!(config.image_gen_base_url.is_none()); + } + + #[test] + fn test_media_config_tts_openai_base_url_serde_roundtrip() { + let config = MediaConfig { + tts_openai_base_url: Some("http://127.0.0.1:8000".to_string()), + ..MediaConfig::default() + }; + let json = serde_json::to_string(&config).unwrap(); + let parsed: MediaConfig = serde_json::from_str(&json).unwrap(); + assert_eq!( + parsed.tts_openai_base_url.as_deref(), + Some("http://127.0.0.1:8000") + ); + } + + #[test] + fn test_media_config_tts_elevenlabs_base_url_serde_roundtrip() { + let config = MediaConfig { + tts_elevenlabs_base_url: Some("http://127.0.0.1:9000".to_string()), + ..MediaConfig::default() + }; + let json = serde_json::to_string(&config).unwrap(); + let parsed: MediaConfig = serde_json::from_str(&json).unwrap(); + assert_eq!( + parsed.tts_elevenlabs_base_url.as_deref(), + Some("http://127.0.0.1:9000") + ); + } + + #[test] + fn test_media_config_image_gen_base_url_serde_roundtrip() { + let config = MediaConfig { + image_gen_base_url: Some("http://127.0.0.1:7000".to_string()), + ..MediaConfig::default() + }; + let json = serde_json::to_string(&config).unwrap(); + let parsed: MediaConfig = serde_json::from_str(&json).unwrap(); + assert_eq!( + parsed.image_gen_base_url.as_deref(), + Some("http://127.0.0.1:7000") + ); + } + + #[test] + fn test_media_config_backward_compat_no_tts_or_image_overrides() { + // Old TOML/JSON without the three new URL override fields must still + // parse with None, thanks to #[serde(default)] on the struct. + let legacy_json = r#"{ + "image_description": true, + "audio_transcription": true, + "video_description": false, + "max_concurrency": 2, + "image_provider": null, + "audio_provider": "openai", + "audio_base_url": null + }"#; + let parsed: MediaConfig = serde_json::from_str(legacy_json).unwrap(); + assert!(parsed.tts_openai_base_url.is_none()); + assert!(parsed.tts_elevenlabs_base_url.is_none()); + assert!(parsed.image_gen_base_url.is_none()); } #[test]