fix(voice): enable GPU detection and Metal acceleration for whisper (#558) (#571)

* fix(device): expand GPU detection with Intel Mac and NVIDIA probes (#558)

Add Intel Mac detection (no Metal GPU for whisper), nvidia-smi probe
for Windows/Linux NVIDIA GPUs, and diagnostic tracing at each
decision point in detect_gpu().

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

* build(cargo): enable whisper-rs Metal feature on macOS (#558)

Add target-specific dependency for macOS that enables the `metal`
feature on whisper-rs, compiling whisper.cpp with Metal GPU support.
Cargo merges features from both declarations so non-macOS builds
are unaffected.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

* fix(whisper): configure GPU params from device profile (#558)

Accept has_gpu and gpu_description in load_engine() and explicitly
set use_gpu and flash_attn on WhisperContextParameters instead of
relying on the compile-time default. Log the selected acceleration
backend at startup.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

* fix(local_ai): pass GPU info to whisper engine load paths (#558)

Thread DeviceProfile has_gpu and gpu_description through both the
bootstrap (startup) and speech (lazy) whisper engine load calls so
the engine can configure Metal or CUDA acceleration at runtime.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

---------

Co-authored-by: Claude Opus 4.6 <noreply@anthropic.com>

Cargo.toml

@@ -101,6 +101,9 @@ wa-rs-proto = { version = "0.2", optional = true, default-features = false }
 wa-rs-ureq-http = { version = "0.2", optional = true }
 wa-rs-tokio-transport = { version = "0.2", optional = true, default-features = false }
 
+[target.'cfg(target_os = "macos")'.dependencies]
+whisper-rs = { version = "0.16", features = ["metal"] }
+
 [target.'cfg(target_os = "linux")'.dependencies]
 landlock = { version = "0.4", optional = true }
 rppal = { version = "0.22", optional = true }

src/openhuman/local_ai/device.rs

@@ -67,22 +67,62 @@ pub fn detect_device_profile() -> DeviceProfile {
 
 /// Best-effort GPU detection.
 ///
-/// Apple Silicon always has a unified GPU (Metal). On other systems we cannot
-/// reliably detect discrete GPUs without heavy platform-specific dependencies,
-/// so we conservatively report unknown.
+/// Apple Silicon always has a unified GPU (Metal). On Windows/Linux, we probe
+/// for NVIDIA GPUs via `nvidia-smi`. On other systems we conservatively report
+/// no GPU.
 fn detect_gpu(cpu_brand: &str, os_name: &str) -> (bool, Option<String>) {
     let brand_lower = cpu_brand.to_ascii_lowercase();
     let os_lower = os_name.to_ascii_lowercase();
 
-    // Apple Silicon detection: brand contains "apple" or we're on macOS with an ARM chip
+    // Apple Silicon detection: brand contains "apple" or we're on macOS with an ARM chip.
     if brand_lower.contains("apple") || (os_lower.contains("mac") && brand_lower.contains("arm")) {
+        tracing::debug!("GPU detected: Apple Silicon (Metal)");
         return (true, Some("Apple Silicon (Metal)".to_string()));
     }
 
-    // Fallback: cannot reliably detect discrete GPU without platform libraries.
+    // Intel Mac: macOS with Intel CPU — no Metal GPU acceleration for whisper.
+    if os_lower.contains("mac") {
+        tracing::debug!("Intel Mac detected — no GPU acceleration available for whisper");
+        return (false, Some("Intel Mac (no Metal GPU)".to_string()));
+    }
+
+    // Windows / Linux: probe for NVIDIA GPU via nvidia-smi.
+    if let Some(desc) = probe_nvidia_smi() {
+        tracing::debug!("GPU detected via nvidia-smi: {desc}");
+        return (true, Some(desc));
+    }
+
+    tracing::debug!("no GPU detected — voice model will use CPU");
     (false, None)
 }
 
+/// Probe for an NVIDIA GPU by running `nvidia-smi --query-gpu=name --format=csv,noheader`.
+/// Returns `Some("NVIDIA <name> (CUDA)")` on success, `None` if nvidia-smi is not available.
+fn probe_nvidia_smi() -> Option<String> {
+    let output = std::process::Command::new("nvidia-smi")
+        .args(["--query-gpu=name", "--format=csv,noheader"])
+        .stdout(std::process::Stdio::piped())
+        .stderr(std::process::Stdio::null())
+        .output()
+        .ok()?;
+
+    if !output.status.success() {
+        return None;
+    }
+
+    let name = String::from_utf8_lossy(&output.stdout)
+        .lines()
+        .next()?
+        .trim()
+        .to_string();
+
+    if name.is_empty() {
+        return None;
+    }
+
+    Some(format!("NVIDIA {name} (CUDA)"))
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;

src/openhuman/local_ai/service/bootstrap.rs

@@ -183,14 +183,17 @@ impl LocalAiService {
         }
 
         // Attempt to load whisper model in-process if configured (blocking I/O).
+        // Pass GPU info from the device profile so whisper can use hardware acceleration.
         if effective_config.local_ai.whisper_in_process {
             if let Ok(model_path) =
                 crate::openhuman::local_ai::paths::resolve_stt_model_path(&effective_config)
             {
                 let model = std::path::PathBuf::from(&model_path);
                 let handle = self.whisper.clone();
+                let gpu = device.has_gpu;
+                let gpu_desc = device.gpu_description.clone();
                 let load_result = tokio::task::spawn_blocking(move || {
-                    super::whisper_engine::load_engine(&handle, &model)
+                    super::whisper_engine::load_engine(&handle, &model, gpu, gpu_desc.as_deref())
                 })
                 .await;
                 match load_result {

src/openhuman/local_ai/service/speech.rs

@@ -53,8 +53,12 @@ impl LocalAiService {
                     debug!(
                         "{LOG_PREFIX} whisper in-process enabled but unloaded; loading model lazily"
                     );
+                    // Detect GPU at lazy-load time so whisper can use acceleration.
+                    let device = crate::openhuman::local_ai::device::detect_device_profile();
+                    let gpu = device.has_gpu;
+                    let gpu_desc = device.gpu_description.clone();
                     let load_result = tokio::task::spawn_blocking(move || {
-                        whisper_engine::load_engine(&handle, &model)
+                        whisper_engine::load_engine(&handle, &model, gpu, gpu_desc.as_deref())
                     })
                     .await
                     .map_err(|e| format!("whisper load task join error: {e}"))?;

src/openhuman/local_ai/service/whisper_engine.rs

@@ -48,9 +48,15 @@ pub fn new_handle() -> WhisperEngineHandle {
     Arc::new(Mutex::new(None))
 }
 
-/// Attempt to load a whisper model into the engine. Returns an error string
-/// if loading fails (e.g. model file missing, unsupported format).
-pub fn load_engine(handle: &WhisperEngineHandle, model_path: &Path) -> Result<(), String> {
+/// Attempt to load a whisper model into the engine, configuring GPU
+/// acceleration based on the detected hardware profile. Returns an error
+/// string if loading fails (e.g. model file missing, unsupported format).
+pub fn load_engine(
+    handle: &WhisperEngineHandle,
+    model_path: &Path,
+    has_gpu: bool,
+    gpu_description: Option<&str>,
+) -> Result<(), String> {
     info!(
         "{LOG_PREFIX} loading whisper model: {}",
         model_path.display()
@@ -60,7 +66,29 @@ pub fn load_engine(handle: &WhisperEngineHandle, model_path: &Path) -> Result<()
         return Err(format!("whisper model not found: {}", model_path.display()));
     }
 
-    let params = WhisperContextParameters::default();
+    let mut params = WhisperContextParameters::default();
+
+    // Explicitly configure GPU acceleration based on device profile.
+    // The default `use_gpu` is `cfg!(feature = "_gpu")` which is only true
+    // when a GPU backend feature (metal, cuda, etc.) is compiled in.
+    params.use_gpu(has_gpu);
+
+    // Enable flash attention when GPU is available — improves throughput
+    // on both Metal and CUDA backends.
+    if has_gpu {
+        params.flash_attn(true);
+    }
+
+    let backend = if has_gpu {
+        gpu_description.unwrap_or("unknown GPU")
+    } else {
+        "CPU (no GPU acceleration)"
+    };
+    info!(
+        "{LOG_PREFIX} whisper acceleration: use_gpu={}, flash_attn={}, backend={}",
+        has_gpu, has_gpu, backend
+    );
+
     let ctx = WhisperContext::new_with_params(model_path.to_str().unwrap_or(""), params)
         .map_err(|e| format!("failed to load whisper model: {e}"))?;
 
@@ -70,7 +98,7 @@ pub fn load_engine(handle: &WhisperEngineHandle, model_path: &Path) -> Result<()
     };
 
     *handle.lock() = Some(engine);
-    info!("{LOG_PREFIX} whisper model loaded successfully");
+    info!("{LOG_PREFIX} whisper model loaded successfully (backend={backend})");
     Ok(())
 }
 
@@ -421,7 +449,7 @@ mod tests {
     #[test]
     fn load_engine_fails_for_missing_model() {
         let handle = new_handle();
-        let result = load_engine(&handle, Path::new("/nonexistent/model.bin"));
+        let result = load_engine(&handle, Path::new("/nonexistent/model.bin"), false, None);
         assert!(result.is_err());
         assert!(!is_loaded(&handle));
     }