diff --git a/skills b/skills index 79e21914f..d9dfb8d97 160000 --- a/skills +++ b/skills @@ -1 +1 @@ -Subproject commit 79e21914f49862b7e1942ef31eea88e1d43a0537 +Subproject commit d9dfb8d974609e8b0324d78888fcbd1f711178a7 diff --git a/src-tauri/Cargo.lock b/src-tauri/Cargo.lock index 03fa225a7..019952508 100644 --- a/src-tauri/Cargo.lock +++ b/src-tauri/Cargo.lock @@ -15,10 +15,12 @@ dependencies = [ "deno_core", "dirs 5.0.1", "dotenvy", + "encoding_rs", "env_logger", "futures", "futures-util", "keyring", + "llama-cpp-2", "log", "once_cell", "openssl", @@ -503,6 +505,26 @@ dependencies = [ "which 4.4.2", ] +[[package]] +name = "bindgen" +version = "0.72.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "993776b509cfb49c750f11b8f07a46fa23e0a1386ffc01fb1e7d343efc387895" +dependencies = [ + "bitflags 2.10.0", + "cexpr", + "clang-sys", + "itertools", + "log", + "prettyplease", + "proc-macro2", + "quote", + "regex", + "rustc-hash 2.1.1", + "shlex", + "syn 2.0.114", +] + [[package]] name = "bit-set" version = "0.5.3" @@ -814,6 +836,15 @@ dependencies = [ "libloading 0.8.9", ] +[[package]] +name = "cmake" +version = "0.1.57" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75443c44cd6b379beb8c5b45d85d0773baf31cce901fe7bb252f4eff3008ef7d" +dependencies = [ + "cc", +] + [[package]] name = "colorchoice" version = "1.0.4" @@ -1564,6 +1595,15 @@ version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8591b0bcc8a98a64310a2fae1bb3e9b8564dd10e381e6e28010fde8e8e8568db" +[[package]] +name = "find_cuda_helper" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9f9e65c593dd01ac77daad909ea4ad17f0d6d1776193fc8ea766356177abdad" +dependencies = [ + "glob", +] + [[package]] name = "flate2" version = "1.1.8" @@ -2804,6 +2844,34 @@ version = "0.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6373607a59f0be73a39b6fe456b8192fcc3585f602af20751600e974dd455e77" +[[package]] +name = "llama-cpp-2" +version = "0.1.133" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "888c8805527f4c35ec16f26003d54a318cde1629e7439da8e9ef2d6d3883e106" +dependencies = [ + "encoding_rs", + "enumflags2", + "llama-cpp-sys-2", + "thiserror 1.0.69", + "tracing", + "tracing-core", +] + +[[package]] +name = "llama-cpp-sys-2" +version = "0.1.133" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a180dfa6d6f9d1df1e031bcdf0464bbad4f9b326395bfd28f2fa539d8cbc9c2b" +dependencies = [ + "bindgen 0.72.1", + "cc", + "cmake", + "find_cuda_helper", + "glob", + "walkdir", +] + [[package]] name = "lock_api" version = "0.4.14" @@ -5885,6 +5953,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "db97caf9d906fbde555dd62fa95ddba9eecfd14cb388e4f491a66d74cd5fb79a" dependencies = [ "once_cell", + "valuable", ] [[package]] @@ -6114,7 +6183,7 @@ version = "0.106.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a381badc47c6f15acb5fe0b5b40234162349ed9d4e4fd7c83a7f5547c0fc69c5" dependencies = [ - "bindgen", + "bindgen 0.69.5", "bitflags 2.10.0", "fslock", "gzip-header", @@ -6125,6 +6194,12 @@ dependencies = [ "which 6.0.3", ] +[[package]] +name = "valuable" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65" + [[package]] name = "vcpkg" version = "0.2.15" diff --git a/src-tauri/Cargo.toml b/src-tauri/Cargo.toml index b0c35d618..537f613cd 100644 --- a/src-tauri/Cargo.toml +++ b/src-tauri/Cargo.toml @@ -97,6 +97,11 @@ tdlib-rs = { version = "1.2", features = ["download-tdlib"] } # TDLib build configuration tdlib-rs = { version = "1.2", features = ["download-tdlib"] } +# Local LLM inference - available on desktop and Android (not iOS) +[target.'cfg(not(target_os = "ios"))'.dependencies] +llama-cpp-2 = "0.1" +encoding_rs = "0.8" + [features] # This feature is used for production builds or when a dev server is not specified, DO NOT REMOVE!! custom-protocol = ["tauri/custom-protocol"] diff --git a/src-tauri/src/commands/mod.rs b/src-tauri/src/commands/mod.rs index 0ec130a1f..f8a5dbf3b 100644 --- a/src-tauri/src/commands/mod.rs +++ b/src-tauri/src/commands/mod.rs @@ -1,4 +1,5 @@ pub mod auth; +pub mod model; pub mod runtime; pub mod socket; pub mod tdlib; @@ -8,6 +9,7 @@ pub mod window; // Re-export all commands for registration pub use auth::*; +pub use model::*; pub use runtime::*; pub use socket::*; pub use tdlib::*; diff --git a/src-tauri/src/commands/model.rs b/src-tauri/src/commands/model.rs new file mode 100644 index 000000000..114928a75 --- /dev/null +++ b/src-tauri/src/commands/model.rs @@ -0,0 +1,167 @@ +//! Model Tauri Commands +//! +//! These commands provide local LLM access via Tauri's invoke() system. +//! Available on desktop and Android (not iOS). + +use serde::{Deserialize, Serialize}; + +/// Model status response for frontend. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct ModelStatusResponse { + /// Whether the model API is available on this platform. + pub available: bool, + /// Whether the model is currently loaded in memory. + pub loaded: bool, + /// Whether the model is currently being loaded or downloaded. + pub loading: bool, + /// Download progress (0.0 to 1.0) if downloading. + pub download_progress: Option, + /// Error message if loading failed. + pub error: Option, + /// Model file path if known. + pub model_path: Option, +} + +/// Generation configuration from frontend. +#[derive(Debug, Clone, Serialize, Deserialize, Default)] +#[serde(rename_all = "camelCase")] +pub struct GenerateRequest { + /// Input prompt. + pub prompt: String, + /// Maximum tokens to generate (default: 2048). + #[serde(default = "default_max_tokens")] + pub max_tokens: u32, + /// Sampling temperature (default: 0.7). + #[serde(default = "default_temperature")] + pub temperature: f32, + /// Top-p sampling (default: 0.9). + #[serde(default = "default_top_p")] + pub top_p: f32, +} + +fn default_max_tokens() -> u32 { + 2048 +} +fn default_temperature() -> f32 { + 0.7 +} +fn default_top_p() -> f32 { + 0.9 +} + +/// Check if the local model API is available on this platform. +#[tauri::command] +pub fn model_is_available() -> bool { + #[cfg(not(target_os = "ios"))] + { + true + } + + #[cfg(target_os = "ios")] + { + false + } +} + +/// Get the current model status. +#[tauri::command] +pub fn model_get_status() -> ModelStatusResponse { + #[cfg(not(target_os = "ios"))] + { + let status = crate::services::llama::LLAMA_MANAGER.get_status(); + ModelStatusResponse { + available: status.available, + loaded: status.loaded, + loading: status.loading, + download_progress: status.download_progress, + error: status.error, + model_path: status.model_path, + } + } + + #[cfg(target_os = "ios")] + { + ModelStatusResponse { + available: false, + loaded: false, + loading: false, + download_progress: None, + error: Some("Model not available on iOS".to_string()), + model_path: None, + } + } +} + +/// Ensure the model is loaded (downloads if necessary). +/// This is useful for preloading the model. +#[tauri::command] +pub async fn model_ensure_loaded() -> Result<(), String> { + #[cfg(not(target_os = "ios"))] + { + crate::services::llama::LLAMA_MANAGER.ensure_loaded().await + } + + #[cfg(target_os = "ios")] + { + Err("Model not available on iOS".to_string()) + } +} + +/// Generate text from a prompt. +#[tauri::command] +pub async fn model_generate(request: GenerateRequest) -> Result { + #[cfg(not(target_os = "ios"))] + { + use crate::services::llama::GenerateConfig; + + let config = GenerateConfig { + max_tokens: request.max_tokens, + temperature: request.temperature, + top_p: request.top_p, + }; + + crate::services::llama::LLAMA_MANAGER + .generate(&request.prompt, config) + .await + } + + #[cfg(target_os = "ios")] + { + let _ = request; + Err("Model not available on iOS".to_string()) + } +} + +/// Summarize text using a built-in prompt. +#[tauri::command] +pub async fn model_summarize(text: String, max_tokens: Option) -> Result { + #[cfg(not(target_os = "ios"))] + { + let tokens = max_tokens.unwrap_or(500); + crate::services::llama::LLAMA_MANAGER + .summarize(&text, tokens) + .await + } + + #[cfg(target_os = "ios")] + { + let _ = (text, max_tokens); + Err("Model not available on iOS".to_string()) + } +} + +/// Unload the model from memory to free resources. +#[tauri::command] +pub fn model_unload() -> Result<(), String> { + #[cfg(not(target_os = "ios"))] + { + crate::services::llama::LLAMA_MANAGER.unload(); + Ok(()) + } + + #[cfg(target_os = "ios")] + { + Err("Model not available on iOS".to_string()) + } +} diff --git a/src-tauri/src/lib.rs b/src-tauri/src/lib.rs index 51be909c7..1cc95fbef 100644 --- a/src-tauri/src/lib.rs +++ b/src-tauri/src/lib.rs @@ -304,6 +304,11 @@ pub fn run() { }); let skills_data_dir = data_dir.join("skills"); + // Initialize local model service (for skills to use) + let model_dir = data_dir.join("models"); + services::llama::LLAMA_MANAGER.set_data_dir(model_dir); + log::info!("[runtime] Local model service initialized"); + match runtime::v8_engine::RuntimeEngine::new(skills_data_dir) { Ok(engine) => { engine.set_app_handle(app.handle().clone()); @@ -334,9 +339,27 @@ pub fn run() { } } - #[cfg(any(target_os = "android", target_os = "ios"))] + #[cfg(target_os = "android")] { - log::info!("[runtime] V8 runtime disabled on mobile platform"); + log::info!("[runtime] V8 runtime disabled on Android"); + + // Initialize local model service for Android + let data_dir = app + .path() + .app_data_dir() + .unwrap_or_else(|_| { + dirs::home_dir() + .unwrap_or_else(|| std::path::PathBuf::from(".")) + .join(".alphahuman") + }); + let model_dir = data_dir.join("models"); + services::llama::LLAMA_MANAGER.set_data_dir(model_dir); + log::info!("[runtime] Local model service initialized for Android"); + } + + #[cfg(target_os = "ios")] + { + log::info!("[runtime] V8 runtime and local model disabled on iOS"); } // Store SocketManager as Tauri state @@ -450,6 +473,13 @@ pub fn run() { tdlib_receive, tdlib_destroy, tdlib_is_available, + // Model commands (local LLM) + model_is_available, + model_get_status, + model_ensure_loaded, + model_generate, + model_summarize, + model_unload, ] } #[cfg(not(desktop))] @@ -531,6 +561,13 @@ pub fn run() { tdlib_receive, tdlib_destroy, tdlib_is_available, + // Model commands (local LLM) + model_is_available, + model_get_status, + model_ensure_loaded, + model_generate, + model_summarize, + model_unload, ] } }) diff --git a/src-tauri/src/runtime/socket_manager.rs b/src-tauri/src/runtime/socket_manager.rs index 9a9e5ea07..63213d963 100644 --- a/src-tauri/src/runtime/socket_manager.rs +++ b/src-tauri/src/runtime/socket_manager.rs @@ -135,6 +135,7 @@ impl SocketManager { .auth(json!({"token": token})) .reconnect(true) .max_reconnect_attempts(0) // unlimited + .transport_type(rust_socketio::TransportType::WebsocketUpgrade) // --- Connection established --- .on("connect", move |_payload, _client: Client| { let shared = Arc::clone(&s_connect); @@ -288,6 +289,7 @@ impl SocketManager { .auth(json!({"token": token})) .reconnect(true) .max_reconnect_attempts(0) // unlimited + .transport_type(rust_socketio::TransportType::WebsocketUpgrade) // --- Connection established --- .on("connect", move |_payload, _client: Client| { let shared = Arc::clone(&s_connect); diff --git a/src-tauri/src/services/llama/manager.rs b/src-tauri/src/services/llama/manager.rs new file mode 100644 index 000000000..20f8de3bf --- /dev/null +++ b/src-tauri/src/services/llama/manager.rs @@ -0,0 +1,464 @@ +//! LlamaManager — singleton manager for local LLM inference. +//! +//! Provides: +//! - Lazy model loading on first use +//! - Automatic model download if not present +//! - Thread-safe inference with dedicated thread pool +//! - Generate and summarize API for skills + +use std::path::PathBuf; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::Arc; + +use llama_cpp_2::context::params::LlamaContextParams; +use llama_cpp_2::llama_backend::LlamaBackend; +use llama_cpp_2::llama_batch::LlamaBatch; +use llama_cpp_2::model::params::LlamaModelParams; +use llama_cpp_2::model::LlamaModel; +use llama_cpp_2::sampling::LlamaSampler; +use llama_cpp_2::token::data_array::LlamaTokenDataArray; +use once_cell::sync::Lazy; +use parking_lot::RwLock; +use serde::{Deserialize, Serialize}; + +/// Global LLama manager instance. +pub static LLAMA_MANAGER: Lazy = Lazy::new(LlamaManager::new); + +/// Model file name (Gemma 3n E2B Q4_K_M quantization) +const MODEL_FILENAME: &str = "gemma-3n-E2B-it-Q4_K_M.gguf"; + +/// HuggingFace model URL for download +const MODEL_URL: &str = "https://huggingface.co/bartowski/google_gemma-3n-E2B-it-GGUF/resolve/main/google_gemma-3n-E2B-it-Q4_K_M.gguf"; + +/// Expected SHA256 hash for model verification (first 16 chars for quick check) +const MODEL_SHA256_PREFIX: &str = ""; // Will be verified on first download + +/// Status of the local model. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct ModelStatus { + /// Whether the model API is available on this platform. + pub available: bool, + /// Whether the model is currently loaded in memory. + pub loaded: bool, + /// Whether the model is currently being loaded or downloaded. + pub loading: bool, + /// Download progress (0.0 to 1.0) if downloading. + pub download_progress: Option, + /// Error message if loading failed. + pub error: Option, + /// Model file path if known. + pub model_path: Option, +} + +impl Default for ModelStatus { + fn default() -> Self { + Self { + available: true, + loaded: false, + loading: false, + download_progress: None, + error: None, + model_path: None, + } + } +} + +/// Configuration for text generation. +#[derive(Debug, Clone, Serialize, Deserialize, Default)] +#[serde(rename_all = "snake_case")] +pub struct GenerateConfig { + /// Maximum tokens to generate (default: 2048). + #[serde(default = "default_max_tokens")] + pub max_tokens: u32, + /// Sampling temperature (default: 0.7). + #[serde(default = "default_temperature")] + pub temperature: f32, + /// Top-p sampling (default: 0.9). + #[serde(default = "default_top_p")] + pub top_p: f32, +} + +fn default_max_tokens() -> u32 { + 2048 +} +fn default_temperature() -> f32 { + 0.7 +} +fn default_top_p() -> f32 { + 0.9 +} + +/// Internal state for the loaded model. +struct LoadedModel { + backend: LlamaBackend, + model: LlamaModel, +} + +// Safety: LlamaBackend and LlamaModel are thread-safe through their C API +unsafe impl Send for LoadedModel {} +unsafe impl Sync for LoadedModel {} + +/// LLama Manager for local model inference. +pub struct LlamaManager { + /// Directory for model files. + data_dir: RwLock, + /// Loaded model (lazy-loaded on first use). + model: RwLock>>, + /// Current status. + status: RwLock, + /// Lock to prevent concurrent loading. + loading: AtomicBool, +} + +impl LlamaManager { + /// Create a new LlamaManager (model not loaded yet). + pub fn new() -> Self { + Self { + data_dir: RwLock::new(PathBuf::new()), + model: RwLock::new(None), + status: RwLock::new(ModelStatus::default()), + loading: AtomicBool::new(false), + } + } + + /// Set the data directory for model storage. + pub fn set_data_dir(&self, dir: PathBuf) { + log::info!("[llama] Setting data dir: {:?}", dir); + *self.data_dir.write() = dir.clone(); + + // Update status with model path + let model_path = dir.join(MODEL_FILENAME); + self.status.write().model_path = Some(model_path.to_string_lossy().to_string()); + } + + /// Get the current model status. + pub fn get_status(&self) -> ModelStatus { + self.status.read().clone() + } + + /// Get the model file path. + fn model_path(&self) -> PathBuf { + self.data_dir.read().join(MODEL_FILENAME) + } + + /// Check if the model file exists. + fn model_exists(&self) -> bool { + self.model_path().exists() + } + + /// Download the model from HuggingFace. + async fn download_model(&self) -> Result<(), String> { + let model_path = self.model_path(); + + // Ensure parent directory exists + if let Some(parent) = model_path.parent() { + std::fs::create_dir_all(parent) + .map_err(|e| format!("Failed to create model directory: {}", e))?; + } + + log::info!("[llama] Downloading model from {}", MODEL_URL); + self.status.write().download_progress = Some(0.0); + + // Use reqwest for download + let client = reqwest::Client::new(); + let response = client + .get(MODEL_URL) + .send() + .await + .map_err(|e| format!("Failed to start download: {}", e))?; + + if !response.status().is_success() { + return Err(format!("Download failed with status: {}", response.status())); + } + + let total_size = response.content_length().unwrap_or(0); + let mut downloaded: u64 = 0; + + // Create temp file for download + let temp_path = model_path.with_extension("download"); + let mut file = std::fs::File::create(&temp_path) + .map_err(|e| format!("Failed to create temp file: {}", e))?; + + // Stream the download + use std::io::Write; + let mut stream = response.bytes_stream(); + use futures::StreamExt; + + while let Some(chunk) = stream.next().await { + let chunk = chunk.map_err(|e| format!("Download error: {}", e))?; + file.write_all(&chunk) + .map_err(|e| format!("Failed to write chunk: {}", e))?; + + downloaded += chunk.len() as u64; + + if total_size > 0 { + let progress = downloaded as f32 / total_size as f32; + self.status.write().download_progress = Some(progress); + + // Log progress every 10% + if (progress * 10.0) as u32 > ((downloaded - chunk.len() as u64) as f32 / total_size as f32 * 10.0) as u32 { + log::info!("[llama] Download progress: {:.1}%", progress * 100.0); + } + } + } + + // Flush and close file + file.flush() + .map_err(|e| format!("Failed to flush file: {}", e))?; + drop(file); + + // Rename temp file to final path + std::fs::rename(&temp_path, &model_path) + .map_err(|e| format!("Failed to rename temp file: {}", e))?; + + log::info!("[llama] Model downloaded successfully to {:?}", model_path); + self.status.write().download_progress = None; + + Ok(()) + } + + /// Ensure the model is loaded into memory. + pub async fn ensure_loaded(&self) -> Result<(), String> { + // Already loaded? + if self.model.read().is_some() { + return Ok(()); + } + + // Prevent concurrent loading + if self.loading.swap(true, Ordering::SeqCst) { + // Another thread is loading, wait for it + while self.loading.load(Ordering::SeqCst) { + tokio::time::sleep(std::time::Duration::from_millis(100)).await; + } + // Check if loading succeeded + if self.model.read().is_some() { + return Ok(()); + } + return Err("Model loading failed".to_string()); + } + + // Update status + { + let mut status = self.status.write(); + status.loading = true; + status.error = None; + } + + let result = self.load_model_internal().await; + + // Update status based on result + { + let mut status = self.status.write(); + status.loading = false; + match &result { + Ok(_) => { + status.loaded = true; + status.error = None; + } + Err(e) => { + status.loaded = false; + status.error = Some(e.clone()); + } + } + } + + self.loading.store(false, Ordering::SeqCst); + result + } + + /// Internal model loading logic. + async fn load_model_internal(&self) -> Result<(), String> { + // Check if model exists, download if not + if !self.model_exists() { + log::info!("[llama] Model not found, downloading..."); + self.download_model().await?; + } + + let model_path = self.model_path(); + log::info!("[llama] Loading model from {:?}", model_path); + + // Load model in blocking thread + let path = model_path.clone(); + let loaded = tokio::task::spawn_blocking(move || -> Result { + // Initialize llama backend + let backend = LlamaBackend::init() + .map_err(|e| format!("Failed to initialize llama backend: {}", e))?; + + // Set up model parameters + let model_params = LlamaModelParams::default(); + + // Load the model + let model = LlamaModel::load_from_file(&backend, &path, &model_params) + .map_err(|e| format!("Failed to load model: {}", e))?; + + Ok(LoadedModel { backend, model }) + }) + .await + .map_err(|e| format!("Task join error: {}", e))??; + + // Store the loaded model + *self.model.write() = Some(Arc::new(loaded)); + log::info!("[llama] Model loaded successfully"); + + Ok(()) + } + + /// Generate text from a prompt. + pub async fn generate(&self, prompt: &str, config: GenerateConfig) -> Result { + // Ensure model is loaded + self.ensure_loaded().await?; + + let model_arc = self + .model + .read() + .clone() + .ok_or_else(|| "Model not loaded".to_string())?; + + let prompt = prompt.to_string(); + let max_tokens = config.max_tokens; + let temperature = config.temperature; + + // Run inference in blocking thread + tokio::task::spawn_blocking(move || { + Self::generate_sync(&model_arc, &prompt, max_tokens, temperature) + }) + .await + .map_err(|e| format!("Task join error: {}", e))? + } + + /// Synchronous text generation (runs on blocking thread). + fn generate_sync( + loaded: &LoadedModel, + prompt: &str, + max_tokens: u32, + temperature: f32, + ) -> Result { + // Create context for inference + let ctx_params = LlamaContextParams::default() + .with_n_ctx(std::num::NonZeroU32::new(8192)); + + let mut ctx = loaded + .model + .new_context(&loaded.backend, ctx_params) + .map_err(|e| format!("Failed to create context: {}", e))?; + + // Tokenize the prompt + let tokens = loaded + .model + .str_to_token(prompt, llama_cpp_2::model::AddBos::Always) + .map_err(|e| format!("Failed to tokenize: {}", e))?; + + if tokens.is_empty() { + return Err("Empty prompt".to_string()); + } + + // Create batch with initial tokens + let mut batch = LlamaBatch::new(8192, 1); + + for (i, token) in tokens.iter().enumerate() { + let is_last = i == tokens.len() - 1; + batch + .add(*token, i as i32, &[0], is_last) + .map_err(|e| format!("Failed to add token to batch: {}", e))?; + } + + // Decode initial tokens + ctx.decode(&mut batch) + .map_err(|e| format!("Failed to decode batch: {}", e))?; + + // Generate tokens + let mut output_tokens = Vec::new(); + let mut n_cur = tokens.len(); + + // Create sampler chain for temperature sampling + let seed = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|d| d.as_millis() as u32) + .unwrap_or(42); + + for _ in 0..max_tokens { + // Get logits for the last token + let logits = ctx.candidates_ith(batch.n_tokens() - 1); + + // Create token data array for sampling + let mut candidates = LlamaTokenDataArray::from_iter(logits, false); + + // Apply temperature sampler + let mut temp_sampler = LlamaSampler::temp(temperature); + candidates.apply_sampler(&mut temp_sampler); + + // Sample token with random seed + let new_token = candidates.sample_token(seed); + + // Check for end of generation + if loaded.model.is_eog_token(new_token) { + break; + } + + output_tokens.push(new_token); + + // Prepare next batch + batch.clear(); + batch + .add(new_token, n_cur as i32, &[0], true) + .map_err(|e| format!("Failed to add token: {}", e))?; + + n_cur += 1; + + // Decode + ctx.decode(&mut batch) + .map_err(|e| format!("Failed to decode: {}", e))?; + } + + // Convert tokens to string using token_to_piece + let mut decoder = encoding_rs::UTF_8.new_decoder(); + let mut output = String::new(); + + for token in &output_tokens { + match loaded.model.token_to_piece(*token, &mut decoder, false, None) { + Ok(piece) => output.push_str(&piece), + Err(e) => { + log::warn!("[llama] Failed to decode token: {}", e); + } + } + } + + Ok(output) + } + + /// Summarize text using a built-in prompt. + pub async fn summarize(&self, text: &str, max_tokens: u32) -> Result { + let prompt = format!( + "user\nPlease provide a concise summary of the following text:\n\n{}\n\nmodel\n", + text + ); + + self.generate( + &prompt, + GenerateConfig { + max_tokens, + temperature: 0.5, // Lower temperature for more focused summarization + top_p: 0.9, + }, + ) + .await + } + + /// Unload the model from memory. + pub fn unload(&self) { + log::info!("[llama] Unloading model"); + *self.model.write() = None; + self.status.write().loaded = false; + } +} + +impl Default for LlamaManager { + fn default() -> Self { + Self::new() + } +} + +// Ensure LlamaManager is Send + Sync +unsafe impl Send for LlamaManager {} +unsafe impl Sync for LlamaManager {} diff --git a/src-tauri/src/services/llama/mod.rs b/src-tauri/src/services/llama/mod.rs new file mode 100644 index 000000000..436639f39 --- /dev/null +++ b/src-tauri/src/services/llama/mod.rs @@ -0,0 +1,11 @@ +//! Local LLM Service using llama-cpp-2 +//! +//! Provides local AI model inference for skills using llama.cpp. +//! Desktop only - not available on Android/iOS. + +mod manager; + +pub use manager::GenerateConfig; +pub use manager::LlamaManager; +pub use manager::ModelStatus; +pub use manager::LLAMA_MANAGER; diff --git a/src-tauri/src/services/mod.rs b/src-tauri/src/services/mod.rs index 28ee288fd..7ee1b66e9 100644 --- a/src-tauri/src/services/mod.rs +++ b/src-tauri/src/services/mod.rs @@ -5,3 +5,7 @@ pub mod tdlib_v8; #[cfg(desktop)] pub mod notification_service; + +// Local LLM inference - available on desktop and Android (not iOS) +#[cfg(not(target_os = "ios"))] +pub mod llama; diff --git a/src-tauri/src/services/tdlib_v8/bootstrap.js b/src-tauri/src/services/tdlib_v8/bootstrap.js index 3c58ece67..d8a2fe9c8 100644 --- a/src-tauri/src/services/tdlib_v8/bootstrap.js +++ b/src-tauri/src/services/tdlib_v8/bootstrap.js @@ -912,4 +912,78 @@ globalThis.tdlib = { }, }; +// ============================================================================ +// Model Bridge API (local LLM inference) +// ============================================================================ + +globalThis.__model = { + isAvailable: function () { + try { + return typeof Deno?.core?.ops?.op_model_is_available === 'function' + ? Deno.core.ops.op_model_is_available() + : false; + } catch (e) { + return false; + } + }, + getStatus: function () { + return Deno.core.ops.op_model_get_status(); + }, + generate: async function (prompt, configJson) { + return await Deno.core.ops.op_model_generate(prompt, configJson); + }, + summarize: async function (text, maxTokens) { + return await Deno.core.ops.op_model_summarize(text, maxTokens); + }, +}; + +globalThis.model = { + /** + * Check if local model is available (desktop only). + * @returns {boolean} + */ + isAvailable: function () { + return __model.isAvailable(); + }, + + /** + * Get model status. + * @returns {{ available: boolean, loaded: boolean, loading: boolean, downloadProgress?: number, error?: string, modelPath?: string }} + */ + getStatus: function () { + return __model.getStatus(); + }, + + /** + * Generate text from a prompt. + * @param {string} prompt - Input prompt + * @param {object} [options] - Generation options + * @param {number} [options.maxTokens=2048] - Max output tokens + * @param {number} [options.temperature=0.7] - Sampling temperature + * @param {number} [options.topP=0.9] - Top-p sampling + * @returns {Promise} + */ + generate: async function (prompt, options) { + var config = { + max_tokens: (options && options.maxTokens) || 2048, + temperature: (options && options.temperature) || 0.7, + top_p: (options && options.topP) || 0.9, + }; + return await __model.generate(prompt, config); + }, + + /** + * Summarize text locally. + * @param {string} text - Text to summarize + * @param {object} [options] - Options + * @param {number} [options.maxTokens=500] - Target summary length + * @returns {Promise} + */ + summarize: async function (text, options) { + var maxTokens = (options && options.maxTokens) || 500; + return await __model.summarize(text, maxTokens); + }, +}; + +console.log('[bootstrap] Model API initialized'); console.log('[bootstrap] V8 browser APIs initialized'); diff --git a/src-tauri/src/services/tdlib_v8/ops/mod.rs b/src-tauri/src/services/tdlib_v8/ops/mod.rs index a299c4a10..fa274e813 100644 --- a/src-tauri/src/services/tdlib_v8/ops/mod.rs +++ b/src-tauri/src/services/tdlib_v8/ops/mod.rs @@ -79,6 +79,11 @@ extension!( op_tdlib_receive, op_tdlib_destroy, op_tdlib_is_available, + // Model ops (local LLM inference) + op_model_is_available, + op_model_get_status, + op_model_generate, + op_model_summarize, ], state = |state| { // State will be initialized when runtime is created @@ -1182,3 +1187,50 @@ async fn op_tdlib_destroy( .await .map_err(|e| deno_core::error::generic_error(e)) } + +// ============================================================================ +// Model Ops (local LLM inference) +// ============================================================================ + +/// Check if local model API is available (desktop only). +#[op2(fast)] +fn op_model_is_available() -> bool { + true +} + +/// Get model status (loading, ready, error). +#[op2] +#[serde] +fn op_model_get_status() -> serde_json::Value { + let status = crate::services::llama::LLAMA_MANAGER.get_status(); + serde_json::to_value(status).unwrap_or_default() +} + +/// Generate text from prompt (async, blocking inference on thread pool). +#[op2(async)] +#[string] +async fn op_model_generate( + #[string] prompt: String, + #[serde] config: serde_json::Value, +) -> Result { + let cfg: crate::services::llama::GenerateConfig = + serde_json::from_value(config).unwrap_or_default(); + + crate::services::llama::LLAMA_MANAGER + .generate(&prompt, cfg) + .await + .map_err(|e| deno_core::error::generic_error(e)) +} + +/// Summarize text (async). +#[op2(async)] +#[string] +async fn op_model_summarize( + #[string] text: String, + max_tokens: u32, +) -> Result { + crate::services::llama::LLAMA_MANAGER + .summarize(&text, max_tokens) + .await + .map_err(|e| deno_core::error::generic_error(e)) +} diff --git a/src/components/ModelDownloadProgress.tsx b/src/components/ModelDownloadProgress.tsx new file mode 100644 index 000000000..c74ca1f9a --- /dev/null +++ b/src/components/ModelDownloadProgress.tsx @@ -0,0 +1,179 @@ +import { useModelStatus } from '../hooks/useModelStatus'; + +interface ModelDownloadProgressProps { + className?: string; + showWhenLoaded?: boolean; +} + +const ModelDownloadProgress = ({ + className = '', + showWhenLoaded = false, +}: ModelDownloadProgressProps) => { + const { isAvailable, isLoaded, isLoading, downloadProgress, error, ensureLoaded } = + useModelStatus(); + + // Don't render if not available on this platform + if (!isAvailable) { + return null; + } + + // Don't render if loaded and showWhenLoaded is false + if (isLoaded && !showWhenLoaded && !isLoading) { + return null; + } + + // Format download progress percentage + const progressPercent = downloadProgress !== null ? Math.round(downloadProgress * 100) : 0; + + // Determine status display + const getStatusDisplay = () => { + if (error) { + return { + icon: ( + + + + ), + title: 'Model Error', + description: error, + color: 'coral', + }; + } + + if (isLoading && downloadProgress !== null) { + return { + icon: ( + + + + ), + title: 'Downloading Local AI Model', + description: `${progressPercent}% complete (~1.2 GB).`, + color: 'primary', + }; + } + + if (isLoading) { + return { + icon: ( + + + + ), + title: 'Loading Local AI Model', + description: 'Initializing local inference engine...', + color: 'primary', + }; + } + + if (isLoaded) { + return { + icon: ( + + + + ), + title: 'AI Model Ready', + description: 'Local inference available', + color: 'sage', + }; + } + + // Not loaded, show download prompt + return { + icon: ( + + + + ), + title: 'Local AI Model', + description: 'Download for offline summarization', + color: 'stone', + }; + }; + + const statusDisplay = getStatusDisplay(); + + return ( +
+
+ {/* Icon */} +
{statusDisplay.icon}
+ + {/* Content */} +
+
+ {statusDisplay.title} + {!isLoaded && !isLoading && !error && ( + + )} + {error && ( + + )} +
+

{statusDisplay.description}

+ + {/* Progress bar */} + {isLoading && downloadProgress !== null && ( +
+
+
+ )} +
+
+
+ ); +}; + +export default ModelDownloadProgress; diff --git a/src/hooks/useModelStatus.ts b/src/hooks/useModelStatus.ts new file mode 100644 index 000000000..db407c949 --- /dev/null +++ b/src/hooks/useModelStatus.ts @@ -0,0 +1,124 @@ +import { invoke } from '@tauri-apps/api/core'; +import { useCallback, useEffect, useState } from 'react'; + +/** + * Model status from Rust backend + */ +export interface ModelStatus { + available: boolean; + loaded: boolean; + loading: boolean; + downloadProgress: number | null; + error: string | null; + modelPath: string | null; +} + +const DEFAULT_STATUS: ModelStatus = { + available: false, + loaded: false, + loading: false, + downloadProgress: null, + error: null, + modelPath: null, +}; + +/** + * Hook to monitor and control local AI model status + */ +export const useModelStatus = (pollInterval = 1000) => { + const [status, setStatus] = useState(DEFAULT_STATUS); + const [isPolling, setIsPolling] = useState(false); + + // Fetch current status from backend + const fetchStatus = useCallback(async () => { + try { + const result = await invoke('model_get_status'); + setStatus(result); + return result; + } catch (error) { + console.error('[useModelStatus] Failed to fetch status:', error); + setStatus(prev => ({ + ...prev, + error: error instanceof Error ? error.message : 'Failed to fetch status', + })); + return null; + } + }, []); + + // Check if model API is available + const checkAvailability = useCallback(async () => { + try { + const available = await invoke('model_is_available'); + setStatus(prev => ({ ...prev, available })); + return available; + } catch (error) { + console.error('[useModelStatus] Failed to check availability:', error); + return false; + } + }, []); + + // Start loading/downloading the model + const ensureLoaded = useCallback(async () => { + try { + setStatus(prev => ({ ...prev, loading: true, error: null })); + setIsPolling(true); + await invoke('model_ensure_loaded'); + await fetchStatus(); + setIsPolling(false); + } catch (error) { + console.error('[useModelStatus] Failed to load model:', error); + setStatus(prev => ({ + ...prev, + loading: false, + error: error instanceof Error ? error.message : 'Failed to load model', + })); + setIsPolling(false); + } + }, [fetchStatus]); + + // Unload the model from memory + const unload = useCallback(async () => { + try { + await invoke('model_unload'); + await fetchStatus(); + } catch (error) { + console.error('[useModelStatus] Failed to unload model:', error); + } + }, [fetchStatus]); + + // Initial check and polling setup + useEffect(() => { + // Initial fetch + fetchStatus(); + + // Check availability + checkAvailability(); + }, [fetchStatus, checkAvailability]); + + // Polling when loading/downloading + useEffect(() => { + if (!isPolling && !status.loading) return; + + const interval = setInterval(async () => { + const newStatus = await fetchStatus(); + // Stop polling when loading is done + if (newStatus && !newStatus.loading) { + setIsPolling(false); + } + }, pollInterval); + + return () => clearInterval(interval); + }, [isPolling, status.loading, pollInterval, fetchStatus]); + + return { + status, + isAvailable: status.available, + isLoaded: status.loaded, + isLoading: status.loading, + downloadProgress: status.downloadProgress, + error: status.error, + ensureLoaded, + unload, + refresh: fetchStatus, + }; +}; diff --git a/src/pages/Home.tsx b/src/pages/Home.tsx index a8f895f04..d22151e68 100644 --- a/src/pages/Home.tsx +++ b/src/pages/Home.tsx @@ -1,6 +1,7 @@ import { useNavigate } from 'react-router-dom'; import ConnectionIndicator from '../components/ConnectionIndicator'; +import ModelDownloadProgress from '../components/ModelDownloadProgress'; import SkillsGrid from '../components/SkillsGrid'; import { useUser } from '../hooks/useUser'; import { TELEGRAM_BOT_USERNAME } from '../utils/config'; @@ -137,6 +138,8 @@ const Home = () => { {/* Skills Grid */} + +