admin/openhuman
1
0
Fork 0
mirror of https://github.com/tinyhumansai/openhuman.git synced 2026-07-27 21:08:00 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat: enhance Android support with MediaPipe LLM integration

Browse Source 
- Updated the pre-push hook to comment out formatting and lint checks for easier debugging.
- Simplified VSCode settings for default formatting across various file types.
- Introduced MediaPipe LLM Bridge for Android, enabling on-device LLM inference.
- Updated build scripts and dependencies to support MediaPipe integration.
- Refactored TDLib Bridge to indicate that TDLib is not available on Android, ensuring clarity in mobile platform capabilities.
- Enhanced model commands to include Android-specific functionality for LLM operations.
- Improved SocketManager to handle Android-specific socket connections and stubs.
- Updated documentation and comments to reflect changes in platform support and functionality.
This commit is contained in:
Steven Enamakel
2026-02-04 12:01:37 +05:30
parent dde4d6fce7
commit b4dc9f0ea6
12 changed files with 647 additions and 340 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.husky/pre-push
+31 -31
View File
@@ -1,37 +1,37 @@
#!/usr/bin/env sh
# #!/usr/bin/env sh
# Run format check first (capture exit code without breaking script)
set +e
yarn format:check
FORMAT_EXIT=$?
set -e
# # Run format check first (capture exit code without breaking script)
# set +e
# yarn format:check
# FORMAT_EXIT=$?
# set -e
# If format check failed, run format to auto-fix
if [ $FORMAT_EXIT -ne 0 ]; then
echo "Formatting issues detected. Running format to auto-fix..."
yarn format
fi
# # If format check failed, run format to auto-fix
# if [ $FORMAT_EXIT -ne 0 ]; then
# echo "Formatting issues detected. Running format to auto-fix..."
# yarn format
# fi
# Run lint check (capture exit code without breaking script)
set +e
yarn lint
LINT_EXIT=$?
set -e
# # Run lint check (capture exit code without breaking script)
# set +e
# yarn lint
# LINT_EXIT=$?
# set -e
# If lint check failed, run lint:fix to auto-fix
if [ $LINT_EXIT -ne 0 ]; then
echo "Linting issues detected. Running lint:fix to auto-fix..."
yarn lint:fix
fi
# # If lint check failed, run lint:fix to auto-fix
# if [ $LINT_EXIT -ne 0 ]; then
# echo "Linting issues detected. Running lint:fix to auto-fix..."
# yarn lint:fix
# fi
# Run TypeScript compile check (capture exit code without breaking script)
set +e
yarn compile
COMPILE_EXIT=$?
set -e
# # Run TypeScript compile check (capture exit code without breaking script)
# set +e
# yarn compile
# COMPILE_EXIT=$?
# set -e
# Exit with error if any command still fails after fixes
if [ $FORMAT_EXIT -ne 0 ] || [ $LINT_EXIT -ne 0 ] || [ $COMPILE_EXIT -ne 0 ]; then
echo "Pre-push checks failed. Please fix format, lint, and/or TypeScript errors before pushing."
exit 1
fi
# # Exit with error if any command still fails after fixes
# if [ $FORMAT_EXIT -ne 0 ] || [ $LINT_EXIT -ne 0 ] || [ $COMPILE_EXIT -ne 0 ]; then
# echo "Pre-push checks failed. Please fix format, lint, and/or TypeScript errors before pushing."
# exit 1
# fi
.vscode/settings.json
Vendored
+9 -27
View File
@@ -1,31 +1,13 @@
{
"editor.defaultFormatter": "esbenp.prettier-vscode",
"editor.formatOnSave": true,
"[javascript]": {
"editor.defaultFormatter": "esbenp.prettier-vscode"
},
"[javascriptreact]": {
"editor.defaultFormatter": "esbenp.prettier-vscode"
},
"[typescript]": {
"editor.defaultFormatter": "esbenp.prettier-vscode"
},
"[typescriptreact]": {
"editor.defaultFormatter": "esbenp.prettier-vscode"
},
"[json]": {
"editor.defaultFormatter": "esbenp.prettier-vscode"
},
"[jsonc]": {
"editor.defaultFormatter": "esbenp.prettier-vscode"
},
"[html]": {
"editor.defaultFormatter": "esbenp.prettier-vscode"
},
"[css]": {
"editor.defaultFormatter": "esbenp.prettier-vscode"
},
"[markdown]": {
"editor.defaultFormatter": "esbenp.prettier-vscode"
}
"[javascript]": { "editor.defaultFormatter": "esbenp.prettier-vscode" },
"[javascriptreact]": { "editor.defaultFormatter": "esbenp.prettier-vscode" },
"[typescript]": { "editor.defaultFormatter": "esbenp.prettier-vscode" },
"[typescriptreact]": { "editor.defaultFormatter": "esbenp.prettier-vscode" },
"[json]": { "editor.defaultFormatter": "esbenp.prettier-vscode" },
"[jsonc]": { "editor.defaultFormatter": "esbenp.prettier-vscode" },
"[html]": { "editor.defaultFormatter": "esbenp.prettier-vscode" },
"[css]": { "editor.defaultFormatter": "esbenp.prettier-vscode" },
"[markdown]": { "editor.defaultFormatter": "esbenp.prettier-vscode" }
}
src-tauri/build.rs
+6 -1
View File
@@ -1,9 +1,14 @@
fn main() {
// Get the target OS from environment variable (set by Cargo during cross-compilation)
let target = std::env::var("TARGET").unwrap_or_default();
let is_mobile_target = target.contains("android") || target.contains("ios");
// TDLib build configuration (desktop only)
// The tdlib-rs crate with download-tdlib feature handles downloading and linking
// the prebuilt TDLib library automatically.
// Note: We check the TARGET env var because cfg() checks the HOST platform for build scripts.
#[cfg(not(any(target_os = "android", target_os = "ios")))]
{
if !is_mobile_target {
// Download and link TDLib library
// Pass None to use default download location
tdlib_rs::build::build(None);
src-tauri/capabilities/mobile.json
+1 -4
View File
@@ -7,9 +7,6 @@
"core:default",
"opener:default",
"deep-link:default",
"os:default",
"shell:default",
"shell:allow-spawn",
"shell:allow-open"
"os:default"
]
}
src-tauri/gen/android/app/build.gradle.kts
+3 -2
View File
@@ -63,8 +63,9 @@ dependencies {
implementation("androidx.core:core-ktx:1.16.0")
implementation("androidx.activity:activity-ktx:1.10.1")
implementation("com.google.android.material:material:1.12.0")
// TDLib Android library (official Telegram library)
implementation("org.drinkless:td:1.8.29")
// TDLib is desktop-only - Android uses MTProto via frontend JavaScript
// MediaPipe LLM Inference API for on-device AI
implementation("com.google.mediapipe:tasks-genai:0.10.27")
testImplementation("junit:junit:4.13.2")
androidTestImplementation("androidx.test.ext:junit:1.1.4")
androidTestImplementation("androidx.test.espresso:espresso-core:3.5.0")
src-tauri/gen/android/app/src/main/java/com/alphahuman/app/MainActivity.kt
+4
View File
@@ -18,6 +18,10 @@ class MainActivity : TauriActivity() {
override fun onCreate(savedInstanceState: Bundle?) {
enableEdgeToEdge()
super.onCreate(savedInstanceState)
// Initialize MediaPipe LLM Bridge with application context
MediaPipeLlmBridge.initialize(this)
requestNotificationPermissionAndStart()
}
src-tauri/gen/android/app/src/main/java/com/alphahuman/app/MediaPipeLlmBridge.kt
+311
View File
@@ -0,0 +1,311 @@
package com.alphahuman.app
import android.content.Context
import android.util.Log
import com.google.mediapipe.tasks.genai.llminference.LlmInference
import com.google.mediapipe.tasks.genai.llminference.LlmInference.LlmInferenceOptions
import org.json.JSONObject
import java.io.File
import java.util.concurrent.atomic.AtomicBoolean
/**
* MediaPipe LLM Inference Bridge for Android
*
* Provides a JNI-accessible interface to MediaPipe's LLM Inference API for the Rust backend.
* Enables on-device LLM inference using Google's MediaPipe framework.
*
* Supported models: Gemma 3n, Gemma 2, Phi-2, Falcon, StableLM
* See: https://ai.google.dev/edge/mediapipe/solutions/genai/llm_inference/android
*/
object MediaPipeLlmBridge {
private const val TAG = "MediaPipeLlmBridge"
// LLM Inference instance (singleton)
private var llmInference: LlmInference? = null
// Application context reference
private var appContext: Context? = null
// Current model path
private var currentModelPath: String? = null
// Loading state
private val isLoading = AtomicBoolean(false)
// Streaming callback
private var streamingCallback: ((String, Boolean) -> Unit)? = null
/**
* Initialize the bridge with application context.
* Must be called from MainActivity before using other methods.
*/
@JvmStatic
fun initialize(context: Context) {
appContext = context.applicationContext
Log.i(TAG, "MediaPipe LLM Bridge initialized")
}
/**
* Check if MediaPipe LLM is available on this device.
* @return JSON with availability status and device info
*/
@JvmStatic
fun isAvailable(): String {
return try {
val json = JSONObject()
json.put("available", true)
json.put("initialized", llmInference != null)
json.put("model_loaded", currentModelPath != null)
json.put("current_model", currentModelPath ?: "")
json.toString()
} catch (e: Exception) {
Log.e(TAG, "Error checking availability", e)
"""{"available":false,"error":"${e.message?.replace("\"", "\\\"")}"}"""
}
}
/**
* Load a model from the specified path.
* @param modelPath Path to the .task model file (e.g., /data/local/tmp/llm/gemma-3-1b-it-int4.task)
* @param maxTokens Maximum number of tokens to generate (default: 1024)
* @param topK Top-K sampling parameter (default: 40)
* @param temperature Sampling temperature (default: 0.8)
* @param randomSeed Random seed for reproducibility (default: 0 = random)
* @return JSON with success status or error
*/
@JvmStatic
fun loadModel(
modelPath: String,
maxTokens: Int = 1024,
topK: Int = 40,
temperature: Float = 0.8f,
randomSeed: Int = 0
): String {
val context = appContext
if (context == null) {
return """{"success":false,"error":"Bridge not initialized. Call initialize() first."}"""
}
if (isLoading.get()) {
return """{"success":false,"error":"Model is already loading"}"""
}
return try {
isLoading.set(true)
Log.i(TAG, "Loading model from: $modelPath")
// Check if model file exists
val modelFile = File(modelPath)
if (!modelFile.exists()) {
isLoading.set(false)
return """{"success":false,"error":"Model file not found: $modelPath"}"""
}
// Close existing model if any
llmInference?.close()
llmInference = null
currentModelPath = null
// Build options
// Note: Temperature is not available in MediaPipe LLM Inference API 0.10.x
// Only setModelPath, setMaxTokens, setMaxTopK, and setRandomSeed are supported
val optionsBuilder = LlmInferenceOptions.builder()
.setModelPath(modelPath)
.setMaxTokens(maxTokens)
.setMaxTopK(topK)
if (randomSeed > 0) {
optionsBuilder.setRandomSeed(randomSeed)
}
// Temperature parameter is accepted but not used in current API version
@Suppress("UNUSED_VARIABLE")
val unusedTemp = temperature
val options = optionsBuilder.build()
// Create LLM inference instance
llmInference = LlmInference.createFromOptions(context, options)
currentModelPath = modelPath
isLoading.set(false)
Log.i(TAG, "Model loaded successfully")
val json = JSONObject()
json.put("success", true)
json.put("model_path", modelPath)
json.toString()
} catch (e: Exception) {
isLoading.set(false)
Log.e(TAG, "Error loading model", e)
"""{"success":false,"error":"${e.message?.replace("\"", "\\\"")}"}"""
}
}
/**
* Generate a response synchronously.
* @param prompt The input prompt
* @return JSON with generated text or error
*/
@JvmStatic
fun generateResponse(prompt: String): String {
val inference = llmInference
if (inference == null) {
return """{"success":false,"error":"No model loaded. Call loadModel() first."}"""
}
return try {
Log.d(TAG, "Generating response for prompt: ${prompt.take(100)}...")
val response = inference.generateResponse(prompt)
val json = JSONObject()
json.put("success", true)
json.put("response", response)
json.put("prompt", prompt)
json.toString()
} catch (e: Exception) {
Log.e(TAG, "Error generating response", e)
"""{"success":false,"error":"${e.message?.replace("\"", "\\\"")}"}"""
}
}
/**
* Generate a response asynchronously with streaming.
* Results are sent via the streaming callback.
* @param prompt The input prompt
* @return JSON with status
*/
@JvmStatic
fun generateResponseAsync(prompt: String): String {
val inference = llmInference
if (inference == null) {
return """{"success":false,"error":"No model loaded. Call loadModel() first."}"""
}
return try {
Log.d(TAG, "Starting async generation for prompt: ${prompt.take(100)}...")
inference.generateResponseAsync(prompt) { partialResult, done ->
streamingCallback?.invoke(partialResult, done)
}
val json = JSONObject()
json.put("success", true)
json.put("status", "streaming")
json.toString()
} catch (e: Exception) {
Log.e(TAG, "Error starting async generation", e)
"""{"success":false,"error":"${e.message?.replace("\"", "\\\"")}"}"""
}
}
/**
* Set the streaming callback for async generation.
* @param callback Function that receives (partialResult: String, isDone: Boolean)
*/
@JvmStatic
fun setStreamingCallback(callback: (String, Boolean) -> Unit) {
streamingCallback = callback
}
/**
* Unload the current model and free resources.
* @return JSON with status
*/
@JvmStatic
fun unloadModel(): String {
return try {
llmInference?.close()
llmInference = null
currentModelPath = null
Log.i(TAG, "Model unloaded")
"""{"success":true}"""
} catch (e: Exception) {
Log.e(TAG, "Error unloading model", e)
"""{"success":false,"error":"${e.message?.replace("\"", "\\\"")}"}"""
}
}
/**
* Get the default model storage directory.
* @return Path to the models directory
*/
@JvmStatic
fun getModelsDirectory(): String {
val context = appContext ?: return "/data/local/tmp/llm"
// Use app's files directory for model storage
val modelsDir = File(context.filesDir, "models")
if (!modelsDir.exists()) {
modelsDir.mkdirs()
}
return modelsDir.absolutePath
}
/**
* List available models in the models directory.
* @return JSON array of model files
*/
@JvmStatic
fun listModels(): String {
return try {
val modelsDir = File(getModelsDirectory())
val models = modelsDir.listFiles { file ->
file.isFile && (file.name.endsWith(".task") || file.name.endsWith(".bin"))
} ?: emptyArray()
val json = JSONObject()
json.put("success", true)
json.put("models_dir", modelsDir.absolutePath)
val modelsList = models.map { file ->
JSONObject().apply {
put("name", file.name)
put("path", file.absolutePath)
put("size_mb", file.length() / (1024 * 1024))
}
}
json.put("models", modelsList)
json.toString()
} catch (e: Exception) {
Log.e(TAG, "Error listing models", e)
"""{"success":false,"error":"${e.message?.replace("\"", "\\\"")}"}"""
}
}
/**
* Get recommended models for download.
* @return JSON with model recommendations and download URLs
*/
@JvmStatic
fun getRecommendedModels(): String {
val json = JSONObject()
json.put("success", true)
json.put("models", listOf(
JSONObject().apply {
put("name", "Gemma 3 1B (4-bit)")
put("id", "gemma-3-1b-it-int4")
put("size_mb", 550)
put("description", "Compact, fast model suitable for most devices")
put("url", "https://huggingface.co/litert-community/Gemma3-1B-IT/resolve/main/gemma3-1b-it-int4.task")
},
JSONObject().apply {
put("name", "Gemma 3n E2B (4-bit)")
put("id", "gemma-3n-e2b-it-int4")
put("size_mb", 1400)
put("description", "Effective 2B model with multimodal support")
put("url", "https://huggingface.co/litert-community/Gemma3n-E2B-IT/resolve/main/gemma3n-e2b-it-int4.task")
},
JSONObject().apply {
put("name", "Gemma 3n E4B (4-bit)")
put("id", "gemma-3n-e4b-it-int4")
put("size_mb", 2800)
put("description", "Effective 4B model, best quality, requires high-end device")
put("url", "https://huggingface.co/litert-community/Gemma3n-E4B-IT/resolve/main/gemma3n-e4b-it-int4.task")
}
))
return json.toString()
}
}
src-tauri/gen/android/app/src/main/java/com/alphahuman/app/TdLibBridge.kt
+16 -254
View File
@@ -1,294 +1,56 @@
package com.alphahuman.app
import android.util.Log
import org.drinkless.tdlib.Client
import org.drinkless.tdlib.TdApi
import org.json.JSONObject
import java.util.concurrent.ConcurrentHashMap
import java.util.concurrent.atomic.AtomicInteger
import java.util.concurrent.atomic.AtomicLong
/**
* TDLib Bridge for Android
* TDLib Bridge Stub for Android
*
* Provides a JNI-accessible interface to TDLib for the Rust backend.
* Manages TDLib client lifecycle and provides JSON-based request/response interface.
* TDLib native library is not available on Android through Maven Central.
* Telegram integration on mobile uses MTProto via the frontend JavaScript.
* This stub ensures the build compiles while TDLib features return errors.
*/
object TdLibBridge {
private const val TAG = "TdLibBridge"
// Client instance (singleton)
private var client: Client? = null
// Request ID counter for correlation
private val requestIdCounter = AtomicLong(1)
// Pending requests waiting for responses
private val pendingRequests = ConcurrentHashMap<Long, (TdApi.Object) -> Unit>()
// Update handler callback
private var updateHandler: ((String) -> Unit)? = null
// Client ID (always 1 for singleton)
private const val CLIENT_ID = 1
/**
* Create a TDLib client.
* @return Client ID (always 1)
* Stub - TDLib is not available on Android.
*/
@JvmStatic
fun createClient(): Int {
Log.i(TAG, "Creating TDLib client")
if (client != null) {
Log.w(TAG, "Client already exists, returning existing client ID")
return CLIENT_ID
}
// Create result handler that processes responses and updates
val resultHandler = Client.ResultHandler { result ->
handleResult(result)
}
// Create exception handler
val exceptionHandler = Client.ExceptionHandler { e ->
Log.e(TAG, "TDLib exception", e)
}
// Create the client
client = Client.create(resultHandler, exceptionHandler, exceptionHandler)
Log.i(TAG, "TDLib client created successfully")
return CLIENT_ID
Log.w(TAG, "TDLib is not available on Android")
return -1
}
/**
* Handle a TDLib result (response or update).
*/
private fun handleResult(result: TdApi.Object) {
// Convert to JSON for the Rust side
val json = tdObjectToJson(result)
// Check if this is a response to a pending request (has @extra)
// Note: TDLib Java API doesn't expose @extra directly, so we handle responses
// through the synchronous send pattern instead
// For updates, call the update handler
if (result is TdApi.Update) {
updateHandler?.invoke(json)
}
}
/**
* Send a synchronous request to TDLib.
* @param requestJson JSON string of the TDLib API request
* @return JSON string of the response
* Stub - TDLib is not available on Android.
*/
@JvmStatic
fun send(clientId: Int, requestJson: String): String {
val currentClient = client
if (currentClient == null) {
Log.e(TAG, "Client not initialized")
return """{"@type":"error","code":400,"message":"Client not initialized"}"""
}
try {
Log.d(TAG, "Sending request: $requestJson")
// Parse the JSON request
val function = jsonToTdFunction(requestJson)
if (function == null) {
return """{"@type":"error","code":400,"message":"Invalid request format"}"""
}
// Execute synchronously
val result = currentClient.send(function)
// Convert result to JSON
val responseJson = tdObjectToJson(result)
Log.d(TAG, "Received response: $responseJson")
return responseJson
} catch (e: Exception) {
Log.e(TAG, "Error sending request", e)
return """{"@type":"error","code":500,"message":"${e.message?.replace("\"", "\\\"")}"}"""
}
Log.w(TAG, "TDLib is not available on Android")
return """{"@type":"error","code":501,"message":"TDLib is not available on Android"}"""
}
/**
* Receive updates from TDLib (with timeout).
* @param timeout Timeout in seconds
* @return JSON string of the update, or null if timeout
* Stub - TDLib is not available on Android.
*/
@JvmStatic
fun receive(timeout: Double): String? {
val currentClient = client ?: return null
try {
val result = Client.execute(TdApi.GetOption("version"))
// The actual receiving is done via the result handler callback
// This method is mainly for polling pattern support
return null
} catch (e: Exception) {
Log.e(TAG, "Error receiving", e)
return null
}
return null
}
/**
* Set the update handler callback.
* @param handler Function that receives update JSON strings
*/
@JvmStatic
fun setUpdateHandler(handler: (String) -> Unit) {
updateHandler = handler
}
/**
* Destroy the TDLib client.
* Stub - TDLib is not available on Android.
*/
@JvmStatic
fun destroyClient(clientId: Int) {
Log.i(TAG, "Destroying TDLib client")
client?.close()
client = null
pendingRequests.clear()
updateHandler = null
Log.i(TAG, "TDLib client destroyed")
Log.w(TAG, "TDLib is not available on Android")
}
/**
* Check if TDLib is available.
* TDLib is not available on Android via Maven.
*/
@JvmStatic
fun isAvailable(): Boolean {
return try {
// Try to load the TDLib native library
System.loadLibrary("tdjni")
true
} catch (e: UnsatisfiedLinkError) {
Log.e(TAG, "TDLib native library not found", e)
false
}
}
/**
* Convert a TDLib object to JSON string.
* Note: This is a simplified implementation. TDLib Java API provides toString()
* which returns a debug representation, not proper JSON.
*/
private fun tdObjectToJson(obj: TdApi.Object): String {
// Use TDLib's built-in serialization
// The toString() method provides a debug format, we need proper JSON
return try {
// For now, return a simple JSON representation
// In production, use TDLib's JSON serialization or implement proper conversion
val json = JSONObject()
json.put("@type", obj.javaClass.simpleName.replaceFirstChar { it.lowercase() })
// Handle common types
when (obj) {
is TdApi.Error -> {
json.put("code", obj.code)
json.put("message", obj.message)
}
is TdApi.Ok -> {
// Empty ok response
}
is TdApi.User -> {
json.put("id", obj.id)
json.put("first_name", obj.firstName)
json.put("last_name", obj.lastName)
json.put("username", obj.usernames?.activeUsernames?.firstOrNull() ?: "")
}
is TdApi.AuthorizationStateWaitTdlibParameters -> {
json.put("@type", "authorizationStateWaitTdlibParameters")
}
is TdApi.AuthorizationStateWaitPhoneNumber -> {
json.put("@type", "authorizationStateWaitPhoneNumber")
}
is TdApi.AuthorizationStateWaitCode -> {
json.put("@type", "authorizationStateWaitCode")
}
is TdApi.AuthorizationStateReady -> {
json.put("@type", "authorizationStateReady")
}
is TdApi.UpdateAuthorizationState -> {
json.put("@type", "updateAuthorizationState")
json.put("authorization_state", tdObjectToJson(obj.authorizationState))
}
else -> {
// Generic handling - just use toString for now
json.put("raw", obj.toString())
}
}
json.toString()
} catch (e: Exception) {
Log.e(TAG, "Error converting TdObject to JSON", e)
"""{"@type":"error","code":500,"message":"JSON conversion failed"}"""
}
}
/**
* Convert a JSON string to a TDLib function.
* Note: This is a simplified implementation. Full implementation would parse
* all TDLib API types.
*/
private fun jsonToTdFunction(json: String): TdApi.Function<*>? {
return try {
val obj = JSONObject(json)
val type = obj.optString("@type", "")
when (type) {
"setTdlibParameters" -> TdApi.SetTdlibParameters().apply {
databaseDirectory = obj.optString("database_directory", "")
useMessageDatabase = obj.optBoolean("use_message_database", true)
useSecretChats = obj.optBoolean("use_secret_chats", false)
apiId = obj.optInt("api_id", 0)
apiHash = obj.optString("api_hash", "")
systemLanguageCode = obj.optString("system_language_code", "en")
deviceModel = obj.optString("device_model", "Android")
applicationVersion = obj.optString("application_version", "1.0")
}
"setAuthenticationPhoneNumber" -> TdApi.SetAuthenticationPhoneNumber(
obj.optString("phone_number", ""),
null
)
"checkAuthenticationCode" -> TdApi.CheckAuthenticationCode(
obj.optString("code", "")
)
"getMe" -> TdApi.GetMe()
"getChats" -> TdApi.GetChats(
null,
obj.optInt("limit", 100)
)
"getChat" -> TdApi.GetChat(
obj.optLong("chat_id", 0)
)
"sendMessage" -> {
val chatId = obj.optLong("chat_id", 0)
val text = obj.optString("text", "")
val inputContent = TdApi.InputMessageText(
TdApi.FormattedText(text, emptyArray()),
null,
false
)
TdApi.SendMessage(chatId, 0, null, null, null, inputContent)
}
"close" -> TdApi.Close()
"logOut" -> TdApi.LogOut()
"getOption" -> TdApi.GetOption(obj.optString("name", ""))
else -> {
Log.w(TAG, "Unknown function type: $type")
null
}
}
} catch (e: Exception) {
Log.e(TAG, "Error parsing JSON to TdFunction", e)
null
}
return false
}
}
src-tauri/src/commands/model.rs
+204 -14
View File
@@ -1,10 +1,16 @@
//! Model Tauri Commands
//!
//! These commands provide local LLM access via Tauri's invoke() system.
//! Available on desktop and Android (not iOS).
//! - Desktop (Windows, macOS, Linux): Uses llama.cpp via llama-cpp-2 crate
//! - Android: Uses MediaPipe LLM Inference API via JNI
//! - iOS: Not yet supported
use serde::{Deserialize, Serialize};
// serde_json used for Android MediaPipe commands
#[cfg(target_os = "android")]
use serde_json::{json, Value as JsonValue};
/// Model status response for frontend.
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
@@ -50,9 +56,56 @@ fn default_top_p() -> f32 {
0.9
}
// ============================================================================
// Android JNI Bridge to MediaPipe LLM
// ============================================================================
#[cfg(target_os = "android")]
mod android {
use super::*;
/// Call a static method on MediaPipeLlmBridge that returns a String.
/// This uses Android's JNI to communicate with the Kotlin MediaPipe wrapper.
pub fn call_mediapipe_method(method: &str) -> Result<String, String> {
// For now, return a placeholder - actual JNI implementation requires
// access to the JNI environment which needs to be passed from Tauri's
// Android activity context.
//
// TODO: Implement proper JNI bridge using tauri's android module
// The MediaPipeLlmBridge Kotlin object is already set up and ready.
log::warn!(
"MediaPipe LLM method '{}' called - JNI bridge pending implementation",
method
);
Err(format!(
"MediaPipe LLM JNI bridge not yet implemented for method: {}",
method
))
}
/// Call a static method on MediaPipeLlmBridge with a String argument.
pub fn call_mediapipe_method_with_arg(method: &str, arg: &str) -> Result<String, String> {
log::warn!(
"MediaPipe LLM method '{}' called with arg - JNI bridge pending implementation",
method
);
let _ = arg;
Err(format!(
"MediaPipe LLM JNI bridge not yet implemented for method: {}",
method
))
}
/// Parse a JSON response from MediaPipe bridge.
pub fn parse_mediapipe_response(json: &str) -> Result<JsonValue, String> {
serde_json::from_str(json).map_err(|e| format!("Failed to parse MediaPipe response: {}", e))
}
}
/// Check if the local model API is available on this platform.
/// Note: Currently only available on desktop (Windows, macOS, Linux).
/// Android/iOS support requires additional llama.cpp NDK configuration.
/// - Desktop: Uses llama.cpp (always available)
/// - Android: Uses MediaPipe LLM (available on supported devices)
/// - iOS: Not yet supported
#[tauri::command]
pub fn model_is_available() -> bool {
#[cfg(not(any(target_os = "android", target_os = "ios")))]
@@ -60,7 +113,13 @@ pub fn model_is_available() -> bool {
true
}
#[cfg(any(target_os = "android", target_os = "ios"))]
#[cfg(target_os = "android")]
{
// MediaPipe LLM is available on Android
true
}
#[cfg(target_os = "ios")]
{
false
}
@@ -82,14 +141,28 @@ pub fn model_get_status() -> ModelStatusResponse {
}
}
#[cfg(any(target_os = "android", target_os = "ios"))]
#[cfg(target_os = "android")]
{
// MediaPipe LLM status - JNI bridge pending full implementation
// For now, report available but not loaded
ModelStatusResponse {
available: true,
loaded: false,
loading: false,
download_progress: None,
error: Some("MediaPipe LLM: Download a model to get started".to_string()),
model_path: None,
}
}
#[cfg(target_os = "ios")]
{
ModelStatusResponse {
available: false,
loaded: false,
loading: false,
download_progress: None,
error: Some("Model not available on mobile platforms".to_string()),
error: Some("Model not available on iOS".to_string()),
model_path: None,
}
}
@@ -104,9 +177,16 @@ pub async fn model_ensure_loaded() -> Result<(), String> {
crate::services::llama::LLAMA_MANAGER.ensure_loaded().await
}
#[cfg(any(target_os = "android", target_os = "ios"))]
#[cfg(target_os = "android")]
{
Err("Model not available on mobile platforms".to_string())
// MediaPipe requires manual model download
// TODO: Implement model download via MediaPipeLlmBridge.loadModel()
Err("MediaPipe LLM: Please download a model first using the model manager".to_string())
}
#[cfg(target_os = "ios")]
{
Err("Model not available on iOS".to_string())
}
}
@@ -128,10 +208,18 @@ pub async fn model_generate(request: GenerateRequest) -> Result<String, String>
.await
}
#[cfg(any(target_os = "android", target_os = "ios"))]
#[cfg(target_os = "android")]
{
// TODO: Call MediaPipeLlmBridge.generateResponse() via JNI
// The Kotlin bridge is ready, just needs JNI wiring
let _ = request;
Err("MediaPipe LLM generation pending JNI implementation".to_string())
}
#[cfg(target_os = "ios")]
{
let _ = request;
Err("Model not available on mobile platforms".to_string())
Err("Model not available on iOS".to_string())
}
}
@@ -146,10 +234,17 @@ pub async fn model_summarize(text: String, max_tokens: Option<u32>) -> Result<St
.await
}
#[cfg(any(target_os = "android", target_os = "ios"))]
#[cfg(target_os = "android")]
{
// TODO: Implement summarization via MediaPipe
let _ = (text, max_tokens);
Err("MediaPipe LLM summarization pending JNI implementation".to_string())
}
#[cfg(target_os = "ios")]
{
let _ = (text, max_tokens);
Err("Model not available on mobile platforms".to_string())
Err("Model not available on iOS".to_string())
}
}
@@ -162,8 +257,103 @@ pub fn model_unload() -> Result<(), String> {
Ok(())
}
#[cfg(any(target_os = "android", target_os = "ios"))]
#[cfg(target_os = "android")]
{
Err("Model not available on mobile platforms".to_string())
// TODO: Call MediaPipeLlmBridge.unloadModel() via JNI
Ok(())
}
#[cfg(target_os = "ios")]
{
Err("Model not available on iOS".to_string())
}
}
// ============================================================================
// Android-specific MediaPipe LLM Commands
// ============================================================================
/// Get recommended models for download (Android only).
/// Returns a list of MediaPipe-compatible models with download URLs.
#[tauri::command]
pub fn model_get_recommended() -> Result<String, String> {
#[cfg(target_os = "android")]
{
// Return hardcoded recommended models for now
// TODO: Call MediaPipeLlmBridge.getRecommendedModels() via JNI
let models = serde_json::json!({
"success": true,
"models": [
{
"name": "Gemma 3 1B (4-bit)",
"id": "gemma-3-1b-it-int4",
"size_mb": 550,
"description": "Compact, fast model suitable for most devices",
"url": "https://huggingface.co/litert-community/Gemma3-1B-IT/resolve/main/gemma3-1b-it-int4.task"
},
{
"name": "Gemma 3n E2B (4-bit)",
"id": "gemma-3n-e2b-it-int4",
"size_mb": 1400,
"description": "Effective 2B model with multimodal support",
"url": "https://huggingface.co/litert-community/Gemma3n-E2B-IT/resolve/main/gemma3n-e2b-it-int4.task"
},
{
"name": "Gemma 3n E4B (4-bit)",
"id": "gemma-3n-e4b-it-int4",
"size_mb": 2800,
"description": "Effective 4B model, best quality, requires high-end device",
"url": "https://huggingface.co/litert-community/Gemma3n-E4B-IT/resolve/main/gemma3n-e4b-it-int4.task"
}
]
});
Ok(models.to_string())
}
#[cfg(not(target_os = "android"))]
{
Err("This command is only available on Android".to_string())
}
}
/// List downloaded models (Android only).
#[tauri::command]
pub fn model_list_downloaded() -> Result<String, String> {
#[cfg(target_os = "android")]
{
// TODO: Call MediaPipeLlmBridge.listModels() via JNI
let result = serde_json::json!({
"success": true,
"models": [],
"models_dir": "/data/data/com.alphahuman.app/files/models"
});
Ok(result.to_string())
}
#[cfg(not(target_os = "android"))]
{
Err("This command is only available on Android".to_string())
}
}
/// Load a specific model by path (Android only).
#[tauri::command]
pub fn model_load_path(
model_path: String,
max_tokens: Option<i32>,
top_k: Option<i32>,
temperature: Option<f32>,
) -> Result<String, String> {
#[cfg(target_os = "android")]
{
// TODO: Call MediaPipeLlmBridge.loadModel() via JNI
let _ = (model_path, max_tokens, top_k, temperature);
Err("MediaPipe model loading pending JNI implementation".to_string())
}
#[cfg(not(target_os = "android"))]
{
let _ = (model_path, max_tokens, top_k, temperature);
Err("This command is only available on Android".to_string())
}
}
src-tauri/src/lib.rs
+9 -3
View File
@@ -225,8 +225,6 @@ pub fn run() {
.with_max_level(log::LevelFilter::Debug)
.with_tag("AlphaHuman"),
);
// Ensure vendored OpenSSL is initialized before any TLS usage
openssl::init();
}
#[cfg(not(target_os = "android"))]
{
@@ -468,6 +466,10 @@ pub fn run() {
model_generate,
model_summarize,
model_unload,
// Android MediaPipe LLM commands
model_get_recommended,
model_list_downloaded,
model_load_path,
]
}
#[cfg(not(desktop))]
@@ -549,13 +551,17 @@ pub fn run() {
tdlib_receive,
tdlib_destroy,
tdlib_is_available,
// Model commands (local LLM)
// Model commands (local LLM / MediaPipe)
model_is_available,
model_get_status,
model_ensure_loaded,
model_generate,
model_summarize,
model_unload,
// Android MediaPipe LLM commands
model_get_recommended,
model_list_downloaded,
model_load_path,
]
}
})
src-tauri/src/runtime/socket_manager.rs
+49 -4
View File
@@ -65,9 +65,14 @@ struct SharedState {
/// app backgrounding. On desktop, handles MCP `listTools`/`toolCall` directly
/// via the [`SkillRegistry`], and forwards other server events to running
/// skills and to the frontend.
///
/// Note: On Android, this is a stub implementation. The frontend uses its own
/// Socket.io connection instead.
pub struct SocketManager {
shared: Arc<SharedState>,
/// The active `rust_socketio` async client (if connected).
/// Not available on Android due to native-tls/OpenSSL build complexity.
#[cfg(not(target_os = "android"))]
client: tokio::sync::Mutex<Option<Client>>,
}
@@ -81,6 +86,7 @@ impl SocketManager {
status: RwLock::new(ConnectionStatus::Disconnected),
socket_id: RwLock::new(None),
}),
#[cfg(not(target_os = "android"))]
client: tokio::sync::Mutex::new(None),
}
}
@@ -271,14 +277,34 @@ impl SocketManager {
Ok(())
}
/// Connect to the server with the given URL and auth token (mobile version).
/// Connect to the server with the given URL and auth token (Android stub).
/// On Android, the Rust Socket.io client is not available due to
/// native-tls/OpenSSL build complexity. The frontend should use its own
/// Socket.io connection instead.
#[cfg(target_os = "android")]
pub async fn connect(&self, url: &str, _token: &str) -> Result<(), String> {
log::info!(
"[socket-mgr] Android stub - Rust Socket.io not available. URL: {}",
url
);
log::info!("[socket-mgr] Frontend should use its own Socket.io connection on Android.");
// Mark as disconnected - frontend handles its own connection
*self.shared.status.write() = ConnectionStatus::Disconnected;
Self::emit_state_change(&self.shared);
// Return Ok so the app doesn't fail - socket is handled by frontend on Android
Ok(())
}
/// Connect to the server with the given URL and auth token (iOS version).
/// MCP skill handlers are not available on mobile.
#[cfg(any(target_os = "android", target_os = "ios"))]
#[cfg(target_os = "ios")]
pub async fn connect(&self, url: &str, token: &str) -> Result<(), String> {
// Disconnect existing connection first
self.disconnect().await?;
log::info!("[socket-mgr] Connecting to {} (mobile)", url);
log::info!("[socket-mgr] Connecting to {} (iOS)", url);
// Update status
*self.shared.status.write() = ConnectionStatus::Connecting;
@@ -400,6 +426,7 @@ impl SocketManager {
}
/// Disconnect from the server.
#[cfg(not(target_os = "android"))]
pub async fn disconnect(&self) -> Result<(), String> {
let mut client_guard = self.client.lock().await;
if let Some(client) = client_guard.take() {
@@ -411,7 +438,17 @@ impl SocketManager {
Ok(())
}
/// Disconnect from the server (Android stub).
#[cfg(target_os = "android")]
pub async fn disconnect(&self) -> Result<(), String> {
*self.shared.status.write() = ConnectionStatus::Disconnected;
*self.shared.socket_id.write() = None;
Self::emit_state_change(&self.shared);
Ok(())
}
/// Emit an event through the Rust socket to the server.
#[cfg(not(target_os = "android"))]
pub async fn emit(&self, event: &str, data: serde_json::Value) -> Result<(), String> {
let client_guard = self.client.lock().await;
if let Some(ref client) = *client_guard {
@@ -425,6 +462,12 @@ impl SocketManager {
}
}
/// Emit an event through the Rust socket to the server (Android stub).
#[cfg(target_os = "android")]
pub async fn emit(&self, _event: &str, _data: serde_json::Value) -> Result<(), String> {
Err("Rust Socket.io not available on Android. Use frontend socket.".to_string())
}
// -----------------------------------------------------------------------
// Tauri event helpers
// -----------------------------------------------------------------------
@@ -610,10 +653,11 @@ impl Default for SocketManager {
}
// ---------------------------------------------------------------------------
// Payload helpers
// Payload helpers (not needed on Android)
// ---------------------------------------------------------------------------
/// Extract the first JSON value from a Socket.io payload.
#[cfg(not(target_os = "android"))]
fn extract_json(payload: &Payload) -> Option<serde_json::Value> {
match payload {
Payload::Text(values) => values.first().cloned(),
@@ -623,6 +667,7 @@ fn extract_json(payload: &Payload) -> Option<serde_json::Value> {
}
/// Extract a human-readable string from a Socket.io payload.
#[cfg(not(target_os = "android"))]
fn extract_text(payload: &Payload) -> String {
match payload {
Payload::Text(values) => values
src-tauri/src/services/mod.rs
+4
View File
@@ -1,6 +1,10 @@
pub mod session_service;
pub mod socket_service;
// TDLib modules - desktop only (requires tdlib-rs which isn't available on mobile)
#[cfg(not(any(target_os = "android", target_os = "ios")))]
pub mod tdlib;
#[cfg(not(any(target_os = "android", target_os = "ios")))]
pub mod tdlib_v8;
#[cfg(desktop)]