From 2570195604ed97543888bdf4a1abcb2ddbeb865f Mon Sep 17 00:00:00 2001 From: "cyrus@tinyhumans.ai" Date: Tue, 31 Mar 2026 18:43:50 +0530 Subject: [PATCH] feat(local-ai): add guided model tier selection by device capability Add tiered model presets (Low/Medium/High) with device-aware recommendations so users can pick a local AI model that fits their machine without editing raw JSON config. Detect RAM, CPU, GPU via sysinfo crate and recommend a tier. Persist selection to config.toml, with env var override and graceful degradation hints on bootstrap failure. - Rust: presets.rs (tier definitions, recommendation logic), device.rs (hardware detection), 3 new RPC methods, env var override, bootstrap hints - Frontend: tier selector UI in Settings > Local AI Model with device info, loading/error states, and "Advanced" toggle for existing controls - Tests: 7 Rust unit tests + comprehensive JSON-RPC E2E test - Also fixes pre-existing lint warning in SkillSetupWizard.tsx Closes #80 --- .claude/memory.md | 9 + .env.example | 7 + Cargo.lock | 84 +- Cargo.toml | 1 + app/src-tauri/Cargo.lock | 2 +- app/src/components/settings/SettingsHome.tsx | 17 + .../settings/panels/DeveloperOptionsPanel.tsx | 16 - .../settings/panels/LocalModelPanel.tsx | 843 +++++++++++------- .../components/skills/SkillSetupWizard.tsx | 5 +- app/src/utils/tauriCommands.ts | 54 ++ src/openhuman/config/schema/load.rs | 22 + src/openhuman/config/schema/local_ai.rs | 3 + src/openhuman/local_ai/device.rs | 111 +++ src/openhuman/local_ai/mod.rs | 4 + src/openhuman/local_ai/presets.rs | 251 ++++++ src/openhuman/local_ai/schemas.rs | 115 +++ src/openhuman/local_ai/service/bootstrap.rs | 24 +- tests/json_rpc_e2e.rs | 122 +++ 18 files changed, 1337 insertions(+), 353 deletions(-) create mode 100644 src/openhuman/local_ai/device.rs create mode 100644 src/openhuman/local_ai/presets.rs diff --git a/.claude/memory.md b/.claude/memory.md index 11523061b..96d9e8c5e 100644 --- a/.claude/memory.md +++ b/.claude/memory.md @@ -25,6 +25,15 @@ Quick reference for anyone starting with Claude on this project. Updated by the - **Ask user when in doubt** — never assume scope or approach - **PRs target upstream** — `tinyhumansai/openhuman` main branch, not fork +## Local AI Presets & Daemon Gotcha + +- **Tier system lives in `src/openhuman/local_ai/presets.rs`** — single source of truth for tier→model ID mapping. To change default models for a release, edit `all_presets()` there. +- **Device detection** uses `sysinfo` crate (`src/openhuman/local_ai/device.rs`). Apple Silicon = GPU always; others = best-effort. +- **`OPENHUMAN_LOCAL_AI_TIER` env var** overrides the selected tier at config load time (in `load.rs`). +- **Frontend tier selector** is in `LocalModelPanel.tsx` under Settings > Local AI Model. Uses `coreRpcClient` to call 3 RPC methods: `local_ai_device_profile`, `local_ai_presets`, `local_ai_apply_preset`. +- **Default config maps to Medium tier** (`gemma3:4b-it-qat`). If someone changes `model_ids.rs` defaults, they should keep `presets.rs` in sync. +- **Daemon binary gotcha** — A daemon process (`openhuman-aarch64-apple-darwin run`) auto-starts on port 7788 and respawns on kill. `yarn tauri dev` reuses it if already running. When adding new RPC methods, you must replace this binary: `cp -f target/debug/openhuman-core app/src-tauri/binaries/openhuman-aarch64-apple-darwin`, then kill the old PID so it respawns with the new binary. + ## Environment - **Core sidecar port** — `7788` (default). Check with `lsof -i :7788`. diff --git a/.env.example b/.env.example index fc2cfcd82..f2bb43a74 100644 --- a/.env.example +++ b/.env.example @@ -88,6 +88,13 @@ OPENHUMAN_PROXY_SCOPE= # [optional] Comma-separated services to proxy OPENHUMAN_PROXY_SERVICES= +# --------------------------------------------------------------------------- +# Local AI model tier +# --------------------------------------------------------------------------- +# [optional] Override selected model tier: low, medium, high +# Applies the corresponding preset at config load time (overrides config.toml). +OPENHUMAN_LOCAL_AI_TIER= + # --------------------------------------------------------------------------- # Local AI binary overrides # --------------------------------------------------------------------------- diff --git a/Cargo.lock b/Cargo.lock index 1ee57c74b..01b85c5fe 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2458,7 +2458,7 @@ dependencies = [ "js-sys", "log", "wasm-bindgen", - "windows-core", + "windows-core 0.62.2", ] [[package]] @@ -3746,6 +3746,15 @@ dependencies = [ "nom 8.0.0", ] +[[package]] +name = "ntapi" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3b335231dfd352ffb0f8017f3b6027a4917f7df785ea2143d8af2adc66980ae" +dependencies = [ + "winapi", +] + [[package]] name = "nu-ansi-term" version = "0.46.0" @@ -3968,6 +3977,7 @@ dependencies = [ "sha2", "shellexpand", "socketioxide", + "sysinfo", "tempfile", "thiserror 2.0.18", "tokio", @@ -6053,6 +6063,19 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "sysinfo" +version = "0.33.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fc858248ea01b66f19d8e8a6d55f41deaf91e9d495246fd01368d99935c6c01" +dependencies = [ + "core-foundation-sys", + "libc", + "memchr", + "ntapi", + "windows", +] + [[package]] name = "tagptr" version = "0.2.0" @@ -7459,19 +7482,52 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" +[[package]] +name = "windows" +version = "0.57.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "12342cb4d8e3b046f3d80effd474a7a02447231330ef77d71daa6fbc40681143" +dependencies = [ + "windows-core 0.57.0", + "windows-targets 0.52.6", +] + +[[package]] +name = "windows-core" +version = "0.57.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2ed2439a290666cd67ecce2b0ffaad89c2a56b976b736e6ece670297897832d" +dependencies = [ + "windows-implement 0.57.0", + "windows-interface 0.57.0", + "windows-result 0.1.2", + "windows-targets 0.52.6", +] + [[package]] name = "windows-core" version = "0.62.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b8e83a14d34d0623b51dce9581199302a221863196a1dde71a7663a4c2be9deb" dependencies = [ - "windows-implement", - "windows-interface", + "windows-implement 0.60.2", + "windows-interface 0.59.3", "windows-link", - "windows-result", + "windows-result 0.4.1", "windows-strings", ] +[[package]] +name = "windows-implement" +version = "0.57.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9107ddc059d5b6fbfbffdfa7a7fe3e22a226def0b2608f72e9d552763d3e1ad7" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + [[package]] name = "windows-implement" version = "0.60.2" @@ -7483,6 +7539,17 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "windows-interface" +version = "0.57.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29bee4b38ea3cde66011baa44dba677c432a78593e202392d1e9070cf2a7fca7" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + [[package]] name = "windows-interface" version = "0.59.3" @@ -7500,6 +7567,15 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" +[[package]] +name = "windows-result" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e383302e8ec8515204254685643de10811af0ed97ea37210dc26fb0032647f8" +dependencies = [ + "windows-targets 0.52.6", +] + [[package]] name = "windows-result" version = "0.4.1" diff --git a/Cargo.toml b/Cargo.toml index cc81b3a32..3e75b6f2c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -67,6 +67,7 @@ rustls = { version = "0.23", features = ["ring"] } rustls-pki-types = "1.14.0" tokio-rustls = "0.26.4" webpki-roots = "1.0.6" +sysinfo = { version = "0.33", default-features = false, features = ["system"] } clap = { version = "4.5", features = ["derive"] } clap_complete = "4.5" lettre = { version = "0.11.19", default-features = false, features = ["builder", "smtp-transport", "rustls-tls"] } diff --git a/app/src-tauri/Cargo.lock b/app/src-tauri/Cargo.lock index 9b07a0ef3..e4906d677 100644 --- a/app/src-tauri/Cargo.lock +++ b/app/src-tauri/Cargo.lock @@ -4,7 +4,7 @@ version = 4 [[package]] name = "OpenHuman" -version = "0.49.31" +version = "0.49.32" dependencies = [ "env_logger", "log", diff --git a/app/src/components/settings/SettingsHome.tsx b/app/src/components/settings/SettingsHome.tsx index f97329adf..253a9b030 100644 --- a/app/src/components/settings/SettingsHome.tsx +++ b/app/src/components/settings/SettingsHome.tsx @@ -271,6 +271,23 @@ const SettingsHome = () => { // onClick: handleViewEncryptionKey, // dangerous: false, // }, + { + id: 'local-model', + title: 'Local AI Model', + description: 'Choose model tier by device capability and manage downloads', + icon: ( + + + + ), + onClick: () => navigateToSettings('local-model'), + dangerous: false, + }, { id: 'team', title: 'Team', diff --git a/app/src/components/settings/panels/DeveloperOptionsPanel.tsx b/app/src/components/settings/panels/DeveloperOptionsPanel.tsx index 40d61275c..89fe40470 100644 --- a/app/src/components/settings/panels/DeveloperOptionsPanel.tsx +++ b/app/src/components/settings/panels/DeveloperOptionsPanel.tsx @@ -35,22 +35,6 @@ const developerItems = [ ), }, - { - id: 'local-model', - title: 'Local Model Runtime', - description: 'Monitor download/load status and test local model calls', - route: 'local-model', - icon: ( - - - - ), - }, { id: 'tauri-commands', title: 'Tauri Command Console', diff --git a/app/src/components/settings/panels/LocalModelPanel.tsx b/app/src/components/settings/panels/LocalModelPanel.tsx index 24536e3ea..90a801b8f 100644 --- a/app/src/components/settings/panels/LocalModelPanel.tsx +++ b/app/src/components/settings/panels/LocalModelPanel.tsx @@ -1,6 +1,7 @@ import { useEffect, useMemo, useState } from 'react'; import { + type ApplyPresetResult, type LocalAiAssetsStatus, type LocalAiDownloadsProgress, type LocalAiEmbeddingResult, @@ -8,12 +9,14 @@ import { type LocalAiStatus, type LocalAiSuggestion, type LocalAiTtsResult, + openhumanLocalAiApplyPreset, openhumanLocalAiAssetsStatus, openhumanLocalAiDownload, openhumanLocalAiDownloadAllAssets, openhumanLocalAiDownloadAsset, openhumanLocalAiDownloadsProgress, openhumanLocalAiEmbed, + openhumanLocalAiPresets, openhumanLocalAiPrompt, openhumanLocalAiStatus, openhumanLocalAiSuggestQuestions, @@ -21,6 +24,7 @@ import { openhumanLocalAiTranscribe, openhumanLocalAiTts, openhumanLocalAiVisionPrompt, + type PresetsResponse, } from '../../../utils/tauriCommands'; import SettingsHeader from '../components/SettingsHeader'; import { useSettingsNavigation } from '../hooks/useSettingsNavigation'; @@ -148,6 +152,13 @@ const LocalModelPanel = () => { const [ttsOutput, setTtsOutput] = useState(null); const [isTtsLoading, setIsTtsLoading] = useState(false); + const [presetsData, setPresetsData] = useState(null); + const [presetsLoading, setPresetsLoading] = useState(true); + const [isApplyingPreset, setIsApplyingPreset] = useState(false); + const [presetError, setPresetError] = useState(''); + const [presetSuccess, setPresetSuccess] = useState(null); + const [showAdvanced, setShowAdvanced] = useState(false); + const progress = useMemo(() => { const downloadProgress = progressFromDownloads(downloads); if (downloadProgress != null) return downloadProgress; @@ -189,8 +200,41 @@ const LocalModelPanel = () => { } }; + const loadPresets = async () => { + setPresetsLoading(true); + try { + const data = await openhumanLocalAiPresets(); + setPresetsData(data); + setPresetError(''); + } catch (err) { + const msg = err instanceof Error ? err.message : 'Failed to load presets'; + console.warn('[LocalModelPanel] failed to load presets:', msg); + setPresetError(msg); + } finally { + setPresetsLoading(false); + } + }; + + const applyPreset = async (tier: string) => { + setIsApplyingPreset(true); + setPresetError(''); + setPresetSuccess(null); + try { + const result = await openhumanLocalAiApplyPreset(tier); + setPresetSuccess(result); + await loadPresets(); + await loadStatus(); + } catch (err) { + const msg = err instanceof Error ? err.message : 'Failed to apply preset'; + setPresetError(msg); + } finally { + setIsApplyingPreset(false); + } + }; + useEffect(() => { void loadStatus(); + void loadPresets(); const timer = setInterval(() => { void loadStatus(); }, 1500); @@ -360,356 +404,497 @@ const LocalModelPanel = () => { } }; + const formatRamGb = (bytes: number): string => { + const gb = bytes / (1024 * 1024 * 1024); + return gb >= 10 ? `${Math.round(gb)} GB` : `${gb.toFixed(1)} GB`; + }; + return (
+ {/* --- Model Tier Selection --- */}
-
-

Runtime Status

- -
+

Model Tier

-
-
- State - - {status ? statusLabel(downloads?.state ?? status.state) : 'Unavailable'} - + {/* Loading / error states */} + {presetsLoading && !presetsData && ( +
+ Loading device info and presets…
- -
-
+ )} + {!presetsLoading && !presetsData && presetError && ( +
+ Could not load presets: {presetError}
+ )} -
- - Progress:{' '} - {isIndeterminateDownload - ? 'Downloading (size unknown)' - : `${Math.round(progress * 100)}%`} - - {downloadedText && {downloadedText}} - {speedText && {speedText}} - {etaText && ETA {etaText}} -
- -
-
-
Provider
-
{status?.provider ?? 'n/a'}
-
-
-
Model
-
{status?.model_id ?? 'n/a'}
-
-
- -
-
-
Backend
-
{status?.active_backend ?? 'cpu'}
-
-
-
Last Latency
-
- {typeof status?.last_latency_ms === 'number' - ? `${status.last_latency_ms} ms` - : 'n/a'} -
-
-
-
Generation TPS
-
- {typeof status?.gen_toks_per_sec === 'number' - ? `${status.gen_toks_per_sec.toFixed(1)} tok/s` - : 'n/a'} -
-
-
- - {status?.model_path && ( -
Artifact: {status.model_path}
- )} - - {status?.backend_reason && ( -
{status.backend_reason}
- )} - {status?.warning &&
{status.warning}
} - {statusError &&
{statusError}
} - -
- - -
-
-
- -
-

Capability Assets

-
-
- Quantization preference: {assets?.quantization ?? 'q4'} -
-
- {[ - { label: 'Chat', key: 'chat' as const, item: assets?.chat }, - { label: 'Vision', key: 'vision' as const, item: assets?.vision }, - { label: 'Embedding', key: 'embedding' as const, item: assets?.embedding }, - { label: 'STT', key: 'stt' as const, item: assets?.stt }, - { label: 'TTS', key: 'tts' as const, item: assets?.tts }, - ].map(({ label, key, item }) => ( -
-
{label}
-
{item?.id ?? 'n/a'}
-
- {statusLabel(item?.state ?? 'idle')} + {/* Device info */} + {presetsData?.device && ( +
+
+
+
RAM
+
+ {formatRamGb(presetsData.device.total_ram_bytes)}
- {item?.path && ( -
{item.path}
- )} -
- ))} -
-
-
- -
-

Test Summarization

-
-