admin/OpenJarvis
1
0
Fork 0
mirror of https://github.com/open-jarvis/OpenJarvis.git synced 2026-07-27 21:05:34 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(desktop): single default model + custom OpenAI-compatible endpoints (#453)

Browse Source
This commit is contained in:
Jon Saad-Falcon
2026-05-30 19:34:51 -07:00
committed by GitHub
parent d9af1eca45
commit 21f1becb4d
6 changed files with 739 additions and 118 deletions
Download Patch File Download Diff File Expand all files Collapse all files
frontend/src-tauri/Cargo.lock
Generated
+28 -10
View File
@@ -1384,7 +1384,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0bb0228f477c0900c880fd78c8759b95c7636dbd7842707f49e132378aa2acdc"
dependencies = [
"heck 0.4.1",
"proc-macro-crate 2.0.2",
"proc-macro-crate 2.0.0",
"proc-macro-error",
"proc-macro2",
"quote",
@@ -2593,7 +2593,7 @@ dependencies = [
[[package]]
name = "openjarvis-desktop"
version = "0.1.0"
version = "1.0.1"
dependencies = [
"dispatch",
"objc",
@@ -2611,6 +2611,7 @@ dependencies = [
"tauri-plugin-single-instance",
"tauri-plugin-updater",
"tokio",
"toml_edit 0.22.27",
]
[[package]]
@@ -3018,11 +3019,10 @@ dependencies = [
[[package]]
name = "proc-macro-crate"
version = "2.0.2"
version = "2.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b00f26d3400549137f92511a46ac1cd8ce37cb5598a96d382381458b992a5d24"
checksum = "7e8366a6159044a37876a2b9817124296703c586a5c92e2c53751fa06d8d43e8"
dependencies = [
"toml_datetime 0.6.3",
"toml_edit 0.20.2",
]
@@ -4766,7 +4766,7 @@ checksum = "185d8ab0dfbb35cf1399a6344d8484209c088f75f8f68230da55d48d95d43e3d"
dependencies = [
"serde",
"serde_spanned 0.6.9",
"toml_datetime 0.6.3",
"toml_datetime 0.6.11",
"toml_edit 0.20.2",
]
@@ -4802,9 +4802,9 @@ dependencies = [
[[package]]
name = "toml_datetime"
version = "0.6.3"
version = "0.6.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7cda73e2f1397b1262d6dfdcef8aafae14d1de7748d66822d3bfeeb6d03e5e4b"
checksum = "22cddaf88f4fbc13c51aebbf5f8eceb5c7c5a9da2ac40a13519eb5b0a0e8f11c"
dependencies = [
"serde",
]
@@ -4834,7 +4834,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1b5bb770da30e5cbfde35a2d7b9b8a2c4b8ef89548a7a6aeab5c9a576e3e7421"
dependencies = [
"indexmap 2.13.0",
"toml_datetime 0.6.3",
"toml_datetime 0.6.11",
"winnow 0.5.40",
]
@@ -4847,10 +4847,22 @@ dependencies = [
"indexmap 2.13.0",
"serde",
"serde_spanned 0.6.9",
"toml_datetime 0.6.3",
"toml_datetime 0.6.11",
"winnow 0.5.40",
]
[[package]]
name = "toml_edit"
version = "0.22.27"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "41fe8c660ae4257887cf66394862d21dbca4a6ddd26f04a3560410406a2f819a"
dependencies = [
"indexmap 2.13.0",
"toml_datetime 0.6.11",
"toml_write",
"winnow 0.7.14",
]
[[package]]
name = "toml_edit"
version = "0.23.10+spec-1.0.0"
@@ -4872,6 +4884,12 @@ dependencies = [
"winnow 0.7.14",
]
[[package]]
name = "toml_write"
version = "0.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5d99f8c9a7727884afe522e9bd5edbfc91a3312b36a77b5fb8926e4c31a41801"
[[package]]
name = "toml_writer"
version = "1.0.6+spec-1.1.0"
frontend/src-tauri/Cargo.toml
+1
View File
@@ -9,6 +9,7 @@ license = "MIT"
tauri-build = { version = "2", features = [] }
[dependencies]
toml_edit = "0.22"
tauri = { version = "2", features = ["tray-icon"] }
tauri-plugin-notification = "2"
tauri-plugin-shell = "2"
frontend/src-tauri/src/lib.rs
+576 -106
View File
@@ -76,29 +76,102 @@ fn total_ram_gb() -> f64 {
8.0
}
/// Return the list of Qwen3.5 models that fit on this machine, smallest first.
fn models_that_fit() -> Vec<&'static str> {
let ram = total_ram_gb();
/// Return the Qwen3.5 models that fit in `ram_gb`, smallest first.
fn models_that_fit_in(ram_gb: f64) -> Vec<&'static str> {
QWEN35_MODELS
.iter()
.filter(|(_, _, min_ram)| ram >= *min_ram)
.filter(|(_, _, min_ram)| ram_gb >= *min_ram)
.map(|(tag, _, _)| *tag)
.collect()
}
/// Pick the default model — prefers STARTUP_MODEL if it fits, otherwise
/// falls back to the third-largest model that fits on this machine.
fn preferred_model() -> &'static str {
let fitting = models_that_fit();
// Prefer STARTUP_MODEL when it fits (fast, good quality)
if fitting.contains(&STARTUP_MODEL) {
return STARTUP_MODEL;
}
/// The default local model: the second-largest Qwen3.5 model that fits in
/// `ram_gb`. Falls back to the only fitting model, or FALLBACK_MODEL if none
/// fit. Deliberately NOT the largest — leaves RAM headroom for the OS/app.
fn default_local_model(ram_gb: f64) -> &'static str {
let fitting = models_that_fit_in(ram_gb);
match fitting.len() {
0 => FALLBACK_MODEL,
1 => fitting[0],
2 => fitting[0],
n => fitting[n - 3], // third-largest
n => fitting[n - 2],
}
}
/// A resolved boot plan derived purely from the inference config + RAM.
/// Pure and side-effect-free so it can be unit-tested without spawning
/// processes or touching the network.
#[derive(Debug, Clone, PartialEq, Eq)]
struct BootPlan {
/// Whether to start and wait for the bundled Ollama.
launch_ollama: bool,
/// The single Ollama model to pull (None for custom endpoints).
model_to_pull: Option<String>,
/// Optional `(engine_key, bare_host)` override for a custom endpoint,
/// e.g. `("lmstudio", "http://localhost:1234")`. Written into
/// ~/.openjarvis/config.toml so `jarvis serve` picks it up.
engine_host: Option<(String, String)>,
/// Args appended after `uv run jarvis serve --port <port>`.
serve_args: Vec<String>,
}
/// Default OpenAI-compatible engine key used when a custom endpoint config
/// omits one (LM Studio is the canonical local server).
const CUSTOM_FALLBACK_ENGINE: &str = "lmstudio";
/// Decide what to launch/pull/serve from the inference config + system RAM.
/// Pure: no I/O, no spawning.
fn boot_plan(cfg: &InferenceConfig, ram_gb: f64) -> BootPlan {
match cfg.kind {
SourceKind::Ollama => {
let model = cfg
.model
.clone()
.unwrap_or_else(|| default_local_model(ram_gb).to_string());
BootPlan {
launch_ollama: true,
model_to_pull: Some(model.clone()),
engine_host: None,
serve_args: vec![
"--engine".into(),
"ollama".into(),
"--model".into(),
model,
"--agent".into(),
"simple".into(),
],
}
}
SourceKind::Custom => {
let engine = cfg
.engine
.clone()
.unwrap_or_else(|| CUSTOM_FALLBACK_ENGINE.to_string());
// Record (engine_key, bare_host) only when a host is configured, so
// boot can write `[engine.<key>] host = ...` into config.toml. An
// empty host is dropped (no override).
let engine_host = cfg
.host
.clone()
.filter(|h| !h.is_empty())
.map(|h| (engine.clone(), h));
// `model` may be empty if the config is malformed; `jarvis serve`
// surfaces a clear error then (there is no universal default model
// for an arbitrary endpoint).
let model = cfg.model.clone().unwrap_or_default();
BootPlan {
launch_ollama: false,
model_to_pull: None,
engine_host,
serve_args: vec![
"--engine".into(),
engine,
"--model".into(),
model,
"--agent".into(),
"simple".into(),
],
}
}
}
}
@@ -343,6 +416,8 @@ struct SetupStatus {
server_ready: bool,
model_ready: bool,
error: Option<String>,
/// "ollama" | "custom" — lets the setup UI relabel the progress steps.
source: String,
}
impl Default for SetupStatus {
@@ -354,6 +429,7 @@ impl Default for SetupStatus {
server_ready: false,
model_ready: false,
error: None,
source: "ollama".into(),
}
}
}
@@ -381,6 +457,28 @@ async fn wait_for_url(url: &str, timeout: Duration) -> bool {
false
}
/// True if a custom OpenAI-compatible endpoint answers at all (any HTTP
/// status counts — even a 404 proves the server is up). `host` is the bare
/// base URL; we probe `<host>/v1/models`.
async fn endpoint_reachable(host: &str, timeout: Duration) -> bool {
let client = match reqwest::Client::builder()
.timeout(Duration::from_secs(3))
.build()
{
Ok(c) => c,
Err(_) => return false,
};
let url = format!("{}/v1/models", host.trim_end_matches('/'));
let deadline = tokio::time::Instant::now() + timeout;
while tokio::time::Instant::now() < deadline {
if client.get(&url).send().await.is_ok() {
return true;
}
tokio::time::sleep(Duration::from_millis(500)).await;
}
false
}
/// Outcome of waiting for `jarvis serve` to become healthy.
///
/// Unlike [`wait_for_url`] this differentiates "server is up but degraded"
@@ -608,80 +706,153 @@ fn format_uv_sync_spawn_error(root: &std::path::Path, uv_bin: &str, err: &str) -
// ---------------------------------------------------------------------------
async fn boot_backend(backend: SharedBackend, status: SharedStatus) {
// Phase 1: Start Ollama
// Decide the inference source (default Ollama) before launching anything.
let cfg = read_inference_config();
let plan = boot_plan(&cfg, total_ram_gb());
{
let mut s = status.lock().await;
s.phase = "ollama".into();
s.detail = "Starting inference engine...".into();
}
// Try the bundled sidecar first, fall back to system ollama
let ollama_child = {
let ollama_bin = resolve_bin("ollama");
let sidecar = tokio::process::Command::new(&ollama_bin)
.arg("serve")
.env("OLLAMA_HOST", format!("127.0.0.1:{}", OLLAMA_PORT))
.stdout(std::process::Stdio::null())
.stderr(std::process::Stdio::null())
.spawn();
match sidecar {
Ok(child) => Some(child),
Err(_) => None,
s.source = match cfg.kind {
SourceKind::Ollama => "ollama",
SourceKind::Custom => "custom",
}
};
if let Some(child) = ollama_child {
backend.lock().await.ollama = Some(ChildHandle { child });
.into();
}
let ollama_url = format!("http://127.0.0.1:{}/api/tags", OLLAMA_PORT);
let ollama_ok = wait_for_url(&ollama_url, Duration::from_secs(30)).await;
// For the Ollama path, the model pull may fall back to FALLBACK_MODEL; we
// record what is actually available here so the serve command below uses
// it instead of the originally-planned tag. None on the custom path.
let mut serve_model_override: Option<String> = None;
if !ollama_ok {
let mut s = status.lock().await;
s.error = Some("Could not start Ollama. Install it from https://ollama.com".into());
return;
}
{
let mut s = status.lock().await;
s.ollama_ready = true;
s.detail = "Inference engine ready.".into();
}
// Phase 2: Pull one small model (qwen3.5:2b) so the app can open fast.
// Remaining models are pulled in the background after the server starts.
{
let mut s = status.lock().await;
s.phase = "model".into();
s.detail = format!("Checking for {}...", STARTUP_MODEL);
}
if !ollama_has_model(STARTUP_MODEL).await {
if plan.launch_ollama {
// Phase 1: Start Ollama
{
let mut s = status.lock().await;
s.detail = format!("Downloading {}... (this may take a minute)", STARTUP_MODEL);
s.phase = "ollama".into();
s.detail = "Starting inference engine...".into();
}
if let Err(e) = pull_model(STARTUP_MODEL).await {
// If the startup model fails, try the tiny fallback
eprintln!("Warning: failed to pull {}: {}", STARTUP_MODEL, e);
if !ollama_has_model(FALLBACK_MODEL).await {
// Try the bundled sidecar first, fall back to system ollama
let ollama_child = {
let ollama_bin = resolve_bin("ollama");
let sidecar = tokio::process::Command::new(&ollama_bin)
.arg("serve")
.env("OLLAMA_HOST", format!("127.0.0.1:{}", OLLAMA_PORT))
.stdout(std::process::Stdio::null())
.stderr(std::process::Stdio::null())
.spawn();
match sidecar {
Ok(child) => Some(child),
Err(_) => None,
}
};
if let Some(child) = ollama_child {
backend.lock().await.ollama = Some(ChildHandle { child });
}
let ollama_url = format!("http://127.0.0.1:{}/api/tags", OLLAMA_PORT);
if !wait_for_url(&ollama_url, Duration::from_secs(30)).await {
let mut s = status.lock().await;
s.error = Some("Could not start Ollama. Install it from https://ollama.com".into());
return;
}
{
let mut s = status.lock().await;
s.ollama_ready = true;
s.detail = "Inference engine ready.".into();
}
// Phase 2: Pull the single default model (see default_local_model /
// boot_plan). We deliberately do NOT pull any others.
let model = plan
.model_to_pull
.clone()
.unwrap_or_else(|| STARTUP_MODEL.to_string());
{
let mut s = status.lock().await;
s.phase = "model".into();
s.detail = format!("Checking for {}...", model);
}
if !ollama_has_model(&model).await {
{
let mut s = status.lock().await;
s.detail = format!("Downloading {}...", FALLBACK_MODEL);
drop(s);
if let Err(e2) = pull_model(FALLBACK_MODEL).await {
let mut s = status.lock().await;
s.error = Some(format!("Failed to download model: {}", e2));
return;
s.detail = format!("Downloading {}... (this may take a minute)", model);
}
if let Err(e) = pull_model(&model).await {
// If the chosen model fails, try the tiny fallback
eprintln!("Warning: failed to pull {}: {}", model, e);
if !ollama_has_model(FALLBACK_MODEL).await {
{
let mut s = status.lock().await;
s.detail = format!("Downloading {}...", FALLBACK_MODEL);
}
if let Err(e2) = pull_model(FALLBACK_MODEL).await {
let mut s = status.lock().await;
s.error = Some(format!("Failed to download model: {}", e2));
return;
}
}
}
}
}
{
let mut s = status.lock().await;
s.model_ready = true;
s.detail = "Model ready.".into();
// The pull may have fallen back to FALLBACK_MODEL; serve and persist
// whatever is actually available now, not the originally-planned tag.
let resolved_model = if ollama_has_model(&model).await {
model
} else {
FALLBACK_MODEL.to_string()
};
serve_model_override = Some(resolved_model.clone());
// Persist the resolved model so Settings shows it and future boots reuse it.
let mut persisted = cfg.clone();
persisted.model = Some(resolved_model);
let _ = write_inference_config(&persisted);
{
let mut s = status.lock().await;
s.model_ready = true;
s.detail = "Model ready.".into();
}
} else {
// Custom OpenAI-compatible endpoint: never start Ollama, never download.
let host = plan
.engine_host
.as_ref()
.map(|(_, v)| v.clone())
.unwrap_or_default();
{
let mut s = status.lock().await;
s.phase = "model".into();
s.detail = format!("Connecting to {}...", host);
}
if host.is_empty() || !endpoint_reachable(&host, Duration::from_secs(15)).await {
let mut s = status.lock().await;
s.error = Some(format!(
"Could not reach your custom inference server at {}. \
Start the server (e.g. LM Studio) and check the URL in Settings, then relaunch.",
if host.is_empty() { "(no URL set)" } else { host.as_str() }
));
return;
}
// Point `jarvis serve` at the user's endpoint by writing the engine
// host into ~/.openjarvis/config.toml (the env var alone is shadowed by
// the engine's non-empty default host in the Python layer).
if let Some((engine, host)) = &plan.engine_host {
if let Err(e) = set_engine_host_in_config(engine, host) {
let mut s = status.lock().await;
s.error = Some(format!("Could not write engine config: {}", e));
return;
}
}
{
let mut s = status.lock().await;
s.ollama_ready = true;
s.model_ready = true;
s.detail = "Connected to custom endpoint.".into();
}
}
// Phase 3: Start jarvis serve
@@ -848,16 +1019,6 @@ async fn boot_backend(backend: SharedBackend, status: SharedStatus) {
}
}
// Start with STARTUP_MODEL (just pulled) or preferred if already available.
let pref = preferred_model();
let startup_model = if ollama_has_model(pref).await {
pref
} else if ollama_has_model(STARTUP_MODEL).await {
STARTUP_MODEL
} else {
FALLBACK_MODEL
};
let root = project_root.as_ref().unwrap();
// Install dependencies automatically (handles fresh clones).
@@ -907,28 +1068,35 @@ async fn boot_backend(backend: SharedBackend, status: SharedStatus) {
{
let mut s = status.lock().await;
s.detail = format!(
"Starting server with {} from {}...",
startup_model,
root.display(),
);
s.detail = format!("Starting API server from {}...", root.display());
}
let mut cmd = tokio::process::Command::new(&uv_bin);
cmd.args([
"run",
"jarvis",
"serve",
"--port",
&JARVIS_PORT.to_string(),
"--model",
startup_model,
"--agent",
"simple",
])
.stdout(std::process::Stdio::null())
.stderr(std::process::Stdio::piped())
.current_dir(root);
let mut serve_argv: Vec<String> = vec![
"run".into(),
"jarvis".into(),
"serve".into(),
"--port".into(),
JARVIS_PORT.to_string(),
];
serve_argv.extend(plan.serve_args.iter().cloned());
// If the Ollama pull fell back to a different tag than planned, serve the
// tag that is actually present. boot_plan always emits `--model` followed
// immediately by its value, so `i + 1` is in bounds.
if let Some(m) = &serve_model_override {
match serve_argv.iter().position(|a| a == "--model") {
Some(i) if i + 1 < serve_argv.len() => serve_argv[i + 1] = m.clone(),
_ => eprintln!(
"Warning: resolved model {:?} could not be applied; \
'--model <value>' not found in serve args {:?}",
m, serve_argv
),
}
}
cmd.args(&serve_argv)
.stdout(std::process::Stdio::null())
.stderr(std::process::Stdio::piped())
.current_dir(root);
// Inject cloud API keys from ~/.openjarvis/cloud-keys.env
for (key, value) in read_cloud_keys() {
@@ -971,11 +1139,16 @@ async fn boot_backend(backend: SharedBackend, status: SharedStatus) {
s.error = Some(format!(
"Jarvis server is running but the inference engine is not available \
(HTTP 503). This usually means the configured model couldn't be loaded.\n\n\
Check the server logs, or run 'uv run jarvis serve --port {} --model {}' \
Check the server logs, or run 'uv run jarvis serve --port {}{}' \
from {} to see the engine error.\n\n\
Server response:\n{}",
JARVIS_PORT,
startup_model,
// Show the args actually passed (after `serve --port <port>`),
// including any post-fallback `--model` override.
match serve_argv.get(5..) {
Some(rest) if !rest.is_empty() => format!(" {}", rest.join(" ")),
_ => String::new(),
},
root.display(),
body.trim(),
));
@@ -1435,6 +1608,58 @@ async fn get_cloud_key_status() -> Result<serde_json::Value, String> {
Ok(serde_json::json!(status))
}
/// Return the current inference-source config for the Settings UI.
#[tauri::command]
async fn get_inference_source() -> Result<InferenceConfig, String> {
Ok(read_inference_config())
}
/// Persist the chosen inference source. `host` is normalized to a bare base
/// URL. For custom endpoints, an optional API key is stored in cloud-keys.env
/// under `<ENGINE>_API_KEY`. Applies on next app launch.
#[tauri::command]
async fn set_inference_source(
kind: String,
model: Option<String>,
host: Option<String>,
engine: Option<String>,
api_key: Option<String>,
) -> Result<(), String> {
let kind = match kind.as_str() {
"ollama" => SourceKind::Ollama,
"custom" => SourceKind::Custom,
other => return Err(format!("Unknown inference source kind: {:?}", other)),
};
let cfg = InferenceConfig {
kind,
model: model.filter(|m| !m.is_empty()),
host: host.map(|h| normalize_host(&h)).filter(|h| !h.is_empty()),
engine: engine.filter(|e| !e.is_empty()),
};
if let SourceKind::Custom = cfg.kind {
if cfg.host.is_none() {
return Err("A server URL is required for a custom endpoint.".into());
}
if cfg.model.as_deref().unwrap_or("").is_empty() {
return Err("A model name is required for a custom endpoint.".into());
}
if let Some(key) = api_key.filter(|k| !k.is_empty()) {
let engine = cfg
.engine
.clone()
.unwrap_or_else(|| CUSTOM_FALLBACK_ENGINE.to_string());
let key_name = format!("{}_API_KEY", engine.to_ascii_uppercase());
// Save the key before persisting the config: if the key can't be
// written, surface it and DON'T record a custom source whose
// credential is missing (which would fail confusingly at runtime).
save_cloud_key(key_name, key)
.await
.map_err(|e| format!("Could not store the API key: {}", e))?;
}
}
write_inference_config(&cfg)
}
/// Pull a model via Ollama (called from frontend download button).
#[tauri::command]
async fn pull_ollama_model(model_name: String) -> Result<serde_json::Value, String> {
@@ -1464,6 +1689,103 @@ async fn delete_ollama_model(model_name: String) -> Result<serde_json::Value, St
Ok(serde_json::json!({"status": "deleted", "model": model_name}))
}
// ---------------------------------------------------------------------------
// Inference-source selection (~/.openjarvis/inference.json)
// ---------------------------------------------------------------------------
#[derive(serde::Serialize, serde::Deserialize, Clone, Copy, Debug, PartialEq, Eq)]
#[serde(rename_all = "lowercase")]
enum SourceKind {
Ollama,
Custom,
}
impl Default for SourceKind {
fn default() -> Self {
SourceKind::Ollama
}
}
#[derive(serde::Serialize, serde::Deserialize, Clone, Debug, Default)]
struct InferenceConfig {
#[serde(default)]
kind: SourceKind,
#[serde(default, skip_serializing_if = "Option::is_none")]
model: Option<String>,
/// Bare base URL (no trailing `/v1`), custom only.
#[serde(default, skip_serializing_if = "Option::is_none")]
host: Option<String>,
/// OpenAI-compatible engine key (e.g. "lmstudio"), custom only.
#[serde(default, skip_serializing_if = "Option::is_none")]
engine: Option<String>,
}
/// Path to the inference-source config (~/.openjarvis/inference.json).
fn inference_config_path() -> std::path::PathBuf {
std::path::PathBuf::from(home_dir())
.join(".openjarvis")
.join("inference.json")
}
/// Parse config text. Any error (missing/garbage) yields the Ollama default —
/// a broken file must never strand the user with no working inference source.
fn parse_inference_config(text: &str) -> InferenceConfig {
serde_json::from_str::<InferenceConfig>(text).unwrap_or_default()
}
/// Read the on-disk inference config, or the Ollama default if absent.
fn read_inference_config() -> InferenceConfig {
match std::fs::read_to_string(inference_config_path()) {
Ok(text) => parse_inference_config(&text),
Err(_) => InferenceConfig::default(),
}
}
/// Write the inference config to disk (pretty JSON).
fn write_inference_config(cfg: &InferenceConfig) -> Result<(), String> {
let path = inference_config_path();
if let Some(parent) = path.parent() {
let _ = std::fs::create_dir_all(parent);
}
let json = serde_json::to_string_pretty(cfg).map_err(|e| e.to_string())?;
std::fs::write(&path, json + "\n").map_err(|e| format!("Failed to save inference config: {}", e))
}
/// Upsert `[engine.<engine>] host = "<host>"` into an existing config.toml
/// string, preserving all other content/formatting. Pure: string in, string out.
fn upsert_engine_host(existing: &str, engine: &str, host: &str) -> Result<String, String> {
let mut doc = existing
.parse::<toml_edit::DocumentMut>()
.map_err(|e| format!("Invalid config.toml: {}", e))?;
doc["engine"][engine]["host"] = toml_edit::value(host);
Ok(doc.to_string())
}
/// Write the custom-endpoint host into ~/.openjarvis/config.toml so
/// `jarvis serve` (which reads that file via load_config) points at it.
/// The `<ENGINE>_HOST` env var is unreliable — it is shadowed by the engine's
/// non-empty default host in the Python layer — so config.toml is the override.
fn set_engine_host_in_config(engine: &str, host: &str) -> Result<(), String> {
let path = std::path::PathBuf::from(home_dir())
.join(".openjarvis")
.join("config.toml");
if let Some(parent) = path.parent() {
let _ = std::fs::create_dir_all(parent);
}
let existing = std::fs::read_to_string(&path).unwrap_or_default();
let updated = upsert_engine_host(&existing, engine, host)?;
std::fs::write(&path, updated).map_err(|e| format!("Failed to write config.toml: {}", e))
}
/// Normalize a user-entered server URL to a bare base host: trim whitespace,
/// drop a trailing `/v1` segment (the engine re-appends its own api prefix),
/// then drop any trailing slash.
fn normalize_host(raw: &str) -> String {
let s = raw.trim().trim_end_matches('/');
let s = s.strip_suffix("/v1").unwrap_or(s);
s.trim_end_matches('/').to_string()
}
/// Check speech backend health.
#[tauri::command]
async fn speech_health(api_url: String) -> Result<serde_json::Value, String> {
@@ -1987,6 +2309,8 @@ pub fn run() {
delete_ollama_model,
save_cloud_key,
get_cloud_key_status,
get_inference_source,
set_inference_source,
toggle_overlay,
hide_overlay,
get_overlay_conversation,
@@ -2009,7 +2333,11 @@ pub fn run() {
#[cfg(test)]
mod tests {
use super::{format_uv_sync_failure, format_uv_sync_spawn_error, uv_sync_stderr_tail};
use super::{
boot_plan, default_local_model, format_uv_sync_failure, format_uv_sync_spawn_error,
normalize_host, parse_inference_config, upsert_engine_host, uv_sync_stderr_tail,
InferenceConfig, SourceKind,
};
use std::path::Path;
#[test]
@@ -2074,4 +2402,146 @@ mod tests {
assert!(msg.contains("/repo"));
assert!(msg.contains("No such file or directory"));
}
#[test]
fn default_local_model_picks_second_largest_that_fits() {
// QWEN35_MODELS min_ram ladder: 4,6,8,12,24,32,96 GB
assert_eq!(default_local_model(4.0), "qwen3.5:0.8b"); // only one fits
assert_eq!(default_local_model(8.0), "qwen3.5:2b"); // fits 0.8/2/4 → 2nd-largest
assert_eq!(default_local_model(16.0), "qwen3.5:4b"); // fits ..9b → 2nd-largest
assert_eq!(default_local_model(32.0), "qwen3.5:27b"); // fits 0.8/2/4/9/27/35b → 2nd-largest is 27b
assert_eq!(default_local_model(128.0), "qwen3.5:35b"); // fits all → 2nd-largest
}
#[test]
fn default_local_model_falls_back_when_nothing_fits() {
assert_eq!(default_local_model(1.0), super::FALLBACK_MODEL);
}
#[test]
fn parse_defaults_to_ollama_when_file_missing_or_garbage() {
assert!(matches!(parse_inference_config("").kind, SourceKind::Ollama));
assert!(matches!(parse_inference_config("not json").kind, SourceKind::Ollama));
}
#[test]
fn parse_reads_custom_endpoint() {
let cfg = parse_inference_config(
r#"{"kind":"custom","model":"qwen2.5-7b","host":"http://localhost:1234","engine":"lmstudio"}"#,
);
assert!(matches!(cfg.kind, SourceKind::Custom));
assert_eq!(cfg.model.as_deref(), Some("qwen2.5-7b"));
assert_eq!(cfg.host.as_deref(), Some("http://localhost:1234"));
assert_eq!(cfg.engine.as_deref(), Some("lmstudio"));
}
#[test]
fn normalize_host_strips_trailing_slash_and_v1() {
assert_eq!(normalize_host("http://localhost:1234/v1"), "http://localhost:1234");
assert_eq!(normalize_host("http://localhost:1234/v1/"), "http://localhost:1234");
assert_eq!(normalize_host("http://localhost:1234/"), "http://localhost:1234");
assert_eq!(normalize_host("http://host:8000"), "http://host:8000");
}
#[test]
fn boot_plan_ollama_launches_and_pulls_one_model() {
let cfg = InferenceConfig { kind: SourceKind::Ollama, ..Default::default() };
let plan = boot_plan(&cfg, 16.0);
assert!(plan.launch_ollama);
assert_eq!(plan.model_to_pull.as_deref(), Some("qwen3.5:4b"));
assert!(plan.engine_host.is_none());
assert!(plan.serve_args.windows(2).any(|w| w == ["--engine", "ollama"]));
assert!(plan.serve_args.windows(2).any(|w| w == ["--model", "qwen3.5:4b"]));
}
#[test]
fn boot_plan_ollama_respects_pinned_model() {
let cfg = InferenceConfig {
kind: SourceKind::Ollama,
model: Some("qwen3.5:9b".into()),
..Default::default()
};
let plan = boot_plan(&cfg, 16.0);
assert_eq!(plan.model_to_pull.as_deref(), Some("qwen3.5:9b"));
}
#[test]
fn boot_plan_custom_skips_ollama_and_sets_engine_host() {
let cfg = InferenceConfig {
kind: SourceKind::Custom,
model: Some("qwen2.5-7b".into()),
host: Some("http://localhost:1234".into()),
engine: Some("lmstudio".into()),
};
let plan = boot_plan(&cfg, 16.0);
assert!(!plan.launch_ollama);
assert!(plan.model_to_pull.is_none());
assert_eq!(
plan.engine_host,
Some(("lmstudio".to_string(), "http://localhost:1234".to_string()))
);
assert!(plan.serve_args.windows(2).any(|w| w == ["--engine", "lmstudio"]));
assert!(plan.serve_args.windows(2).any(|w| w == ["--model", "qwen2.5-7b"]));
}
#[test]
fn boot_plan_custom_defaults_engine_to_lmstudio() {
let cfg = InferenceConfig {
kind: SourceKind::Custom,
model: Some("m".into()),
host: Some("http://h:1".into()),
engine: None,
};
let plan = boot_plan(&cfg, 16.0);
assert_eq!(plan.engine_host.as_ref().unwrap().0, "lmstudio");
assert!(plan.serve_args.windows(2).any(|w| w == ["--engine", "lmstudio"]));
}
#[test]
fn boot_plan_custom_omits_engine_host_when_no_host() {
// No configured host → don't set engine_host (no override to write).
let cfg = InferenceConfig {
kind: SourceKind::Custom,
model: Some("m".into()),
host: None,
engine: Some("lmstudio".into()),
};
let plan = boot_plan(&cfg, 16.0);
assert!(plan.engine_host.is_none());
}
#[test]
fn boot_plan_ollama_uses_fallback_model_on_low_ram() {
// Below the smallest model's min_ram → default_local_model → FALLBACK_MODEL.
let cfg = InferenceConfig { kind: SourceKind::Ollama, ..Default::default() };
let plan = boot_plan(&cfg, 1.0);
assert_eq!(plan.model_to_pull.as_deref(), Some(super::FALLBACK_MODEL));
}
#[test]
fn upsert_engine_host_writes_into_empty_config() {
let out = upsert_engine_host("", "lmstudio", "http://localhost:1234").unwrap();
let doc: toml_edit::DocumentMut = out.parse().unwrap();
assert_eq!(
doc["engine"]["lmstudio"]["host"].as_str(),
Some("http://localhost:1234")
);
}
#[test]
fn upsert_engine_host_preserves_existing_content() {
let existing = "[intelligence]\ndefault_model = \"keep-me\"\n";
let out = upsert_engine_host(existing, "vllm", "http://host:8000").unwrap();
let doc: toml_edit::DocumentMut = out.parse().unwrap();
assert_eq!(doc["intelligence"]["default_model"].as_str(), Some("keep-me"));
assert_eq!(doc["engine"]["vllm"]["host"].as_str(), Some("http://host:8000"));
}
#[test]
fn upsert_engine_host_updates_existing_host() {
let existing = "[engine.lmstudio]\nhost = \"http://old:1\"\n";
let out = upsert_engine_host(existing, "lmstudio", "http://new:2").unwrap();
let doc: toml_edit::DocumentMut = out.parse().unwrap();
assert_eq!(doc["engine"]["lmstudio"]["host"].as_str(), Some("http://new:2"));
}
}
frontend/src/components/SetupScreen.tsx
+8 -1
View File
@@ -146,7 +146,14 @@ export function SetupScreen({ onReady }: { onReady: () => void }) {
{/* Steps */}
<div className="flex flex-col gap-2 mb-8">
{STEPS.map((step) => (
{(status?.source === 'custom'
? [
{ key: 'ollama_ready' as const, label: 'Inference Engine', icon: Cpu, detail: 'Connecting to your server...' },
{ key: 'model_ready' as const, label: 'Endpoint', icon: Database, detail: 'Checking endpoint...' },
{ key: 'server_ready' as const, label: 'API Server', icon: Server, detail: 'Starting server...' },
]
: STEPS
).map((step) => (
<StepRow
key={step.key}
icon={step.icon}
frontend/src/lib/api.ts
+44
View File
@@ -69,6 +69,7 @@ export interface SetupStatus {
server_ready: boolean;
model_ready: boolean;
error: string | null;
source?: 'ollama' | 'custom'; // drives source-aware setup labels
}
export async function getSetupStatus(): Promise<SetupStatus | null> {
@@ -983,3 +984,46 @@ export async function denyAction(actionId: string): Promise<void> {
const res = await fetch(`${getBase()}/v1/approvals/${actionId}/deny`, { method: 'POST' });
if (!res.ok) throw new Error(`Failed: ${res.status}`);
}
// ---------------------------------------------------------------------------
// Inference source (desktop only)
// ---------------------------------------------------------------------------
export type InferenceSource = {
kind: 'ollama' | 'custom';
model?: string;
host?: string;
engine?: string;
};
export async function getInferenceSource(): Promise<InferenceSource> {
if (isTauri()) {
try {
const { invoke } = await import('@tauri-apps/api/core');
return await invoke<InferenceSource>('get_inference_source');
} catch (e: any) {
throw new Error(e?.message ?? e ?? 'Failed to read inference source');
}
}
return { kind: 'ollama' };
}
export async function setInferenceSource(
src: InferenceSource & { apiKey?: string },
): Promise<void> {
if (!isTauri()) throw new Error('Inference source is configurable in the desktop app only.');
try {
const { invoke } = await import('@tauri-apps/api/core');
await invoke<void>('set_inference_source', {
kind: src.kind,
model: src.model ?? null,
host: src.host ?? null,
engine: src.engine ?? null,
apiKey: src.apiKey ?? null,
});
} catch (e: any) {
// Surface the backend's actionable error strings (e.g. "A server URL is
// required…", "Could not store the API key…") as proper Error instances.
throw new Error(e?.message ?? e ?? 'Failed to save inference source');
}
}
frontend/src/pages/SettingsPage.tsx
+82 -1
View File
@@ -19,7 +19,7 @@ import {
RefreshCw,
} from 'lucide-react';
import { useAppStore, type ThemeMode } from '../lib/store';
import { checkHealth, fetchSpeechHealth, getMemoryStats } from '../lib/api';
import { checkHealth, fetchSpeechHealth, getMemoryStats, getInferenceSource, setInferenceSource, type InferenceSource } from '../lib/api';
import { isAutoUpdateDisabled, setAutoUpdateDisabled } from '../components/Desktop/UpdateChecker';
function OllamaModelList() {
@@ -162,6 +162,35 @@ export function SettingsPage() {
try { return parseInt(localStorage.getItem('openjarvis-memory-max-tokens') || '2048'); } catch { return 2048; }
});
const [srcKind, setSrcKind] = useState<InferenceSource['kind']>('ollama');
const [customHost, setCustomHost] = useState('http://localhost:1234/v1');
const [customModel, setCustomModel] = useState('');
const [customEngine, setCustomEngine] = useState('lmstudio');
const [customKey, setCustomKey] = useState('');
const [srcMsg, setSrcMsg] = useState('');
useEffect(() => {
getInferenceSource().then((s) => {
setSrcKind(s.kind);
if (s.host) setCustomHost(s.host);
if (s.model) setCustomModel(s.model);
if (s.engine) setCustomEngine(s.engine);
}).catch(() => {});
}, []);
const saveSource = useCallback(async () => {
try {
if (srcKind === 'custom') {
await setInferenceSource({ kind: 'custom', host: customHost, model: customModel, engine: customEngine, apiKey: customKey || undefined });
} else {
await setInferenceSource({ kind: 'ollama' });
}
setSrcMsg('Saved — restart the app to apply.');
} catch (e: any) {
setSrcMsg(e?.message ?? 'Failed to save.');
}
}, [srcKind, customHost, customModel, customEngine, customKey]);
useEffect(() => {
checkHealth().then(setHealthy);
fetchSpeechHealth()
@@ -318,6 +347,58 @@ export function SettingsPage() {
</SettingRow>
</Section>
{/* Inference source */}
<Section title="Inference source">
<SettingRow label="Source" description="Where the app runs models. Applies after restart.">
<select
value={srcKind}
onChange={(e) => { setSrcKind(e.target.value as InferenceSource['kind']); setSrcMsg(''); }}
className="text-sm px-3 py-1.5 rounded-lg outline-none w-56"
style={{ background: 'var(--color-bg-secondary)', color: 'var(--color-text)', border: '1px solid var(--color-border)' }}
>
<option value="ollama">Bundled Ollama (default)</option>
<option value="custom">Custom OpenAI-compatible server</option>
</select>
</SettingRow>
{srcKind === 'custom' && (
<>
<SettingRow label="Server URL" description="e.g. LM Studio: http://localhost:1234/v1">
<input type="text" value={customHost} onChange={(e) => { setCustomHost(e.target.value); setSrcMsg(''); }} placeholder="http://localhost:1234/v1"
className="text-sm px-3 py-1.5 rounded-lg outline-none w-56"
style={{ background: 'var(--color-bg-secondary)', color: 'var(--color-text)', border: '1px solid var(--color-border)' }} />
</SettingRow>
<SettingRow label="Model" description="Model id served by your endpoint">
<input type="text" value={customModel} onChange={(e) => { setCustomModel(e.target.value); setSrcMsg(''); }} placeholder="qwen2.5-7b-instruct"
className="text-sm px-3 py-1.5 rounded-lg outline-none w-56"
style={{ background: 'var(--color-bg-secondary)', color: 'var(--color-text)', border: '1px solid var(--color-border)' }} />
</SettingRow>
<SettingRow label="Server type" description="OpenAI-compatible engine">
<select value={customEngine} onChange={(e) => { setCustomEngine(e.target.value); setSrcMsg(''); }}
className="text-sm px-3 py-1.5 rounded-lg outline-none w-56"
style={{ background: 'var(--color-bg-secondary)', color: 'var(--color-text)', border: '1px solid var(--color-border)' }}>
<option value="lmstudio">LM Studio</option>
<option value="vllm">vLLM</option>
<option value="sglang">SGLang</option>
<option value="llamacpp">llama.cpp</option>
<option value="mlx">MLX</option>
</select>
</SettingRow>
<SettingRow label="API key (optional)" description="Only if your server requires one">
<input type="password" value={customKey} onChange={(e) => { setCustomKey(e.target.value); setSrcMsg(''); }} placeholder="leave blank if none"
className="text-sm px-3 py-1.5 rounded-lg outline-none w-56"
style={{ background: 'var(--color-bg-secondary)', color: 'var(--color-text)', border: '1px solid var(--color-border)' }} />
</SettingRow>
</>
)}
<SettingRow label="" description={srcMsg}>
<button onClick={saveSource}
className="text-sm px-3 py-1.5 rounded-lg outline-none cursor-pointer"
style={{ background: 'var(--color-accent, var(--color-bg-tertiary))', color: 'var(--color-text)', border: '1px solid var(--color-border)' }}>
Save inference source
</button>
</SettingRow>
</Section>
{/* Models */}
<Section title="Models">
<SettingRow label="Local models (Ollama)" description="Models available for local inference">