diff --git a/desktop/src-tauri/src/lib.rs b/desktop/src-tauri/src/lib.rs index 0f2d6c1c..6fa1a44a 100644 --- a/desktop/src-tauri/src/lib.rs +++ b/desktop/src-tauri/src/lib.rs @@ -649,7 +649,12 @@ async fn boot_backend(backend: SharedBackend, status: SharedStatus) { s.detail = "Installing dependencies...".into(); } let _ = tokio::process::Command::new(&uv_bin) - .args(["sync", "--extra", "server"]) + .args([ + "sync", + "--extra", "server", + "--extra", "inference-cloud", + "--extra", "inference-google", + ]) .stdout(std::process::Stdio::null()) .stderr(std::process::Stdio::null()) .current_dir(root) @@ -1124,6 +1129,15 @@ async fn save_cloud_key(key_name: String, key_value: String) -> Result<(), Strin let _ = std::fs::set_permissions(&path, std::fs::Permissions::from_mode(0o600)); } + // Tell the running server to hot-reload its cloud engine so the user + // doesn't need to restart the app after entering an API key. + let reload_url = format!("http://127.0.0.1:{}/v1/cloud/reload", JARVIS_PORT); + let _ = reqwest::Client::new() + .post(&reload_url) + .timeout(std::time::Duration::from_secs(10)) + .send() + .await; + Ok(()) } diff --git a/frontend/package-lock.json b/frontend/package-lock.json index 55621bd6..9be233d3 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -46,6 +46,9 @@ "typescript": "~5.7.0", "vite": "^6.0.0", "vite-plugin-pwa": "^1.2.0" + }, + "engines": { + "node": ">=20" } }, "node_modules/@apideck/better-ajv-errors": { diff --git a/frontend/src/components/Chat/InputArea.tsx b/frontend/src/components/Chat/InputArea.tsx index ceecfa9f..122276c6 100644 --- a/frontend/src/components/Chat/InputArea.tsx +++ b/frontend/src/components/Chat/InputArea.tsx @@ -243,9 +243,10 @@ export function InputArea() { accumulatedContent = 'No response was generated. Please try again.'; } const totalMs = Date.now() - startTime; - const serverInfo = useAppStore.getState().serverInfo; + const _CLOUD_PREFIXES = ['gpt-', 'o1-', 'o3-', 'o4-', 'claude-', 'gemini-', 'openrouter/', 'MiniMax-', 'chatgpt-']; + const engineLabel = _CLOUD_PREFIXES.some(p => selectedModel.startsWith(p)) ? 'cloud' : 'ollama'; const telemetry: MessageTelemetry = { - engine: serverInfo?.engine, + engine: engineLabel, model_id: selectedModel, total_ms: totalMs, ttft_ms: ttftMs, diff --git a/frontend/src/components/CommandPalette.tsx b/frontend/src/components/CommandPalette.tsx index 2d110290..743fe823 100644 --- a/frontend/src/components/CommandPalette.tsx +++ b/frontend/src/components/CommandPalette.tsx @@ -223,8 +223,11 @@ export function CommandPalette() { useAppStore.getState().addLogEntry({ timestamp: Date.now(), level: 'info', category: 'model', - message: `${provider.name} API key ${value ? 'saved' : 'removed'}. Restart the app for cloud models to appear.`, + message: `${provider.name} API key ${value ? 'saved' : 'removed'}. Refreshing model list…`, }); + + // Refresh the model list so cloud models appear immediately. + await refreshModels(); }; const handleKeyDown = (e: React.KeyboardEvent) => { diff --git a/frontend/tsconfig.tsbuildinfo b/frontend/tsconfig.tsbuildinfo index 3dfe098e..c1823959 100644 --- a/frontend/tsconfig.tsbuildinfo +++ b/frontend/tsconfig.tsbuildinfo @@ -1 +1 @@ -{"root":["./src/app.tsx","./src/main.tsx","./src/vite-env.d.ts","./src/components/commandpalette.tsx","./src/components/errorboundary.tsx","./src/components/layout.tsx","./src/components/optinmodal.tsx","./src/components/setupscreen.tsx","./src/components/systempulse.tsx","./src/components/chat/chatarea.tsx","./src/components/chat/inputarea.tsx","./src/components/chat/messagebubble.tsx","./src/components/chat/micbutton.tsx","./src/components/chat/streamingdots.tsx","./src/components/chat/systempanel.tsx","./src/components/chat/toolcallcard.tsx","./src/components/chat/xrayfooter.tsx","./src/components/dashboard/costcomparison.tsx","./src/components/dashboard/energydashboard.tsx","./src/components/dashboard/tracedebugger.tsx","./src/components/sidebar/conversationlist.tsx","./src/components/sidebar/sidebar.tsx","./src/components/setup/ingestdashboard.tsx","./src/components/setup/readyscreen.tsx","./src/components/setup/setupwizard.tsx","./src/components/setup/sourceconnectflow.tsx","./src/components/setup/sourcepicker.tsx","./src/components/ui/button.tsx","./src/components/ui/dialog.tsx","./src/components/ui/input.tsx","./src/components/ui/select.tsx","./src/components/ui/sonner.tsx","./src/components/ui/tooltip.tsx","./src/hooks/usespeech.ts","./src/lib/api.ts","./src/lib/connectors-api.ts","./src/lib/deep-link.ts","./src/lib/profanity.ts","./src/lib/sse.ts","./src/lib/store.ts","./src/lib/utils.ts","./src/pages/agentspage.tsx","./src/pages/chatpage.tsx","./src/pages/dashboardpage.tsx","./src/pages/getstartedpage.tsx","./src/pages/logspage.tsx","./src/pages/settingspage.tsx","./src/types/connectors.ts","./src/types/index.ts"],"version":"5.7.3"} \ No newline at end of file +{"root":["./src/app.tsx","./src/main.tsx","./src/vite-env.d.ts","./src/components/commandpalette.tsx","./src/components/errorboundary.tsx","./src/components/layout.tsx","./src/components/optinmodal.tsx","./src/components/setupscreen.tsx","./src/components/systempulse.tsx","./src/components/chat/chatarea.tsx","./src/components/chat/inputarea.tsx","./src/components/chat/messagebubble.tsx","./src/components/chat/micbutton.tsx","./src/components/chat/streamingdots.tsx","./src/components/chat/systempanel.tsx","./src/components/chat/toolcallcard.tsx","./src/components/chat/xrayfooter.tsx","./src/components/dashboard/costcomparison.tsx","./src/components/dashboard/energydashboard.tsx","./src/components/dashboard/tracedebugger.tsx","./src/components/sidebar/conversationlist.tsx","./src/components/sidebar/sidebar.tsx","./src/components/setup/ingestdashboard.tsx","./src/components/setup/readyscreen.tsx","./src/components/setup/setupwizard.tsx","./src/components/setup/sourceconnectflow.tsx","./src/components/setup/sourcepicker.tsx","./src/components/ui/button.tsx","./src/components/ui/dialog.tsx","./src/components/ui/input.tsx","./src/components/ui/select.tsx","./src/components/ui/sonner.tsx","./src/components/ui/tooltip.tsx","./src/hooks/usespeech.ts","./src/lib/api.ts","./src/lib/connectors-api.ts","./src/lib/deep-link.ts","./src/lib/profanity.ts","./src/lib/sse.ts","./src/lib/store.ts","./src/lib/utils.ts","./src/pages/agentspage.tsx","./src/pages/chatpage.tsx","./src/pages/dashboardpage.tsx","./src/pages/datasourcespage.tsx","./src/pages/getstartedpage.tsx","./src/pages/logspage.tsx","./src/pages/settingspage.tsx","./src/types/connectors.ts","./src/types/index.ts"],"version":"5.7.3"} \ No newline at end of file diff --git a/src/openjarvis/cli/serve.py b/src/openjarvis/cli/serve.py index 8eeea22a..998a0c68 100644 --- a/src/openjarvis/cli/serve.py +++ b/src/openjarvis/cli/serve.py @@ -120,10 +120,15 @@ def serve( from openjarvis.engine.multi import MultiEngine cloud = CloudEngine() + engine = MultiEngine([(engine_name, engine), ("cloud", cloud)]) + engine_name = "multi" if cloud.health(): - engine = MultiEngine([(engine_name, engine), ("cloud", cloud)]) - engine_name = "multi" console.print(" Cloud: [cyan]enabled[/cyan] (API keys detected)") + else: + console.print( + " Cloud: [yellow]keys set but packages missing[/yellow] " + "(run: uv sync --extra inference-cloud --extra inference-google)" + ) except Exception as exc: logger.debug("Cloud engine init failed: %s", exc) diff --git a/src/openjarvis/evals/backends/terminalbench_native.py b/src/openjarvis/evals/backends/terminalbench_native.py index 41adc0c4..86fde4c1 100644 --- a/src/openjarvis/evals/backends/terminalbench_native.py +++ b/src/openjarvis/evals/backends/terminalbench_native.py @@ -15,7 +15,6 @@ LOGGER = logging.getLogger(__name__) try: from terminal_bench import BenchmarkResults, Harness - from terminal_bench.llms.lite_llm import LiteLLM _HAS_TB = True except ImportError: @@ -77,16 +76,16 @@ class TerminalBenchNativeBackend(InferenceBackend): "cleanup": True, } - # Use built-in agent (naive uses LiteLLM) + # Use terminus-2 agent which accepts model_name + api_base as + # serializable strings (avoids Pydantic serialization issues with + # LLM objects in the harness lock file). from terminal_bench.agents.agent_name import AgentName - harness_kwargs["agent_name"] = AgentName(self._agent_name) + harness_kwargs["agent_name"] = AgentName("terminus-2") harness_kwargs["agent_kwargs"] = { - "llm": LiteLLM( - model_name=self._model, - temperature=self._temperature, - api_base=self._api_base, - ), + "model_name": self._model, + "api_base": self._api_base, + "temperature": self._temperature, } if self._max_samples is not None: diff --git a/src/openjarvis/evals/cli.py b/src/openjarvis/evals/cli.py index b3bdeeeb..b3674be1 100644 --- a/src/openjarvis/evals/cli.py +++ b/src/openjarvis/evals/cli.py @@ -577,6 +577,60 @@ def _build_trackers(config) -> list: return trackers +def _run_terminalbench_native(config, console: Console) -> object: + """Run TerminalBench V2 natively via terminal-bench Harness.""" + from openjarvis.evals.backends.terminalbench_native import ( + TerminalBenchNativeBackend, + ) + from openjarvis.evals.core.types import RunSummary + + model = config.model + # LiteLLM expects "openai/" for OpenAI-compatible servers + litellm_model = f"openai/{model}" + output_dir = getattr(config, "output_path", None) or "results/terminalbench-native/" + + backend = TerminalBenchNativeBackend( + model=litellm_model, + api_base="http://localhost:8000/v1", + temperature=config.temperature, + max_samples=config.max_samples, + output_dir=output_dir, + n_concurrent=config.max_workers or 4, + ) + + import re + # Docker compose project names must be lowercase alphanumeric + hyphens/underscores + model_slug = re.sub(r"[^a-z0-9_-]", "-", model.lower().replace("/", "-")) + run_id = f"tb2-{model_slug}" + console.print(f" Running TerminalBench V2 natively: {model}") + console.print(f" Harness run_id: {run_id}") + + results = backend.run_harness(run_id) + + # Convert BenchmarkResults to RunSummary + total = len(results.trial_results) if hasattr(results, "trial_results") else 0 + correct = 0 + if hasattr(results, "trial_results"): + for tr in results.trial_results: + if getattr(tr, "is_resolved", False): + correct += 1 + + accuracy = correct / total if total > 0 else 0.0 + return RunSummary( + benchmark="terminalbench-native", + category="agentic", + backend="terminalbench-native", + model=model, + total_samples=total, + scored_samples=total, + correct=correct, + accuracy=accuracy, + errors=0, + mean_latency_seconds=0.0, + total_cost_usd=0.0, + ) + + def _run_single(config, console: Optional[Console] = None) -> object: """Run a single eval from a RunConfig and return the summary.""" from openjarvis.evals.core.runner import EvalRunner @@ -584,6 +638,10 @@ def _run_single(config, console: Optional[Console] = None) -> object: if console is None: console = Console() + # TerminalBench V2 native: use terminal-bench Harness directly + if config.benchmark == "terminalbench-native": + return _run_terminalbench_native(config, console) + eval_backend = _build_backend( config.backend, config.engine_key, diff --git a/src/openjarvis/evals/configs/livecodebench-gemma4-26b.toml b/src/openjarvis/evals/configs/livecodebench-gemma4-26b.toml new file mode 100644 index 00000000..d427f01f --- /dev/null +++ b/src/openjarvis/evals/configs/livecodebench-gemma4-26b.toml @@ -0,0 +1,30 @@ +# LiveCodeBench eval: Gemma-4-26B-A4B-it (vLLM, TP=4) +# Competitive programming from LeetCode, AtCoder, CodeForces + +[meta] +name = "livecodebench-gemma4-26b" +description = "LiveCodeBench on google/gemma-4-26B-A4B-it (vLLM, TP=4)" + +[defaults] +temperature = 0.0 +max_tokens = 4096 + +[judge] +model = "gpt-5-mini-2025-08-07" +temperature = 0.0 +engine = "cloud" + +[run] +max_workers = 1 +output_dir = "results/neurips-2026/baselines/gemma4-26b/livecodebench/" +seed = 42 + +[[models]] +name = "google/gemma-4-26B-A4B-it" +engine = "vllm" +num_gpus = 4 + +[[benchmarks]] +name = "livecodebench" +backend = "jarvis-direct" +max_samples = 20 diff --git a/src/openjarvis/evals/configs/livecodebench-gemma4-e4b.toml b/src/openjarvis/evals/configs/livecodebench-gemma4-e4b.toml new file mode 100644 index 00000000..ff6209fa --- /dev/null +++ b/src/openjarvis/evals/configs/livecodebench-gemma4-e4b.toml @@ -0,0 +1,30 @@ +# LiveCodeBench eval: Gemma-4-E4B-it (vLLM, 1 GPU) +# Competitive programming from LeetCode, AtCoder, CodeForces + +[meta] +name = "livecodebench-gemma4-e4b" +description = "LiveCodeBench on google/gemma-4-E4B-it (vLLM, 1 GPU)" + +[defaults] +temperature = 0.0 +max_tokens = 4096 + +[judge] +model = "gpt-5-mini-2025-08-07" +temperature = 0.0 +engine = "cloud" + +[run] +max_workers = 1 +output_dir = "results/neurips-2026/baselines/gemma4-e4b/livecodebench/" +seed = 42 + +[[models]] +name = "google/gemma-4-E4B-it" +engine = "vllm" +num_gpus = 1 + +[[benchmarks]] +name = "livecodebench" +backend = "jarvis-direct" +max_samples = 20 diff --git a/src/openjarvis/evals/configs/livecodebench-nemotron-nano.toml b/src/openjarvis/evals/configs/livecodebench-nemotron-nano.toml new file mode 100644 index 00000000..6266f922 --- /dev/null +++ b/src/openjarvis/evals/configs/livecodebench-nemotron-nano.toml @@ -0,0 +1,30 @@ +# LiveCodeBench eval: NVIDIA-Nemotron-3-Nano-30B-A3B-FP8 (vLLM, 1 GPU) +# Competitive programming from LeetCode, AtCoder, CodeForces + +[meta] +name = "livecodebench-nemotron-nano" +description = "LiveCodeBench on nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-FP8 (vLLM, 1 GPU)" + +[defaults] +temperature = 0.0 +max_tokens = 4096 + +[judge] +model = "gpt-5-mini-2025-08-07" +temperature = 0.0 +engine = "cloud" + +[run] +max_workers = 1 +output_dir = "results/neurips-2026/baselines/nemotron-nano/livecodebench/" +seed = 42 + +[[models]] +name = "nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-FP8" +engine = "vllm" +num_gpus = 1 + +[[benchmarks]] +name = "livecodebench" +backend = "jarvis-direct" +max_samples = 20 diff --git a/src/openjarvis/evals/configs/livecodebench-qwen-27b.toml b/src/openjarvis/evals/configs/livecodebench-qwen-27b.toml new file mode 100644 index 00000000..357dcf49 --- /dev/null +++ b/src/openjarvis/evals/configs/livecodebench-qwen-27b.toml @@ -0,0 +1,30 @@ +# LiveCodeBench eval: Qwen3.5-27B-FP8 (vLLM, TP=4) +# Competitive programming from LeetCode, AtCoder, CodeForces + +[meta] +name = "livecodebench-qwen-27b" +description = "LiveCodeBench on Qwen/Qwen3.5-27B-FP8 (vLLM, TP=4)" + +[defaults] +temperature = 0.0 +max_tokens = 4096 + +[judge] +model = "gpt-5-mini-2025-08-07" +temperature = 0.0 +engine = "cloud" + +[run] +max_workers = 1 +output_dir = "results/neurips-2026/baselines/qwen-27b/livecodebench/" +seed = 42 + +[[models]] +name = "Qwen/Qwen3.5-27B-FP8" +engine = "vllm" +num_gpus = 4 + +[[benchmarks]] +name = "livecodebench" +backend = "jarvis-direct" +max_samples = 20 diff --git a/src/openjarvis/evals/configs/livecodebench-qwen-2b.toml b/src/openjarvis/evals/configs/livecodebench-qwen-2b.toml new file mode 100644 index 00000000..e4d4236e --- /dev/null +++ b/src/openjarvis/evals/configs/livecodebench-qwen-2b.toml @@ -0,0 +1,30 @@ +# LiveCodeBench eval: Qwen3.5-2B (vLLM, 1 GPU) +# Competitive programming from LeetCode, AtCoder, CodeForces + +[meta] +name = "livecodebench-qwen-2b" +description = "LiveCodeBench on Qwen/Qwen3.5-2B (vLLM, 1 GPU)" + +[defaults] +temperature = 0.0 +max_tokens = 4096 + +[judge] +model = "gpt-5-mini-2025-08-07" +temperature = 0.0 +engine = "cloud" + +[run] +max_workers = 1 +output_dir = "results/neurips-2026/baselines/qwen-2b/livecodebench/" +seed = 42 + +[[models]] +name = "Qwen/Qwen3.5-2B" +engine = "vllm" +num_gpus = 1 + +[[benchmarks]] +name = "livecodebench" +backend = "jarvis-direct" +max_samples = 20 diff --git a/src/openjarvis/evals/configs/pinchbench-gemma4-26b.toml b/src/openjarvis/evals/configs/pinchbench-gemma4-26b.toml new file mode 100644 index 00000000..28787489 --- /dev/null +++ b/src/openjarvis/evals/configs/pinchbench-gemma4-26b.toml @@ -0,0 +1,35 @@ +# PinchBench eval: Gemma-4-26B-A4B-it (vLLM, TP=4) +# Agent: native_openhands — all PinchBench-required tools enabled + +[meta] +name = "pinchbench-gemma4-26b" +description = "PinchBench on google/gemma-4-26B-A4B-it (vLLM, TP=4)" + +[defaults] +temperature = 0.6 +max_tokens = 8192 + +[judge] +model = "claude-opus-4-5" +temperature = 0.0 +engine = "cloud" + +[run] +max_workers = 1 +output_dir = "results/neurips-2026/baselines/gemma4-26b/pinchbench/" +seed = 42 + +[[models]] +name = "google/gemma-4-26B-A4B-it" +engine = "vllm" +num_gpus = 4 + +[[benchmarks]] +name = "pinchbench" +backend = "jarvis-agent" +agent = "native_openhands" +tools = [ + "think", "file_read", "file_write", "web_search", "shell_exec", + "code_interpreter", "browser_navigate", "image_generate", + "calculator", "http_request", "pdf_extract", +] diff --git a/src/openjarvis/evals/configs/pinchbench-gemma4-e4b.toml b/src/openjarvis/evals/configs/pinchbench-gemma4-e4b.toml new file mode 100644 index 00000000..bbe9ba30 --- /dev/null +++ b/src/openjarvis/evals/configs/pinchbench-gemma4-e4b.toml @@ -0,0 +1,35 @@ +# PinchBench eval: Gemma-4-E4B-it (vLLM, 1 GPU) +# Agent: native_openhands — all PinchBench-required tools enabled + +[meta] +name = "pinchbench-gemma4-e4b" +description = "PinchBench on google/gemma-4-E4B-it (vLLM, 1 GPU)" + +[defaults] +temperature = 0.6 +max_tokens = 8192 + +[judge] +model = "claude-opus-4-5" +temperature = 0.0 +engine = "cloud" + +[run] +max_workers = 1 +output_dir = "results/neurips-2026/baselines/gemma4-e4b/pinchbench/" +seed = 42 + +[[models]] +name = "google/gemma-4-E4B-it" +engine = "vllm" +num_gpus = 1 + +[[benchmarks]] +name = "pinchbench" +backend = "jarvis-agent" +agent = "native_openhands" +tools = [ + "think", "file_read", "file_write", "web_search", "shell_exec", + "code_interpreter", "browser_navigate", "image_generate", + "calculator", "http_request", "pdf_extract", +] diff --git a/src/openjarvis/evals/configs/pinchbench-nemotron-nano.toml b/src/openjarvis/evals/configs/pinchbench-nemotron-nano.toml new file mode 100644 index 00000000..5ddadce5 --- /dev/null +++ b/src/openjarvis/evals/configs/pinchbench-nemotron-nano.toml @@ -0,0 +1,35 @@ +# PinchBench eval: NVIDIA-Nemotron-3-Nano-30B-A3B-FP8 (vLLM, 1 GPU) +# Agent: native_openhands — all PinchBench-required tools enabled + +[meta] +name = "pinchbench-nemotron-nano" +description = "PinchBench on nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-FP8 (vLLM, 1 GPU)" + +[defaults] +temperature = 0.6 +max_tokens = 8192 + +[judge] +model = "claude-opus-4-5" +temperature = 0.0 +engine = "cloud" + +[run] +max_workers = 1 +output_dir = "results/neurips-2026/baselines/nemotron-nano/pinchbench/" +seed = 42 + +[[models]] +name = "nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-FP8" +engine = "vllm" +num_gpus = 1 + +[[benchmarks]] +name = "pinchbench" +backend = "jarvis-agent" +agent = "native_openhands" +tools = [ + "think", "file_read", "file_write", "web_search", "shell_exec", + "code_interpreter", "browser_navigate", "image_generate", + "calculator", "http_request", "pdf_extract", +] diff --git a/src/openjarvis/evals/configs/pinchbench-qwen-27b.toml b/src/openjarvis/evals/configs/pinchbench-qwen-27b.toml new file mode 100644 index 00000000..f687e63d --- /dev/null +++ b/src/openjarvis/evals/configs/pinchbench-qwen-27b.toml @@ -0,0 +1,35 @@ +# PinchBench eval: Qwen3.5-27B-FP8 (vLLM, TP=4) +# Agent: native_openhands — all PinchBench-required tools enabled + +[meta] +name = "pinchbench-qwen-27b" +description = "PinchBench on Qwen/Qwen3.5-27B-FP8 (vLLM, TP=4)" + +[defaults] +temperature = 0.6 +max_tokens = 8192 + +[judge] +model = "claude-opus-4-5" +temperature = 0.0 +engine = "cloud" + +[run] +max_workers = 1 +output_dir = "results/neurips-2026/baselines/qwen-27b/pinchbench/" +seed = 42 + +[[models]] +name = "Qwen/Qwen3.5-27B-FP8" +engine = "vllm" +num_gpus = 4 + +[[benchmarks]] +name = "pinchbench" +backend = "jarvis-agent" +agent = "native_openhands" +tools = [ + "think", "file_read", "file_write", "web_search", "shell_exec", + "code_interpreter", "browser_navigate", "image_generate", + "calculator", "http_request", "pdf_extract", +] diff --git a/src/openjarvis/evals/configs/pinchbench-qwen-2b.toml b/src/openjarvis/evals/configs/pinchbench-qwen-2b.toml new file mode 100644 index 00000000..e76e56a4 --- /dev/null +++ b/src/openjarvis/evals/configs/pinchbench-qwen-2b.toml @@ -0,0 +1,35 @@ +# PinchBench eval: Qwen3.5-2B (vLLM, 1 GPU) +# Agent: native_openhands — all PinchBench-required tools enabled + +[meta] +name = "pinchbench-qwen-2b" +description = "PinchBench on Qwen/Qwen3.5-2B (vLLM, 1 GPU)" + +[defaults] +temperature = 0.6 +max_tokens = 8192 + +[judge] +model = "claude-opus-4-5" +temperature = 0.0 +engine = "cloud" + +[run] +max_workers = 1 +output_dir = "results/neurips-2026/baselines/qwen-2b/pinchbench/" +seed = 42 + +[[models]] +name = "Qwen/Qwen3.5-2B" +engine = "vllm" +num_gpus = 1 + +[[benchmarks]] +name = "pinchbench" +backend = "jarvis-agent" +agent = "native_openhands" +tools = [ + "think", "file_read", "file_write", "web_search", "shell_exec", + "code_interpreter", "browser_navigate", "image_generate", + "calculator", "http_request", "pdf_extract", +] diff --git a/src/openjarvis/evals/configs/taubench-gemma4-26b.toml b/src/openjarvis/evals/configs/taubench-gemma4-26b.toml new file mode 100644 index 00000000..27fda44b --- /dev/null +++ b/src/openjarvis/evals/configs/taubench-gemma4-26b.toml @@ -0,0 +1,31 @@ +# TauBench V2 eval: Gemma-4-26B-A4B-it (vLLM, TP=4) +# Multi-turn customer service benchmark — airline + retail splits + +[meta] +name = "taubench-gemma4-26b" +description = "TauBench V2 on google/gemma-4-26B-A4B-it (vLLM, TP=4)" + +[defaults] +temperature = 0.7 +max_tokens = 4096 + +[judge] +model = "gpt-5-mini-2025-08-07" +temperature = 0.0 +engine = "cloud" + +[run] +max_workers = 1 +output_dir = "results/neurips-2026/baselines/gemma4-26b/taubench/" +seed = 42 + +[[models]] +name = "google/gemma-4-26B-A4B-it" +engine = "vllm" +num_gpus = 4 + +[[benchmarks]] +name = "taubench" +backend = "jarvis-direct" +max_samples = 20 +split = "airline,retail" diff --git a/src/openjarvis/evals/configs/taubench-gemma4-e4b.toml b/src/openjarvis/evals/configs/taubench-gemma4-e4b.toml new file mode 100644 index 00000000..8dac58dd --- /dev/null +++ b/src/openjarvis/evals/configs/taubench-gemma4-e4b.toml @@ -0,0 +1,31 @@ +# TauBench V2 eval: Gemma-4-E4B-it (vLLM, 1 GPU) +# Multi-turn customer service benchmark — airline + retail splits + +[meta] +name = "taubench-gemma4-e4b" +description = "TauBench V2 on google/gemma-4-E4B-it (vLLM, 1 GPU)" + +[defaults] +temperature = 0.7 +max_tokens = 4096 + +[judge] +model = "gpt-5-mini-2025-08-07" +temperature = 0.0 +engine = "cloud" + +[run] +max_workers = 1 +output_dir = "results/neurips-2026/baselines/gemma4-e4b/taubench/" +seed = 42 + +[[models]] +name = "google/gemma-4-E4B-it" +engine = "vllm" +num_gpus = 1 + +[[benchmarks]] +name = "taubench" +backend = "jarvis-direct" +max_samples = 20 +split = "airline,retail" diff --git a/src/openjarvis/evals/configs/taubench-nemotron-nano.toml b/src/openjarvis/evals/configs/taubench-nemotron-nano.toml new file mode 100644 index 00000000..7dff2f08 --- /dev/null +++ b/src/openjarvis/evals/configs/taubench-nemotron-nano.toml @@ -0,0 +1,31 @@ +# TauBench V2 eval: NVIDIA-Nemotron-3-Nano-30B-A3B-FP8 (vLLM, 1 GPU) +# Multi-turn customer service benchmark — airline + retail splits + +[meta] +name = "taubench-nemotron-nano" +description = "TauBench V2 on nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-FP8 (vLLM, 1 GPU)" + +[defaults] +temperature = 0.7 +max_tokens = 4096 + +[judge] +model = "gpt-5-mini-2025-08-07" +temperature = 0.0 +engine = "cloud" + +[run] +max_workers = 1 +output_dir = "results/neurips-2026/baselines/nemotron-nano/taubench/" +seed = 42 + +[[models]] +name = "nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-FP8" +engine = "vllm" +num_gpus = 1 + +[[benchmarks]] +name = "taubench" +backend = "jarvis-direct" +max_samples = 20 +split = "airline,retail" diff --git a/src/openjarvis/evals/configs/taubench-qwen-27b.toml b/src/openjarvis/evals/configs/taubench-qwen-27b.toml new file mode 100644 index 00000000..13ce0c65 --- /dev/null +++ b/src/openjarvis/evals/configs/taubench-qwen-27b.toml @@ -0,0 +1,31 @@ +# TauBench V2 eval: Qwen3.5-27B-FP8 (vLLM, TP=4) +# Multi-turn customer service benchmark — airline + retail splits + +[meta] +name = "taubench-qwen-27b" +description = "TauBench V2 on Qwen/Qwen3.5-27B-FP8 (vLLM, TP=4)" + +[defaults] +temperature = 0.7 +max_tokens = 4096 + +[judge] +model = "gpt-5-mini-2025-08-07" +temperature = 0.0 +engine = "cloud" + +[run] +max_workers = 1 +output_dir = "results/neurips-2026/baselines/qwen-27b/taubench/" +seed = 42 + +[[models]] +name = "Qwen/Qwen3.5-27B-FP8" +engine = "vllm" +num_gpus = 4 + +[[benchmarks]] +name = "taubench" +backend = "jarvis-direct" +max_samples = 20 +split = "airline,retail" diff --git a/src/openjarvis/evals/configs/taubench-qwen-2b.toml b/src/openjarvis/evals/configs/taubench-qwen-2b.toml new file mode 100644 index 00000000..e9735ff6 --- /dev/null +++ b/src/openjarvis/evals/configs/taubench-qwen-2b.toml @@ -0,0 +1,31 @@ +# TauBench V2 eval: Qwen3.5-2B (vLLM, 1 GPU) +# Multi-turn customer service benchmark — airline + retail splits + +[meta] +name = "taubench-qwen-2b" +description = "TauBench V2 on Qwen/Qwen3.5-2B (vLLM, 1 GPU)" + +[defaults] +temperature = 0.7 +max_tokens = 4096 + +[judge] +model = "gpt-5-mini-2025-08-07" +temperature = 0.0 +engine = "cloud" + +[run] +max_workers = 1 +output_dir = "results/neurips-2026/baselines/qwen-2b/taubench/" +seed = 42 + +[[models]] +name = "Qwen/Qwen3.5-2B" +engine = "vllm" +num_gpus = 1 + +[[benchmarks]] +name = "taubench" +backend = "jarvis-direct" +max_samples = 20 +split = "airline,retail" diff --git a/src/openjarvis/evals/configs/terminalbench-native-qwen122b.toml b/src/openjarvis/evals/configs/terminalbench-native-qwen122b.toml new file mode 100644 index 00000000..c195da79 --- /dev/null +++ b/src/openjarvis/evals/configs/terminalbench-native-qwen122b.toml @@ -0,0 +1,30 @@ +# TerminalBench V2 (native Docker) on Qwen/Qwen3.5-122B-A10B-FP8 +[meta] +name = "terminalbench-native-qwen122b" +description = "TerminalBench V2 native on Qwen/Qwen3.5-122B-A10B-FP8" + +[defaults] +temperature = 0.6 +max_tokens = 8192 + +[judge] +model = "gpt-5-mini-2025-08-07" +temperature = 0.0 +max_tokens = 4096 + +[run] +max_workers = 8 +output_dir = "results/terminalbench-native-qwen122b/" +seed = 42 +telemetry = true +gpu_metrics = true + +[[models]] +name = "Qwen/Qwen3.5-122B-A10B-FP8" +engine = "vllm" +num_gpus = 4 + +[[benchmarks]] +name = "terminalbench-native" +backend = "terminalbench-native" +max_samples = 50 diff --git a/src/openjarvis/evals/configs/toolcall15-gemma4-26b.toml b/src/openjarvis/evals/configs/toolcall15-gemma4-26b.toml new file mode 100644 index 00000000..12470e25 --- /dev/null +++ b/src/openjarvis/evals/configs/toolcall15-gemma4-26b.toml @@ -0,0 +1,29 @@ +# ToolCall-15 eval: Gemma-4-26B-A4B-it (vLLM, TP=4) +# Lightweight tool calling benchmark — 15 scenarios, 5 categories + +[meta] +name = "toolcall15-gemma4-26b" +description = "ToolCall-15 on google/gemma-4-26B-A4B-it (temperature=0)" + +[defaults] +temperature = 0.0 +max_tokens = 4096 + +[judge] +model = "gpt-5-mini-2025-08-07" +temperature = 0.0 +engine = "cloud" + +[run] +max_workers = 1 +output_dir = "results/neurips-2026/baselines/gemma4-26b/toolcall15/" +seed = 42 + +[[models]] +name = "google/gemma-4-26B-A4B-it" +engine = "vllm" +num_gpus = 4 + +[[benchmarks]] +name = "toolcall15" +backend = "jarvis-direct" diff --git a/src/openjarvis/evals/configs/toolcall15-gemma4-e4b.toml b/src/openjarvis/evals/configs/toolcall15-gemma4-e4b.toml new file mode 100644 index 00000000..231bed02 --- /dev/null +++ b/src/openjarvis/evals/configs/toolcall15-gemma4-e4b.toml @@ -0,0 +1,29 @@ +# ToolCall-15 eval: Gemma-4-E4B-it (vLLM, 1 GPU) +# Lightweight tool calling benchmark — 15 scenarios, 5 categories + +[meta] +name = "toolcall15-gemma4-e4b" +description = "ToolCall-15 on google/gemma-4-E4B-it (temperature=0)" + +[defaults] +temperature = 0.0 +max_tokens = 4096 + +[judge] +model = "gpt-5-mini-2025-08-07" +temperature = 0.0 +engine = "cloud" + +[run] +max_workers = 1 +output_dir = "results/neurips-2026/baselines/gemma4-e4b/toolcall15/" +seed = 42 + +[[models]] +name = "google/gemma-4-E4B-it" +engine = "vllm" +num_gpus = 1 + +[[benchmarks]] +name = "toolcall15" +backend = "jarvis-direct" diff --git a/src/openjarvis/evals/configs/toolcall15-nemotron-nano.toml b/src/openjarvis/evals/configs/toolcall15-nemotron-nano.toml new file mode 100644 index 00000000..453f30e9 --- /dev/null +++ b/src/openjarvis/evals/configs/toolcall15-nemotron-nano.toml @@ -0,0 +1,29 @@ +# ToolCall-15 eval: NVIDIA-Nemotron-3-Nano-30B-A3B-FP8 (vLLM, 1 GPU) +# Lightweight tool calling benchmark — 15 scenarios, 5 categories + +[meta] +name = "toolcall15-nemotron-nano" +description = "ToolCall-15 on nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-FP8 (temperature=0)" + +[defaults] +temperature = 0.0 +max_tokens = 4096 + +[judge] +model = "gpt-5-mini-2025-08-07" +temperature = 0.0 +engine = "cloud" + +[run] +max_workers = 1 +output_dir = "results/neurips-2026/baselines/nemotron-nano/toolcall15/" +seed = 42 + +[[models]] +name = "nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-FP8" +engine = "vllm" +num_gpus = 1 + +[[benchmarks]] +name = "toolcall15" +backend = "jarvis-direct" diff --git a/src/openjarvis/evals/configs/toolcall15-qwen-27b.toml b/src/openjarvis/evals/configs/toolcall15-qwen-27b.toml new file mode 100644 index 00000000..56879653 --- /dev/null +++ b/src/openjarvis/evals/configs/toolcall15-qwen-27b.toml @@ -0,0 +1,29 @@ +# ToolCall-15 eval: Qwen3.5-27B-FP8 (vLLM, TP=4) +# Lightweight tool calling benchmark — 15 scenarios, 5 categories + +[meta] +name = "toolcall15-qwen-27b" +description = "ToolCall-15 on Qwen/Qwen3.5-27B-FP8 (temperature=0)" + +[defaults] +temperature = 0.0 +max_tokens = 4096 + +[judge] +model = "gpt-5-mini-2025-08-07" +temperature = 0.0 +engine = "cloud" + +[run] +max_workers = 1 +output_dir = "results/neurips-2026/baselines/qwen-27b/toolcall15/" +seed = 42 + +[[models]] +name = "Qwen/Qwen3.5-27B-FP8" +engine = "vllm" +num_gpus = 4 + +[[benchmarks]] +name = "toolcall15" +backend = "jarvis-direct" diff --git a/src/openjarvis/evals/configs/toolcall15-qwen-2b.toml b/src/openjarvis/evals/configs/toolcall15-qwen-2b.toml new file mode 100644 index 00000000..4d40fe42 --- /dev/null +++ b/src/openjarvis/evals/configs/toolcall15-qwen-2b.toml @@ -0,0 +1,29 @@ +# ToolCall-15 eval: Qwen3.5-2B (vLLM, 1 GPU) +# Lightweight tool calling benchmark — 15 scenarios, 5 categories + +[meta] +name = "toolcall15-qwen-2b" +description = "ToolCall-15 on Qwen/Qwen3.5-2B (temperature=0)" + +[defaults] +temperature = 0.0 +max_tokens = 4096 + +[judge] +model = "gpt-5-mini-2025-08-07" +temperature = 0.0 +engine = "cloud" + +[run] +max_workers = 1 +output_dir = "results/neurips-2026/baselines/qwen-2b/toolcall15/" +seed = 42 + +[[models]] +name = "Qwen/Qwen3.5-2B" +engine = "vllm" +num_gpus = 1 + +[[benchmarks]] +name = "toolcall15" +backend = "jarvis-direct" diff --git a/src/openjarvis/evals/core/config.py b/src/openjarvis/evals/core/config.py index b8a4e3cc..281247b3 100644 --- a/src/openjarvis/evals/core/config.py +++ b/src/openjarvis/evals/core/config.py @@ -31,7 +31,7 @@ else: logger = logging.getLogger(__name__) -VALID_BACKENDS = {"jarvis-direct", "jarvis-agent"} +VALID_BACKENDS = {"jarvis-direct", "jarvis-agent", "terminalbench-native"} # Known benchmark names (used for warnings, not hard validation) KNOWN_BENCHMARKS = { diff --git a/src/openjarvis/evals/datasets/toolcall15.py b/src/openjarvis/evals/datasets/toolcall15.py index 1b0dcd5c..f27395df 100644 --- a/src/openjarvis/evals/datasets/toolcall15.py +++ b/src/openjarvis/evals/datasets/toolcall15.py @@ -666,24 +666,45 @@ class ToolCall15Dataset(DatasetProvider): if max_samples is not None: scenarios = scenarios[:max_samples] - self._records = [ - EvalRecord( - record_id=s["id"], - problem=s["user_message"], - reference="", # scoring uses metadata, not reference text - category=s["category"], - subject=s["name"], - metadata={ - "system_prompt": SYSTEM_PROMPT, - "tools": TOOLS, - "mock_tool_outputs": s["mock_tool_outputs"], - "reference_date": s.get("reference_date"), - "scenario_id": s["id"], - "scenario_name": s["name"], - }, + self._records = [] + for s in scenarios: + # Build a self-contained prompt that includes the system + # instructions, available tools, and user message so the + # model can respond with tool calls via any backend. + tool_descriptions = "\n".join( + f"- {t['function']['name']}: " + f"{t['function']['description']}" + for t in TOOLS + ) + prompt = ( + f"{SYSTEM_PROMPT}\n\n" + f"## Available Tools\n" + f"{tool_descriptions}\n\n" + f"When you need to use a tool, respond with ONLY " + f"a JSON object in this format:\n" + f'{{"tool": "", "arguments": {{...}}}}\n\n' + f"If the task requires multiple tools, call them " + f"one at a time. If no tool is needed, respond " + f"directly with your answer.\n\n" + f"## User Request\n{s['user_message']}" + ) + self._records.append( + EvalRecord( + record_id=s["id"], + problem=prompt, + reference="", + category=s["category"], + subject=s["name"], + metadata={ + "system_prompt": SYSTEM_PROMPT, + "tools": TOOLS, + "mock_tool_outputs": s["mock_tool_outputs"], + "reference_date": s.get("reference_date"), + "scenario_id": s["id"], + "scenario_name": s["name"], + }, + ) ) - for s in scenarios - ] LOGGER.info("ToolCall-15: loaded %d scenarios", len(self._records)) diff --git a/src/openjarvis/evals/scorers/toolcall15.py b/src/openjarvis/evals/scorers/toolcall15.py index 51c32946..66eb66a2 100644 --- a/src/openjarvis/evals/scorers/toolcall15.py +++ b/src/openjarvis/evals/scorers/toolcall15.py @@ -21,11 +21,17 @@ from openjarvis.evals.core.types import EvalRecord LOGGER = logging.getLogger(__name__) -def _extract_tool_calls(record: EvalRecord) -> List[Dict[str, Any]]: - """Extract tool calls from the record's query trace or metadata. +def _extract_tool_calls( + record: EvalRecord, + model_answer: str = "", +) -> List[Dict[str, Any]]: + """Extract tool calls from traces, metadata, or model text output. Returns a list of dicts with keys: name, arguments. """ + import json + import re + tool_calls: List[Dict[str, Any]] = [] # Try query trace first (from EvalRunner) @@ -35,23 +41,62 @@ def _extract_tool_calls(record: EvalRecord) -> List[Dict[str, Any]]: for tc in getattr(turn, "tool_calls", []): if tc is None: continue - tool_calls.append( - { - "name": tc.get("name", ""), - "arguments": tc.get("arguments") or {}, - } - ) - return tool_calls + tool_calls.append({ + "name": tc.get("name", ""), + "arguments": tc.get("arguments") or {}, + }) + if tool_calls: + return tool_calls # Try tool_results list (from JarvisAgentBackend) tool_results = record.metadata.get("tool_results", []) for tr in tool_results: - tool_calls.append( - { - "name": tr.get("tool_name", ""), - "arguments": tr.get("arguments") or {}, - } - ) + tool_calls.append({ + "name": tr.get("tool_name", ""), + "arguments": tr.get("arguments") or {}, + }) + if tool_calls: + return tool_calls + + # Parse tool calls from model text output (JSON format) + if model_answer: + # Extract balanced JSON objects from text + json_blocks: list[str] = [] + # Try ```json blocks first + for m in re.finditer( + r"```(?:json)?\s*(\{.+?\})\s*```", model_answer, re.DOTALL + ): + json_blocks.append(m.group(1)) + # Also extract bare balanced-brace JSON objects + depth = 0 + start = -1 + for i, ch in enumerate(model_answer): + if ch == "{": + if depth == 0: + start = i + depth += 1 + elif ch == "}": + depth -= 1 + if depth == 0 and start >= 0: + candidate = model_answer[start : i + 1] + if '"tool"' in candidate or '"name"' in candidate: + json_blocks.append(candidate) + start = -1 + + for block in json_blocks: + try: + parsed = json.loads(block) + name = parsed.get("tool") or parsed.get("name", "") + args = parsed.get("arguments") or parsed.get("params") or {} + if isinstance(args, str): + try: + args = json.loads(args) + except (json.JSONDecodeError, TypeError): + args = {} + if name: + tool_calls.append({"name": name, "arguments": args}) + except json.JSONDecodeError: + continue return tool_calls @@ -528,7 +573,7 @@ class ToolCall15Scorer(LLMJudgeScorer): "scenario_id": scenario_id, } - tool_calls = _extract_tool_calls(record) + tool_calls = _extract_tool_calls(record, model_answer) points, reason = scorer_fn(tool_calls, model_answer) diff --git a/src/openjarvis/learning/optimize/store.py b/src/openjarvis/learning/optimize/store.py index 8841384f..7d1eb7ef 100644 --- a/src/openjarvis/learning/optimize/store.py +++ b/src/openjarvis/learning/optimize/store.py @@ -92,7 +92,7 @@ class OptimizationStore: def __init__(self, db_path: Union[str, Path]) -> None: self._db_path = str(db_path) - self._conn = sqlite3.connect(self._db_path) + self._conn = sqlite3.connect(self._db_path, check_same_thread=False) self._conn.execute("PRAGMA journal_mode=WAL") self._conn.execute(_CREATE_RUNS) self._conn.execute(_CREATE_TRIALS) diff --git a/src/openjarvis/security/audit.py b/src/openjarvis/security/audit.py index 06a68d30..888fe160 100644 --- a/src/openjarvis/security/audit.py +++ b/src/openjarvis/security/audit.py @@ -39,7 +39,7 @@ class AuditLogger: from openjarvis.security.file_utils import secure_create secure_create(self._db_path) - self._conn = sqlite3.connect(str(self._db_path)) + self._conn = sqlite3.connect(str(self._db_path), check_same_thread=False) self._conn.execute( """ CREATE TABLE IF NOT EXISTS security_events ( diff --git a/src/openjarvis/server/cloud_router.py b/src/openjarvis/server/cloud_router.py new file mode 100644 index 00000000..c71623c3 --- /dev/null +++ b/src/openjarvis/server/cloud_router.py @@ -0,0 +1,389 @@ +"""Direct cloud API router — bypasses the engine system entirely. + +Reads API keys from ~/.openjarvis/cloud-keys.env at request time so +it works even when the server was started without cloud keys in its +environment. Uses httpx directly so no cloud SDK packages are required. +""" + +from __future__ import annotations + +import json +import os +from collections.abc import AsyncIterator +from pathlib import Path +from typing import Any, Sequence + +import httpx + +from openjarvis.core.types import Message + +# --------------------------------------------------------------------------- +# Key / provider detection +# --------------------------------------------------------------------------- + +_CLOUD_ENV_FILE = Path.home() / ".openjarvis" / "cloud-keys.env" + +_OPENAI_PREFIXES = ("gpt-", "o1-", "o3-", "o4-", "chatgpt-") +_ANTHROPIC_PREFIXES = ("claude-",) +_GOOGLE_PREFIXES = ("gemini-",) +_MINIMAX_PREFIXES = ("MiniMax-",) + + +def _load_keys() -> dict[str, str]: + """Read cloud-keys.env from disk every call so live updates are picked up.""" + keys: dict[str, str] = {} + # File first, then fall back to process environment + if _CLOUD_ENV_FILE.exists(): + for raw in _CLOUD_ENV_FILE.read_text().splitlines(): + line = raw.strip() + if line and not line.startswith("#") and "=" in line: + k, v = line.split("=", 1) + keys[k.strip()] = v.strip() + # Process env can override (e.g. during testing) + for name in ( + "OPENAI_API_KEY", + "ANTHROPIC_API_KEY", + "GEMINI_API_KEY", + "GOOGLE_API_KEY", + "OPENROUTER_API_KEY", + "MINIMAX_API_KEY", + ): + val = os.environ.get(name) + if val: + keys[name] = val + return keys + + +def get_provider(model: str) -> str | None: + """Return the provider for a model name, or None if it's a local model.""" + if any(model.startswith(p) for p in _OPENAI_PREFIXES): + return "openai" + if any(model.startswith(p) for p in _ANTHROPIC_PREFIXES): + return "anthropic" + if any(model.startswith(p) for p in _GOOGLE_PREFIXES): + return "google" + if any(model.startswith(p) for p in _MINIMAX_PREFIXES): + return "minimax" + if "/" in model: # openrouter format: "meta-llama/llama-3-8b" + return "openrouter" + return None + + +def is_cloud_model(model: str) -> bool: + """Return True if the model is served by a cloud provider.""" + return get_provider(model) is not None + + +# --------------------------------------------------------------------------- +# Message conversion +# --------------------------------------------------------------------------- + + +def _to_openai_msgs(messages: Sequence[Message]) -> list[dict[str, Any]]: + out = [] + for m in messages: + role = m.role.value if hasattr(m.role, "value") else str(m.role) + out.append({"role": role, "content": m.content or ""}) + return out + + +def _to_anthropic_msgs( + messages: Sequence[Message], +) -> tuple[str, list[dict[str, Any]]]: + """Return (system_text, chat_messages) in Anthropic format.""" + system_text = "" + chat: list[dict[str, Any]] = [] + for m in messages: + role = m.role.value if hasattr(m.role, "value") else str(m.role) + if role == "system": + system_text = m.content or "" + else: + # Anthropic only allows "user" and "assistant" + ar = "user" if role != "assistant" else "assistant" + chat.append({"role": ar, "content": m.content or ""}) + return system_text, chat + + +def _to_google_contents(messages: Sequence[Message]) -> list[dict[str, Any]]: + """Convert to Google Gemini content format.""" + contents = [] + for m in messages: + role = m.role.value if hasattr(m.role, "value") else str(m.role) + if role == "system": + # Gemini doesn't have a system role in the contents array; + # prepend as a user message. + contents.append({"role": "user", "parts": [{"text": m.content or ""}]}) + contents.append({"role": "model", "parts": [{"text": "Understood."}]}) + elif role == "assistant": + contents.append({"role": "model", "parts": [{"text": m.content or ""}]}) + else: + contents.append({"role": "user", "parts": [{"text": m.content or ""}]}) + return contents + + +# --------------------------------------------------------------------------- +# Streaming generators +# --------------------------------------------------------------------------- + + +async def _stream_openai( + model: str, + messages: Sequence[Message], + temperature: float, + max_tokens: int, + base_url: str = "https://api.openai.com/v1", + api_key_name: str = "OPENAI_API_KEY", +) -> AsyncIterator[str]: + keys = _load_keys() + api_key = keys.get(api_key_name, "") + if not api_key: + raise ValueError(f"{api_key_name} not set — add it in the Cloud Models tab") + + payload = { + "model": model, + "messages": _to_openai_msgs(messages), + "temperature": temperature, + "max_tokens": max_tokens, + "stream": True, + } + + async with httpx.AsyncClient(timeout=180) as client: + async with client.stream( + "POST", + f"{base_url}/chat/completions", + json=payload, + headers={ + "Authorization": f"Bearer {api_key}", + "Content-Type": "application/json", + }, + ) as resp: + resp.raise_for_status() + async for line in resp.aiter_lines(): + if not line.startswith("data: "): + continue + data = line[6:].strip() + if data == "[DONE]": + break + try: + chunk = json.loads(data) + delta = chunk["choices"][0]["delta"].get("content") or "" + if delta: + yield delta + except Exception: + pass + + +async def _stream_anthropic( + model: str, + messages: Sequence[Message], + temperature: float, + max_tokens: int, +) -> AsyncIterator[str]: + keys = _load_keys() + api_key = keys.get("ANTHROPIC_API_KEY", "") + if not api_key: + raise ValueError("ANTHROPIC_API_KEY not set — add it in the Cloud Models tab") + + system_text, chat_msgs = _to_anthropic_msgs(messages) + payload: dict[str, Any] = { + "model": model, + "messages": chat_msgs, + "max_tokens": max_tokens, + "temperature": temperature, + "stream": True, + } + if system_text: + payload["system"] = system_text + + async with httpx.AsyncClient(timeout=180) as client: + async with client.stream( + "POST", + "https://api.anthropic.com/v1/messages", + json=payload, + headers={ + "x-api-key": api_key, + "anthropic-version": "2023-06-01", + "Content-Type": "application/json", + }, + ) as resp: + resp.raise_for_status() + async for line in resp.aiter_lines(): + if not line.startswith("data: "): + continue + data = line[6:].strip() + try: + event = json.loads(data) + if event.get("type") == "content_block_delta": + text = event.get("delta", {}).get("text", "") + if text: + yield text + except Exception: + pass + + +async def _stream_google( + model: str, + messages: Sequence[Message], + temperature: float, + max_tokens: int, +) -> AsyncIterator[str]: + keys = _load_keys() + api_key = keys.get("GEMINI_API_KEY") or keys.get("GOOGLE_API_KEY", "") + if not api_key: + raise ValueError("GEMINI_API_KEY not set — add it in the Cloud Models tab") + + contents = _to_google_contents(messages) + payload: dict[str, Any] = { + "contents": contents, + "generationConfig": { + "temperature": temperature, + "maxOutputTokens": max_tokens, + }, + } + + url = ( + f"https://generativelanguage.googleapis.com/v1beta/models/" + f"{model}:streamGenerateContent?alt=sse&key={api_key}" + ) + + async with httpx.AsyncClient(timeout=180) as client: + async with client.stream("POST", url, json=payload) as resp: + resp.raise_for_status() + async for line in resp.aiter_lines(): + if not line.startswith("data: "): + continue + data = line[6:].strip() + try: + chunk = json.loads(data) + parts = ( + chunk.get("candidates", [{}])[0] + .get("content", {}) + .get("parts", []) + ) + for part in parts: + text = part.get("text", "") + if text: + yield text + except Exception: + pass + + +# --------------------------------------------------------------------------- +# Local (Ollama) direct streaming — bypasses engine routing entirely +# --------------------------------------------------------------------------- + + +def _ollama_host() -> str: + return os.environ.get("OLLAMA_HOST", "http://localhost:11434").rstrip("/") + + +async def stream_local( + model: str, + messages: Sequence[Message], + temperature: float = 0.7, + max_tokens: int = 1024, +) -> AsyncIterator[str]: + """Stream tokens directly from Ollama, bypassing the engine system.""" + payload = { + "model": model, + "messages": _to_openai_msgs(messages), + "stream": True, + # Disable extended thinking (Qwen3.5 etc.) — when enabled all tokens + # go into the 'thinking' field and 'content' stays empty. + "think": False, + "options": { + "temperature": temperature, + "num_predict": max_tokens, + }, + } + host = _ollama_host() + async with httpx.AsyncClient(timeout=300) as client: + async with client.stream("POST", f"{host}/api/chat", json=payload) as resp: + resp.raise_for_status() + async for line in resp.aiter_lines(): + if not line: + continue + try: + data = json.loads(line) + token = data.get("message", {}).get("content", "") + if token: + yield token + if data.get("done"): + break + except Exception: + pass + + +async def list_local_models() -> list[str]: + """Return Ollama model names directly from the Ollama API.""" + host = _ollama_host() + try: + async with httpx.AsyncClient(timeout=10) as client: + resp = await client.get(f"{host}/api/tags") + resp.raise_for_status() + data = resp.json() + return [m["name"] for m in data.get("models", [])] + except Exception: + return [] + + +# --------------------------------------------------------------------------- +# Public entry point +# --------------------------------------------------------------------------- + + +async def stream_cloud( + model: str, + messages: Sequence[Message], + temperature: float = 0.7, + max_tokens: int = 1024, +) -> AsyncIterator[str]: + """Stream tokens from a cloud provider for the given model.""" + provider = get_provider(model) + + if provider == "openai": + async for token in _stream_openai(model, messages, temperature, max_tokens): + yield token + + elif provider == "anthropic": + async for token in _stream_anthropic(model, messages, temperature, max_tokens): + yield token + + elif provider == "google": + async for token in _stream_google(model, messages, temperature, max_tokens): + yield token + + elif provider == "openrouter": + keys = _load_keys() + api_key = keys.get("OPENROUTER_API_KEY", "") + if not api_key: + raise ValueError( + "OPENROUTER_API_KEY not set — add it in the Cloud Models tab" + ) + async for token in _stream_openai( + model, + messages, + temperature, + max_tokens, + base_url="https://openrouter.ai/api/v1", + api_key_name="OPENROUTER_API_KEY", + ): + yield token + + elif provider == "minimax": + keys = _load_keys() + api_key = keys.get("MINIMAX_API_KEY", "") + if not api_key: + raise ValueError("MINIMAX_API_KEY not set — add it in the Cloud Models tab") + async for token in _stream_openai( + model, + messages, + temperature, + max_tokens, + base_url="https://api.minimax.io/v1", + api_key_name="MINIMAX_API_KEY", + ): + yield token + + else: + raise ValueError(f"Unknown cloud provider for model: {model!r}") diff --git a/src/openjarvis/server/routes.py b/src/openjarvis/server/routes.py index 7a8d473c..214c1661 100644 --- a/src/openjarvis/server/routes.py +++ b/src/openjarvis/server/routes.py @@ -305,9 +305,19 @@ async def _handle_stream( complexity_info=None, ): """Stream response using SSE format.""" + from openjarvis.server.cloud_router import ( + is_cloud_model, + stream_cloud, + stream_local, + ) + messages = _to_messages(req.messages) chunk_id = f"chatcmpl-{uuid.uuid4().hex[:12]}" + # Route directly to the right backend — bypasses engine routing entirely + # so broken MultiEngine state can never misdirect requests. + use_cloud = is_cloud_model(model) + async def generate(): # Send role chunk first first_chunk = ChatCompletionChunk( @@ -322,13 +332,45 @@ async def _handle_stream( yield f"data: {first_chunk.model_dump_json()}\n\n" try: - # Stream content - async for token in engine.stream( - messages, - model=model, - temperature=req.temperature, - max_tokens=req.max_tokens, - ): + # Cloud models → direct cloud API (reads keys from disk). + # Local models → engine.stream() first so mock engines work in + # tests. Fall back to stream_local() only when the engine would + # mis-route the request to a cloud backend (MultiEngine routing + # confusion), which is detected by checking the routed engine's + # is_cloud attribute. + if use_cloud: + token_iter = stream_cloud( + model, messages, req.temperature, req.max_tokens + ) + else: + # Use engine.stream() by default (preserves mock-engine + # compatibility in tests). Only fall back to stream_local() + # when a real MultiEngine would mis-route the local model to a + # cloud backend — detected via isinstance so mocks are not + # accidentally matched. + _use_local_fallback = False + try: + from openjarvis.engine.multi import MultiEngine + + _inner = getattr(engine, "_inner", engine) + if isinstance(_inner, MultiEngine): + _routed = _inner._engine_for(model) + if _routed is not None and getattr(_routed, "is_cloud", False): + _use_local_fallback = True + except Exception: + pass + if _use_local_fallback: + token_iter = stream_local( + model, messages, req.temperature, req.max_tokens + ) + else: + token_iter = engine.stream( + messages, + model=model, + temperature=req.temperature, + max_tokens=req.max_tokens, + ) + async for token in token_iter: chunk = ChatCompletionChunk( id=chunk_id, model=model, @@ -380,17 +422,11 @@ async def _handle_stream( ) finish_dict = _json.loads(finish_data.model_dump_json()) - # Pull usage from the engine if it tracked it during streaming - raw_engine = engine - # Unwrap InstrumentedEngine if present - if hasattr(raw_engine, "_inner"): - raw_engine = raw_engine._inner - # Unwrap MultiEngine if present - if hasattr(raw_engine, "_engine_for"): - raw_engine = raw_engine._engine_for(model) - stream_usage = getattr(raw_engine, "_last_stream_usage", None) - if isinstance(stream_usage, dict) and stream_usage.get("total_tokens", 0) > 0: - finish_dict["usage"] = stream_usage + # Tag the finish chunk with the correct engine label. + # We use the routing decision (use_cloud) directly rather than + # unwrapping the engine chain, which can be in a broken state. + finish_dict.setdefault("telemetry", {}) + finish_dict["telemetry"]["engine"] = "cloud" if use_cloud else "ollama" if complexity_info is not None: finish_dict["complexity"] = complexity_info.model_dump() @@ -407,9 +443,22 @@ async def _handle_stream( @router.get("/v1/models") async def list_models(request: Request) -> ModelListResponse: - """List available models from the engine.""" + """List locally installed models (Ollama). + + Cloud models are not included here — they live in the Cloud Models tab + of the UI and are selected there, not from this endpoint. + """ + from openjarvis.server.cloud_router import is_cloud_model, list_local_models + + # Prefer engine.list_models() so mock engines work in tests. + # Filter out any cloud model IDs that may appear via MultiEngine. + # Fall back to direct Ollama query only when the engine returns nothing. engine = request.app.state.engine - model_ids = engine.list_models() + all_ids = engine.list_models() + model_ids = [m for m in all_ids if not is_cloud_model(m)] + if not model_ids: + model_ids = await list_local_models() + return ModelListResponse( data=[ModelObject(id=mid) for mid in model_ids], ) @@ -487,6 +536,63 @@ async def delete_model(model_name: str, request: Request): return {"status": "deleted", "model": model_name} +@router.post("/v1/cloud/reload") +async def reload_cloud_engine(request: Request): + """Hot-reload cloud API keys and (re-)initialize the cloud engine. + + Called by the desktop app immediately after the user saves a cloud API + key so that cloud models become available without a full app restart. + """ + import os + from pathlib import Path + + # Re-read ~/.openjarvis/cloud-keys.env and update the running process env. + keys_path = Path.home() / ".openjarvis" / "cloud-keys.env" + if keys_path.exists(): + for raw_line in keys_path.read_text().splitlines(): + line = raw_line.strip() + if line and not line.startswith("#") and "=" in line: + k, v = line.split("=", 1) + os.environ[k.strip()] = v.strip() + + # Try to build a fresh CloudEngine. + try: + from openjarvis.engine.cloud import CloudEngine + from openjarvis.engine.multi import MultiEngine + + cloud = CloudEngine() + if not cloud.health(): + return { + "status": "no_cloud", + "message": "No cloud models available (check API keys)", + } + except Exception as exc: + return {"status": "error", "message": str(exc)} + + # Locate the innermost engine, working through InstrumentedEngine layers. + outer = request.app.state.engine + inner = getattr(outer, "_inner", outer) + + if isinstance(inner, MultiEngine): + # Replace or insert the cloud entry in the existing MultiEngine. + new_engines = [(k, e) for k, e in inner._engines if k != "cloud"] + new_engines.append(("cloud", cloud)) + inner._engines = new_engines + inner._refresh_map() + else: + # Wrap the existing engine (which may be security-wrapped) with a new + # MultiEngine that includes the cloud engine. + engine_name = getattr(request.app.state, "engine_name", "local") + new_multi = MultiEngine([(engine_name, inner), ("cloud", cloud)]) + if hasattr(outer, "_inner"): + outer._inner = new_multi + else: + request.app.state.engine = new_multi + request.app.state.engine_name = "multi" + + return {"status": "ok", "message": "Cloud engine reloaded"} + + @router.get("/v1/savings") async def savings(request: Request): """Return savings summary compared to cloud providers. diff --git a/src/openjarvis/telemetry/aggregator.py b/src/openjarvis/telemetry/aggregator.py index f4beee43..ead6c5ed 100644 --- a/src/openjarvis/telemetry/aggregator.py +++ b/src/openjarvis/telemetry/aggregator.py @@ -89,7 +89,7 @@ class TelemetryAggregator: def __init__(self, db_path: str | Path) -> None: self._db_path = str(db_path) - self._conn = sqlite3.connect(self._db_path) + self._conn = sqlite3.connect(self._db_path, check_same_thread=False) self._conn.row_factory = sqlite3.Row @staticmethod diff --git a/src/openjarvis/tools/storage/knowledge_graph.py b/src/openjarvis/tools/storage/knowledge_graph.py index 4890de61..9daa68db 100644 --- a/src/openjarvis/tools/storage/knowledge_graph.py +++ b/src/openjarvis/tools/storage/knowledge_graph.py @@ -60,7 +60,7 @@ class KnowledgeGraphMemory: ) -> None: self._db_path = Path(db_path) self._db_path.parent.mkdir(parents=True, exist_ok=True) - self._conn = sqlite3.connect(str(self._db_path)) + self._conn = sqlite3.connect(str(self._db_path), check_same_thread=False) self._create_tables() def _create_tables(self) -> None: