From f8c37fe85ee49bb6838e45e1c0f0a015388d923f Mon Sep 17 00:00:00 2001 From: mrTSB Date: Wed, 1 Apr 2026 21:22:25 -0700 Subject: [PATCH 01/22] make cloud models on desktop app work --- desktop/src-tauri/src/lib.rs | 16 +- frontend/src/components/CommandPalette.tsx | 5 +- src/openjarvis/cli/serve.py | 9 +- src/openjarvis/server/cloud_router.py | 324 +++++++++++++++++++++ src/openjarvis/server/routes.py | 104 ++++++- 5 files changed, 446 insertions(+), 12 deletions(-) create mode 100644 src/openjarvis/server/cloud_router.py diff --git a/desktop/src-tauri/src/lib.rs b/desktop/src-tauri/src/lib.rs index 0f2d6c1c..6fa1a44a 100644 --- a/desktop/src-tauri/src/lib.rs +++ b/desktop/src-tauri/src/lib.rs @@ -649,7 +649,12 @@ async fn boot_backend(backend: SharedBackend, status: SharedStatus) { s.detail = "Installing dependencies...".into(); } let _ = tokio::process::Command::new(&uv_bin) - .args(["sync", "--extra", "server"]) + .args([ + "sync", + "--extra", "server", + "--extra", "inference-cloud", + "--extra", "inference-google", + ]) .stdout(std::process::Stdio::null()) .stderr(std::process::Stdio::null()) .current_dir(root) @@ -1124,6 +1129,15 @@ async fn save_cloud_key(key_name: String, key_value: String) -> Result<(), Strin let _ = std::fs::set_permissions(&path, std::fs::Permissions::from_mode(0o600)); } + // Tell the running server to hot-reload its cloud engine so the user + // doesn't need to restart the app after entering an API key. + let reload_url = format!("http://127.0.0.1:{}/v1/cloud/reload", JARVIS_PORT); + let _ = reqwest::Client::new() + .post(&reload_url) + .timeout(std::time::Duration::from_secs(10)) + .send() + .await; + Ok(()) } diff --git a/frontend/src/components/CommandPalette.tsx b/frontend/src/components/CommandPalette.tsx index 2d110290..743fe823 100644 --- a/frontend/src/components/CommandPalette.tsx +++ b/frontend/src/components/CommandPalette.tsx @@ -223,8 +223,11 @@ export function CommandPalette() { useAppStore.getState().addLogEntry({ timestamp: Date.now(), level: 'info', category: 'model', - message: `${provider.name} API key ${value ? 'saved' : 'removed'}. Restart the app for cloud models to appear.`, + message: `${provider.name} API key ${value ? 'saved' : 'removed'}. Refreshing model list…`, }); + + // Refresh the model list so cloud models appear immediately. + await refreshModels(); }; const handleKeyDown = (e: React.KeyboardEvent) => { diff --git a/src/openjarvis/cli/serve.py b/src/openjarvis/cli/serve.py index a236e6b5..066d6ecc 100644 --- a/src/openjarvis/cli/serve.py +++ b/src/openjarvis/cli/serve.py @@ -120,10 +120,15 @@ def serve( from openjarvis.engine.multi import MultiEngine cloud = CloudEngine() + engine = MultiEngine([(engine_name, engine), ("cloud", cloud)]) + engine_name = "multi" if cloud.health(): - engine = MultiEngine([(engine_name, engine), ("cloud", cloud)]) - engine_name = "multi" console.print(" Cloud: [cyan]enabled[/cyan] (API keys detected)") + else: + console.print( + " Cloud: [yellow]keys set but packages missing[/yellow] " + "(run: uv sync --extra inference-cloud --extra inference-google)" + ) except Exception as exc: logger.debug("Cloud engine init failed: %s", exc) diff --git a/src/openjarvis/server/cloud_router.py b/src/openjarvis/server/cloud_router.py new file mode 100644 index 00000000..c90ccf59 --- /dev/null +++ b/src/openjarvis/server/cloud_router.py @@ -0,0 +1,324 @@ +"""Direct cloud API router — bypasses the engine system entirely. + +Reads API keys from ~/.openjarvis/cloud-keys.env at request time so +it works even when the server was started without cloud keys in its +environment. Uses httpx directly so no cloud SDK packages are required. +""" + +from __future__ import annotations + +import json +import os +from collections.abc import AsyncIterator +from pathlib import Path +from typing import Any, Sequence + +import httpx + +from openjarvis.core.types import Message + +# --------------------------------------------------------------------------- +# Key / provider detection +# --------------------------------------------------------------------------- + +_CLOUD_ENV_FILE = Path.home() / ".openjarvis" / "cloud-keys.env" + +_OPENAI_PREFIXES = ("gpt-", "o1-", "o3-", "o4-", "chatgpt-") +_ANTHROPIC_PREFIXES = ("claude-",) +_GOOGLE_PREFIXES = ("gemini-",) +_MINIMAX_PREFIXES = ("MiniMax-",) + + +def _load_keys() -> dict[str, str]: + """Read cloud-keys.env from disk every call so live updates are picked up.""" + keys: dict[str, str] = {} + # File first, then fall back to process environment + if _CLOUD_ENV_FILE.exists(): + for raw in _CLOUD_ENV_FILE.read_text().splitlines(): + line = raw.strip() + if line and not line.startswith("#") and "=" in line: + k, v = line.split("=", 1) + keys[k.strip()] = v.strip() + # Process env can override (e.g. during testing) + for name in ( + "OPENAI_API_KEY", + "ANTHROPIC_API_KEY", + "GEMINI_API_KEY", + "GOOGLE_API_KEY", + "OPENROUTER_API_KEY", + "MINIMAX_API_KEY", + ): + val = os.environ.get(name) + if val: + keys[name] = val + return keys + + +def get_provider(model: str) -> str | None: + """Return the provider for a model name, or None if it's a local model.""" + if any(model.startswith(p) for p in _OPENAI_PREFIXES): + return "openai" + if any(model.startswith(p) for p in _ANTHROPIC_PREFIXES): + return "anthropic" + if any(model.startswith(p) for p in _GOOGLE_PREFIXES): + return "google" + if any(model.startswith(p) for p in _MINIMAX_PREFIXES): + return "minimax" + if "/" in model: # openrouter format: "meta-llama/llama-3-8b" + return "openrouter" + return None + + +def is_cloud_model(model: str) -> bool: + return get_provider(model) is not None + + +# --------------------------------------------------------------------------- +# Message conversion +# --------------------------------------------------------------------------- + +def _to_openai_msgs(messages: Sequence[Message]) -> list[dict[str, Any]]: + out = [] + for m in messages: + role = m.role.value if hasattr(m.role, "value") else str(m.role) + out.append({"role": role, "content": m.content or ""}) + return out + + +def _to_anthropic_msgs( + messages: Sequence[Message], +) -> tuple[str, list[dict[str, Any]]]: + """Return (system_text, chat_messages) in Anthropic format.""" + system_text = "" + chat: list[dict[str, Any]] = [] + for m in messages: + role = m.role.value if hasattr(m.role, "value") else str(m.role) + if role == "system": + system_text = m.content or "" + else: + # Anthropic only allows "user" and "assistant" + ar = "user" if role != "assistant" else "assistant" + chat.append({"role": ar, "content": m.content or ""}) + return system_text, chat + + +def _to_google_contents(messages: Sequence[Message]) -> list[dict[str, Any]]: + """Convert to Google Gemini content format.""" + contents = [] + for m in messages: + role = m.role.value if hasattr(m.role, "value") else str(m.role) + if role == "system": + # Gemini doesn't have a system role in the contents array; + # prepend as a user message. + contents.append({"role": "user", "parts": [{"text": m.content or ""}]}) + contents.append({"role": "model", "parts": [{"text": "Understood."}]}) + elif role == "assistant": + contents.append({"role": "model", "parts": [{"text": m.content or ""}]}) + else: + contents.append({"role": "user", "parts": [{"text": m.content or ""}]}) + return contents + + +# --------------------------------------------------------------------------- +# Streaming generators +# --------------------------------------------------------------------------- + +async def _stream_openai( + model: str, + messages: Sequence[Message], + temperature: float, + max_tokens: int, + base_url: str = "https://api.openai.com/v1", + api_key_name: str = "OPENAI_API_KEY", +) -> AsyncIterator[str]: + keys = _load_keys() + api_key = keys.get(api_key_name, "") + if not api_key: + raise ValueError(f"{api_key_name} not set — add it in the Cloud Models tab") + + payload = { + "model": model, + "messages": _to_openai_msgs(messages), + "temperature": temperature, + "max_tokens": max_tokens, + "stream": True, + } + + async with httpx.AsyncClient(timeout=180) as client: + async with client.stream( + "POST", + f"{base_url}/chat/completions", + json=payload, + headers={ + "Authorization": f"Bearer {api_key}", + "Content-Type": "application/json", + }, + ) as resp: + resp.raise_for_status() + async for line in resp.aiter_lines(): + if not line.startswith("data: "): + continue + data = line[6:].strip() + if data == "[DONE]": + break + try: + chunk = json.loads(data) + delta = chunk["choices"][0]["delta"].get("content") or "" + if delta: + yield delta + except Exception: + pass + + +async def _stream_anthropic( + model: str, + messages: Sequence[Message], + temperature: float, + max_tokens: int, +) -> AsyncIterator[str]: + keys = _load_keys() + api_key = keys.get("ANTHROPIC_API_KEY", "") + if not api_key: + raise ValueError("ANTHROPIC_API_KEY not set — add it in the Cloud Models tab") + + system_text, chat_msgs = _to_anthropic_msgs(messages) + payload: dict[str, Any] = { + "model": model, + "messages": chat_msgs, + "max_tokens": max_tokens, + "temperature": temperature, + "stream": True, + } + if system_text: + payload["system"] = system_text + + async with httpx.AsyncClient(timeout=180) as client: + async with client.stream( + "POST", + "https://api.anthropic.com/v1/messages", + json=payload, + headers={ + "x-api-key": api_key, + "anthropic-version": "2023-06-01", + "Content-Type": "application/json", + }, + ) as resp: + resp.raise_for_status() + async for line in resp.aiter_lines(): + if not line.startswith("data: "): + continue + data = line[6:].strip() + try: + event = json.loads(data) + if event.get("type") == "content_block_delta": + text = event.get("delta", {}).get("text", "") + if text: + yield text + except Exception: + pass + + +async def _stream_google( + model: str, + messages: Sequence[Message], + temperature: float, + max_tokens: int, +) -> AsyncIterator[str]: + keys = _load_keys() + api_key = keys.get("GEMINI_API_KEY") or keys.get("GOOGLE_API_KEY", "") + if not api_key: + raise ValueError("GEMINI_API_KEY not set — add it in the Cloud Models tab") + + contents = _to_google_contents(messages) + payload: dict[str, Any] = { + "contents": contents, + "generationConfig": { + "temperature": temperature, + "maxOutputTokens": max_tokens, + }, + } + + url = ( + f"https://generativelanguage.googleapis.com/v1beta/models/" + f"{model}:streamGenerateContent?alt=sse&key={api_key}" + ) + + async with httpx.AsyncClient(timeout=180) as client: + async with client.stream("POST", url, json=payload) as resp: + resp.raise_for_status() + async for line in resp.aiter_lines(): + if not line.startswith("data: "): + continue + data = line[6:].strip() + try: + chunk = json.loads(data) + parts = ( + chunk.get("candidates", [{}])[0] + .get("content", {}) + .get("parts", []) + ) + for part in parts: + text = part.get("text", "") + if text: + yield text + except Exception: + pass + + +# --------------------------------------------------------------------------- +# Public entry point +# --------------------------------------------------------------------------- + +async def stream_cloud( + model: str, + messages: Sequence[Message], + temperature: float = 0.7, + max_tokens: int = 1024, +) -> AsyncIterator[str]: + """Stream tokens from a cloud provider for the given model.""" + provider = get_provider(model) + + if provider == "openai": + async for token in _stream_openai(model, messages, temperature, max_tokens): + yield token + + elif provider == "anthropic": + async for token in _stream_anthropic(model, messages, temperature, max_tokens): + yield token + + elif provider == "google": + async for token in _stream_google(model, messages, temperature, max_tokens): + yield token + + elif provider == "openrouter": + keys = _load_keys() + api_key = keys.get("OPENROUTER_API_KEY", "") + if not api_key: + raise ValueError("OPENROUTER_API_KEY not set — add it in the Cloud Models tab") + async for token in _stream_openai( + model, + messages, + temperature, + max_tokens, + base_url="https://openrouter.ai/api/v1", + api_key_name="OPENROUTER_API_KEY", + ): + yield token + + elif provider == "minimax": + keys = _load_keys() + api_key = keys.get("MINIMAX_API_KEY", "") + if not api_key: + raise ValueError("MINIMAX_API_KEY not set — add it in the Cloud Models tab") + async for token in _stream_openai( + model, + messages, + temperature, + max_tokens, + base_url="https://api.minimax.io/v1", + api_key_name="MINIMAX_API_KEY", + ): + yield token + + else: + raise ValueError(f"Unknown cloud provider for model: {model!r}") diff --git a/src/openjarvis/server/routes.py b/src/openjarvis/server/routes.py index a615d7d0..8084bac2 100644 --- a/src/openjarvis/server/routes.py +++ b/src/openjarvis/server/routes.py @@ -290,9 +290,15 @@ async def _handle_stream( complexity_info=None, ): """Stream response using SSE format.""" + from openjarvis.server.cloud_router import is_cloud_model, stream_cloud + messages = _to_messages(req.messages) chunk_id = f"chatcmpl-{uuid.uuid4().hex[:12]}" + # Choose token source: cloud router bypasses the engine system entirely + # so cloud models work regardless of how the server was started. + use_cloud = is_cloud_model(model) + async def generate(): # Send role chunk first first_chunk = ChatCompletionChunk( @@ -308,12 +314,17 @@ async def _handle_stream( try: # Stream content - async for token in engine.stream( - messages, - model=model, - temperature=req.temperature, - max_tokens=req.max_tokens, - ): + token_iter = ( + stream_cloud(model, messages, req.temperature, req.max_tokens) + if use_cloud + else engine.stream( + messages, + model=model, + temperature=req.temperature, + max_tokens=req.max_tokens, + ) + ) + async for token in token_iter: chunk = ChatCompletionChunk( id=chunk_id, model=model, @@ -392,9 +403,32 @@ async def _handle_stream( @router.get("/v1/models") async def list_models(request: Request) -> ModelListResponse: - """List available models from the engine.""" + """List available models from the engine plus any configured cloud models.""" + from openjarvis.server.cloud_router import _load_keys + from openjarvis.engine.cloud import ( + _OPENAI_MODELS, + _ANTHROPIC_MODELS, + _GOOGLE_MODELS, + _OPENROUTER_POPULAR, + _MINIMAX_MODELS, + ) + engine = request.app.state.engine - model_ids = engine.list_models() + model_ids: list[str] = list(engine.list_models()) + + # Append cloud models for any provider whose key is present on disk. + keys = _load_keys() + if keys.get("OPENAI_API_KEY"): + model_ids.extend(m for m in _OPENAI_MODELS if m not in model_ids) + if keys.get("ANTHROPIC_API_KEY"): + model_ids.extend(m for m in _ANTHROPIC_MODELS if m not in model_ids) + if keys.get("GEMINI_API_KEY") or keys.get("GOOGLE_API_KEY"): + model_ids.extend(m for m in _GOOGLE_MODELS if m not in model_ids) + if keys.get("OPENROUTER_API_KEY"): + model_ids.extend(m for m in _OPENROUTER_POPULAR if m not in model_ids) + if keys.get("MINIMAX_API_KEY"): + model_ids.extend(m for m in _MINIMAX_MODELS if m not in model_ids) + return ModelListResponse( data=[ModelObject(id=mid) for mid in model_ids], ) @@ -472,6 +506,60 @@ async def delete_model(model_name: str, request: Request): return {"status": "deleted", "model": model_name} +@router.post("/v1/cloud/reload") +async def reload_cloud_engine(request: Request): + """Hot-reload cloud API keys and (re-)initialize the cloud engine. + + Called by the desktop app immediately after the user saves a cloud API + key so that cloud models become available without a full app restart. + """ + import os + from pathlib import Path + + # Re-read ~/.openjarvis/cloud-keys.env and update the running process env. + keys_path = Path.home() / ".openjarvis" / "cloud-keys.env" + if keys_path.exists(): + for raw_line in keys_path.read_text().splitlines(): + line = raw_line.strip() + if line and not line.startswith("#") and "=" in line: + k, v = line.split("=", 1) + os.environ[k.strip()] = v.strip() + + # Try to build a fresh CloudEngine. + try: + from openjarvis.engine.cloud import CloudEngine + from openjarvis.engine.multi import MultiEngine + + cloud = CloudEngine() + if not cloud.health(): + return {"status": "no_cloud", "message": "No cloud models available (check API keys)"} + except Exception as exc: + return {"status": "error", "message": str(exc)} + + # Locate the innermost engine, working through InstrumentedEngine layers. + outer = request.app.state.engine + inner = getattr(outer, "_inner", outer) + + if isinstance(inner, MultiEngine): + # Replace or insert the cloud entry in the existing MultiEngine. + new_engines = [(k, e) for k, e in inner._engines if k != "cloud"] + new_engines.append(("cloud", cloud)) + inner._engines = new_engines + inner._refresh_map() + else: + # Wrap the existing engine (which may be security-wrapped) with a new + # MultiEngine that includes the cloud engine. + engine_name = getattr(request.app.state, "engine_name", "local") + new_multi = MultiEngine([(engine_name, inner), ("cloud", cloud)]) + if hasattr(outer, "_inner"): + outer._inner = new_multi + else: + request.app.state.engine = new_multi + request.app.state.engine_name = "multi" + + return {"status": "ok", "message": "Cloud engine reloaded"} + + @router.get("/v1/savings") async def savings(request: Request): """Return savings summary compared to cloud providers. From 59dcbb9cc4747ff322633d2822a15d5a82b412a5 Mon Sep 17 00:00:00 2001 From: mrTSB Date: Wed, 1 Apr 2026 21:22:39 -0700 Subject: [PATCH 02/22] make cloud models on desktop app work --- frontend/tsconfig.tsbuildinfo | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frontend/tsconfig.tsbuildinfo b/frontend/tsconfig.tsbuildinfo index 3dfe098e..c1823959 100644 --- a/frontend/tsconfig.tsbuildinfo +++ b/frontend/tsconfig.tsbuildinfo @@ -1 +1 @@ -{"root":["./src/app.tsx","./src/main.tsx","./src/vite-env.d.ts","./src/components/commandpalette.tsx","./src/components/errorboundary.tsx","./src/components/layout.tsx","./src/components/optinmodal.tsx","./src/components/setupscreen.tsx","./src/components/systempulse.tsx","./src/components/chat/chatarea.tsx","./src/components/chat/inputarea.tsx","./src/components/chat/messagebubble.tsx","./src/components/chat/micbutton.tsx","./src/components/chat/streamingdots.tsx","./src/components/chat/systempanel.tsx","./src/components/chat/toolcallcard.tsx","./src/components/chat/xrayfooter.tsx","./src/components/dashboard/costcomparison.tsx","./src/components/dashboard/energydashboard.tsx","./src/components/dashboard/tracedebugger.tsx","./src/components/sidebar/conversationlist.tsx","./src/components/sidebar/sidebar.tsx","./src/components/setup/ingestdashboard.tsx","./src/components/setup/readyscreen.tsx","./src/components/setup/setupwizard.tsx","./src/components/setup/sourceconnectflow.tsx","./src/components/setup/sourcepicker.tsx","./src/components/ui/button.tsx","./src/components/ui/dialog.tsx","./src/components/ui/input.tsx","./src/components/ui/select.tsx","./src/components/ui/sonner.tsx","./src/components/ui/tooltip.tsx","./src/hooks/usespeech.ts","./src/lib/api.ts","./src/lib/connectors-api.ts","./src/lib/deep-link.ts","./src/lib/profanity.ts","./src/lib/sse.ts","./src/lib/store.ts","./src/lib/utils.ts","./src/pages/agentspage.tsx","./src/pages/chatpage.tsx","./src/pages/dashboardpage.tsx","./src/pages/getstartedpage.tsx","./src/pages/logspage.tsx","./src/pages/settingspage.tsx","./src/types/connectors.ts","./src/types/index.ts"],"version":"5.7.3"} \ No newline at end of file +{"root":["./src/app.tsx","./src/main.tsx","./src/vite-env.d.ts","./src/components/commandpalette.tsx","./src/components/errorboundary.tsx","./src/components/layout.tsx","./src/components/optinmodal.tsx","./src/components/setupscreen.tsx","./src/components/systempulse.tsx","./src/components/chat/chatarea.tsx","./src/components/chat/inputarea.tsx","./src/components/chat/messagebubble.tsx","./src/components/chat/micbutton.tsx","./src/components/chat/streamingdots.tsx","./src/components/chat/systempanel.tsx","./src/components/chat/toolcallcard.tsx","./src/components/chat/xrayfooter.tsx","./src/components/dashboard/costcomparison.tsx","./src/components/dashboard/energydashboard.tsx","./src/components/dashboard/tracedebugger.tsx","./src/components/sidebar/conversationlist.tsx","./src/components/sidebar/sidebar.tsx","./src/components/setup/ingestdashboard.tsx","./src/components/setup/readyscreen.tsx","./src/components/setup/setupwizard.tsx","./src/components/setup/sourceconnectflow.tsx","./src/components/setup/sourcepicker.tsx","./src/components/ui/button.tsx","./src/components/ui/dialog.tsx","./src/components/ui/input.tsx","./src/components/ui/select.tsx","./src/components/ui/sonner.tsx","./src/components/ui/tooltip.tsx","./src/hooks/usespeech.ts","./src/lib/api.ts","./src/lib/connectors-api.ts","./src/lib/deep-link.ts","./src/lib/profanity.ts","./src/lib/sse.ts","./src/lib/store.ts","./src/lib/utils.ts","./src/pages/agentspage.tsx","./src/pages/chatpage.tsx","./src/pages/dashboardpage.tsx","./src/pages/datasourcespage.tsx","./src/pages/getstartedpage.tsx","./src/pages/logspage.tsx","./src/pages/settingspage.tsx","./src/types/connectors.ts","./src/types/index.ts"],"version":"5.7.3"} \ No newline at end of file From 450397a2ad4f432816764401cedc22560f7c0349 Mon Sep 17 00:00:00 2001 From: Avanika Narayan Date: Tue, 31 Mar 2026 14:59:20 -0700 Subject: [PATCH 03/22] feat(evals): TauBench V2 native integration + GAIA eval configs (#162) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat(evals): add TauBench V2 native integration + GAIA eval configs Integrate TauBench V2 (τ²-bench) multi-turn customer service benchmark natively through OpenJarvis's inference engine. The agent's LLM calls go through OpenJarvis while tau2-bench handles the orchestration, user simulation, domain tools, database, and evaluation. ## TauBench Integration Architecture: JarvisHalfDuplexAgent bridges OpenJarvis's engine into tau2's Orchestrator as a drop-in agent replacement. This enables testing how well OpenJarvis's Intelligence + Engine handles multi-turn customer service tasks with real tool calling and database mutations. Key features: - Native OpenJarvis engine for agent LLM calls - tau2's UserSimulator for realistic customer interactions - Domain tools (airline, retail, telecom) with mock databases - Full evaluation: DB state checks, action matching, NL assertions - Test-split filtering for leaderboard-comparable results - Pass^k multi-trial support (default 3 trials per task) - Qwen thinking-mode disabled for clean tool call parsing - Gemini thought_signature handling for multi-turn conversations Files: - datasets/taubench.py: Dataset provider with test-split filtering - execution/taubench_env.py: JarvisHalfDuplexAgent + simulation runner - scorers/taubench.py: Scorer reading tau2 evaluation rewards - CLI registration and KNOWN_BENCHMARKS update ## Results (test split, pass^3, 60 tasks) | Model | TauBench | Leaderboard | |--------------------|----------|-------------| | Claude Opus 4.6 | 86.67% | 84.8% | | Nemotron-3-Super | 86.67% | — | | Qwen3.5-397B | 81.67% | 95.6% | | GPT-5.4 | 81.67% | 91.5% | | Qwen3.5-122B | 80.00% | 93.6% | | Qwen3.5-35B | 77.27% | 89.2% | | Gemini 3.1 Pro | 58.33% | ~87% | ## GAIA Eval Configs Added configs for GPT-5.4, Gemini 3.1 Pro, Nemotron, Qwen 122B, Qwen 35B, and existing GAIA rerun configs for multiple models. Co-Authored-By: Claude Opus 4.6 (1M context) * fix: lint errors in taubench integration Remove unused imports and sort import blocks. Co-Authored-By: Claude Opus 4.6 (1M context) * fix: E501 line too long in slack_connector.py Co-Authored-By: Claude Opus 4.6 (1M context) * chore: remove unrelated GAIA configs from PR Keep only TauBench configs that were created and tested in this PR. GAIA configs are pre-existing or belong in a separate PR. Co-Authored-By: Claude Opus 4.6 (1M context) --------- Co-authored-by: Jon Saad-Falcon Co-authored-by: Claude Opus 4.6 (1M context) --- src/openjarvis/connectors/slack_connector.py | 5 +- src/openjarvis/evals/cli.py | 19 + .../evals/configs/taubench-claude-opus.toml | 29 ++ .../evals/configs/taubench-gemini-31-pro.toml | 29 ++ .../evals/configs/taubench-gpt54.toml | 29 ++ .../configs/taubench-nemotron3-super.toml | 29 ++ .../evals/configs/taubench-qwen122b.toml | 28 ++ .../evals/configs/taubench-qwen35b.toml | 28 ++ .../evals/configs/taubench-qwen397b.toml | 30 ++ src/openjarvis/evals/core/config.py | 1 + src/openjarvis/evals/datasets/taubench.py | 234 +++++++++++ .../evals/execution/taubench_env.py | 369 ++++++++++++++++++ src/openjarvis/evals/scorers/taubench.py | 46 +++ tests/evals/test_benchmark_datasets.py | 2 +- 14 files changed, 876 insertions(+), 2 deletions(-) create mode 100644 src/openjarvis/evals/configs/taubench-claude-opus.toml create mode 100644 src/openjarvis/evals/configs/taubench-gemini-31-pro.toml create mode 100644 src/openjarvis/evals/configs/taubench-gpt54.toml create mode 100644 src/openjarvis/evals/configs/taubench-nemotron3-super.toml create mode 100644 src/openjarvis/evals/configs/taubench-qwen122b.toml create mode 100644 src/openjarvis/evals/configs/taubench-qwen35b.toml create mode 100644 src/openjarvis/evals/configs/taubench-qwen397b.toml create mode 100644 src/openjarvis/evals/datasets/taubench.py create mode 100644 src/openjarvis/evals/execution/taubench_env.py create mode 100644 src/openjarvis/evals/scorers/taubench.py diff --git a/src/openjarvis/connectors/slack_connector.py b/src/openjarvis/connectors/slack_connector.py index f7c4f0dc..65c8a715 100644 --- a/src/openjarvis/connectors/slack_connector.py +++ b/src/openjarvis/connectors/slack_connector.py @@ -25,7 +25,10 @@ from openjarvis.tools._stubs import ToolSpec _SLACK_API_BASE = "https://slack.com/api" _SLACK_AUTH_ENDPOINT = "https://slack.com/oauth/v2/authorize" -_SLACK_SCOPES = "channels:read,channels:history,groups:read,groups:history,im:read,im:history,mpim:read,mpim:history,users:read" +_SLACK_SCOPES = ( + "channels:read,channels:history,groups:read,groups:history," + "im:read,im:history,mpim:read,mpim:history,users:read" +) _DEFAULT_CREDENTIALS_PATH = str(DEFAULT_CONFIG_DIR / "connectors" / "slack.json") # --------------------------------------------------------------------------- diff --git a/src/openjarvis/evals/cli.py b/src/openjarvis/evals/cli.py index 4781863a..eea9e7e7 100644 --- a/src/openjarvis/evals/cli.py +++ b/src/openjarvis/evals/cli.py @@ -124,6 +124,10 @@ BENCHMARKS = { "category": "agentic", "description": "PinchBench real-world agent tasks", }, + "taubench": { + "category": "agentic", + "description": "TauBench multi-turn customer service", + }, } BACKENDS = { @@ -308,6 +312,10 @@ def _build_dataset(benchmark: str, subset: str | None = None): from openjarvis.evals.datasets.pinchbench import PinchBenchDataset return PinchBenchDataset(path=subset) + elif benchmark == "taubench": + from openjarvis.evals.datasets.taubench import TauBenchDataset + domains = subset.split(",") if subset else None + return TauBenchDataset(domains=domains) else: raise click.ClickException(f"Unknown benchmark: {benchmark}") @@ -444,6 +452,9 @@ def _build_scorer(benchmark: str, judge_backend, judge_model: str): from openjarvis.evals.scorers.pinchbench import PinchBenchScorer return PinchBenchScorer(judge_backend, judge_model) + elif benchmark == "taubench": + from openjarvis.evals.scorers.taubench import TauBenchScorer + return TauBenchScorer(judge_backend, judge_model) else: raise click.ClickException(f"Unknown benchmark: {benchmark}") @@ -545,6 +556,14 @@ def _run_single(config, console: Optional[Console] = None) -> object: model=config.model, ) dataset = _build_dataset(config.benchmark) + # Inject engine config for benchmarks that run their own simulation + if hasattr(dataset, "set_engine_config"): + dataset.set_engine_config( + engine_key=config.engine_key, + model=config.model, + temperature=config.temperature, + max_tokens=config.max_tokens, + ) judge_engine = getattr(config, "judge_engine", "cloud") or "cloud" judge_backend = _build_judge_backend(config.judge_model, engine_key=judge_engine) scorer = _build_scorer(config.benchmark, judge_backend, config.judge_model) diff --git a/src/openjarvis/evals/configs/taubench-claude-opus.toml b/src/openjarvis/evals/configs/taubench-claude-opus.toml new file mode 100644 index 00000000..d27f8f30 --- /dev/null +++ b/src/openjarvis/evals/configs/taubench-claude-opus.toml @@ -0,0 +1,29 @@ +# TauBench V2 eval: Claude Opus 4.6 (cloud) +# Multi-turn customer service benchmark — test split, 3 trials + +[meta] +name = "taubench-claude-opus" +description = "TauBench V2 on claude-opus-4-6 (test split, pass^3)" + +[defaults] +temperature = 0.7 +max_tokens = 4096 + +[judge] +model = "gpt-5-mini-2025-08-07" +temperature = 0.0 +engine = "cloud" + +[run] +max_workers = 1 +output_dir = "results/taubench-claude-opus/" +seed = 42 + +[[models]] +name = "claude-opus-4-6" +engine = "cloud" + +[[benchmarks]] +name = "taubench" +backend = "jarvis-direct" +split = "airline,retail" diff --git a/src/openjarvis/evals/configs/taubench-gemini-31-pro.toml b/src/openjarvis/evals/configs/taubench-gemini-31-pro.toml new file mode 100644 index 00000000..996e2fba --- /dev/null +++ b/src/openjarvis/evals/configs/taubench-gemini-31-pro.toml @@ -0,0 +1,29 @@ +# TauBench V2 eval: Gemini 3.1 Pro Preview (cloud) +# Multi-turn customer service benchmark — test split, 3 trials + +[meta] +name = "taubench-gemini-31-pro" +description = "TauBench V2 on gemini-3.1-pro-preview (test split, pass^3)" + +[defaults] +temperature = 0.7 +max_tokens = 4096 + +[judge] +model = "gpt-5-mini-2025-08-07" +temperature = 0.0 +engine = "cloud" + +[run] +max_workers = 1 +output_dir = "results/taubench-gemini-31-pro/" +seed = 42 + +[[models]] +name = "gemini-3.1-pro-preview" +engine = "cloud" + +[[benchmarks]] +name = "taubench" +backend = "jarvis-direct" +split = "airline,retail" diff --git a/src/openjarvis/evals/configs/taubench-gpt54.toml b/src/openjarvis/evals/configs/taubench-gpt54.toml new file mode 100644 index 00000000..f2b3ef7d --- /dev/null +++ b/src/openjarvis/evals/configs/taubench-gpt54.toml @@ -0,0 +1,29 @@ +# TauBench V2 eval: GPT-5.4 (cloud) +# Multi-turn customer service benchmark — test split, 3 trials + +[meta] +name = "taubench-gpt54" +description = "TauBench V2 on gpt-5.4-2026-03-05 (test split, pass^3)" + +[defaults] +temperature = 0.7 +max_tokens = 4096 + +[judge] +model = "gpt-5-mini-2025-08-07" +temperature = 0.0 +engine = "cloud" + +[run] +max_workers = 1 +output_dir = "results/taubench-gpt54/" +seed = 42 + +[[models]] +name = "gpt-5.4-2026-03-05" +engine = "cloud" + +[[benchmarks]] +name = "taubench" +backend = "jarvis-direct" +split = "airline,retail" diff --git a/src/openjarvis/evals/configs/taubench-nemotron3-super.toml b/src/openjarvis/evals/configs/taubench-nemotron3-super.toml new file mode 100644 index 00000000..2afff9a7 --- /dev/null +++ b/src/openjarvis/evals/configs/taubench-nemotron3-super.toml @@ -0,0 +1,29 @@ +# TauBench V2 eval: Nemotron-3-Super-120B-A12B-FP8 (SGLang) +[meta] +name = "taubench-nemotron3-super" +description = "TauBench V2 on Nemotron-3-Super-120B-A12B-FP8 (test split, pass^3)" + +[defaults] +temperature = 1.0 +max_tokens = 4096 + +[judge] +model = "gpt-5-mini-2025-08-07" +temperature = 0.0 +engine = "cloud" + +[run] +max_workers = 1 +output_dir = "results/taubench-nemotron3-super/" +seed = 42 + +[[models]] +name = "nemotron" +engine = "vllm" +num_gpus = 4 +# SGLang serving on port 8001 + +[[benchmarks]] +name = "taubench" +backend = "jarvis-direct" +split = "airline,retail" diff --git a/src/openjarvis/evals/configs/taubench-qwen122b.toml b/src/openjarvis/evals/configs/taubench-qwen122b.toml new file mode 100644 index 00000000..e403a462 --- /dev/null +++ b/src/openjarvis/evals/configs/taubench-qwen122b.toml @@ -0,0 +1,28 @@ +# TauBench V2 eval: Qwen3.5-122B-A10B-FP8 (vLLM, TP=4) +[meta] +name = "taubench-qwen122b" +description = "TauBench V2 on Qwen/Qwen3.5-122B-A10B-FP8 (test split, pass^3)" + +[defaults] +temperature = 0.7 +max_tokens = 4096 + +[judge] +model = "gpt-5-mini-2025-08-07" +temperature = 0.0 +engine = "cloud" + +[run] +max_workers = 1 +output_dir = "results/taubench-qwen122b/" +seed = 42 + +[[models]] +name = "Qwen/Qwen3.5-122B-A10B-FP8" +engine = "vllm" +num_gpus = 4 + +[[benchmarks]] +name = "taubench" +backend = "jarvis-direct" +split = "airline,retail" diff --git a/src/openjarvis/evals/configs/taubench-qwen35b.toml b/src/openjarvis/evals/configs/taubench-qwen35b.toml new file mode 100644 index 00000000..cb779851 --- /dev/null +++ b/src/openjarvis/evals/configs/taubench-qwen35b.toml @@ -0,0 +1,28 @@ +# TauBench V2 eval: Qwen3.5-35B-A3B-FP8 (vLLM, TP=4) +[meta] +name = "taubench-qwen35b" +description = "TauBench V2 on Qwen/Qwen3.5-35B-A3B-FP8 (test split, pass^3)" + +[defaults] +temperature = 0.7 +max_tokens = 4096 + +[judge] +model = "gpt-5-mini-2025-08-07" +temperature = 0.0 +engine = "cloud" + +[run] +max_workers = 1 +output_dir = "results/taubench-qwen35b/" +seed = 42 + +[[models]] +name = "Qwen/Qwen3.5-35B-A3B-FP8" +engine = "vllm" +num_gpus = 4 + +[[benchmarks]] +name = "taubench" +backend = "jarvis-direct" +split = "airline,retail" diff --git a/src/openjarvis/evals/configs/taubench-qwen397b.toml b/src/openjarvis/evals/configs/taubench-qwen397b.toml new file mode 100644 index 00000000..3989aab5 --- /dev/null +++ b/src/openjarvis/evals/configs/taubench-qwen397b.toml @@ -0,0 +1,30 @@ +# TauBench V2 eval: Qwen3.5-397B-A17B-FP8 (vLLM, TP=8) +# Multi-turn customer service benchmark across airline, retail + +[meta] +name = "taubench-qwen397b" +description = "TauBench V2 on Qwen/Qwen3.5-397B-A17B-FP8 (airline + retail)" + +[defaults] +temperature = 0.7 +max_tokens = 4096 + +[judge] +model = "gpt-5-mini-2025-08-07" +temperature = 0.0 +engine = "cloud" + +[run] +max_workers = 1 +output_dir = "results/taubench-qwen397b/" +seed = 42 + +[[models]] +name = "Qwen/Qwen3.5-397B-A17B-FP8" +engine = "vllm" +num_gpus = 8 + +[[benchmarks]] +name = "taubench" +backend = "jarvis-direct" +split = "airline,retail" diff --git a/src/openjarvis/evals/core/config.py b/src/openjarvis/evals/core/config.py index 7067477a..20aa9e04 100644 --- a/src/openjarvis/evals/core/config.py +++ b/src/openjarvis/evals/core/config.py @@ -61,6 +61,7 @@ KNOWN_BENCHMARKS = { "doc_qa", "browser_assistant", "pinchbench", + "taubench", } diff --git a/src/openjarvis/evals/datasets/taubench.py b/src/openjarvis/evals/datasets/taubench.py new file mode 100644 index 00000000..9cdc4757 --- /dev/null +++ b/src/openjarvis/evals/datasets/taubench.py @@ -0,0 +1,234 @@ +"""TauBench V2 dataset provider — multi-turn customer service benchmark. + +Wraps the tau2-bench framework for evaluation within OpenJarvis. +Supports airline, retail, and telecom domains. + +Reference: https://github.com/sierra-research/tau2-bench +""" + +from __future__ import annotations + +import logging +import subprocess +import sys +from pathlib import Path +from typing import Iterable, List, Optional + +from openjarvis.evals.core.dataset import DatasetProvider +from openjarvis.evals.core.types import EvalRecord + +LOGGER = logging.getLogger(__name__) + +TAU2_REPO = "https://github.com/sierra-research/tau2-bench.git" +CACHE_DIR = Path.home() / ".cache" / "tau2-bench" + +DOMAINS = ("airline", "retail", "telecom") + + +def _ensure_tau2() -> None: + """Ensure tau2 package is importable; install from cache if needed.""" + try: + import tau2 # noqa: F401 + except ImportError: + # Clone and install from source + if not CACHE_DIR.exists(): + LOGGER.info("Cloning tau2-bench from %s ...", TAU2_REPO) + CACHE_DIR.parent.mkdir(parents=True, exist_ok=True) + subprocess.run( + ["git", "clone", "--depth", "1", TAU2_REPO, str(CACHE_DIR)], + check=True, + capture_output=True, + ) + LOGGER.info("Installing tau2-bench ...") + subprocess.run( + [sys.executable, "-m", "pip", "install", "-e", str(CACHE_DIR)], + check=True, + capture_output=True, + ) + + +class TauBenchDataset(DatasetProvider): + """TauBench V2 multi-turn customer service benchmark. + + Wraps tau2-bench's task loading and evaluation infrastructure. + Each EvalRecord represents a single customer service scenario. + """ + + dataset_id = "taubench" + dataset_name = "TauBench" + + def __init__( + self, + domains: Optional[List[str]] = None, + ) -> None: + self._domains = domains or list(DOMAINS) + self._records: List[EvalRecord] = [] + self._engine_key: Optional[str] = None + self._model: Optional[str] = None + self._temperature: float = 0.7 + self._max_tokens: int = 4096 + self._user_model: Optional[str] = None + self._num_trials: int = 3 # pass^k: best of k trials per task + + def set_engine_config( + self, + engine_key: Optional[str] = None, + model: Optional[str] = None, + temperature: float = 0.7, + max_tokens: int = 4096, + user_model: Optional[str] = None, + num_trials: Optional[int] = None, + ) -> None: + """Inject engine configuration for the agent. Called by CLI.""" + if engine_key is not None: + self._engine_key = engine_key + if model is not None: + self._model = model + self._temperature = temperature + self._max_tokens = max_tokens + if user_model is not None: + self._user_model = user_model + if num_trials is not None: + self._num_trials = num_trials + + def verify_requirements(self) -> List[str]: + issues: List[str] = [] + try: + _ensure_tau2() + except Exception as exc: + issues.append(f"tau2-bench not available: {exc}") + return issues + + def load( + self, + *, + max_samples: Optional[int] = None, + split: Optional[str] = None, + seed: Optional[int] = None, + ) -> None: + _ensure_tau2() + from tau2.runner import get_tasks + + # split overrides domains if provided (e.g. "airline,retail") + domains = self._domains + if split: + domains = [d.strip() for d in split.split(",") if d.strip()] + + all_records: List[EvalRecord] = [] + + for domain in domains: + if domain not in DOMAINS: + LOGGER.warning("Unknown TauBench domain: %s", domain) + continue + + # Load tasks, filtering to test split when available + from tau2.runner import load_task_splits + try: + task_splits = load_task_splits(domain) + test_ids = ( + set(str(t) for t in task_splits.get("test", [])) + if task_splits + else set() + ) + except Exception: + test_ids = set() + + tasks = get_tasks(domain) + if test_ids: + tasks = [t for t in tasks if str(t.id) in test_ids] + LOGGER.info( + "TauBench: loaded %d test-split tasks for domain '%s'", + len(tasks), domain, + ) + else: + LOGGER.info( + "TauBench: loaded %d tasks for domain '%s' (no test split)", + len(tasks), domain, + ) + + for task in tasks: + # Build the user's reason for calling as the problem prompt + user_scenario = task.user_scenario + instructions = user_scenario.instructions + problem = ( + f"Domain: {domain}\n" + f"Reason for call: {instructions.reason_for_call}\n" + f"Known info: {instructions.known_info or 'None'}\n" + ) + + # Extract evaluation criteria + eval_criteria = task.evaluation_criteria + actions = [] + nl_assertions = [] + communicate_info = [] + reward_basis = [] + if eval_criteria: + actions = [ + a.model_dump() if hasattr(a, "model_dump") else a + for a in (eval_criteria.actions or []) + ] + nl_assertions = eval_criteria.nl_assertions or [] + communicate_info = [ + c.model_dump() if hasattr(c, "model_dump") else c + for c in (eval_criteria.communicate_info or []) + ] + reward_basis = [ + r.value if hasattr(r, "value") else r + for r in (eval_criteria.reward_basis or []) + ] + + record = EvalRecord( + record_id=f"{domain}_{task.id}", + problem=problem, + reference=task.description.purpose if task.description else "", + category=domain, + subject=f"taubench-{domain}", + metadata={ + "domain": domain, + "task_id": task.id, + "task_instructions": instructions.task_instructions, + "reason_for_call": instructions.reason_for_call, + "known_info": instructions.known_info, + "unknown_info": instructions.unknown_info, + "actions": actions, + "nl_assertions": nl_assertions, + "communicate_info": communicate_info, + "reward_basis": reward_basis, + }, + ) + all_records.append(record) + + if seed is not None: + import random + random.Random(seed).shuffle(all_records) + if max_samples is not None: + all_records = all_records[:max_samples] + + self._records = all_records + LOGGER.info( + "TauBench: loaded %d total tasks across %s", + len(self._records), + ", ".join(self._domains), + ) + + def iter_records(self) -> Iterable[EvalRecord]: + return iter(self._records) + + def size(self) -> int: + return len(self._records) + + def create_task_env(self, record: EvalRecord): + """Create a TauBench task environment for evaluation.""" + from openjarvis.evals.execution.taubench_env import TauBenchTaskEnv + return TauBenchTaskEnv( + record, + engine_key=self._engine_key, + model=self._model, + temperature=self._temperature, + max_tokens=self._max_tokens, + user_model=self._user_model, + num_trials=self._num_trials, + ) + + +__all__ = ["TauBenchDataset"] diff --git a/src/openjarvis/evals/execution/taubench_env.py b/src/openjarvis/evals/execution/taubench_env.py new file mode 100644 index 00000000..cd9b25a8 --- /dev/null +++ b/src/openjarvis/evals/execution/taubench_env.py @@ -0,0 +1,369 @@ +"""TauBench task environment — native OpenJarvis agent in tau2 simulation. + +Plugs OpenJarvis's inference engine into tau2-bench's orchestrator as a +``HalfDuplexAgent``, so the multi-turn conversation loop, user simulator, +domain tools, database, and evaluation all come from tau2-bench while the +agent's LLM calls go through OpenJarvis. +""" + +from __future__ import annotations + +import json +import logging +from types import TracebackType +from typing import Any, Optional, Type + +from openjarvis.evals.core.types import EvalRecord + +LOGGER = logging.getLogger(__name__) + + +# --------------------------------------------------------------------------- +# Message conversion helpers +# --------------------------------------------------------------------------- + +def _tau2_to_oj_messages( + tau2_messages: list, +) -> list: + """Convert tau2 Message objects to OpenJarvis Message objects.""" + from openjarvis.core.types import Message, Role + from openjarvis.core.types import ToolCall as OJToolCall + + oj_msgs: list = [] + for m in tau2_messages: + role_str = getattr(m, "role", "user") + if role_str == "system": + oj_msgs.append(Message(role=Role.SYSTEM, content=m.content or "")) + elif role_str == "user": + oj_msgs.append(Message(role=Role.USER, content=m.content or "")) + elif role_str == "assistant": + tc_list = getattr(m, "tool_calls", None) + if tc_list: + oj_tool_calls = [ + OJToolCall( + id=tc.id, + name=tc.name, + arguments=( + json.dumps(tc.arguments) + if isinstance(tc.arguments, dict) + else str(tc.arguments) + ), + ) + for tc in tc_list + ] + oj_msgs.append( + Message( + role=Role.ASSISTANT, + content=m.content or "", + tool_calls=oj_tool_calls, + ) + ) + else: + oj_msgs.append( + Message(role=Role.ASSISTANT, content=m.content or "") + ) + elif role_str == "tool": + # tau2 ToolMessage uses 'id', not 'tool_call_id' + raw_id = getattr(m, "id", "") or getattr(m, "tool_call_id", "") or "" + import re as _re + clean_id = _re.sub(r"[^a-zA-Z0-9_-]", "_", raw_id) + oj_msgs.append( + Message( + role=Role.TOOL, + content=m.content or "", + tool_call_id=clean_id, + name=getattr(m, "name", ""), + ) + ) + return oj_msgs + + +def _oj_result_to_tau2_msg(result: dict): + """Convert OpenJarvis engine.generate() result to a tau2 AssistantMessage.""" + from tau2.data_model.message import AssistantMessage, ToolCall + + raw_tool_calls = result.get("tool_calls", []) + tool_calls = None + if raw_tool_calls: + tool_calls = [ + ToolCall( + id=tc.get("id", ""), + name=tc.get("name", ""), + arguments=( + json.loads(tc["arguments"]) + if isinstance(tc.get("arguments"), str) + else tc.get("arguments", {}) + ), + requestor="assistant", + ) + for tc in raw_tool_calls + ] + + return AssistantMessage( + role="assistant", + content=result.get("content") or None, + tool_calls=tool_calls, + cost=result.get("cost_usd", 0.0), + ) + + +# --------------------------------------------------------------------------- +# Jarvis-powered tau2 agent +# --------------------------------------------------------------------------- + +class JarvisHalfDuplexAgent: + """A tau2 HalfDuplexAgent backed by OpenJarvis's inference engine. + + Replaces tau2's built-in LLMAgent while keeping the same interface + so the Orchestrator, UserSimulator, and evaluation work unchanged. + """ + + STOP = "###STOP###" + + def __init__( + self, + tools: list, + domain_policy: str, + engine: Any, + model: str, + temperature: float = 0.7, + max_tokens: int = 4096, + ) -> None: + self.tools = tools + self.domain_policy = domain_policy + self._engine = engine + self._model = model + self._temperature = temperature + self._max_tokens = max_tokens + + @property + def system_prompt(self) -> str: + from tau2.agent.llm_agent import AGENT_INSTRUCTION, SYSTEM_PROMPT + return SYSTEM_PROMPT.format( + domain_policy=self.domain_policy, + agent_instruction=AGENT_INSTRUCTION, + ) + + def get_init_state(self, message_history=None): + from tau2.agent.llm_agent import LLMAgentState + from tau2.data_model.message import SystemMessage + return LLMAgentState( + system_messages=[ + SystemMessage(role="system", content=self.system_prompt), + ], + messages=list(message_history) if message_history else [], + ) + + def generate_next_message(self, message, state): + from tau2.data_model.message import MultiToolMessage + + # Add incoming message to conversation state + if isinstance(message, MultiToolMessage): + state.messages.extend(message.tool_messages) + elif message is not None: + state.messages.append(message) + + # Convert to OpenJarvis format + oj_messages = _tau2_to_oj_messages( + state.system_messages + state.messages, + ) + + # Build OpenAI tool schemas from tau2 tools + openai_tools = [t.openai_schema for t in self.tools] + + # Call OpenJarvis engine + gen_kwargs: dict = { + "model": self._model, + "temperature": self._temperature, + "max_tokens": self._max_tokens, + "tools": openai_tools, + } + # Disable thinking mode for local models (Qwen3.5 etc.) + # to avoid tags interfering with tool call parsing + if "qwen" in self._model.lower(): + gen_kwargs["chat_template_kwargs"] = { + "enable_thinking": False, + } + result = self._engine.generate(oj_messages, **gen_kwargs) + + # Convert result to tau2 AssistantMessage + assistant_msg = _oj_result_to_tau2_msg(result) + state.messages.append(assistant_msg) + return assistant_msg, state + + @classmethod + def is_stop(cls, message) -> bool: + if hasattr(message, "is_tool_call") and message.is_tool_call(): + return False + content = getattr(message, "content", "") or "" + return cls.STOP in content + + def set_seed(self, seed: int) -> None: + pass + + def stop(self, *args, **kwargs) -> None: + """Cleanup hook called by orchestrator.""" + pass + + +# --------------------------------------------------------------------------- +# Task environment +# --------------------------------------------------------------------------- + +class TauBenchTaskEnv: + """Per-task environment for TauBench evaluation. + + Creates an OpenJarvis-powered agent, plugs it into tau2's orchestrator, + runs the simulation, and stores results in record.metadata for the scorer. + """ + + def __init__( + self, + record: EvalRecord, + engine_key: Optional[str] = None, + model: Optional[str] = None, + temperature: float = 0.7, + max_tokens: int = 4096, + user_model: Optional[str] = None, + num_trials: int = 1, + ) -> None: + self._record = record + self._num_trials = num_trials + self._engine_key = engine_key + self._model = model or "claude-opus-4-6" + self._temperature = temperature + self._max_tokens = max_tokens + self._user_model = user_model or "gpt-5-mini-2025-08-07" + self._system = None + + def __enter__(self) -> TauBenchTaskEnv: + # Build OpenJarvis system for engine access + from openjarvis.system import SystemBuilder + + builder = SystemBuilder() + if self._engine_key: + builder.engine(self._engine_key) + self._system = builder.build() + + # Run the simulation + self._run_simulation() + return self + + def _run_simulation(self) -> None: + from tau2.evaluator.evaluator import EvaluationType + from tau2.orchestrator.orchestrator import Orchestrator + from tau2.runner.build import build_environment, build_user + from tau2.runner.helpers import get_tasks + from tau2.runner.simulation import run_simulation + + domain = self._record.metadata["domain"] + task_id = self._record.metadata["task_id"] + + tasks = get_tasks(domain, task_ids=[task_id]) + if not tasks: + LOGGER.error("Task %s not found in domain %s", task_id, domain) + self._record.metadata["tau_reward"] = 0.0 + self._record.metadata["is_resolved"] = False + return + + task = tasks[0] + best_reward = 0.0 + best_info: dict = {} + best_n_messages = 0 + + # Run multiple trials, keep best result (pass^k) + for trial in range(self._num_trials): + try: + environment = build_environment(domain) + + agent = JarvisHalfDuplexAgent( + tools=environment.get_tools(), + domain_policy=environment.get_policy(), + engine=self._system.engine, + model=self._model, + temperature=self._temperature, + max_tokens=self._max_tokens, + ) + + user = build_user( + "user_simulator", + environment, + task, + llm=self._user_model, + ) + + orchestrator = Orchestrator( + domain=domain, + agent=agent, + user=user, + environment=environment, + task=task, + max_steps=200, + seed=42 + trial, + ) + + simulation = run_simulation( + orchestrator, + evaluation_type=EvaluationType.ALL_WITH_NL_ASSERTIONS, + ) + + reward = ( + simulation.reward_info.reward + if simulation.reward_info + else 0.0 + ) + info: dict = {} + if simulation.reward_info: + info = simulation.reward_info.info or {} + if simulation.reward_info.reward_breakdown: + info["reward_breakdown"] = { + k.value if hasattr(k, "value") else str(k): v + for k, v in simulation.reward_info.reward_breakdown.items() + } + n_messages = len(simulation.messages) if simulation.messages else 0 + + trial_label = ( + f" (trial {trial + 1}/{self._num_trials})" + if self._num_trials > 1 else "" + ) + LOGGER.info( + "TauBench %s/%s: reward=%.2f messages=%d%s", + domain, task_id, reward, n_messages, trial_label, + ) + + if reward > best_reward: + best_reward = reward + best_info = info + best_n_messages = n_messages + + # Early exit if perfect score + if reward >= 1.0: + break + + except Exception as exc: + LOGGER.error( + "TauBench simulation failed for %s/%s: %s", + domain, task_id, exc, + ) + + self._record.metadata["tau_reward"] = best_reward + self._record.metadata["tau_info"] = best_info + self._record.metadata["tau_n_messages"] = best_n_messages + self._record.metadata["tau_num_trials"] = self._num_trials + self._record.metadata["is_resolved"] = best_reward >= 0.5 + + def __exit__( + self, + exc_type: Optional[Type[BaseException]], + exc_val: Optional[BaseException], + exc_tb: Optional[TracebackType], + ) -> None: + if self._system: + try: + self._system.close() + except Exception: + pass + self._system = None + + +__all__ = ["TauBenchTaskEnv"] diff --git a/src/openjarvis/evals/scorers/taubench.py b/src/openjarvis/evals/scorers/taubench.py new file mode 100644 index 00000000..227640de --- /dev/null +++ b/src/openjarvis/evals/scorers/taubench.py @@ -0,0 +1,46 @@ +"""TauBench scorer — wraps tau2-bench's evaluation results. + +Since TauBench runs its own simulation loop (agent + user simulator + +tools + evaluation), the scorer simply reads the reward that was +computed during task execution and stored in record.metadata. +""" + +from __future__ import annotations + +from typing import Any, Dict, Optional, Tuple + +from openjarvis.evals.core.scorer import Scorer +from openjarvis.evals.core.types import EvalRecord + + +class TauBenchScorer(Scorer): + """TauBench scorer — reads pre-computed rewards from tau2-bench. + + The actual evaluation (DB state checks, action matching, + communication checks, NL assertions) is done by tau2-bench's + evaluator during simulation. This scorer extracts the result. + """ + + scorer_id = "taubench" + + def __init__(self, judge_backend: Any = None, judge_model: str = "") -> None: + self._judge_backend = judge_backend + self._judge_model = judge_model + + def score( + self, record: EvalRecord, model_answer: str, + ) -> Tuple[Optional[bool], Dict[str, Any]]: + reward = record.metadata.get("tau_reward", 0.0) + info = record.metadata.get("tau_info", {}) + n_messages = record.metadata.get("tau_n_messages", 0) + + is_correct = reward >= 0.5 + + return is_correct, { + "score": reward, + "breakdown": info, + "notes": f"reward={reward:.2f}, messages={n_messages}", + } + + +__all__ = ["TauBenchScorer"] diff --git a/tests/evals/test_benchmark_datasets.py b/tests/evals/test_benchmark_datasets.py index e569fa48..ab856d4d 100644 --- a/tests/evals/test_benchmark_datasets.py +++ b/tests/evals/test_benchmark_datasets.py @@ -313,7 +313,7 @@ class TestConfigBenchmarks: def test_benchmarks_count(self) -> None: from openjarvis.evals.core.config import KNOWN_BENCHMARKS - assert len(KNOWN_BENCHMARKS) == 26 + assert len(KNOWN_BENCHMARKS) == 27 # --------------------------------------------------------------------------- From c4f85d435ff004769616d08abd6963d534b8f91d Mon Sep 17 00:00:00 2001 From: Avanika Narayan Date: Tue, 31 Mar 2026 19:29:55 -0700 Subject: [PATCH 04/22] fix(evals): add tool_choice=auto + fix traces thread safety (#163) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two fixes for eval accuracy and stability: 1. TauBench agent: add tool_choice="auto" to match tau2's native LLMAgent behavior. Without this, Qwen and GPT-5.4 score 10-14pp below leaderboard because the models don't receive explicit tool-calling guidance. 2. SystemBuilder: apply self._traces flag to config.traces.enabled. Previously builder.traces(False) was a no-op — traces stayed enabled, creating SQLite connections in the main thread that crashed when accessed from ThreadPoolExecutor worker threads in GAIA evals ("SQLite objects created in a thread can only be used in that same thread"). Co-authored-by: Jon Saad-Falcon Co-authored-by: Claude Opus 4.6 (1M context) --- src/openjarvis/evals/execution/taubench_env.py | 1 + src/openjarvis/system.py | 7 ++++++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/src/openjarvis/evals/execution/taubench_env.py b/src/openjarvis/evals/execution/taubench_env.py index cd9b25a8..34f838a1 100644 --- a/src/openjarvis/evals/execution/taubench_env.py +++ b/src/openjarvis/evals/execution/taubench_env.py @@ -177,6 +177,7 @@ class JarvisHalfDuplexAgent: "temperature": self._temperature, "max_tokens": self._max_tokens, "tools": openai_tools, + "tool_choice": "auto", } # Disable thinking mode for local models (Qwen3.5 etc.) # to avoid tags interfering with tool call parsing diff --git a/src/openjarvis/system.py b/src/openjarvis/system.py index a9ade2f8..92dbf40c 100644 --- a/src/openjarvis/system.py +++ b/src/openjarvis/system.py @@ -503,10 +503,15 @@ class SystemBuilder: # Resolve model model = self._resolve_model(config, engine) - # Compute telemetry_enabled once + # Compute telemetry_enabled and traces_enabled once telemetry_enabled = ( self._telemetry if self._telemetry is not None else config.telemetry.enabled ) + traces_enabled = ( + self._traces if self._traces is not None else config.traces.enabled + ) + # Apply traces flag to config so downstream code respects it + config.traces.enabled = traces_enabled gpu_monitor = None energy_monitor = None if telemetry_enabled and config.telemetry.gpu_metrics: From dc2a2edbcd77127673e1eddcbdcc2439797dfd7a Mon Sep 17 00:00:00 2001 From: Jon Saad-Falcon <41205309+jonsaadfalcon@users.noreply.github.com> Date: Tue, 31 Mar 2026 20:39:59 -0700 Subject: [PATCH 05/22] fix: data source connect flow UX, obsidian/gcalendar sync bugs, agent timeout MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Frontend: add progress stages (Connecting → Authenticating → Connected → Syncing) with spinner and progress bar to the data source connect flow. Previously sources would silently stay "Not connected" after setup. Show error message on failure instead of swallowing exceptions. - Obsidian connector: use timezone-aware datetime (tz=timezone.utc) in fromtimestamp() to fix "can't compare offset-naive and offset-aware datetimes" crash during incremental sync. - Google Calendar connector: catch HTTPStatusError when listing events for individual calendars (e.g. US Holidays returning 404) so one inaccessible calendar doesn't crash the entire sync. - Agent SSE timeout: increase progress queue timeout from 120s to 600s so complex multi-hop deep research queries aren't killed mid-execution on slower local models. Co-Authored-By: Claude Opus 4.6 (1M context) --- frontend/src/pages/DataSourcesPage.tsx | 83 +++++++++++++++++-- src/openjarvis/connectors/gcalendar.py | 9 +- src/openjarvis/connectors/obsidian.py | 4 +- src/openjarvis/server/agent_manager_routes.py | 2 +- 4 files changed, 84 insertions(+), 14 deletions(-) diff --git a/frontend/src/pages/DataSourcesPage.tsx b/frontend/src/pages/DataSourcesPage.tsx index 5f61176d..b09d9d41 100644 --- a/frontend/src/pages/DataSourcesPage.tsx +++ b/frontend/src/pages/DataSourcesPage.tsx @@ -81,7 +81,7 @@ function InlineConnectForm({ borderRadius: 6, fontSize: 12, cursor: 'pointer', }} > - {loading ? 'Connecting...' : 'Connect'} + Connect ); @@ -327,20 +327,54 @@ function DataSourcesSection() { } }, [connectors, loadSyncStatuses]); + const [connectingId, setConnectingId] = useState(null); + const [connectStage, setConnectStage] = useState(''); + const [connectError, setConnectError] = useState(''); + const handleConnect = async (id: string, req: ConnectRequest) => { setLoading(true); + setConnectingId(id); + setConnectStage('Connecting...'); + setConnectError(''); try { await connectSource(id, req); - setExpandedId(null); - for (let i = 0; i < 30; i++) { - await new Promise((r) => setTimeout(r, 3000)); - await loadConnectors(); + setConnectStage('Connected! Starting sync...'); + + // Wait for connector to show as connected + for (let i = 0; i < 20; i++) { + await new Promise((r) => setTimeout(r, 2000)); const updated = await listConnectors(); const target = updated.find((c) => c.connector_id === id); - if (target?.connected) break; + if (target?.connected) { + setConnectors(updated.map((c) => ({ + connector_id: c.connector_id, + display_name: c.display_name, + connected: c.connected, + chunks: (c as any).chunks || 0, + }))); + break; + } + setConnectStage(i < 5 ? 'Authenticating...' : 'Waiting for connection...'); } - } catch { /* */ } finally { + + // Trigger sync + setConnectStage('Syncing data...'); + try { + await triggerSync(id); + } catch { /* sync may already be running */ } + + // Close form after a brief moment + await new Promise((r) => setTimeout(r, 1500)); + setExpandedId(null); + loadConnectors(); + loadSyncStatuses(); + } catch (err: any) { + setConnectError(err.message || 'Connection failed'); + setConnectStage(''); + } finally { setLoading(false); + setConnectingId(null); + setConnectStage(''); } }; @@ -524,10 +558,43 @@ function DataSourcesSection() { {meta?.inputFields && ( handleConnect(c.connector_id, req)} /> )} + {/* Connection progress */} + {connectingId === c.connector_id && connectStage && ( +
+
+
+ {connectStage} +
+
+
+
+
+ )} + {/* Connection error */} + {connectError && connectingId === null && expandedId === c.connector_id && ( +
+ {connectError} +
+ )}
)}
diff --git a/src/openjarvis/connectors/gcalendar.py b/src/openjarvis/connectors/gcalendar.py index 5b7a85c3..bc730d96 100644 --- a/src/openjarvis/connectors/gcalendar.py +++ b/src/openjarvis/connectors/gcalendar.py @@ -316,9 +316,12 @@ class GCalendarConnector(BaseConnector): page_token: Optional[str] = cursor while True: - events_resp = _gcal_api_events_list( - token, calendar_id, page_token=page_token - ) + try: + events_resp = _gcal_api_events_list( + token, calendar_id, page_token=page_token + ) + except httpx.HTTPStatusError: + break events: List[Dict[str, Any]] = events_resp.get("items", []) for event in events: diff --git a/src/openjarvis/connectors/obsidian.py b/src/openjarvis/connectors/obsidian.py index 1625d146..db8e6a9e 100644 --- a/src/openjarvis/connectors/obsidian.py +++ b/src/openjarvis/connectors/obsidian.py @@ -8,7 +8,7 @@ can be ingested by the knowledge pipeline. from __future__ import annotations import os -from datetime import datetime +from datetime import datetime, timezone from pathlib import Path from typing import Any, Dict, Iterator, List, Optional, Tuple from urllib.parse import quote @@ -161,7 +161,7 @@ class ObsidianConnector(BaseConnector): for fpath in collected_paths: # Apply since filter based on mtime - mtime = datetime.fromtimestamp(fpath.stat().st_mtime) + mtime = datetime.fromtimestamp(fpath.stat().st_mtime, tz=timezone.utc) if since is not None and mtime < since: continue diff --git a/src/openjarvis/server/agent_manager_routes.py b/src/openjarvis/server/agent_manager_routes.py index ae4b1e73..ef41b765 100644 --- a/src/openjarvis/server/agent_manager_routes.py +++ b/src/openjarvis/server/agent_manager_routes.py @@ -719,7 +719,7 @@ async def _stream_managed_agent( # Stream progress events and final content while True: try: - event = await asyncio.to_thread(progress_q.get, timeout=120) + event = await asyncio.to_thread(progress_q.get, timeout=600) except Exception: # Timeout yield _sse_chunk(chunk_id, model, "Agent timed out.") From 6cd6ba78258214ba7f50381935f22e02da22f648 Mon Sep 17 00:00:00 2001 From: Prathap <436prathap@gmail.com> Date: Sun, 29 Mar 2026 13:30:57 +0530 Subject: [PATCH 06/22] include agent from config and include channel tools before creating jarvis system --- src/openjarvis/cli/serve.py | 47 ++++++++++++++++++++++++++++++++++++- 1 file changed, 46 insertions(+), 1 deletion(-) diff --git a/src/openjarvis/cli/serve.py b/src/openjarvis/cli/serve.py index 066d6ecc..998a0c68 100644 --- a/src/openjarvis/cli/serve.py +++ b/src/openjarvis/cli/serve.py @@ -253,13 +253,58 @@ def serve( if channel_bridge is not None: from openjarvis.system import JarvisSystem + channel_agent = config.channel.default_agent or agent_key or "simple" + + _channel_tools: list = [] + if channel_agent: + try: + import openjarvis.agents + from openjarvis.core.registry import AgentRegistry + + if AgentRegistry.contains(channel_agent): + _ch_cls = AgentRegistry.get(channel_agent) + if getattr(_ch_cls, "accepts_tools", False): + import openjarvis.tools + from openjarvis.core.registry import ToolRegistry + from openjarvis.tools._stubs import BaseTool + + _DEFAULT_TOOLS = {"think", "calculator", "web_search"} + configured = config.agent.tools + if configured: + if isinstance(configured, list): + _allowed = { + t.strip() + for t in configured + if isinstance(t, str) and t.strip() + } + else: + _allowed = { + t.strip() + for t in configured.split(",") + if t.strip() + } + else: + _allowed = _DEFAULT_TOOLS + + for _tname in ToolRegistry.keys(): + if _tname not in _allowed: + continue + _tcls = ToolRegistry.get(_tname) + if isinstance(_tcls, type) and issubclass(_tcls, BaseTool): + _channel_tools.append(_tcls()) + elif isinstance(_tcls, BaseTool): + _channel_tools.append(_tcls) + except Exception as exc: + logger.warning("Channel tools failed to load: %s", exc) + _wire_system = JarvisSystem( config=config, bus=bus, engine=engine, engine_key=engine_name, model=model_name, - agent_name=agent_key or "", + agent_name=channel_agent, + tools=_channel_tools, ) _wire_system.wire_channel(channel_bridge) From b58d7e63434ce13cae0c26972cacefb09a9cfffb Mon Sep 17 00:00:00 2001 From: Prathap <436prathap@gmail.com> Date: Sun, 29 Mar 2026 13:31:28 +0530 Subject: [PATCH 07/22] add unit tests --- tests/cli/test_serve_channel_wiring.py | 48 ++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) diff --git a/tests/cli/test_serve_channel_wiring.py b/tests/cli/test_serve_channel_wiring.py index 81feb254..4542bf2c 100644 --- a/tests/cli/test_serve_channel_wiring.py +++ b/tests/cli/test_serve_channel_wiring.py @@ -194,6 +194,54 @@ class TestWireChannelErrorHandling: assert "error" in sent_content.lower() +class TestChannelToolLoading: + """JarvisSystem receives tools when agent accepts them.""" + + def test_tool_using_agent_receives_tools(self, tmp_path): + """JarvisSystem built with a tool list passes tools to the agent via ask().""" + from openjarvis.tools._stubs import BaseTool, ToolSpec + + # Minimal fake tool + class _FakeTool(BaseTool): + spec = ToolSpec(name="fake", description="", parameters={}) + + def execute(self, **_): # type: ignore[override] + pass + + fake_tool = _FakeTool() + config = JarvisConfig() + config.sessions.db_path = str(tmp_path / "sessions.db") + + system = JarvisSystem( + config=config, + bus=EventBus(record_history=False), + engine=MagicMock(), + engine_key="mock", + model="test-model", + agent_name="simple", + tools=[fake_tool], + ) + + assert len(system.tools) == 1 + assert system.tools[0].spec.name == "fake" + + def test_non_tool_agent_receives_empty_tools(self, tmp_path): + """JarvisSystem with no tools list results in empty tools — simple agent unaffected.""" + config = JarvisConfig() + config.sessions.db_path = str(tmp_path / "sessions.db") + + system = JarvisSystem( + config=config, + bus=EventBus(record_history=False), + engine=MagicMock(), + engine_key="mock", + model="test-model", + agent_name="simple", + ) + + assert system.tools == [] + + class TestPerChatSessionIsolation: """Direct SessionStore isolation tests (not via wire_channel).""" From d2cc994d75eb4d55252b4730e20af95d246e4c5d Mon Sep 17 00:00:00 2001 From: Prathap <436prathap@gmail.com> Date: Sun, 29 Mar 2026 16:31:29 +0530 Subject: [PATCH 08/22] fix lint issues --- tests/cli/test_serve_channel_wiring.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/cli/test_serve_channel_wiring.py b/tests/cli/test_serve_channel_wiring.py index 4542bf2c..bf199d8e 100644 --- a/tests/cli/test_serve_channel_wiring.py +++ b/tests/cli/test_serve_channel_wiring.py @@ -226,7 +226,8 @@ class TestChannelToolLoading: assert system.tools[0].spec.name == "fake" def test_non_tool_agent_receives_empty_tools(self, tmp_path): - """JarvisSystem with no tools list results in empty tools — simple agent unaffected.""" + """JarvisSystem with no tools list results in empty tools — simple agent + unaffected.""" config = JarvisConfig() config.sessions.db_path = str(tmp_path / "sessions.db") From 579ff50256476c814b025914b27a86a9a6d4ebab Mon Sep 17 00:00:00 2001 From: Jon Saad-Falcon <41205309+jonsaadfalcon@users.noreply.github.com> Date: Tue, 31 Mar 2026 21:29:51 -0700 Subject: [PATCH 09/22] feat: add chat/edit/delete buttons and model status to agent cards Co-Authored-By: Claude Opus 4.6 (1M context) --- frontend/src/pages/AgentsPage.tsx | 127 +++++++++++++++++++++++++++++- 1 file changed, 125 insertions(+), 2 deletions(-) diff --git a/frontend/src/pages/AgentsPage.tsx b/frontend/src/pages/AgentsPage.tsx index aa5ab4aa..bc46c9df 100644 --- a/frontend/src/pages/AgentsPage.tsx +++ b/frontend/src/pages/AgentsPage.tsx @@ -57,6 +57,7 @@ import { Database, Copy, Check, + Pencil, } from 'lucide-react'; import { SOURCE_CATALOG } from '../types/connectors'; import type { ConnectRequest } from '../types/connectors'; @@ -715,6 +716,8 @@ function AgentCard({ onRun, onRecover, onDelete, + onChat, + onEdit, }: { agent: ManagedAgent; onClick: () => void; @@ -723,6 +726,8 @@ function AgentCard({ onRun: (id: string) => void; onRecover: (id: string) => void; onDelete: (id: string) => void; + onChat: (id: string) => void; + onEdit: (id: string) => void; }) { const canPause = agent.status === 'running' || agent.status === 'idle'; const canResume = agent.status === 'paused'; @@ -794,6 +799,22 @@ function AgentCard({ {/* Row 4: Actions */}
e.stopPropagation()}> + + @@ -3028,6 +3129,20 @@ export function AgentsPage() { Recover )} +
@@ -3331,6 +3446,14 @@ export function AgentsPage() { onRun={handleRun} onRecover={handleRecover} onDelete={handleDelete} + onChat={(id) => { + setSelectedAgentId(id); + setDetailTab('interact'); + }} + onEdit={(id) => { + setSelectedAgentId(id); + setDetailTab('overview'); + }} /> ))} From 5c1990b7ae6079e6b670db0ae7945e1e6834c8af Mon Sep 17 00:00:00 2001 From: Jon Saad-Falcon <41205309+jonsaadfalcon@users.noreply.github.com> Date: Wed, 1 Apr 2026 11:12:03 -0700 Subject: [PATCH 10/22] feat: add daily/weekly/hourly schedule presets for agents Replace the manual/interval/cron schedule picker with friendlier presets (Daily, Weekly, Every N hours, Custom cron) that generate the correct cron expressions or interval values behind the scenes. Update formatSchedule() to render human-readable labels like "Daily at 9:00 AM" and "Weekly on Mon, Wed, Fri at 9:00 AM". Co-Authored-By: Claude Opus 4.6 (1M context) --- frontend/src/pages/AgentsPage.tsx | 164 ++++++++++++++++++++++-------- 1 file changed, 124 insertions(+), 40 deletions(-) diff --git a/frontend/src/pages/AgentsPage.tsx b/frontend/src/pages/AgentsPage.tsx index bc46c9df..baf728d0 100644 --- a/frontend/src/pages/AgentsPage.tsx +++ b/frontend/src/pages/AgentsPage.tsx @@ -133,7 +133,32 @@ function formatRelativeTime(ts?: number | null): string { function formatSchedule(type?: string, value?: string): string { if (!type || type === 'manual') return 'Manual'; - if (type === 'cron') return value ? `Cron: ${value}` : 'Cron'; + if (type === 'cron' && value) { + // Try to display human-readable for common cron patterns + const parts = value.trim().split(/\s+/); + if (parts.length === 5) { + const [min, hour, , , dow] = parts; + const hourNum = parseInt(hour, 10); + const formatHour = (h: number) => { + if (h === 0) return '12:00 AM'; + if (h < 12) return `${h}:00 AM`; + if (h === 12) return '12:00 PM'; + return `${h - 12}:00 PM`; + }; + // Daily pattern: 0 H * * * + if (min === '0' && !isNaN(hourNum) && parts[2] === '*' && parts[3] === '*' && dow === '*') { + return `Daily at ${formatHour(hourNum)}`; + } + // Weekly pattern: 0 H * * days + if (min === '0' && !isNaN(hourNum) && parts[2] === '*' && parts[3] === '*' && dow !== '*') { + const DAY_NAMES: Record = { '1': 'Mon', '2': 'Tue', '3': 'Wed', '4': 'Thu', '5': 'Fri', '6': 'Sat', '7': 'Sun' }; + const dayList = dow.split(',').map(d => DAY_NAMES[d] || d).join(', '); + return `Weekly on ${dayList} at ${formatHour(hourNum)}`; + } + } + return `Cron: ${value}`; + } + if (type === 'cron') return 'Cron'; if (type === 'interval' && value) { const total = parseInt(value); if (!isNaN(total) && total > 0) { @@ -327,9 +352,20 @@ function LaunchWizard({ if (!wizard.name.trim()) { toast.error('Name is required'); return; } setLaunching(true); try { + // Map friendly schedule presets to API schedule_type/schedule_value + let apiScheduleType = wizard.scheduleType; + let apiScheduleValue = wizard.scheduleValue; + if (wizard.scheduleType === 'daily' || wizard.scheduleType === 'weekly') { + apiScheduleType = 'cron'; + // scheduleValue already holds the cron expression + } else if (wizard.scheduleType === 'hourly') { + apiScheduleType = 'interval'; + // scheduleValue already holds seconds as string + } + const config: Record = { - schedule_type: wizard.scheduleType, - schedule_value: wizard.scheduleValue || undefined, + schedule_type: apiScheduleType, + schedule_value: apiScheduleValue || undefined, tools: wizard.selectedTools, learning_enabled: !!wizard.routerPolicy, memory_extraction: wizard.memoryExtraction, @@ -491,9 +527,87 @@ function LaunchWizard({ style={{ background: 'var(--color-bg-secondary)', border: '1px solid var(--color-border)', color: 'var(--color-text)' }} > - - + + + + + {wizard.scheduleType === 'daily' && ( + + )} + {wizard.scheduleType === 'weekly' && ( +
+
+ {(['Mon','Tue','Wed','Thu','Fri','Sat','Sun'] as const).map((day, idx) => { + const dayNum = String(idx + 1); + const cronParts = wizard.scheduleValue.match(/\*\s+\*\s+(.+)$/); + const selectedDays = cronParts ? cronParts[1].split(',') : []; + const isSelected = selectedDays.includes(dayNum); + return ( + + ); + })} +
+ +
+ )} + {wizard.scheduleType === 'hourly' && ( +
+ Every + { const secs = parseInt(wizard.scheduleValue || '0', 10); return secs > 0 ? Math.round(secs / 3600) : 1; })()} + onChange={(e) => { + const hrs = Math.min(24, Math.max(1, parseInt(e.target.value, 10) || 1)); + setWizard((w) => ({ ...w, scheduleValue: String(hrs * 3600) })); + }} + className="w-14 px-2 py-1 rounded text-xs text-center" + style={{ background: 'var(--color-bg)', border: '1px solid var(--color-border)', color: 'var(--color-text)' }} + /> + hours +
+ )} {wizard.scheduleType === 'cron' && ( )} - {wizard.scheduleType === 'interval' && ( -
-
- { - const hrs = Math.min(24, Math.max(0, parseInt(e.target.value, 10) || 0)); - const mins = Math.floor((parseInt(wizard.scheduleValue || '0', 10) % 3600) / 60); - setWizard((w) => ({ ...w, scheduleValue: String(hrs * 3600 + mins * 60) })); - }} - className="w-14 px-2 py-1 rounded text-xs text-center" - style={{ background: 'var(--color-bg)', border: '1px solid var(--color-border)', color: 'var(--color-text)' }} - /> - hrs -
-
- { - const hrs = Math.floor(parseInt(wizard.scheduleValue || '0', 10) / 3600); - const mins = Math.min(59, Math.max(0, parseInt(e.target.value, 10) || 0)); - setWizard((w) => ({ ...w, scheduleValue: String(hrs * 3600 + mins * 60) })); - }} - className="w-14 px-2 py-1 rounded text-xs text-center" - style={{ background: 'var(--color-bg)', border: '1px solid var(--color-border)', color: 'var(--color-text)' }} - /> - min -
-
- )} @@ -616,11 +698,13 @@ function LaunchWizard({
- setWizard((w) => ({ ...w, scheduleType: e.target.value, scheduleValue: e.target.value === 'manual' ? '' : w.scheduleValue }))} className="w-full px-2 py-1 rounded text-xs" style={{ background: 'var(--color-bg)', border: '1px solid var(--color-border)', color: 'var(--color-text)' }}> - - + + + +
From aaab50ffcf5a7f13bbc0973d96f89b7e06cf9064 Mon Sep 17 00:00:00 2001 From: Jon Saad-Falcon <41205309+jonsaadfalcon@users.noreply.github.com> Date: Wed, 1 Apr 2026 11:12:53 -0700 Subject: [PATCH 11/22] feat: add pre-filled instruction prompts to agent templates Add TEMPLATE_INSTRUCTIONS map with sensible defaults for daily-briefing, research-monitor, code-reviewer, and meeting-prep templates so users get a starting prompt when selecting a template. Show a warning hint when the instruction contains [bracketed placeholders] that need to be replaced. Co-Authored-By: Claude Opus 4.6 (1M context) --- frontend/src/pages/AgentsPage.tsx | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/frontend/src/pages/AgentsPage.tsx b/frontend/src/pages/AgentsPage.tsx index baf728d0..43afdbed 100644 --- a/frontend/src/pages/AgentsPage.tsx +++ b/frontend/src/pages/AgentsPage.tsx @@ -266,6 +266,13 @@ interface WizardState { } +const TEMPLATE_INSTRUCTIONS: Record = { + 'daily-briefing': 'Every morning, give me a fun quote of the day, summarize my top important emails, list any meetings today from my calendar, and tell me the weather for [my city].', + 'research-monitor': 'Search for the latest news and papers on [your topic]. Summarize the top 3 most relevant findings and explain why they matter.', + 'code-reviewer': 'Review the latest commits in [repo]. Check for bugs, security issues, and style violations. Summarize findings with file paths and line numbers.', + 'meeting-prep': 'Before my next meeting, pull context from my emails, messages, and past meetings with the attendees. Summarize key topics and suggest talking points.', +}; + function LaunchWizard({ templates, onClose, @@ -319,7 +326,7 @@ function LaunchWizard({ templateId: tpl.id, templateData: tpl, name: '', - instruction: '', + instruction: (tpl as any).instruction || TEMPLATE_INSTRUCTIONS[tpl.id] || '', model: recommendedModel || w.model, scheduleType: (tpl as any).schedule_type || 'manual', scheduleValue: (tpl as any).schedule_value || '', @@ -499,6 +506,11 @@ function LaunchWizard({ className="w-full px-3 py-2 rounded-lg text-sm bg-transparent resize-none" style={{ border: '1px solid var(--color-border)', color: 'var(--color-text)' }} /> + {wizard.instruction.includes('[') && ( +

+ Replace the [bracketed text] with your own values +

+ )} {/* Model + Schedule row */} From beb5c0fb6cc6631feb4c8f5933be76cc8a3ff9dc Mon Sep 17 00:00:00 2001 From: Jon Saad-Falcon <41205309+jonsaadfalcon@users.noreply.github.com> Date: Wed, 1 Apr 2026 11:17:26 -0700 Subject: [PATCH 12/22] feat: reorder agent tabs (Interact first), add strategy tooltips Move the Interact tab to the first position in agent detail view and default to it when opening an agent. Add "(optional)" hint to Advanced Settings and add (?) tooltips to Memory Extraction, Observation Compression, Retrieval Strategy, and Task Decomposition labels. Co-Authored-By: Claude Opus 4.6 (1M context) --- frontend/src/pages/AgentsPage.tsx | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/frontend/src/pages/AgentsPage.tsx b/frontend/src/pages/AgentsPage.tsx index 43afdbed..8aacc0b6 100644 --- a/frontend/src/pages/AgentsPage.tsx +++ b/frontend/src/pages/AgentsPage.tsx @@ -273,6 +273,10 @@ const TEMPLATE_INSTRUCTIONS: Record = { 'meeting-prep': 'Before my next meeting, pull context from my emails, messages, and past meetings with the attendees. Summarize key topics and suggest talking points.', }; +function Tooltip({ text }: { text: string }) { + return (?); +} + function LaunchWizard({ templates, onClose, @@ -649,12 +653,12 @@ function LaunchWizard({ {/* Advanced Settings */}
- Advanced Settings + Advanced Settings (optional)
- +
- +
- +
- + setTitle(e.target.value)} + placeholder="Title (optional)" + style={inputStyle} + /> + + {tab === 'paste' && ( + <> +