diff --git a/docs/javascripts/leaderboard.js b/docs/javascripts/leaderboard.js index b1a04598..7d2ffc41 100644 --- a/docs/javascripts/leaderboard.js +++ b/docs/javascripts/leaderboard.js @@ -5,6 +5,10 @@ var SUPABASE_ANON_KEY = "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiJzdXBhYmFzZSIsInJlZiI6Im10YnRncHd6cmJvc3R3ZWFhbnByIiwicm9sZSI6ImFub24iLCJpYXQiOjE3NzMxODk0OTQsImV4cCI6MjA4ODc2NTQ5NH0._xMlqCfljtXpwPj54H-ghxfLFO-jiq4W2WhpU8vVL1c"; + var PAGE_SIZE = 50; + var allRows = []; + var currentPage = 0; + function escapeHtml(s) { var el = document.createElement("span"); el.textContent = s; @@ -19,6 +23,67 @@ return n.toLocaleString(); } + function totalPages() { + return Math.max(1, Math.ceil(allRows.length / PAGE_SIZE)); + } + + function renderPage() { + var tbody = document.getElementById("leaderboard-body"); + if (!tbody) return; + + var start = currentPage * PAGE_SIZE; + var end = Math.min(start + PAGE_SIZE, allRows.length); + var pageRows = allRows.slice(start, end); + + var html = ""; + for (var j = 0; j < pageRows.length; j++) { + var rank = start + j + 1; + var rankClass = rank <= 3 ? " lb-rank-" + rank : ""; + var medal = + rank === 1 ? "\uD83E\uDD47" : rank === 2 ? "\uD83E\uDD48" : rank === 3 ? "\uD83E\uDD49" : ""; + var row = pageRows[j]; + html += + "" + + '' + (medal || rank) + "" + + '' + escapeHtml(row.display_name) + "" + + '$' + Number(row.dollar_savings || 0).toFixed(4) + "" + + '' + Number(row.energy_wh_saved || 0).toFixed(2) + "" + + '' + fmtLarge(Number(row.flops_saved || 0)) + "" + + '' + Number(row.total_calls || 0).toLocaleString() + "" + + '' + Number(row.total_tokens || 0).toLocaleString() + "" + + ""; + } + tbody.innerHTML = html; + + renderPagination(); + } + + function renderPagination() { + var container = document.getElementById("leaderboard-pagination"); + if (!container) return; + + var pages = totalPages(); + if (pages <= 1) { + container.innerHTML = ""; + return; + } + + var prevDisabled = currentPage === 0; + var nextDisabled = currentPage >= pages - 1; + + container.innerHTML = + '" + + 'Page ' + (currentPage + 1) + " of " + pages + "" + + '"; + + var prevBtn = document.getElementById("lb-prev"); + var nextBtn = document.getElementById("lb-next"); + if (prevBtn) prevBtn.onclick = function () { if (currentPage > 0) { currentPage--; renderPage(); } }; + if (nextBtn) nextBtn.onclick = function () { if (currentPage < pages - 1) { currentPage++; renderPage(); } }; + } + function loadLeaderboard() { var tbody = document.getElementById("leaderboard-body"); if (!tbody) return; @@ -32,7 +97,7 @@ fetch( SUPABASE_URL + - "/rest/v1/savings_entries?select=display_name,dollar_savings,energy_wh_saved,flops_saved,total_calls,total_tokens&order=dollar_savings.desc&limit=100", + "/rest/v1/savings_entries?select=display_name,dollar_savings,energy_wh_saved,flops_saved,total_calls,total_tokens&order=dollar_savings.desc&limit=1000", { headers: { apikey: SUPABASE_ANON_KEY, @@ -52,6 +117,9 @@ return; } + allRows = rows; + currentPage = 0; + var totalMembers = rows.length; var totalDollars = 0; var totalRequests = 0; @@ -72,25 +140,7 @@ if (elRequests) elRequests.textContent = totalRequests.toLocaleString(); if (elTokens) elTokens.textContent = fmtLarge(totalTokens); - var html = ""; - for (var j = 0; j < rows.length; j++) { - var rank = j + 1; - var rankClass = rank <= 3 ? " lb-rank-" + rank : ""; - var medal = - rank === 1 ? "\uD83E\uDD47" : rank === 2 ? "\uD83E\uDD48" : rank === 3 ? "\uD83E\uDD49" : ""; - var row = rows[j]; - html += - "" + - '' + (medal || rank) + "" + - '' + escapeHtml(row.display_name) + "" + - '$' + Number(row.dollar_savings || 0).toFixed(4) + "" + - '' + Number(row.energy_wh_saved || 0).toFixed(2) + "" + - '' + fmtLarge(Number(row.flops_saved || 0)) + "" + - '' + Number(row.total_calls || 0).toLocaleString() + "" + - '' + Number(row.total_tokens || 0).toLocaleString() + "" + - ""; - } - tbody.innerHTML = html; + renderPage(); }) .catch(function (err) { tbody.innerHTML = @@ -101,7 +151,6 @@ }); } - // Run on page load and refresh every 60s if (document.getElementById("leaderboard-body")) { loadLeaderboard(); setInterval(loadLeaderboard, 60000); diff --git a/docs/leaderboard.md b/docs/leaderboard.md index 747e5234..f676dfb1 100644 --- a/docs/leaderboard.md +++ b/docs/leaderboard.md @@ -52,6 +52,8 @@ See how the OpenJarvis community saves money, energy, and compute by running AI +
+

*Dollar savings estimates assume local open-source models (e.g. Qwen, Nemotron, Kimi) produce roughly the same number of tokens per request, on average, as closed-source cloud models.

diff --git a/docs/stylesheets/extra.css b/docs/stylesheets/extra.css index 1b56a47e..71ac65d2 100644 --- a/docs/stylesheets/extra.css +++ b/docs/stylesheets/extra.css @@ -491,6 +491,41 @@ 0 1px 2px 1px rgba(0, 0, 0, 0.3); } +/* Leaderboard pagination */ +.lb-pagination { + display: flex; + align-items: center; + justify-content: center; + gap: 16px; + margin: 20px 0 8px; +} + +.lb-page-btn { + padding: 6px 16px; + border-radius: 6px; + border: 1px solid var(--md-default-fg-color--lighter, #ccc); + background: var(--md-code-bg-color, #f5f5f5); + color: var(--md-default-fg-color, #333); + font-size: 13px; + font-weight: 500; + cursor: pointer; + transition: opacity 0.15s; +} + +.lb-page-btn:hover:not([disabled]) { + opacity: 0.8; +} + +.lb-page-btn[disabled] { + opacity: 0.35; + cursor: default; +} + +.lb-page-info { + font-size: 13px; + color: var(--md-default-fg-color--light, #666); +} + /* Responsive: hide placeholder on mobile */ @media (max-width: 768px) { .DocSearch-Button-Placeholder { diff --git a/instr.md b/instr.md new file mode 100644 index 00000000..4eb6d013 --- /dev/null +++ b/instr.md @@ -0,0 +1,417 @@ +# Agent Runtime Manual Test Plan + +**Branch:** `main` +**PR Reference:** [#32](https://github.com/open-jarvis/OpenJarvis/pull/32) + +--- + +## Setup + +```bash +git checkout main && git pull +uv sync --extra dev +``` + +Create `~/.openjarvis/config.toml`: + +```toml +[engine] +type = "cloud" + +[intelligence] +default_model = "Qwen/Qwen3.5-35B-A3B" + +[engine.cloud] +provider = "openai" +api_key = "sk-..." +``` + +For every test case, record: **Pass / Fail / Partial / Blocked**, what you actually saw, and screenshots for any UI issues. + +--- + +## Part 1: CLI (`jarvis agents`) + +### 1.1 Commands exist + +| # | Test | Expected | +|---|------|----------| +| 1 | `jarvis agents --help` | Shows all subcommands: `launch`, `start`, `stop`, `run`, `status`, `logs`, `daemon`, `watch`, `recover`, `errors`, `ask`, `instruct`, `messages`, `list`, `info`, `create`, `pause`, `resume`, `delete`, `bind`, `channels`, `search`, `templates`, `tasks` | + +### 1.2 Agent lifecycle: create → run → pause → resume → delete + +| # | Test | Expected | +|---|------|----------| +| 2 | `jarvis agents launch` | Wizard: template list → name/schedule/tools/budget/learning prompts → creates agent, prints ID | +| 3 | `jarvis agents list` | Agent appears, status=`idle` | +| 4 | `jarvis agents status` | Table: name, status dot, schedule, last run, runs=0, cost=$0 | +| 5 | `jarvis agents run ` | Prints progress then "Tick complete. Status: idle, runs: 1" | +| 6 | `jarvis agents status` | runs=1, last run time updated | +| 7 | `jarvis agents pause ` then `status` | Status shows `paused` | +| 8 | `jarvis agents resume ` then `status` | Status back to `idle` | +| 9 | `jarvis agents delete ` then `list` | Agent gone (soft-deleted/archived, not in list) | + +### 1.3 Agent creation variants + +| # | Test | Expected | +|---|------|----------| +| 10 | `jarvis agents create "Test Agent"` | Creates agent by name, prints ID | +| 11 | `jarvis agents create --template ` | Creates from template, inherits template config | +| 12 | `jarvis agents launch` → pick a template | Wizard pre-fills config from template | +| 13 | `jarvis agents launch` → pick "Custom Agent" | Wizard starts with blank config | +| 14 | `jarvis agents templates` | Lists built-in + user templates with descriptions | + +### 1.4 Scheduling + +| # | Test | Expected | +|---|------|----------| +| 15 | Create agent with `schedule_type=interval`, `schedule_value=30` | Created | +| 16 | `jarvis agents start ` | "Agent registered with scheduler" | +| 17 | `jarvis agents stop ` | "Agent deregistered from scheduler" | +| 18 | `jarvis agents daemon` | Starts, prints agent count, blocks. Ctrl+C → "Daemon stopped." clean exit | +| 19 | Create agent with `schedule_type=cron`, `schedule_value="*/5 * * * *"` | Created | +| 20 | `jarvis agents start ` (cron agent) | Registered, next fire time displayed or logged | +| 21 | Create agent with `schedule_type=manual` then `start ` | Agent registered but never auto-fires | + +### 1.5 Interaction: ask / instruct / messages + +| # | Test | Expected | +|---|------|----------| +| 22 | `jarvis agents ask "What is 2+2?"` | Runs tick, prints agent response inline | +| 23 | `jarvis agents messages ` | Shows user→agent ask + agent→user response | +| 24 | `jarvis agents instruct "Focus on ML papers"` | "Instruction queued for next tick" | +| 25 | `jarvis agents messages ` | Queued instruction shows `[queued]`, status=pending | +| 26 | `jarvis agents run ` then `messages ` | Queued message now delivered, status changes from pending | +| 27 | `jarvis agents ask ""` (empty message) | Graceful error or rejection, no crash | +| 28 | `jarvis agents instruct ` with very long message (>1000 chars) | Accepted and stored correctly | + +### 1.6 Error recovery & monitoring + +| # | Test | Expected | +|---|------|----------| +| 29 | `jarvis agents errors` | Lists agents in error/needs_attention/stalled/budget_exceeded (or empty table) | +| 30 | `jarvis agents recover ` (on errored agent) | Restores checkpoint, status → `idle` | +| 31 | `jarvis agents recover ` (on idle agent) | Clear message: "Agent is not in error state" or similar | +| 32 | `jarvis agents logs ` | Recent traces with tick IDs and timestamps | +| 33 | `jarvis agents logs ` | Clear error: "Agent not found" | +| 34 | `jarvis agents watch` (then run a tick in another terminal) | Events stream live: AGENT_TICK_START, AGENT_TICK_END visible. Ctrl+C to stop. | +| 35 | `jarvis agents watch ` | Same, filtered to one agent only | +| 36 | `jarvis agents watch` then Ctrl+C | Clean exit, no traceback, no hanging threads | + +### 1.7 Agent info & inspection + +| # | Test | Expected | +|---|------|----------| +| 37 | `jarvis agents info ` | Shows agent type, status, memory snippet, tasks, channels, config details | +| 38 | `jarvis agents tasks ` | Lists tasks with statuses (or empty state) | +| 39 | `jarvis agents channels ` | Lists channel bindings (or empty state) | +| 40 | `jarvis agents search "keyword"` | Searches across agent traces, returns relevant results | + +### 1.8 Edge cases & invalid input + +| # | Test | Expected | +|---|------|----------| +| 41 | `jarvis agents run ` | Clear error: "Agent not found" — no Python traceback | +| 42 | `jarvis agents pause ` twice | Second pause is no-op or clear message, no crash | +| 43 | `jarvis agents resume ` (already idle) | No-op or clear message, no crash | +| 44 | `jarvis agents run ` while another tick is running | Concurrency guard: "Agent is already running" error | +| 45 | `jarvis agents delete ` then `run ` | Clear error about deleted/archived agent | +| 46 | Create agent with invalid cron expression | Rejected with clear validation error | +| 47 | Create agent with negative budget | Rejected or clamped to 0 | + +### 1.9 CLI aesthetics + +| # | Check | Expected | +|---|-------|----------| +| 48 | `status` table formatting | Columns aligned, readable at 80-char terminal width | +| 49 | Error messages (run with no engine configured) | Clear human-readable message, no Python tracebacks | +| 50 | `launch` wizard prompts | Clear labels, sensible defaults, no confusing jargon | +| 51 | `watch` event stream | Color-coded, event type + agent name visible, timestamps | +| 52 | `list` table with 0 agents | "No agents found" or empty table — not a crash | +| 53 | `list` table with 10+ agents | Table remains readable, no column overflow | +| 54 | All commands with `--help` | Every subcommand has a help string | + +--- + +## Part 2: Web Frontend + +### 2.0 Setup + +```bash +# Terminal 1 # Terminal 2 +uv run jarvis serve cd frontend && npm install && npm run dev +``` + +Open http://localhost:5173, navigate to **Agents** page via sidebar. + +### 2.1 Navigation & routing + +| # | Test | Expected | +|---|------|----------| +| 55 | Click "Agents" in sidebar | AgentsPage renders, URL is `/agents` | +| 56 | Direct navigation to `/agents` | Page loads correctly (no blank screen) | +| 57 | Browser back/forward after visiting agent detail | Navigation works, state preserved | + +### 2.2 List view + +| # | Test | Expected | +|---|------|----------| +| 58 | Page loads with backend running | No console errors, agent list renders | +| 59 | Page loads with backend **down** | User-visible error message (not blank white screen), no console exceptions | +| 60 | Agent cards | Name, color status dot, schedule description, last run time, runs count, cost | +| 61 | "Run Now" button | Triggers tick, card updates (runs count increments, last run time updates) | +| 62 | Pause/Resume button | Toggles status, dot color changes immediately | +| 63 | Agent list auto-refresh | After running a tick via CLI, the web list eventually reflects the updated state | +| 64 | 10+ agents in list | Cards render without performance issues, scroll works | + +### 2.3 Launch wizard + +| # | Test | Expected | +|---|------|----------| +| 65 | Click "Launch Agent" | Modal appears: Step 1 template picker with templates + "Custom Agent" option | +| 66 | Templates load from API | Template cards display with names and descriptions | +| 67 | Select template → Next → Step 2 | Config form: name (pre-filled from template), schedule_type dropdown, schedule_value, tools checkboxes, budget, learning toggle (off) | +| 68 | Select "Custom Agent" → Next → Step 2 | Config form with blank name, no pre-filled values | +| 69 | Next → Step 3 | Review summary of all config values | +| 70 | Click Launch | Agent created, modal closes, new agent appears in list | +| 71 | Back button at Step 2 | Returns to Step 1, template selection preserved | +| 72 | Back button at Step 3 | Returns to Step 2, all form inputs preserved | +| 73 | Launch with empty name | Inline error: "Agent name is required" — modal stays open | +| 74 | Launch with all tools selected | All tools included in review and in created agent config | +| 75 | Click outside modal / press Escape | Modal closes (or stays open — document behavior) | +| 76 | Schedule type = "Manual" | schedule_value input is disabled/hidden | +| 77 | Schedule type = "Cron" | schedule_value placeholder shows cron example | +| 78 | Schedule type = "Interval" | schedule_value placeholder shows seconds example | + +### 2.4 Detail view (click an agent) + +| # | Test | Expected | +|---|------|----------| +| 79 | Click agent card | Detail view opens with tabbed interface | +| 80 | **Overview** tab | Stat cards (Total Runs, Success Rate, Total Cost), config display, channels list, action buttons | +| 81 | **Overview** action buttons | Run Now, Pause, Resume visible and functional | +| 82 | **Interact** tab | Chat message list, textarea, "Immediate" and "Queue" send buttons | +| 83 | Send immediate message | Appears in chat with user styling, agent responds after tick | +| 84 | Send queued message | Shows with "queued" badge, status=pending | +| 85 | Send empty message | Button disabled or graceful rejection — no empty message sent | +| 86 | Rapid-fire send (click Send multiple times quickly) | No duplicate messages, no race condition errors | +| 87 | Chat auto-scroll | New messages scroll into view automatically | +| 88 | **Tasks** tab | Task list with status badges (completed=green, failed=red, active=blue, pending=gray) | +| 89 | **Tasks** tab (no tasks) | Empty state: "No tasks assigned." | +| 90 | **Memory** tab | summary_memory text displayed in readable format | +| 91 | **Memory** tab (no memory) | Empty state: "Agent has no stored memory yet." | +| 92 | **Learning** tab | Toggle switch (read-only, off by default), placeholder text for future events | +| 93 | **Logs** tab | Placeholder / empty state message (not a crash or blank) | +| 94 | Tab switching — rapid clicks | All 6 tabs render instantly, no layout shift, no flash of wrong content | + +### 2.5 Error states + +| # | Test | Expected | +|---|------|----------| +| 95 | Agent in `error` status | Red status dot/badge, "Recover" button visible | +| 96 | Click Recover | Status resets to `idle`, dot turns green | +| 97 | Agent in `needs_attention` status | Amber badge visible | +| 98 | Agent in `budget_exceeded` status | Orange badge visible | +| 99 | Agent in `stalled` status | Yellow badge visible | +| 100 | Backend goes down while page is open | Next refresh/action shows error — not silent failure | +| 101 | Delete agent → confirm it disappears from list | Agent removed from list immediately (or on next refresh) | +| 102 | Delete agent (no confirmation dialog in web) | **Document:** Is instant delete OK or should there be a confirm? | + +### 2.6 Overflow menu + +| # | Test | Expected | +|---|------|----------| +| 103 | Click "..." menu on agent card | Dropdown with Delete + other options | +| 104 | Click Delete from menu | Agent deleted, list updates | +| 105 | Click outside dropdown | Dropdown closes | + +### 2.7 Web aesthetics & UX + +| # | Check | Expected | +|---|-------|----------| +| 106 | Status dot colors | idle=#22c55e, running=#3b82f6, paused=#6b7280, error=#ef4444, needs_attention=#f59e0b, budget_exceeded=#f97316, stalled=#eab308 | +| 107 | Launch wizard spacing/alignment | Modal centered, steps clearly numbered, form inputs aligned, no overlap | +| 108 | Detail view tab switching | Instant, no layout shift or flash | +| 109 | Interact tab chat feel | Messages visually distinct (user=right vs agent=left or different colors), auto-scroll, clear input area | +| 110 | Responsive at 1024px width | No overflow or cut-off content, agent cards reflow | +| 111 | Responsive at 1440px width | Proper use of space, no excessive stretching | +| 112 | Responsive at 768px width (tablet) | Still usable, no broken layout | +| 113 | Empty states | "No agents yet" + CTA button / "No messages" / "No tasks" — not blank white space | +| 114 | Loading states | "Loading agents..." shown during fetch, spinner or skeleton | +| 115 | Page title / browser tab | Meaningful title (not just "Vite App") | +| 116 | Console errors | Zero console errors during normal usage flow | + +--- + +## Part 3: Desktop App + +### 3.0 Setup + +```bash +# Terminal 1 # Terminal 2 +uv run jarvis serve cd desktop && npm install && npm run tauri dev +``` + +Navigate to the **Agents** tab. + +### 3.1 Functionality + +| # | Test | Expected | +|---|------|----------| +| 117 | Left panel: agent list | Status dots, schedule descriptions, last run times | +| 118 | Click agent → right panel | Tabbed detail view (Overview, Interact, Tasks, Memory, Learning, Logs) | +| 119 | No agent selected | Right panel shows "Select an agent to view details" | +| 120 | "Launch Agent" button | Opens wizard, same 3-step flow as web | +| 121 | Launch wizard → Create agent | Agent appears in left panel list | +| 122 | **Overview** tab | Key-value stats (Status, Agent Type, Schedule, Last Run, Total Runs, Total Cost, Budget) + action buttons (Run Now, Pause, Resume, Recover) | +| 123 | **Interact** tab | Chat UI, mode toggle (immediate/queued), Enter shortcut sends message | +| 124 | Send immediate message | Response appears in chat | +| 125 | Send queued message | Shows as pending | +| 126 | **Tasks** tab | Task list with colored status badges + created-at timestamps | +| 127 | **Memory** tab | summary_memory in monospace font | +| 128 | **Learning** tab | Shows enabled/disabled status + placeholder text | +| 129 | **Logs** tab | Placeholder: "Log streaming not yet connected." | +| 130 | Auto-refresh | Agent list refreshes on ~10s interval (verify with CLI-triggered state change) | +| 131 | Delete agent via desktop | Confirmation dialog appears, agent removed on confirm | + +### 3.2 Desktop edge cases + +| # | Test | Expected | +|---|------|----------| +| 132 | Backend not running → open desktop app | Error state shown, not a crash | +| 133 | Backend dies while desktop is open | Graceful degradation on next action/refresh | +| 134 | Selected agent deleted via CLI → desktop refreshes | Selected agent deselects, list updates | + +### 3.3 Desktop aesthetics + +| # | Check | Expected | +|---|-------|----------| +| 135 | Catppuccin color scheme consistent | idle=#a6e3a1, running=#89b4fa, paused=#6c7086, error=#f38ba8, needs_attention=#fab387, stalled=#f9e2af | +| 136 | Left/right panel split | Resizable or fixed at reasonable ratio, no overlap | +| 137 | Tab switching | Smooth, no flicker | +| 138 | Launch wizard modal | Properly overlays content, dismissible with Escape or outside click | +| 139 | Text readability | Font sizes consistent, sufficient contrast against dark background | +| 140 | Window resize | Layout adapts, no overflow or clipping | +| 141 | Status badge consistency with web | Same statuses map to same semantic colors (green=idle, blue=running, etc.) | + +--- + +## Part 4: API Backend (Direct) + +### 4.1 REST endpoint smoke tests + +Run with `uv run jarvis serve` and test via curl or Postman. + +| # | Test | Expected | +|---|------|----------| +| 142 | `GET /v1/managed-agents` | 200, returns `[]` or agent list JSON | +| 143 | `POST /v1/managed-agents` with valid body | 200/201, returns created agent JSON with `id` | +| 144 | `POST /v1/managed-agents` with empty body | 422 or 400 with validation error | +| 145 | `GET /v1/managed-agents/` | 200, returns single agent | +| 146 | `GET /v1/managed-agents/` | 404, returns error JSON | +| 147 | `POST /v1/managed-agents//run` | 200, tick executes | +| 148 | `POST /v1/managed-agents//pause` | 200, status changes to paused | +| 149 | `POST /v1/managed-agents//resume` | 200, status changes to idle | +| 150 | `POST /v1/managed-agents//recover` | 200 if errored, appropriate error if not | +| 151 | `DELETE /v1/managed-agents/` | 200, agent archived | +| 152 | `GET /v1/templates` | 200, returns template list | +| 153 | `POST /v1/templates//instantiate` | 200, creates agent from template | +| 154 | `GET /v1/agents/errors` | 200, returns list of problem agents | +| 155 | `GET /v1/agents/health` | 200, returns health summary | + +### 4.2 Message endpoints + +| # | Test | Expected | +|---|------|----------| +| 156 | `POST /v1/managed-agents//messages` with `{"content":"hi","direction":"user_to_agent","mode":"immediate"}` | 200, message stored | +| 157 | `GET /v1/managed-agents//messages` | 200, returns message list | +| 158 | `POST /v1/managed-agents//messages` with `{"content":"","direction":"user_to_agent","mode":"immediate"}` | 422 or graceful handling | +| 159 | `POST /v1/managed-agents//messages` with `{"content":"cmd","direction":"user_to_agent","mode":"queued"}` | 200, message has status=pending | + +### 4.3 Task & channel endpoints + +| # | Test | Expected | +|---|------|----------| +| 160 | `GET /v1/managed-agents//tasks` | 200, returns task list | +| 161 | `POST /v1/managed-agents//tasks` | 200, creates task | +| 162 | `GET /v1/managed-agents//channels` | 200, returns channel bindings | +| 163 | `GET /v1/managed-agents//state` | 200, returns full agent state | + +### 4.4 WebSocket events + +| # | Test | Expected | +|---|------|----------| +| 164 | Connect to `ws://localhost:8222/v1/agents/events` | Connection established | +| 165 | Trigger a tick → observe WS messages | Receive AGENT_TICK_START and AGENT_TICK_END events | +| 166 | Connect with `?agent_id=` filter | Only events for that agent | +| 167 | Disconnect cleanly | No server error logs | + +--- + +## Part 5: Cross-Platform Consistency + +| # | Test | Expected | +|---|------|----------| +| 168 | Create agent via CLI → check web + desktop | Same name, status, config everywhere | +| 169 | Run tick via CLI → check web + desktop | Run count and last run time update in both UIs | +| 170 | Send message via web Interact → check CLI `messages` | Same content, direction, mode | +| 171 | Pause via desktop → check CLI `status` + web | `paused` everywhere | +| 172 | Delete via web → check CLI `list` + desktop | Gone everywhere | +| 173 | Create via web wizard → check CLI `list` + desktop | Agent visible in all three | +| 174 | Recover via CLI → check web + desktop | Status back to idle in all UIs | +| 175 | Send queued message via CLI `instruct` → check web Interact | Message shows with pending/queued status | +| 176 | Multiple agents created from different clients | All agents appear correctly in all views | + +--- + +## Part 6: Stress & Concurrency + +| # | Test | Expected | +|---|------|----------| +| 177 | Run tick on same agent from two terminals simultaneously | Concurrency guard blocks second tick: "Agent is already running" | +| 178 | Create 20+ agents → check list performance | All clients render list without lag | +| 179 | Rapidly pause/resume same agent | All state transitions correct, no stuck states | +| 180 | Run daemon + manual `run` at same time | No double-ticking, concurrency guard holds | +| 181 | Delete agent while tick is in progress | Tick completes or fails gracefully, agent ends up archived | + +--- + +## Part 7: Deferred Features (Placeholder Verification) + +Confirm these show placeholders (not crashes): + +| # | Feature | CLI | Web | Desktop | +|---|---------|-----|-----|---------| +| 182 | Budget enforcement | `run` still works even if cost > budget | No enforcement, budget is display-only | Same | +| 183 | Stall detection | No automatic stall detection fires | N/A | N/A | +| 184 | Learning event timeline | `Learning` tab shows placeholder text | Same | Same | +| 185 | Logs trace replay | `Logs` tab shows placeholder text | Same | Same | +| 186 | `POST /v1/skills` | N/A | N/A | Returns `"not_implemented"` | +| 187 | `POST /v1/optimize/runs` | N/A | N/A | Returns placeholder `run_id` | +| 188 | `GET /v1/feedback/stats` | N/A | N/A | Returns `{total: 0, mean_score: 0.0}` | + +--- + +## Deliverables + +**1. Test results** — Spreadsheet with columns: #, Status (Pass/Fail/Partial/Blocked), Actual Behavior, Screenshot (for UI issues). + +**2. Bug list** — Each bug: steps to reproduce, expected vs actual, severity (Critical/Major/Minor), screenshot. + +**3. UX & aesthetics feedback** — Is the launch wizard clear? Are status colors distinguishable? Does the Interact tab feel like chat? Is CLI output readable? Are error messages helpful? Is the delete-without-confirm behavior in web acceptable? + +**4. API error handling audit** — Document all cases where the frontend silently swallows errors (currently: agent list fetch, interact tab sends). Recommend which should show user-visible errors. + +**5. Deferred features check** — Confirm placeholder items in Part 7 show graceful stubs (not crashes or blank screens). + +--- + +## Notes + +- Backend (`jarvis serve`) must be running for web and desktop (default port 8222). +- Without an engine configured, `run`/`ask` will error — document whether the error message is clear. +- `daemon` and `watch` block — Ctrl+C to exit. +- Web frontend API client has unused functions (`updateManagedAgent`, `createAgentTask`, `fetchAgentState`, `fetchErrorAgents`) — not a bug, but note for future. +- Desktop API client is missing some endpoints that the web client has (`fetchAgentChannels`, `fetchAgentState`, `fetchErrorAgents`) — may affect feature parity. +- Frontend has **no automated tests** — all testing is manual per this plan. +- Both frontends silently catch API errors (`.catch(() => {})`) — this is a known UX gap to evaluate. diff --git a/scripts/GRID_SEARCH_GUIDE.md b/scripts/GRID_SEARCH_GUIDE.md new file mode 100644 index 00000000..98e8d5dd --- /dev/null +++ b/scripts/GRID_SEARCH_GUIDE.md @@ -0,0 +1,350 @@ +# Eval Grid Search — 4× A100 Runbook + +Complete guide for running the OpenJarvis evaluation grid search across **5 models × 4 engines × 5 agents × 15 benchmarks** on a 4× NVIDIA A100 (80 GB) node. + +## Grid Dimensions + +| Dimension | Values | +|-----------|--------| +| **Models** | GPT-OSS-120B, Qwen3.5-122B-FP8, Qwen3.5-397B-GGUF, Kimi-K2.5-GGUF, GLM-5-GGUF | +| **Engines** | vLLM, SGLang, llama.cpp, Ollama | +| **Agents** | simple, orchestrator, native_react, native_openhands, rlm | +| **Benchmarks** | supergpqa, gpqa, mmlu-pro, math500, natural-reasoning, hle, simpleqa, wildchat, ipw, gaia, frames, swebench, swefficiency, terminalbench, terminalbench-native | +| **Samples** | 5 per benchmark | + +**Total: 750 experiment cells** (each model only runs on its compatible engines). + +| Model | Compatible Engines | +|-------|--------------------| +| openai/gpt-oss-120b | vllm, sglang | +| Qwen/Qwen3.5-122B-A10B-FP8 | vllm, sglang | +| unsloth/Qwen3.5-397B-A17B-GGUF | llamacpp, ollama | +| unsloth/Kimi-K2.5-GGUF | llamacpp, ollama | +| unsloth/GLM-5-GGUF | llamacpp, ollama | + +--- + +## Phase 1: Environment Setup + +```bash +# Clone and install +cd ~/gabebo # or wherever your workspace lives +git clone OpenJarvis && cd OpenJarvis +uv sync --extra dev + +# Install eval dependencies +uv pip install openai datasets huggingface-hub terminal-bench + +# Load API keys (needed for LLM judge — gpt-5-mini) +source .env + +# Log in to HuggingFace (needed for gated datasets) +huggingface-cli login +``` + +> **SGLang note:** SGLang requires Python ≤ 3.12 (FlashInfer/outlines_core fail on 3.13). +> If your base env is Python 3.13, create a separate conda env: +> ```bash +> conda create -n sglang python=3.11 -y && conda activate sglang +> pip install "sglang[all]" +> ``` + +--- + +## Phase 2: Download Models + +```bash +# HuggingFace weights (vLLM / SGLang) +hf download openai/gpt-oss-120b --local-dir ~/models/gpt-oss-120b +hf download Qwen/Qwen3.5-122B-A10B-FP8 --local-dir ~/models/Qwen3.5-122B-A10B-FP8 + +# GGUF quantizations (llama.cpp / Ollama) +# Qwen3.5-397B — UD-Q4_K_XL fits in 4× A100 (~214 GB) +hf download unsloth/Qwen3.5-397B-A17B-GGUF \ + --include "Q4_K_M/*.gguf" \ + --local-dir ~/models/Qwen3.5-397B-A17B-GGUF + +# Kimi-K2.5 — UD-IQ2_XXS to fit (~240 GB usable) +hf download unsloth/Kimi-K2.5-GGUF \ + --include "UD-IQ2_XXS/*.gguf" \ + --local-dir ~/models/Kimi-K2.5-GGUF + +# GLM-5 — UD-IQ2_XXS to fit +hf download unsloth/GLM-5-GGUF \ + --include "UD-IQ2_XXS/*.gguf" \ + --local-dir ~/models/GLM-5-GGUF +``` + +### Verify downloads + +```bash +# HuggingFace weights — check for config.json and safetensors shards +ls ~/models/gpt-oss-120b/config.json +ls ~/models/Qwen3.5-122B-A10B-FP8/config.json + +# GGUF files — check they exist and aren't zero-byte +find ~/models/Qwen3.5-397B-A17B-GGUF -name "*.gguf" -exec ls -lh {} \; +find ~/models/Kimi-K2.5-GGUF -name "*.gguf" -exec ls -lh {} \; +find ~/models/GLM-5-GGUF -name "*.gguf" -exec ls -lh {} \; +``` + +--- + +## Phase 3: Run the Grid (One Model at a Time) + +Serve a model, run all agents/benchmarks for it, then kill the server and swap. + +### 3A. GPT-OSS-120B via vLLM + +```bash +# Terminal 1: Start vLLM server +vllm serve openai/gpt-oss-120b \ + --tensor-parallel-size 2 \ + --port 8000 \ + --max-model-len 8192 +``` + +```bash +# Terminal 2: Wait for "Uvicorn running" then run grid +cd OpenJarvis && source .env +uv run python scripts/run_grid_search.py \ + --model "gpt-oss" --engine vllm -n 5 +# When done, Ctrl-C the vLLM server +``` + +### 3B. GPT-OSS-120B via SGLang + +```bash +# Terminal 1: Start SGLang server +python -m sglang.launch_server \ + --model-path openai/gpt-oss-120b \ + --tp 2 \ + --port 30000 +``` + +```bash +# Terminal 2: Run grid +uv run python scripts/run_grid_search.py \ + --model "gpt-oss" --engine sglang --resume -n 5 +# Kill server when done +``` + +### 3C. Qwen3.5-122B-FP8 via vLLM + +```bash +# Terminal 1 +vllm serve Qwen/Qwen3.5-122B-A10B-FP8 \ + --tensor-parallel-size 2 \ + --port 8000 \ + --max-model-len 8192 \ + --quantization fp8 +``` + +```bash +# Terminal 2 +uv run python scripts/run_grid_search.py \ + --model "Qwen3.5-122B" --engine vllm --resume -n 5 +``` + +### 3D. Qwen3.5-122B-FP8 via SGLang + +```bash +# Terminal 1 +python -m sglang.launch_server \ + --model-path Qwen/Qwen3.5-122B-A10B-FP8 \ + --tp 2 \ + --port 30000 \ + --quantization fp8 +``` + +```bash +# Terminal 2 +uv run python scripts/run_grid_search.py \ + --model "Qwen3.5-122B" --engine sglang --resume -n 5 +``` + +### 3E. Qwen3.5-397B GGUF via llama.cpp + +```bash +# Terminal 1: Start llama.cpp server with all 4 GPUs +./llama.cpp/build/bin/llama-server \ + -m ~/models/Qwen3.5-397B-A17B-GGUF/Q4_K_M/Qwen3.5-397B-A17B-Q4_K_M-00001-of-00005.gguf \ + --n-gpu-layers 99 \ + --tensor-split 1,1,1,1 \ + --port 8080 \ + --ctx-size 8192 +``` + +```bash +# Terminal 2 +uv run python scripts/run_grid_search.py \ + --model "Qwen3.5-397B" --engine llamacpp --resume -n 5 +``` + +### 3F. Qwen3.5-397B GGUF via Ollama + +```bash +# Create an Ollama modelfile +cat > /tmp/Qwen3.5-397B.Modelfile << 'EOF' +FROM ~/models/Qwen3.5-397B-A17B-GGUF/Q4_K_M/Qwen3.5-397B-A17B-Q4_K_M-00001-of-00005.gguf +EOF + +# Import into Ollama +ollama create qwen3.5-397b -f /tmp/Qwen3.5-397B.Modelfile + +# Ollama serves automatically on port 11434 +uv run python scripts/run_grid_search.py \ + --model "Qwen3.5-397B" --engine ollama --resume -n 5 + +# Unload when done +ollama stop qwen3.5-397b +``` + +### 3G. Kimi-K2.5 GGUF via llama.cpp + +```bash +# Terminal 1 +./llama.cpp/build/bin/llama-server \ + -m ~/models/Kimi-K2.5-GGUF/UD-IQ2_XXS/Kimi-K2.5-UD-IQ2_XXS-00001-of-00005.gguf \ + --n-gpu-layers 99 \ + --tensor-split 1,1,1,1 \ + --port 8080 \ + --ctx-size 8192 +``` + +```bash +# Terminal 2 +uv run python scripts/run_grid_search.py \ + --model "Kimi-K2.5" --engine llamacpp --resume -n 5 +``` + +### 3H. Kimi-K2.5 GGUF via Ollama + +```bash +cat > /tmp/Kimi-K2.5.Modelfile << 'EOF' +FROM ~/models/Kimi-K2.5-GGUF/UD-IQ2_XXS/Kimi-K2.5-UD-IQ2_XXS-00001-of-00005.gguf +EOF + +ollama create kimi-k2.5 -f /tmp/Kimi-K2.5.Modelfile + +uv run python scripts/run_grid_search.py \ + --model "Kimi-K2.5" --engine ollama --resume -n 5 + +ollama stop kimi-k2.5 +``` + +### 3I. GLM-5 GGUF via llama.cpp + +```bash +# Terminal 1 +./llama.cpp/build/bin/llama-server \ + -m ~/models/GLM-5-GGUF/UD-IQ2_XXS/GLM-5-UD-IQ2_XXS-00001-of-00005.gguf \ + --n-gpu-layers 99 \ + --tensor-split 1,1,1,1 \ + --port 8080 \ + --ctx-size 8192 +``` + +```bash +# Terminal 2 +uv run python scripts/run_grid_search.py \ + --model "GLM-5" --engine llamacpp --resume -n 5 +``` + +### 3J. GLM-5 GGUF via Ollama + +```bash +cat > /tmp/GLM-5.Modelfile << 'EOF' +FROM ~/models/GLM-5-GGUF/UD-IQ2_XXS/GLM-5-UD-IQ2_XXS-00001-of-00005.gguf +EOF + +ollama create glm-5 -f /tmp/GLM-5.Modelfile + +uv run python scripts/run_grid_search.py \ + --model "GLM-5" --engine ollama --resume -n 5 + +ollama stop glm-5 +``` + +--- + +## Phase 4: Recover Failed Runs + +If some runs fail (missing packages, engine not reachable, etc.), fix the issue, then: + +```bash +# Delete error summaries so --resume retries them +find results/grid-search -name "*.summary.json" \ + -exec grep -l '"error"' {} \; -delete + +# Re-run with --resume (only retries deleted/missing summaries) +uv run python scripts/run_grid_search.py --resume -n 5 +``` + +### Common Failures and Fixes + +| Error | Cause | Fix | +|-------|-------|-----| +| `No inference engine available` | `openai` package missing (LLM judge can't init) | `uv pip install openai` | +| `No module named 'datasets'` | Missing HF datasets package | `uv pip install datasets` | +| `terminal-bench package required` | Missing terminal-bench | `uv pip install terminal-bench` | +| `Dataset doesn't exist on the Hub` | Gated dataset or HF auth needed | `huggingface-cli login`, accept terms on HF website | +| `IPW data directory not found` | IPW uses local data, not HuggingFace | Place files in `src/openjarvis/evals/data/ipw/` | +| `natural-reasoning` 0 samples | Field name mismatch in dataset loader | Patch `src/openjarvis/evals/datasets/natural_reasoning.py` | + +--- + +## Phase 5: Analyze Results + +```bash +# Preview what ran +uv run python scripts/run_grid_search.py --dry-run --resume + +# Consolidated results (appended after each run) +cat results/grid-search/grid-results.jsonl + +# Per-run summaries +find results/grid-search -name "*.summary.json" | head -20 +cat results/grid-search/openai-gpt-oss-120b/vllm/simple/supergpqa.summary.json + +# Count completed vs failed +echo "Completed:" && find results/grid-search -name "*.summary.json" \ + -exec grep -L '"error"' {} \; | wc -l +echo "Failed:" && find results/grid-search -name "*.summary.json" \ + -exec grep -l '"error"' {} \; | wc -l +``` + +--- + +## Useful Flags + +```bash +# Preview the full matrix without running +uv run python scripts/run_grid_search.py --dry-run + +# Filter to a single model + engine +uv run python scripts/run_grid_search.py --model "gpt-oss" --engine vllm -n 5 + +# Filter to a single agent or benchmark +uv run python scripts/run_grid_search.py --agent native_react --benchmark supergpqa -n 5 + +# Increase sample count +uv run python scripts/run_grid_search.py -n 50 + +# Verbose logging +uv run python scripts/run_grid_search.py -v --resume -n 5 +``` + +--- + +## GPU Assignment Tips + +Use `CUDA_VISIBLE_DEVICES` to pin servers to specific GPUs: + +```bash +# Run vLLM on GPUs 0,1 and llama.cpp on GPUs 2,3 simultaneously +CUDA_VISIBLE_DEVICES=0,1 vllm serve openai/gpt-oss-120b --tensor-parallel-size 2 --port 8000 +CUDA_VISIBLE_DEVICES=2,3 ./llama.cpp/build/bin/llama-server -m model.gguf --n-gpu-layers 99 --port 8080 +``` + +This lets you run two model servers in parallel on different GPU pairs.