admin/OpenJarvis
1
0
Fork 0
mirror of https://github.com/open-jarvis/OpenJarvis.git synced 2026-07-31 03:12:16 +00:00
Code Issues Packages Projects Releases Wiki Activity

adding pagination and limiting to 50 per sign up on leaderboard

Browse Source
This commit is contained in:
Gabriel Bo
2026-03-16 15:58:39 -07:00
parent 59b5295b20
commit 14733433b0
5 changed files with 874 additions and 21 deletions
Download Patch File Download Diff File Expand all files Collapse all files
docs/javascripts/leaderboard.js
+70 -21
View File
@@ -5,6 +5,10 @@
var SUPABASE_ANON_KEY =
"eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiJzdXBhYmFzZSIsInJlZiI6Im10YnRncHd6cmJvc3R3ZWFhbnByIiwicm9sZSI6ImFub24iLCJpYXQiOjE3NzMxODk0OTQsImV4cCI6MjA4ODc2NTQ5NH0._xMlqCfljtXpwPj54H-ghxfLFO-jiq4W2WhpU8vVL1c";
var PAGE_SIZE = 50;
var allRows = [];
var currentPage = 0;
function escapeHtml(s) {
var el = document.createElement("span");
el.textContent = s;
@@ -19,6 +23,67 @@
return n.toLocaleString();
}
function totalPages() {
return Math.max(1, Math.ceil(allRows.length / PAGE_SIZE));
}
function renderPage() {
var tbody = document.getElementById("leaderboard-body");
if (!tbody) return;
var start = currentPage * PAGE_SIZE;
var end = Math.min(start + PAGE_SIZE, allRows.length);
var pageRows = allRows.slice(start, end);
var html = "";
for (var j = 0; j < pageRows.length; j++) {
var rank = start + j + 1;
var rankClass = rank <= 3 ? " lb-rank-" + rank : "";
var medal =
rank === 1 ? "\uD83E\uDD47" : rank === 2 ? "\uD83E\uDD48" : rank === 3 ? "\uD83E\uDD49" : "";
var row = pageRows[j];
html +=
"<tr>" +
'<td><span class="lb-rank' + rankClass + '">' + (medal || rank) + "</span></td>" +
'<td class="lb-name">' + escapeHtml(row.display_name) + "</td>" +
'<td class="lb-number">$' + Number(row.dollar_savings || 0).toFixed(4) + "</td>" +
'<td class="lb-number">' + Number(row.energy_wh_saved || 0).toFixed(2) + "</td>" +
'<td class="lb-number">' + fmtLarge(Number(row.flops_saved || 0)) + "</td>" +
'<td class="lb-number">' + Number(row.total_calls || 0).toLocaleString() + "</td>" +
'<td class="lb-number">' + Number(row.total_tokens || 0).toLocaleString() + "</td>" +
"</tr>";
}
tbody.innerHTML = html;
renderPagination();
}
function renderPagination() {
var container = document.getElementById("leaderboard-pagination");
if (!container) return;
var pages = totalPages();
if (pages <= 1) {
container.innerHTML = "";
return;
}
var prevDisabled = currentPage === 0;
var nextDisabled = currentPage >= pages - 1;
container.innerHTML =
'<button class="lb-page-btn"' + (prevDisabled ? " disabled" : "") + ' id="lb-prev">' +
"\u2190 Prev</button>" +
'<span class="lb-page-info">Page ' + (currentPage + 1) + " of " + pages + "</span>" +
'<button class="lb-page-btn"' + (nextDisabled ? " disabled" : "") + ' id="lb-next">' +
"Next \u2192</button>";
var prevBtn = document.getElementById("lb-prev");
var nextBtn = document.getElementById("lb-next");
if (prevBtn) prevBtn.onclick = function () { if (currentPage > 0) { currentPage--; renderPage(); } };
if (nextBtn) nextBtn.onclick = function () { if (currentPage < pages - 1) { currentPage++; renderPage(); } };
}
function loadLeaderboard() {
var tbody = document.getElementById("leaderboard-body");
if (!tbody) return;
@@ -32,7 +97,7 @@
fetch(
SUPABASE_URL +
"/rest/v1/savings_entries?select=display_name,dollar_savings,energy_wh_saved,flops_saved,total_calls,total_tokens&order=dollar_savings.desc&limit=100",
"/rest/v1/savings_entries?select=display_name,dollar_savings,energy_wh_saved,flops_saved,total_calls,total_tokens&order=dollar_savings.desc&limit=1000",
{
headers: {
apikey: SUPABASE_ANON_KEY,
@@ -52,6 +117,9 @@
return;
}
allRows = rows;
currentPage = 0;
var totalMembers = rows.length;
var totalDollars = 0;
var totalRequests = 0;
@@ -72,25 +140,7 @@
if (elRequests) elRequests.textContent = totalRequests.toLocaleString();
if (elTokens) elTokens.textContent = fmtLarge(totalTokens);
var html = "";
for (var j = 0; j < rows.length; j++) {
var rank = j + 1;
var rankClass = rank <= 3 ? " lb-rank-" + rank : "";
var medal =
rank === 1 ? "\uD83E\uDD47" : rank === 2 ? "\uD83E\uDD48" : rank === 3 ? "\uD83E\uDD49" : "";
var row = rows[j];
html +=
"<tr>" +
'<td><span class="lb-rank' + rankClass + '">' + (medal || rank) + "</span></td>" +
'<td class="lb-name">' + escapeHtml(row.display_name) + "</td>" +
'<td class="lb-number">$' + Number(row.dollar_savings || 0).toFixed(4) + "</td>" +
'<td class="lb-number">' + Number(row.energy_wh_saved || 0).toFixed(2) + "</td>" +
'<td class="lb-number">' + fmtLarge(Number(row.flops_saved || 0)) + "</td>" +
'<td class="lb-number">' + Number(row.total_calls || 0).toLocaleString() + "</td>" +
'<td class="lb-number">' + Number(row.total_tokens || 0).toLocaleString() + "</td>" +
"</tr>";
}
tbody.innerHTML = html;
renderPage();
})
.catch(function (err) {
tbody.innerHTML =
@@ -101,7 +151,6 @@
});
}
// Run on page load and refresh every 60s
if (document.getElementById("leaderboard-body")) {
loadLeaderboard();
setInterval(loadLeaderboard, 60000);
docs/leaderboard.md
+2
View File
@@ -52,6 +52,8 @@ See how the OpenJarvis community saves money, energy, and compute by running AI
</table>
</div>
<div id="leaderboard-pagination" class="lb-pagination"></div>
<p style="font-size:12px;opacity:0.6;margin-top:12px">
*Dollar savings estimates assume local open-source models (e.g. Qwen, Nemotron, Kimi) produce roughly the same number of tokens per request, on average, as closed-source cloud models.
</p>
docs/stylesheets/extra.css
+35
View File
@@ -491,6 +491,41 @@
0 1px 2px 1px rgba(0, 0, 0, 0.3);
}
/* Leaderboard pagination */
.lb-pagination {
display: flex;
align-items: center;
justify-content: center;
gap: 16px;
margin: 20px 0 8px;
}
.lb-page-btn {
padding: 6px 16px;
border-radius: 6px;
border: 1px solid var(--md-default-fg-color--lighter, #ccc);
background: var(--md-code-bg-color, #f5f5f5);
color: var(--md-default-fg-color, #333);
font-size: 13px;
font-weight: 500;
cursor: pointer;
transition: opacity 0.15s;
}
.lb-page-btn:hover:not([disabled]) {
opacity: 0.8;
}
.lb-page-btn[disabled] {
opacity: 0.35;
cursor: default;
}
.lb-page-info {
font-size: 13px;
color: var(--md-default-fg-color--light, #666);
}
/* Responsive: hide placeholder on mobile */
@media (max-width: 768px) {
.DocSearch-Button-Placeholder {
instr.md
+417
View File
@@ -0,0 +1,417 @@
# Agent Runtime Manual Test Plan
**Branch:** `main`
**PR Reference:** [#32](https://github.com/open-jarvis/OpenJarvis/pull/32)
---
## Setup
```bash
git checkout main && git pull
uv sync --extra dev
```
Create `~/.openjarvis/config.toml`:
```toml
[engine]
type = "cloud"
[intelligence]
default_model = "Qwen/Qwen3.5-35B-A3B"
[engine.cloud]
provider = "openai"
api_key = "sk-..."
```
For every test case, record: **Pass / Fail / Partial / Blocked**, what you actually saw, and screenshots for any UI issues.
---
## Part 1: CLI (`jarvis agents`)
### 1.1 Commands exist
| # | Test | Expected |
|---|------|----------|
| 1 | `jarvis agents --help` | Shows all subcommands: `launch`, `start`, `stop`, `run`, `status`, `logs`, `daemon`, `watch`, `recover`, `errors`, `ask`, `instruct`, `messages`, `list`, `info`, `create`, `pause`, `resume`, `delete`, `bind`, `channels`, `search`, `templates`, `tasks` |
### 1.2 Agent lifecycle: create → run → pause → resume → delete
| # | Test | Expected |
|---|------|----------|
| 2 | `jarvis agents launch` | Wizard: template list → name/schedule/tools/budget/learning prompts → creates agent, prints ID |
| 3 | `jarvis agents list` | Agent appears, status=`idle` |
| 4 | `jarvis agents status` | Table: name, status dot, schedule, last run, runs=0, cost=$0 |
| 5 | `jarvis agents run <id>` | Prints progress then "Tick complete. Status: idle, runs: 1" |
| 6 | `jarvis agents status` | runs=1, last run time updated |
| 7 | `jarvis agents pause <id>` then `status` | Status shows `paused` |
| 8 | `jarvis agents resume <id>` then `status` | Status back to `idle` |
| 9 | `jarvis agents delete <id>` then `list` | Agent gone (soft-deleted/archived, not in list) |
### 1.3 Agent creation variants
| # | Test | Expected |
|---|------|----------|
| 10 | `jarvis agents create "Test Agent"` | Creates agent by name, prints ID |
| 11 | `jarvis agents create --template <template_name>` | Creates from template, inherits template config |
| 12 | `jarvis agents launch` → pick a template | Wizard pre-fills config from template |
| 13 | `jarvis agents launch` → pick "Custom Agent" | Wizard starts with blank config |
| 14 | `jarvis agents templates` | Lists built-in + user templates with descriptions |
### 1.4 Scheduling
| # | Test | Expected |
|---|------|----------|
| 15 | Create agent with `schedule_type=interval`, `schedule_value=30` | Created |
| 16 | `jarvis agents start <id>` | "Agent registered with scheduler" |
| 17 | `jarvis agents stop <id>` | "Agent deregistered from scheduler" |
| 18 | `jarvis agents daemon` | Starts, prints agent count, blocks. Ctrl+C → "Daemon stopped." clean exit |
| 19 | Create agent with `schedule_type=cron`, `schedule_value="*/5 * * * *"` | Created |
| 20 | `jarvis agents start <id>` (cron agent) | Registered, next fire time displayed or logged |
| 21 | Create agent with `schedule_type=manual` then `start <id>` | Agent registered but never auto-fires |
### 1.5 Interaction: ask / instruct / messages
| # | Test | Expected |
|---|------|----------|
| 22 | `jarvis agents ask <id> "What is 2+2?"` | Runs tick, prints agent response inline |
| 23 | `jarvis agents messages <id>` | Shows user→agent ask + agent→user response |
| 24 | `jarvis agents instruct <id> "Focus on ML papers"` | "Instruction queued for next tick" |
| 25 | `jarvis agents messages <id>` | Queued instruction shows `[queued]`, status=pending |
| 26 | `jarvis agents run <id>` then `messages <id>` | Queued message now delivered, status changes from pending |
| 27 | `jarvis agents ask <id> ""` (empty message) | Graceful error or rejection, no crash |
| 28 | `jarvis agents instruct <id>` with very long message (>1000 chars) | Accepted and stored correctly |
### 1.6 Error recovery & monitoring
| # | Test | Expected |
|---|------|----------|
| 29 | `jarvis agents errors` | Lists agents in error/needs_attention/stalled/budget_exceeded (or empty table) |
| 30 | `jarvis agents recover <id>` (on errored agent) | Restores checkpoint, status → `idle` |
| 31 | `jarvis agents recover <id>` (on idle agent) | Clear message: "Agent is not in error state" or similar |
| 32 | `jarvis agents logs <id>` | Recent traces with tick IDs and timestamps |
| 33 | `jarvis agents logs <nonexistent_id>` | Clear error: "Agent not found" |
| 34 | `jarvis agents watch` (then run a tick in another terminal) | Events stream live: AGENT_TICK_START, AGENT_TICK_END visible. Ctrl+C to stop. |
| 35 | `jarvis agents watch <id>` | Same, filtered to one agent only |
| 36 | `jarvis agents watch` then Ctrl+C | Clean exit, no traceback, no hanging threads |
### 1.7 Agent info & inspection
| # | Test | Expected |
|---|------|----------|
| 37 | `jarvis agents info <id>` | Shows agent type, status, memory snippet, tasks, channels, config details |
| 38 | `jarvis agents tasks <id>` | Lists tasks with statuses (or empty state) |
| 39 | `jarvis agents channels <id>` | Lists channel bindings (or empty state) |
| 40 | `jarvis agents search "keyword"` | Searches across agent traces, returns relevant results |
### 1.8 Edge cases & invalid input
| # | Test | Expected |
|---|------|----------|
| 41 | `jarvis agents run <nonexistent_id>` | Clear error: "Agent not found" — no Python traceback |
| 42 | `jarvis agents pause <id>` twice | Second pause is no-op or clear message, no crash |
| 43 | `jarvis agents resume <id>` (already idle) | No-op or clear message, no crash |
| 44 | `jarvis agents run <id>` while another tick is running | Concurrency guard: "Agent is already running" error |
| 45 | `jarvis agents delete <id>` then `run <id>` | Clear error about deleted/archived agent |
| 46 | Create agent with invalid cron expression | Rejected with clear validation error |
| 47 | Create agent with negative budget | Rejected or clamped to 0 |
### 1.9 CLI aesthetics
| # | Check | Expected |
|---|-------|----------|
| 48 | `status` table formatting | Columns aligned, readable at 80-char terminal width |
| 49 | Error messages (run with no engine configured) | Clear human-readable message, no Python tracebacks |
| 50 | `launch` wizard prompts | Clear labels, sensible defaults, no confusing jargon |
| 51 | `watch` event stream | Color-coded, event type + agent name visible, timestamps |
| 52 | `list` table with 0 agents | "No agents found" or empty table — not a crash |
| 53 | `list` table with 10+ agents | Table remains readable, no column overflow |
| 54 | All commands with `--help` | Every subcommand has a help string |
---
## Part 2: Web Frontend
### 2.0 Setup
```bash
# Terminal 1 # Terminal 2
uv run jarvis serve cd frontend && npm install && npm run dev
```
Open http://localhost:5173, navigate to **Agents** page via sidebar.
### 2.1 Navigation & routing
| # | Test | Expected |
|---|------|----------|
| 55 | Click "Agents" in sidebar | AgentsPage renders, URL is `/agents` |
| 56 | Direct navigation to `/agents` | Page loads correctly (no blank screen) |
| 57 | Browser back/forward after visiting agent detail | Navigation works, state preserved |
### 2.2 List view
| # | Test | Expected |
|---|------|----------|
| 58 | Page loads with backend running | No console errors, agent list renders |
| 59 | Page loads with backend **down** | User-visible error message (not blank white screen), no console exceptions |
| 60 | Agent cards | Name, color status dot, schedule description, last run time, runs count, cost |
| 61 | "Run Now" button | Triggers tick, card updates (runs count increments, last run time updates) |
| 62 | Pause/Resume button | Toggles status, dot color changes immediately |
| 63 | Agent list auto-refresh | After running a tick via CLI, the web list eventually reflects the updated state |
| 64 | 10+ agents in list | Cards render without performance issues, scroll works |
### 2.3 Launch wizard
| # | Test | Expected |
|---|------|----------|
| 65 | Click "Launch Agent" | Modal appears: Step 1 template picker with templates + "Custom Agent" option |
| 66 | Templates load from API | Template cards display with names and descriptions |
| 67 | Select template → Next → Step 2 | Config form: name (pre-filled from template), schedule_type dropdown, schedule_value, tools checkboxes, budget, learning toggle (off) |
| 68 | Select "Custom Agent" → Next → Step 2 | Config form with blank name, no pre-filled values |
| 69 | Next → Step 3 | Review summary of all config values |
| 70 | Click Launch | Agent created, modal closes, new agent appears in list |
| 71 | Back button at Step 2 | Returns to Step 1, template selection preserved |
| 72 | Back button at Step 3 | Returns to Step 2, all form inputs preserved |
| 73 | Launch with empty name | Inline error: "Agent name is required" — modal stays open |
| 74 | Launch with all tools selected | All tools included in review and in created agent config |
| 75 | Click outside modal / press Escape | Modal closes (or stays open — document behavior) |
| 76 | Schedule type = "Manual" | schedule_value input is disabled/hidden |
| 77 | Schedule type = "Cron" | schedule_value placeholder shows cron example |
| 78 | Schedule type = "Interval" | schedule_value placeholder shows seconds example |
### 2.4 Detail view (click an agent)
| # | Test | Expected |
|---|------|----------|
| 79 | Click agent card | Detail view opens with tabbed interface |
| 80 | **Overview** tab | Stat cards (Total Runs, Success Rate, Total Cost), config display, channels list, action buttons |
| 81 | **Overview** action buttons | Run Now, Pause, Resume visible and functional |
| 82 | **Interact** tab | Chat message list, textarea, "Immediate" and "Queue" send buttons |
| 83 | Send immediate message | Appears in chat with user styling, agent responds after tick |
| 84 | Send queued message | Shows with "queued" badge, status=pending |
| 85 | Send empty message | Button disabled or graceful rejection — no empty message sent |
| 86 | Rapid-fire send (click Send multiple times quickly) | No duplicate messages, no race condition errors |
| 87 | Chat auto-scroll | New messages scroll into view automatically |
| 88 | **Tasks** tab | Task list with status badges (completed=green, failed=red, active=blue, pending=gray) |
| 89 | **Tasks** tab (no tasks) | Empty state: "No tasks assigned." |
| 90 | **Memory** tab | summary_memory text displayed in readable format |
| 91 | **Memory** tab (no memory) | Empty state: "Agent has no stored memory yet." |
| 92 | **Learning** tab | Toggle switch (read-only, off by default), placeholder text for future events |
| 93 | **Logs** tab | Placeholder / empty state message (not a crash or blank) |
| 94 | Tab switching — rapid clicks | All 6 tabs render instantly, no layout shift, no flash of wrong content |
### 2.5 Error states
| # | Test | Expected |
|---|------|----------|
| 95 | Agent in `error` status | Red status dot/badge, "Recover" button visible |
| 96 | Click Recover | Status resets to `idle`, dot turns green |
| 97 | Agent in `needs_attention` status | Amber badge visible |
| 98 | Agent in `budget_exceeded` status | Orange badge visible |
| 99 | Agent in `stalled` status | Yellow badge visible |
| 100 | Backend goes down while page is open | Next refresh/action shows error — not silent failure |
| 101 | Delete agent → confirm it disappears from list | Agent removed from list immediately (or on next refresh) |
| 102 | Delete agent (no confirmation dialog in web) | **Document:** Is instant delete OK or should there be a confirm? |
### 2.6 Overflow menu
| # | Test | Expected |
|---|------|----------|
| 103 | Click "..." menu on agent card | Dropdown with Delete + other options |
| 104 | Click Delete from menu | Agent deleted, list updates |
| 105 | Click outside dropdown | Dropdown closes |
### 2.7 Web aesthetics & UX
| # | Check | Expected |
|---|-------|----------|
| 106 | Status dot colors | idle=#22c55e, running=#3b82f6, paused=#6b7280, error=#ef4444, needs_attention=#f59e0b, budget_exceeded=#f97316, stalled=#eab308 |
| 107 | Launch wizard spacing/alignment | Modal centered, steps clearly numbered, form inputs aligned, no overlap |
| 108 | Detail view tab switching | Instant, no layout shift or flash |
| 109 | Interact tab chat feel | Messages visually distinct (user=right vs agent=left or different colors), auto-scroll, clear input area |
| 110 | Responsive at 1024px width | No overflow or cut-off content, agent cards reflow |
| 111 | Responsive at 1440px width | Proper use of space, no excessive stretching |
| 112 | Responsive at 768px width (tablet) | Still usable, no broken layout |
| 113 | Empty states | "No agents yet" + CTA button / "No messages" / "No tasks" — not blank white space |
| 114 | Loading states | "Loading agents..." shown during fetch, spinner or skeleton |
| 115 | Page title / browser tab | Meaningful title (not just "Vite App") |
| 116 | Console errors | Zero console errors during normal usage flow |
---
## Part 3: Desktop App
### 3.0 Setup
```bash
# Terminal 1 # Terminal 2
uv run jarvis serve cd desktop && npm install && npm run tauri dev
```
Navigate to the **Agents** tab.
### 3.1 Functionality
| # | Test | Expected |
|---|------|----------|
| 117 | Left panel: agent list | Status dots, schedule descriptions, last run times |
| 118 | Click agent → right panel | Tabbed detail view (Overview, Interact, Tasks, Memory, Learning, Logs) |
| 119 | No agent selected | Right panel shows "Select an agent to view details" |
| 120 | "Launch Agent" button | Opens wizard, same 3-step flow as web |
| 121 | Launch wizard → Create agent | Agent appears in left panel list |
| 122 | **Overview** tab | Key-value stats (Status, Agent Type, Schedule, Last Run, Total Runs, Total Cost, Budget) + action buttons (Run Now, Pause, Resume, Recover) |
| 123 | **Interact** tab | Chat UI, mode toggle (immediate/queued), Enter shortcut sends message |
| 124 | Send immediate message | Response appears in chat |
| 125 | Send queued message | Shows as pending |
| 126 | **Tasks** tab | Task list with colored status badges + created-at timestamps |
| 127 | **Memory** tab | summary_memory in monospace font |
| 128 | **Learning** tab | Shows enabled/disabled status + placeholder text |
| 129 | **Logs** tab | Placeholder: "Log streaming not yet connected." |
| 130 | Auto-refresh | Agent list refreshes on ~10s interval (verify with CLI-triggered state change) |
| 131 | Delete agent via desktop | Confirmation dialog appears, agent removed on confirm |
### 3.2 Desktop edge cases
| # | Test | Expected |
|---|------|----------|
| 132 | Backend not running → open desktop app | Error state shown, not a crash |
| 133 | Backend dies while desktop is open | Graceful degradation on next action/refresh |
| 134 | Selected agent deleted via CLI → desktop refreshes | Selected agent deselects, list updates |
### 3.3 Desktop aesthetics
| # | Check | Expected |
|---|-------|----------|
| 135 | Catppuccin color scheme consistent | idle=#a6e3a1, running=#89b4fa, paused=#6c7086, error=#f38ba8, needs_attention=#fab387, stalled=#f9e2af |
| 136 | Left/right panel split | Resizable or fixed at reasonable ratio, no overlap |
| 137 | Tab switching | Smooth, no flicker |
| 138 | Launch wizard modal | Properly overlays content, dismissible with Escape or outside click |
| 139 | Text readability | Font sizes consistent, sufficient contrast against dark background |
| 140 | Window resize | Layout adapts, no overflow or clipping |
| 141 | Status badge consistency with web | Same statuses map to same semantic colors (green=idle, blue=running, etc.) |
---
## Part 4: API Backend (Direct)
### 4.1 REST endpoint smoke tests
Run with `uv run jarvis serve` and test via curl or Postman.
| # | Test | Expected |
|---|------|----------|
| 142 | `GET /v1/managed-agents` | 200, returns `[]` or agent list JSON |
| 143 | `POST /v1/managed-agents` with valid body | 200/201, returns created agent JSON with `id` |
| 144 | `POST /v1/managed-agents` with empty body | 422 or 400 with validation error |
| 145 | `GET /v1/managed-agents/<id>` | 200, returns single agent |
| 146 | `GET /v1/managed-agents/<bad_id>` | 404, returns error JSON |
| 147 | `POST /v1/managed-agents/<id>/run` | 200, tick executes |
| 148 | `POST /v1/managed-agents/<id>/pause` | 200, status changes to paused |
| 149 | `POST /v1/managed-agents/<id>/resume` | 200, status changes to idle |
| 150 | `POST /v1/managed-agents/<id>/recover` | 200 if errored, appropriate error if not |
| 151 | `DELETE /v1/managed-agents/<id>` | 200, agent archived |
| 152 | `GET /v1/templates` | 200, returns template list |
| 153 | `POST /v1/templates/<id>/instantiate` | 200, creates agent from template |
| 154 | `GET /v1/agents/errors` | 200, returns list of problem agents |
| 155 | `GET /v1/agents/health` | 200, returns health summary |
### 4.2 Message endpoints
| # | Test | Expected |
|---|------|----------|
| 156 | `POST /v1/managed-agents/<id>/messages` with `{"content":"hi","direction":"user_to_agent","mode":"immediate"}` | 200, message stored |
| 157 | `GET /v1/managed-agents/<id>/messages` | 200, returns message list |
| 158 | `POST /v1/managed-agents/<id>/messages` with `{"content":"","direction":"user_to_agent","mode":"immediate"}` | 422 or graceful handling |
| 159 | `POST /v1/managed-agents/<id>/messages` with `{"content":"cmd","direction":"user_to_agent","mode":"queued"}` | 200, message has status=pending |
### 4.3 Task & channel endpoints
| # | Test | Expected |
|---|------|----------|
| 160 | `GET /v1/managed-agents/<id>/tasks` | 200, returns task list |
| 161 | `POST /v1/managed-agents/<id>/tasks` | 200, creates task |
| 162 | `GET /v1/managed-agents/<id>/channels` | 200, returns channel bindings |
| 163 | `GET /v1/managed-agents/<id>/state` | 200, returns full agent state |
### 4.4 WebSocket events
| # | Test | Expected |
|---|------|----------|
| 164 | Connect to `ws://localhost:8222/v1/agents/events` | Connection established |
| 165 | Trigger a tick → observe WS messages | Receive AGENT_TICK_START and AGENT_TICK_END events |
| 166 | Connect with `?agent_id=<id>` filter | Only events for that agent |
| 167 | Disconnect cleanly | No server error logs |
---
## Part 5: Cross-Platform Consistency
| # | Test | Expected |
|---|------|----------|
| 168 | Create agent via CLI → check web + desktop | Same name, status, config everywhere |
| 169 | Run tick via CLI → check web + desktop | Run count and last run time update in both UIs |
| 170 | Send message via web Interact → check CLI `messages` | Same content, direction, mode |
| 171 | Pause via desktop → check CLI `status` + web | `paused` everywhere |
| 172 | Delete via web → check CLI `list` + desktop | Gone everywhere |
| 173 | Create via web wizard → check CLI `list` + desktop | Agent visible in all three |
| 174 | Recover via CLI → check web + desktop | Status back to idle in all UIs |
| 175 | Send queued message via CLI `instruct` → check web Interact | Message shows with pending/queued status |
| 176 | Multiple agents created from different clients | All agents appear correctly in all views |
---
## Part 6: Stress & Concurrency
| # | Test | Expected |
|---|------|----------|
| 177 | Run tick on same agent from two terminals simultaneously | Concurrency guard blocks second tick: "Agent is already running" |
| 178 | Create 20+ agents → check list performance | All clients render list without lag |
| 179 | Rapidly pause/resume same agent | All state transitions correct, no stuck states |
| 180 | Run daemon + manual `run` at same time | No double-ticking, concurrency guard holds |
| 181 | Delete agent while tick is in progress | Tick completes or fails gracefully, agent ends up archived |
---
## Part 7: Deferred Features (Placeholder Verification)
Confirm these show placeholders (not crashes):
| # | Feature | CLI | Web | Desktop |
|---|---------|-----|-----|---------|
| 182 | Budget enforcement | `run` still works even if cost > budget | No enforcement, budget is display-only | Same |
| 183 | Stall detection | No automatic stall detection fires | N/A | N/A |
| 184 | Learning event timeline | `Learning` tab shows placeholder text | Same | Same |
| 185 | Logs trace replay | `Logs` tab shows placeholder text | Same | Same |
| 186 | `POST /v1/skills` | N/A | N/A | Returns `"not_implemented"` |
| 187 | `POST /v1/optimize/runs` | N/A | N/A | Returns placeholder `run_id` |
| 188 | `GET /v1/feedback/stats` | N/A | N/A | Returns `{total: 0, mean_score: 0.0}` |
---
## Deliverables
**1. Test results** — Spreadsheet with columns: #, Status (Pass/Fail/Partial/Blocked), Actual Behavior, Screenshot (for UI issues).
**2. Bug list** — Each bug: steps to reproduce, expected vs actual, severity (Critical/Major/Minor), screenshot.
**3. UX & aesthetics feedback** — Is the launch wizard clear? Are status colors distinguishable? Does the Interact tab feel like chat? Is CLI output readable? Are error messages helpful? Is the delete-without-confirm behavior in web acceptable?
**4. API error handling audit** — Document all cases where the frontend silently swallows errors (currently: agent list fetch, interact tab sends). Recommend which should show user-visible errors.
**5. Deferred features check** — Confirm placeholder items in Part 7 show graceful stubs (not crashes or blank screens).
---
## Notes
- Backend (`jarvis serve`) must be running for web and desktop (default port 8222).
- Without an engine configured, `run`/`ask` will error — document whether the error message is clear.
- `daemon` and `watch` block — Ctrl+C to exit.
- Web frontend API client has unused functions (`updateManagedAgent`, `createAgentTask`, `fetchAgentState`, `fetchErrorAgents`) — not a bug, but note for future.
- Desktop API client is missing some endpoints that the web client has (`fetchAgentChannels`, `fetchAgentState`, `fetchErrorAgents`) — may affect feature parity.
- Frontend has **no automated tests** — all testing is manual per this plan.
- Both frontends silently catch API errors (`.catch(() => {})`) — this is a known UX gap to evaluate.
scripts/GRID_SEARCH_GUIDE.md
+350
View File
@@ -0,0 +1,350 @@
# Eval Grid Search — 4× A100 Runbook
Complete guide for running the OpenJarvis evaluation grid search across **5 models × 4 engines × 5 agents × 15 benchmarks** on a 4× NVIDIA A100 (80 GB) node.
## Grid Dimensions
| Dimension | Values |
|-----------|--------|
| **Models** | GPT-OSS-120B, Qwen3.5-122B-FP8, Qwen3.5-397B-GGUF, Kimi-K2.5-GGUF, GLM-5-GGUF |
| **Engines** | vLLM, SGLang, llama.cpp, Ollama |
| **Agents** | simple, orchestrator, native_react, native_openhands, rlm |
| **Benchmarks** | supergpqa, gpqa, mmlu-pro, math500, natural-reasoning, hle, simpleqa, wildchat, ipw, gaia, frames, swebench, swefficiency, terminalbench, terminalbench-native |
| **Samples** | 5 per benchmark |
**Total: 750 experiment cells** (each model only runs on its compatible engines).
| Model | Compatible Engines |
|-------|--------------------|
| openai/gpt-oss-120b | vllm, sglang |
| Qwen/Qwen3.5-122B-A10B-FP8 | vllm, sglang |
| unsloth/Qwen3.5-397B-A17B-GGUF | llamacpp, ollama |
| unsloth/Kimi-K2.5-GGUF | llamacpp, ollama |
| unsloth/GLM-5-GGUF | llamacpp, ollama |
---
## Phase 1: Environment Setup
```bash
# Clone and install
cd ~/gabebo # or wherever your workspace lives
git clone <repo-url> OpenJarvis && cd OpenJarvis
uv sync --extra dev
# Install eval dependencies
uv pip install openai datasets huggingface-hub terminal-bench
# Load API keys (needed for LLM judge — gpt-5-mini)
source .env
# Log in to HuggingFace (needed for gated datasets)
huggingface-cli login
```
> **SGLang note:** SGLang requires Python ≤ 3.12 (FlashInfer/outlines_core fail on 3.13).
> If your base env is Python 3.13, create a separate conda env:
> ```bash
> conda create -n sglang python=3.11 -y && conda activate sglang
> pip install "sglang[all]"
> ```
---
## Phase 2: Download Models
```bash
# HuggingFace weights (vLLM / SGLang)
hf download openai/gpt-oss-120b --local-dir ~/models/gpt-oss-120b
hf download Qwen/Qwen3.5-122B-A10B-FP8 --local-dir ~/models/Qwen3.5-122B-A10B-FP8
# GGUF quantizations (llama.cpp / Ollama)
# Qwen3.5-397B — UD-Q4_K_XL fits in 4× A100 (~214 GB)
hf download unsloth/Qwen3.5-397B-A17B-GGUF \
--include "Q4_K_M/*.gguf" \
--local-dir ~/models/Qwen3.5-397B-A17B-GGUF
# Kimi-K2.5 — UD-IQ2_XXS to fit (~240 GB usable)
hf download unsloth/Kimi-K2.5-GGUF \
--include "UD-IQ2_XXS/*.gguf" \
--local-dir ~/models/Kimi-K2.5-GGUF
# GLM-5 — UD-IQ2_XXS to fit
hf download unsloth/GLM-5-GGUF \
--include "UD-IQ2_XXS/*.gguf" \
--local-dir ~/models/GLM-5-GGUF
```
### Verify downloads
```bash
# HuggingFace weights — check for config.json and safetensors shards
ls ~/models/gpt-oss-120b/config.json
ls ~/models/Qwen3.5-122B-A10B-FP8/config.json
# GGUF files — check they exist and aren't zero-byte
find ~/models/Qwen3.5-397B-A17B-GGUF -name "*.gguf" -exec ls -lh {} \;
find ~/models/Kimi-K2.5-GGUF -name "*.gguf" -exec ls -lh {} \;
find ~/models/GLM-5-GGUF -name "*.gguf" -exec ls -lh {} \;
```
---
## Phase 3: Run the Grid (One Model at a Time)
Serve a model, run all agents/benchmarks for it, then kill the server and swap.
### 3A. GPT-OSS-120B via vLLM
```bash
# Terminal 1: Start vLLM server
vllm serve openai/gpt-oss-120b \
--tensor-parallel-size 2 \
--port 8000 \
--max-model-len 8192
```
```bash
# Terminal 2: Wait for "Uvicorn running" then run grid
cd OpenJarvis && source .env
uv run python scripts/run_grid_search.py \
--model "gpt-oss" --engine vllm -n 5
# When done, Ctrl-C the vLLM server
```
### 3B. GPT-OSS-120B via SGLang
```bash
# Terminal 1: Start SGLang server
python -m sglang.launch_server \
--model-path openai/gpt-oss-120b \
--tp 2 \
--port 30000
```
```bash
# Terminal 2: Run grid
uv run python scripts/run_grid_search.py \
--model "gpt-oss" --engine sglang --resume -n 5
# Kill server when done
```
### 3C. Qwen3.5-122B-FP8 via vLLM
```bash
# Terminal 1
vllm serve Qwen/Qwen3.5-122B-A10B-FP8 \
--tensor-parallel-size 2 \
--port 8000 \
--max-model-len 8192 \
--quantization fp8
```
```bash
# Terminal 2
uv run python scripts/run_grid_search.py \
--model "Qwen3.5-122B" --engine vllm --resume -n 5
```
### 3D. Qwen3.5-122B-FP8 via SGLang
```bash
# Terminal 1
python -m sglang.launch_server \
--model-path Qwen/Qwen3.5-122B-A10B-FP8 \
--tp 2 \
--port 30000 \
--quantization fp8
```
```bash
# Terminal 2
uv run python scripts/run_grid_search.py \
--model "Qwen3.5-122B" --engine sglang --resume -n 5
```
### 3E. Qwen3.5-397B GGUF via llama.cpp
```bash
# Terminal 1: Start llama.cpp server with all 4 GPUs
./llama.cpp/build/bin/llama-server \
-m ~/models/Qwen3.5-397B-A17B-GGUF/Q4_K_M/Qwen3.5-397B-A17B-Q4_K_M-00001-of-00005.gguf \
--n-gpu-layers 99 \
--tensor-split 1,1,1,1 \
--port 8080 \
--ctx-size 8192
```
```bash
# Terminal 2
uv run python scripts/run_grid_search.py \
--model "Qwen3.5-397B" --engine llamacpp --resume -n 5
```
### 3F. Qwen3.5-397B GGUF via Ollama
```bash
# Create an Ollama modelfile
cat > /tmp/Qwen3.5-397B.Modelfile << 'EOF'
FROM ~/models/Qwen3.5-397B-A17B-GGUF/Q4_K_M/Qwen3.5-397B-A17B-Q4_K_M-00001-of-00005.gguf
EOF
# Import into Ollama
ollama create qwen3.5-397b -f /tmp/Qwen3.5-397B.Modelfile
# Ollama serves automatically on port 11434
uv run python scripts/run_grid_search.py \
--model "Qwen3.5-397B" --engine ollama --resume -n 5
# Unload when done
ollama stop qwen3.5-397b
```
### 3G. Kimi-K2.5 GGUF via llama.cpp
```bash
# Terminal 1
./llama.cpp/build/bin/llama-server \
-m ~/models/Kimi-K2.5-GGUF/UD-IQ2_XXS/Kimi-K2.5-UD-IQ2_XXS-00001-of-00005.gguf \
--n-gpu-layers 99 \
--tensor-split 1,1,1,1 \
--port 8080 \
--ctx-size 8192
```
```bash
# Terminal 2
uv run python scripts/run_grid_search.py \
--model "Kimi-K2.5" --engine llamacpp --resume -n 5
```
### 3H. Kimi-K2.5 GGUF via Ollama
```bash
cat > /tmp/Kimi-K2.5.Modelfile << 'EOF'
FROM ~/models/Kimi-K2.5-GGUF/UD-IQ2_XXS/Kimi-K2.5-UD-IQ2_XXS-00001-of-00005.gguf
EOF
ollama create kimi-k2.5 -f /tmp/Kimi-K2.5.Modelfile
uv run python scripts/run_grid_search.py \
--model "Kimi-K2.5" --engine ollama --resume -n 5
ollama stop kimi-k2.5
```
### 3I. GLM-5 GGUF via llama.cpp
```bash
# Terminal 1
./llama.cpp/build/bin/llama-server \
-m ~/models/GLM-5-GGUF/UD-IQ2_XXS/GLM-5-UD-IQ2_XXS-00001-of-00005.gguf \
--n-gpu-layers 99 \
--tensor-split 1,1,1,1 \
--port 8080 \
--ctx-size 8192
```
```bash
# Terminal 2
uv run python scripts/run_grid_search.py \
--model "GLM-5" --engine llamacpp --resume -n 5
```
### 3J. GLM-5 GGUF via Ollama
```bash
cat > /tmp/GLM-5.Modelfile << 'EOF'
FROM ~/models/GLM-5-GGUF/UD-IQ2_XXS/GLM-5-UD-IQ2_XXS-00001-of-00005.gguf
EOF
ollama create glm-5 -f /tmp/GLM-5.Modelfile
uv run python scripts/run_grid_search.py \
--model "GLM-5" --engine ollama --resume -n 5
ollama stop glm-5
```
---
## Phase 4: Recover Failed Runs
If some runs fail (missing packages, engine not reachable, etc.), fix the issue, then:
```bash
# Delete error summaries so --resume retries them
find results/grid-search -name "*.summary.json" \
-exec grep -l '"error"' {} \; -delete
# Re-run with --resume (only retries deleted/missing summaries)
uv run python scripts/run_grid_search.py --resume -n 5
```
### Common Failures and Fixes
| Error | Cause | Fix |
|-------|-------|-----|
| `No inference engine available` | `openai` package missing (LLM judge can't init) | `uv pip install openai` |
| `No module named 'datasets'` | Missing HF datasets package | `uv pip install datasets` |
| `terminal-bench package required` | Missing terminal-bench | `uv pip install terminal-bench` |
| `Dataset doesn't exist on the Hub` | Gated dataset or HF auth needed | `huggingface-cli login`, accept terms on HF website |
| `IPW data directory not found` | IPW uses local data, not HuggingFace | Place files in `src/openjarvis/evals/data/ipw/` |
| `natural-reasoning` 0 samples | Field name mismatch in dataset loader | Patch `src/openjarvis/evals/datasets/natural_reasoning.py` |
---
## Phase 5: Analyze Results
```bash
# Preview what ran
uv run python scripts/run_grid_search.py --dry-run --resume
# Consolidated results (appended after each run)
cat results/grid-search/grid-results.jsonl
# Per-run summaries
find results/grid-search -name "*.summary.json" | head -20
cat results/grid-search/openai-gpt-oss-120b/vllm/simple/supergpqa.summary.json
# Count completed vs failed
echo "Completed:" && find results/grid-search -name "*.summary.json" \
-exec grep -L '"error"' {} \; | wc -l
echo "Failed:" && find results/grid-search -name "*.summary.json" \
-exec grep -l '"error"' {} \; | wc -l
```
---
## Useful Flags
```bash
# Preview the full matrix without running
uv run python scripts/run_grid_search.py --dry-run
# Filter to a single model + engine
uv run python scripts/run_grid_search.py --model "gpt-oss" --engine vllm -n 5
# Filter to a single agent or benchmark
uv run python scripts/run_grid_search.py --agent native_react --benchmark supergpqa -n 5
# Increase sample count
uv run python scripts/run_grid_search.py -n 50
# Verbose logging
uv run python scripts/run_grid_search.py -v --resume -n 5
```
---
## GPU Assignment Tips
Use `CUDA_VISIBLE_DEVICES` to pin servers to specific GPUs:
```bash
# Run vLLM on GPUs 0,1 and llama.cpp on GPUs 2,3 simultaneously
CUDA_VISIBLE_DEVICES=0,1 vllm serve openai/gpt-oss-120b --tensor-parallel-size 2 --port 8000
CUDA_VISIBLE_DEVICES=2,3 ./llama.cpp/build/bin/llama-server -m model.gguf --n-gpu-layers 99 --port 8080
```
This lets you run two model servers in parallel on different GPU pairs.