From efd395d9ade8b8ebd1eaed068815467d938e446d Mon Sep 17 00:00:00 2001 From: Garry Tan Date: Sat, 18 Apr 2026 08:26:10 +0800 Subject: [PATCH] =?UTF-8?q?test(bench):=20memory=20=E2=80=94=2010=20in-fli?= =?UTF-8?q?ght=20subagents,=20single-proc=20vs=2010-proc=20cost?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Measures resident memory for keeping 10 subagents in flight. Minions: one worker process, concurrency=10 with handlers that park on a promise — sample RSS of the test process via process.memoryUsage(). OpenClaw: 10 parallel `openclaw agent --local` processes, sum their RSS via `ps -o rss=`. Handlers are cheap sleeps, no LLM — we want harness memory, not LLM client state. Budget: $0. --- test/e2e/bench-vs-openclaw/memory.bench.ts | 225 +++++++++++++++++++++ 1 file changed, 225 insertions(+) create mode 100644 test/e2e/bench-vs-openclaw/memory.bench.ts diff --git a/test/e2e/bench-vs-openclaw/memory.bench.ts b/test/e2e/bench-vs-openclaw/memory.bench.ts new file mode 100644 index 000000000..c27d25ebc --- /dev/null +++ b/test/e2e/bench-vs-openclaw/memory.bench.ts @@ -0,0 +1,225 @@ +/** + * Memory bench: resident memory cost of keeping 10 subagents in flight. + * + * Minions side: one worker process with concurrency=10 runs 10 sleepy + * handlers in parallel. RSS is measured on the test/worker process via + * `process.memoryUsage().rss`. + * + * OpenClaw side: 10 parallel `openclaw agent --local` spawns. Each is + * its own process with its own runtime, auth, plugins. Total RSS = + * sum of all 10 via `ps -o rss=`. + * + * Handlers are intentionally cheap (sleep, no LLM) so we measure the + * *harness* memory cost, not LLM client state. + * + * Budget: $0 (no LLM calls). + * + * Run: DATABASE_URL=... bun test test/e2e/bench-vs-openclaw/memory.bench.ts + */ + +import { describe, test, expect, beforeAll, afterAll, beforeEach } from 'bun:test'; +import { spawn, execFileSync } from 'node:child_process'; +import { hasDatabase, setupDB, teardownDB, getConn, getEngine } from '../helpers.ts'; +import { PostgresEngine } from '../../../src/core/postgres-engine.ts'; +import { MinionQueue } from '../../../src/core/minions/queue.ts'; +import { MinionWorker } from '../../../src/core/minions/worker.ts'; +import { runMigrations } from '../../../src/core/migrate.ts'; +import { BENCH_PROMPT } from './harness.ts'; + +const skip = !hasDatabase(); +const describeBench = skip ? describe.skip : describe; + +if (skip) { + console.log('Skipping memory bench (DATABASE_URL not set)'); +} + +const IN_FLIGHT = 10; + +function rssMB(): number { + return Math.round(process.memoryUsage().rss / (1024 * 1024)); +} + +/** Sum RSS of pids via ps. Returns MB. Missing pids count as 0. */ +function pidsRssMB(pids: number[]): number { + if (pids.length === 0) return 0; + try { + const out = execFileSync('ps', ['-o', 'rss=', '-p', pids.join(',')], { + encoding: 'utf-8', + }); + const kbSum = out + .split('\n') + .map((l) => parseInt(l.trim(), 10)) + .filter((n) => !Number.isNaN(n)) + .reduce((a, b) => a + b, 0); + return Math.round(kbSum / 1024); + } catch { + return 0; + } +} + +describeBench('Bench: Memory (RSS with 10 subagents in flight)', () => { + beforeAll(async () => { + await setupDB(); + await runMigrations(getEngine()); + }); + + afterAll(async () => { + await teardownDB(); + }); + + beforeEach(async () => { + await getConn().unsafe( + `TRUNCATE minion_attachments, minion_inbox, minion_jobs RESTART IDENTITY CASCADE`, + ); + }); + + test(`Minions: worker + ${IN_FLIGHT} in-flight handlers, single-process RSS`, async () => { + const url = process.env.DATABASE_URL!; + const engine = new PostgresEngine(); + await engine.connect({ engine: 'postgres', database_url: url, poolSize: 16 }); + + try { + const queue = new MinionQueue(engine); + const worker = new MinionWorker(engine, { + concurrency: IN_FLIGHT, + pollInterval: 25, + lockDuration: 60_000, + stalledInterval: 60_000, + }); + + let active = 0; + let peakActive = 0; + const rssSamples: number[] = []; + const release: Array<() => void> = []; + + worker.register('bench-mem', async () => { + active++; + peakActive = Math.max(peakActive, active); + await new Promise((resolve) => { + release.push(resolve); + }); + active--; + return { ok: true }; + }); + + const baselineRssMB = rssMB(); + const startP = worker.start(); + + // Submit 10 jobs, let them all get claimed and sit + const ids: number[] = []; + for (let i = 0; i < IN_FLIGHT; i++) { + const job = await queue.add('bench-mem', { i }); + ids.push(job.id); + } + + // Wait for all 10 to be claimed + const deadline = Date.now() + 10_000; + while (Date.now() < deadline && peakActive < IN_FLIGHT) { + await new Promise((r) => setTimeout(r, 50)); + } + expect(peakActive).toBe(IN_FLIGHT); + + // Sample RSS 5× while all 10 are in flight + for (let i = 0; i < 5; i++) { + rssSamples.push(rssMB()); + await new Promise((r) => setTimeout(r, 200)); + } + const peakInFlightRssMB = Math.max(...rssSamples); + const deltaRssMB = peakInFlightRssMB - baselineRssMB; + + // Release all handlers + for (const r of release) r(); + + // Wait for completion + const doneDeadline = Date.now() + 5000; + while (Date.now() < doneDeadline) { + const rows = await getConn().unsafe<{ n: number }[]>( + `SELECT count(*)::int AS n FROM minion_jobs WHERE status = 'completed' AND id = ANY($1)`, + [ids], + ); + if (rows[0].n === IN_FLIGHT) break; + await new Promise((r) => setTimeout(r, 50)); + } + + worker.stop(); + await startP; + + console.log( + `\n[minions-memory] baselineRssMB=${baselineRssMB} peakInFlightRssMB=${peakInFlightRssMB} deltaMB=${deltaRssMB} inFlight=${IN_FLIGHT} processes=1`, + ); + } finally { + await engine.disconnect(); + } + }, 60_000); + + test(`OpenClaw: ${IN_FLIGHT} parallel --local spawns, summed RSS via ps`, async () => { + // Spawn 10 OC processes in parallel. Track pids. Sample summed RSS + // a few times while they're all alive, then kill them. + const children: ReturnType[] = []; + const pids: number[] = []; + + for (let i = 0; i < IN_FLIGHT; i++) { + const proc = spawn( + 'openclaw', + ['agent', '--agent', 'main', '--local', '--message', BENCH_PROMPT, '--timeout', '120'], + { env: process.env, stdio: ['ignore', 'pipe', 'pipe'] }, + ); + children.push(proc); + if (typeof proc.pid === 'number') pids.push(proc.pid); + } + + // Wait for all 10 to actually be running (RSS > 0 in ps) + const settleDeadline = Date.now() + 15_000; + let aliveCount = 0; + while (Date.now() < settleDeadline) { + const alive = pids.filter((pid) => { + try { + process.kill(pid, 0); + return true; + } catch { + return false; + } + }); + aliveCount = alive.length; + if (aliveCount === IN_FLIGHT) break; + await new Promise((r) => setTimeout(r, 200)); + } + + // Sample summed RSS while all are (hopefully) mid-dispatch + const rssSamples: number[] = []; + for (let i = 0; i < 5; i++) { + rssSamples.push(pidsRssMB(pids)); + await new Promise((r) => setTimeout(r, 400)); + } + + const peakSumRssMB = Math.max(...rssSamples); + const meanSumRssMB = Math.round(rssSamples.reduce((a, b) => a + b, 0) / rssSamples.length); + + // Cleanup: kill all survivors + for (const proc of children) { + try { + proc.kill('SIGKILL'); + } catch {} + } + await Promise.all( + children.map( + (proc) => + new Promise((resolve) => { + if (proc.exitCode !== null) return resolve(); + proc.on('close', () => resolve()); + setTimeout(() => resolve(), 3000); + }), + ), + ); + + console.log( + `\n[openclaw-memory] aliveAtSample=${aliveCount}/${IN_FLIGHT} peakSumRssMB=${peakSumRssMB} meanSumRssMB=${meanSumRssMB} rssSamples=[${rssSamples.join(',')}] processes=${IN_FLIGHT}`, + ); + + // Headline: memory scales with number of processes. At least 5 should + // have been alive long enough to sample; if OC failed to spawn we'd + // see 0 RSS everywhere. + expect(aliveCount).toBeGreaterThanOrEqual(5); + expect(peakSumRssMB).toBeGreaterThan(0); + }, 60_000); +});