From b81c598f732f7583e2eb67e804db806a3d10df37 Mon Sep 17 00:00:00 2001 From: Wintermute Date: Thu, 14 May 2026 23:40:30 +0000 Subject: [PATCH] fix: use RssAnon instead of VmRSS for watchdog threshold MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit process.memoryUsage().rss returns VmRSS which includes file-backed mmap'd pages. On repos with large git packfiles (96K+ pages), git operations inflate VmRSS to 7GB+ while actual heap usage is ~100MB. The kernel reclaims these pages under memory pressure — they're cache. Replace with /proc/self/status RssAnon + RssShmem which measures only anonymous pages (heap, stack, anonymous mmap). This is the memory that actually matters for OOM risk. Falls back to process.memoryUsage().rss on non-Linux. Before: watchdog triggers every autopilot cycle (7GB VmRSS > 4GB threshold) After: watchdog only triggers on real memory growth (~100MB << 4GB threshold) Related: #1002 (supervisor crash-count fix for the same symptom) --- src/core/minions/worker.ts | 29 ++++++++++++++++++++++++++++- 1 file changed, 28 insertions(+), 1 deletion(-) diff --git a/src/core/minions/worker.ts b/src/core/minions/worker.ts index 43b70fccb..82468805a 100644 --- a/src/core/minions/worker.ts +++ b/src/core/minions/worker.ts @@ -22,6 +22,33 @@ import { calculateBackoff } from './backoff.ts'; import { randomUUID } from 'crypto'; import { EventEmitter } from 'events'; import { evaluateQuietHours, type QuietHoursConfig } from './quiet-hours.ts'; +import { readFileSync } from 'fs'; + +/** + * Read accurate RSS from /proc/self/status (RssAnon + RssShmem). + * + * `process.memoryUsage().rss` returns VmRSS which includes file-backed mmap'd + * pages (e.g. git packfiles). On a 96K-page brain repo, git operations can + * inflate VmRSS to 7GB+ while actual heap usage is ~100MB. The kernel reclaims + * file-backed pages under memory pressure — they're cache, not real usage. + * + * RssAnon = anonymous pages (heap, stack, anonymous mmap). This is the memory + * the process actually allocated and would cause OOM if it grew unbounded. + * RssShmem = shared anonymous pages (IPC, tmpfs). Usually 0 for workers. + * + * Falls back to process.memoryUsage().rss on non-Linux or read errors. + */ +function getAccurateRss(): number { + try { + const status = readFileSync('/proc/self/status', 'utf8'); + const anonKb = parseInt(status.match(/RssAnon:\s+(\d+)/)?.[1] ?? '0', 10); + const shmemKb = parseInt(status.match(/RssShmem:\s+(\d+)/)?.[1] ?? '0', 10); + if (anonKb > 0) return (anonKb + shmemKb) * 1024; // return bytes + } catch { + // Non-Linux or /proc unavailable + } + return process.memoryUsage().rss; +} /** Reason payload emitted with `'unhealthy'` when self-health-check trips. * CLI layer (jobs.ts:work) subscribes and decides whether to call process.exit. */ @@ -93,7 +120,7 @@ export class MinionWorker extends EventEmitter { maxStalledCount: opts?.maxStalledCount ?? 1, pollInterval: opts?.pollInterval ?? 5000, maxRssMb: opts?.maxRssMb ?? 0, - getRss: opts?.getRss ?? (() => process.memoryUsage().rss), + getRss: opts?.getRss ?? getAccurateRss, rssCheckInterval: opts?.rssCheckInterval ?? 60000, healthCheckInterval: opts?.healthCheckInterval ?? 60000, stallWarnAfterMs: opts?.stallWarnAfterMs ?? 5 * 60_000,