mirror of
https://github.com/garrytan/gbrain.git
synced 2026-07-27 21:19:18 +00:00
* fix(minions): self-identifying RSS watchdog + cgroup-aware default + pooler-reap self-heal (#1678) Problem 1: distinct WORKER_EXIT_RSS_WATCHDOG exit code + cause-keyed supervisor breaker (bypasses the stable-run reset that hid the 400x/24h loop) + rss_watchdog audit bucket + 80% soft-warn; cgroup-aware resolveDefaultMaxRssMb replaces the flat 2048 default at every spawn site. Problem 2: CONNECTION_ENDED classified retryable; postgres-engine sql getter throws a retryable error on a reaped instance pool instead of the misleading module-singleton fallthrough; promoteDelayed reconnect-retry; claim recovers on the next poll tick (no double-claim); lock-renewal tick reconnect-once dep. * feat(cycle): surface silent extract_atoms backlog + bounded --drain + fix lint clobbering the shared DB connection (#1678) Problem 3: extract_atoms_backlog doctor check + pack_gated skip marker + shared countExtractAtomsBacklog; `gbrain dream --phase extract_atoms --drain [--window N]` single-hold bounded drain (same cycleLockIdFor, rediscover each batch, reports remaining, exits non-zero while work remains). Also fixes a real production bug found via E2E: the cycle lint phase's resolveLintContentSanity created + disconnected a module-style engine that nulled the shared db singleton mid-cycle, breaking every later phase with "connect() has not been called". Lint now reuses the caller's live engine (cycle + Minion handlers thread it; standalone CLI keeps the create-own path). * chore: bump version and changelog (v0.41.39.0) Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com> * fix(#1678): pre-landing review — route transaction/withReservedConnection through the sql getter + drain treats failed count as incomplete Codex adversarial review findings: - #2: transaction(), withReservedConnection(), and one other site bypassed the v0.42.2.0 sql-getter self-heal via `this._sql || db.getConnection()`, so a reaped instance pool fell through to the module singleton there. Route all three through `this.sql` so they throw the retryable instance-pool error and recover consistently (MinionQueue.transaction hits this). - #4: `gbrain dream --drain` treated a null backlog count (query failure) as success via `remaining ?? 0`; now null exits EXIT_DRAIN_INCOMPLETE so automation never believes an unverified backlog drained. - #1 (claim orphan) + #3 (PGLite drain lock) documented as follow-ups in TODOS. * docs: document v0.42.2.0 #1678 modules + behavior in CLAUDE.md Adds Key Files entries for worker-exit-codes.ts, rss-default.ts, and extract-atoms-drain.ts, plus v0.42.2.0 annotations on worker.ts, child-worker-supervisor.ts, lock-renewal-tick.ts, and dream.ts. Regenerated llms-full.txt to match (test/build-llms.test.ts gate). Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com> * chore: re-version v0.42.2.0 → v0.42.5.0 across VERSION/package.json/CHANGELOG/docs/comments Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com> --------- Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
99 lines
3.6 KiB
TypeScript
99 lines
3.6 KiB
TypeScript
/**
|
|
* issue #1678 — cgroup-aware auto-sized RSS watchdog default.
|
|
*
|
|
* The load-bearing case (Codex #5): a tiny cgroup limit on a huge host must
|
|
* win, so the watchdog cap sits BELOW the real ceiling and the graceful drain
|
|
* beats the kernel OOM-killer. Plain os.totalmem() would pick a 16GB cap on a
|
|
* 4GB-limited container and re-break into a silent SIGKILL.
|
|
*/
|
|
|
|
import { describe, it, expect } from 'bun:test';
|
|
import {
|
|
resolveDefaultMaxRssMb,
|
|
describeDefaultMaxRss,
|
|
readCgroupMemLimitBytes,
|
|
RSS_DEFAULT_FLOOR_MB,
|
|
RSS_DEFAULT_CEIL_MB,
|
|
} from '../src/core/minions/rss-default.ts';
|
|
|
|
const GB = 1024 * 1024 * 1024;
|
|
|
|
describe('resolveDefaultMaxRssMb — clamp', () => {
|
|
it('8GB host (no cgroup) → floor 4096', () => {
|
|
expect(resolveDefaultMaxRssMb({ totalMemBytes: 8 * GB, cgroupLimitBytes: null })).toBe(4096);
|
|
});
|
|
|
|
it('16GB host → 8192 (0.5x, inside the band)', () => {
|
|
expect(resolveDefaultMaxRssMb({ totalMemBytes: 16 * GB, cgroupLimitBytes: null })).toBe(8192);
|
|
});
|
|
|
|
it('32GB host → ceil 16384', () => {
|
|
expect(resolveDefaultMaxRssMb({ totalMemBytes: 32 * GB, cgroupLimitBytes: null })).toBe(16384);
|
|
});
|
|
|
|
it('126GB host → ceil 16384 (the incident box)', () => {
|
|
expect(resolveDefaultMaxRssMb({ totalMemBytes: 126 * GB, cgroupLimitBytes: null })).toBe(16384);
|
|
});
|
|
|
|
it('result always within [floor, ceil] for huge hosts', () => {
|
|
const mb = resolveDefaultMaxRssMb({ totalMemBytes: 1024 * GB, cgroupLimitBytes: null });
|
|
expect(mb).toBeGreaterThanOrEqual(RSS_DEFAULT_FLOOR_MB);
|
|
expect(mb).toBeLessThanOrEqual(RSS_DEFAULT_CEIL_MB);
|
|
});
|
|
});
|
|
|
|
describe('resolveDefaultMaxRssMb — cgroup limit wins (Codex #5)', () => {
|
|
it('4GB cgroup on a 126GB host → cap stays BELOW the 4GB ceiling', () => {
|
|
const d = describeDefaultMaxRss({ totalMemBytes: 126 * GB, cgroupLimitBytes: 4 * GB });
|
|
expect(d.source).toBe('cgroup-limited');
|
|
expect(d.basisMb).toBe(4096);
|
|
// 0.5x4096 = 2048, below the 4096 floor but the floor must NOT push the cap
|
|
// up to/above the real 4GB ceiling — that would defeat drain-before-OOM.
|
|
expect(d.mb).toBeLessThan(4096);
|
|
expect(d.mb).toBe(2048);
|
|
});
|
|
|
|
it('8GB cgroup on a big host → 4096 (0.5x), source cgroup-limited', () => {
|
|
const d = describeDefaultMaxRss({ totalMemBytes: 64 * GB, cgroupLimitBytes: 8 * GB });
|
|
expect(d.mb).toBe(4096);
|
|
expect(d.source).toBe('cgroup-limited');
|
|
});
|
|
|
|
it('cgroup limit >= host RAM reads as host (unlimited sentinel collapses via min)', () => {
|
|
const d = describeDefaultMaxRss({ totalMemBytes: 16 * GB, cgroupLimitBytes: 9_223_372_036_854_771_712 });
|
|
expect(d.source).toBe('host');
|
|
expect(d.mb).toBe(8192);
|
|
});
|
|
});
|
|
|
|
describe('readCgroupMemLimitBytes', () => {
|
|
it('cgroup v2 "max" → null (no enforced limit)', () => {
|
|
const read = (p: string) => {
|
|
if (p === '/sys/fs/cgroup/memory.max') return 'max\n';
|
|
throw new Error('ENOENT');
|
|
};
|
|
expect(readCgroupMemLimitBytes(read)).toBeNull();
|
|
});
|
|
|
|
it('cgroup v2 numeric → that value', () => {
|
|
const read = (p: string) => {
|
|
if (p === '/sys/fs/cgroup/memory.max') return String(4 * GB) + '\n';
|
|
throw new Error('ENOENT');
|
|
};
|
|
expect(readCgroupMemLimitBytes(read)).toBe(4 * GB);
|
|
});
|
|
|
|
it('falls back to cgroup v1 when v2 unreadable', () => {
|
|
const read = (p: string) => {
|
|
if (p === '/sys/fs/cgroup/memory/memory.limit_in_bytes') return String(2 * GB);
|
|
throw new Error('ENOENT');
|
|
};
|
|
expect(readCgroupMemLimitBytes(read)).toBe(2 * GB);
|
|
});
|
|
|
|
it('neither file present → null', () => {
|
|
const read = () => { throw new Error('ENOENT'); };
|
|
expect(readCgroupMemLimitBytes(read)).toBeNull();
|
|
});
|
|
});
|