import { Worker } from 'node:worker_threads' import { delay } from './async.mjs' import { killAll, treeOf } from './proc.mjs' /** * Memory ceiling of one launched browser. A worker thread samples the summed * VmRSS of every process below the browser's root pid every 50 ms and, above * the ceiling, SIGKILLs that tree (the shared host must never reach the * kernel OOM killer). The sampler lives in its own thread so a busy main * event loop cannot delay it. */ export const CEILING_BYTES = 4 * 1024 ** 3 export const SAMPLE_INTERVAL_MS = 50 export const WATCHDOG_CAVEAT = 'The 4 GiB RSS watchdog is not a guaranteed memory limit: it samples ' + 'every 50 ms, so a faster allocation can overshoot it before the kill; ' + 'summed VmRSS counts shared pages once per process; a process outside ' + "the browser's tree is not seen." /** Shared Int32 slots, sizes in KiB. */ export const SLOT = { currentKib: 0, currentMaxKib: 1, peakKib: 2, peakMaxKib: 3, samples: 4, stopped: 5, resetRequested: 6, stoppedAtKib: 7, currentContentKib: 8, peakContentKib: 9, } const SLOT_COUNT = 10 const RESET_WAIT_MS = 1000 const POLL_MS = 5 export const watchdogFacts = () => ({ ceilingBytes: CEILING_BYTES, sampleIntervalMs: SAMPLE_INTERVAL_MS, measure: 'summed VmRSS (/proc//status) of the browser process tree; ' + 'content = renderer / Web Content / WPEWebProcess processes only', onCeiling: 'SIGKILL of the tree, run recorded as stopped_by_ceiling', caveat: WATCHDOG_CAVEAT, }) const kib = (stats, slot) => Atomics.load(stats, slot) * 1024 function snapshotOf(stats) { return { currentBytes: kib(stats, SLOT.currentKib), currentMaxProcessBytes: kib(stats, SLOT.currentMaxKib), peakBytes: kib(stats, SLOT.peakKib), peakMaxProcessBytes: kib(stats, SLOT.peakMaxKib), currentContentBytes: kib(stats, SLOT.currentContentKib), peakContentBytes: kib(stats, SLOT.peakContentKib), samples: Atomics.load(stats, SLOT.samples), stopped: Atomics.load(stats, SLOT.stopped) === 1, stoppedAtBytes: kib(stats, SLOT.stoppedAtKib), } } async function waitUntil(test, timeoutMs, what) { const end = performance.now() + timeoutMs while (!test()) { if (performance.now() > end) throw new Error(`watchdog: ${what}`) await delay(POLL_MS) } } /** Starts a peak window at the next sample (the sampler owns the write). */ async function resetPeak(stats) { Atomics.store(stats, SLOT.resetRequested, 1) const done = () => Atomics.load(stats, SLOT.resetRequested) === 0 await waitUntil(done, RESET_WAIT_MS, 'peak reset not acknowledged') } /** * `stopped` resolves when the sampler killed the tree, or when the sampler * itself died: the tree is then killed here, so no browser runs unwatched. */ function stoppedPromise(worker, rootPid, stats, isStopping) { return new Promise((resolve) => { let settled = false const settle = (message) => { settled = true resolve(message) } const failed = (error) => { if (settled || isStopping()) return if (Atomics.load(stats, SLOT.stopped) === 1) { return settle({ type: 'ceiling', totalBytes: kib(stats, SLOT.stoppedAtKib), }) } killAll(treeOf(rootPid)) settle({ type: 'watchdog_failed', error: String(error) }) } worker.on('message', (m) => m.type === 'ceiling' && settle(m)) worker.on('error', failed) worker.on('exit', (code) => failed(`sampler exited (${code})`)) }) } export function startWatchdog({ rootPid, ceilingBytes = CEILING_BYTES }) { const shared = new SharedArrayBuffer(SLOT_COUNT * 4) const stats = new Int32Array(shared) const worker = new Worker(new URL('./watchdog-worker.mjs', import.meta.url), { workerData: { rootPid, ceilingBytes, shared }, }) let stopping = false const sampled = () => Atomics.load(stats, SLOT.samples) > 0 return { ready: waitUntil(sampled, RESET_WAIT_MS, 'no first sample'), stopped: stoppedPromise(worker, rootPid, stats, () => stopping), snapshot: () => snapshotOf(stats), resetPeak: () => resetPeak(stats), stop: () => { stopping = true return worker.terminate() }, } }