Skip to content

The agent that grows with you

fix(desktop-e2e): stop the gateways Playwright runs leave behind, including a killed run's

Teknium29d13bd134c5
diff --git a/apps/desktop/e2e/core/harness.ts b/apps/desktop/e2e/core/harness.ts
index 82d447171d0..edae0ca9778 100644
--- a/apps/desktop/e2e/core/harness.ts
+++ b/apps/desktop/e2e/core/harness.ts
@@ -18,8 +18,9 @@ import { DatabaseSync } from 'node:sqlite'
import { _electron, type ElectronApplication, expect, type Page } from '@playwright/test'
import { resolveElectronBinary } from '../electron-binary'
+import { stopSandboxGatewayDaemons } from '../run-tmp-reaper'
-import { isSandboxProcess, procfsCensus, type ProcInfo, readProc, sandboxProcessesOf } from './process-census'
+import { type ProcInfo, sandboxProcessesOf } from './process-census'
export type { ProcInfo } from './process-census'
@@ -86,73 +87,6 @@ export function createCoreSandbox(label: string): CoreSandbox {
}
}
-function sleepSync(ms: number): void {
- Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, ms)
-}
-
-function processHomeUnder(pid: number, root: string, hermesHome: string): boolean {
- if (!procfsCensus) {
- return isSandboxProcess(pid, hermesHome)
- }
-
- const info = readProc(pid)
-
- return Boolean(info?.cmdline) && info!.environ.split('\0').some(entry => entry.startsWith(`HERMES_HOME=${root}`))
-}
-
-/**
- * SIGTERM (then SIGKILL after 15 s) every daemon recorded in a sandbox
- * `gateway.lock` (launch home and each profile home). The pid is only
- * signalled while its environment still names this sandbox, so a recycled pid
- * is never touched.
- */
-function stopSandboxGatewayDaemons(root: string, hermesHome: string): void {
- const profilesDir = path.join(hermesHome, 'profiles')
-
- const homes = [
- hermesHome,
- ...(fs.existsSync(profilesDir) ? fs.readdirSync(profilesDir).map(name => path.join(profilesDir, name)) : [])
- ]
-
- const pids = new Set<number>()
-
- for (const home of homes) {
- try {
- const pid = Number(JSON.parse(fs.readFileSync(path.join(home, 'gateway.lock'), 'utf8'))?.pid)
-
- if (Number.isInteger(pid) && pid > 0 && processHomeUnder(pid, root, hermesHome)) {
- pids.add(pid)
- }
- } catch {
- // No lock (remote-only spec, never booted) or an empty one: nothing to stop.
- }
- }
-
- const signal = (sig: NodeJS.Signals) => {
- for (const pid of pids) {
- try {
- process.kill(pid, sig)
- } catch {
- pids.delete(pid)
- }
- }
- }
-
- signal('SIGTERM')
-
- for (const deadline = Date.now() + 15_000; pids.size && Date.now() < deadline; ) {
- sleepSync(100)
-
- for (const pid of [...pids]) {
- if (!processHomeUnder(pid, root, hermesHome)) {
- pids.delete(pid)
- }
- }
- }
-
- signal('SIGKILL')
-}
-
/**
* Sandbox config: only the scripted provider. The approval prompts under test
* come from Hermes's own detector.
diff --git a/apps/desktop/e2e/fixtures.ts b/apps/desktop/e2e/fixtures.ts
index 3ce674c2ba6..57003cbb29b 100644
--- a/apps/desktop/e2e/fixtures.ts
+++ b/apps/desktop/e2e/fixtures.ts
@@ -31,6 +31,7 @@ import { writeEnvFile, writeMockProviderConfig } from '../../../tests-js/scripts
import { type MockServerOptions, startMockServer } from '../../../tests-js/scripts/mock-server'
import { resolveElectronBinary } from './electron-binary'
+import { stopSandboxGatewayDaemons } from './run-tmp-reaper'
import { installErrorBannerGuard } from './test'
const DESKTOP_ROOT = path.resolve(import.meta.dirname, '..')
@@ -150,6 +151,10 @@ export function createSandbox(prefix: string): Sandbox {
hermesHome,
userDataDir,
cleanup: () => {
+ // `app.close()` leaves the detached gateway the app dialled; alive, it
+ // would also rewrite this home right after the rmSync below.
+ stopSandboxGatewayDaemons(root, hermesHome)
+
try {
fs.rmSync(root, { recursive: true, force: true })
} catch {
diff --git a/apps/desktop/e2e/run-tmp-reaper.ts b/apps/desktop/e2e/run-tmp-reaper.ts
new file mode 100644
index 00000000000..1db0f43569b
--- /dev/null
+++ b/apps/desktop/e2e/run-tmp-reaper.ts
@@ -0,0 +1,187 @@
+/**
+ * Stop the backends E2E runs start, at three points.
+ *
+ * Desktop dials its backend through `hermes gateway ensure`, which spawns a detached
+ * `gateway run` that outlives the app on purpose (production attach mode). Closing the app
+ * therefore never stops it, and an orphan rewrites its sandbox HERMES_HOME after the
+ * fixture's rmSync, so the leaked run root reappears. Each layer below covers a teardown the
+ * one before it cannot reach:
+ *
+ * - per sandbox, `stopSandboxGatewayDaemons`: the daemons a sandbox's `gateway.lock` files
+ * record (every platform);
+ * - per run, at runner exit: every process whose environment names this run's root (a spec
+ * whose fixture never returned, profile gateways outside the lock homes);
+ * - per host, at the next runner's start: every process left under a run root whose runner
+ * is dead (SIGKILLed by a CI or agent timeout, so neither teardown nor the exit hook ran).
+ *
+ * The run-level layers read /proc (Linux only; elsewhere they are no-ops). They only signal a
+ * process whose environment carries BOTH `HERMES_E2E_RUN_TMP=<root>` and a HERMES_HOME under
+ * that root, so a developer's own gateway (no run root) is never a candidate, and a root whose
+ * runner is alive (a concurrent run) or unrecorded (an older harness) is left alone.
+ */
+
+import * as fs from 'node:fs'
+import * as path from 'node:path'
+
+import { isSandboxProcess, procfsCensus, readProc } from './core/process-census'
+
+const OWNER_FILE = '.runner.json'
+
+function sleepSync(ms: number): void {
+ Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, ms)
+}
+
+/**
+ * SIGTERM `pids`, then SIGKILL whatever `stillOurs` still claims after `graceMs`. Re-checking
+ * `stillOurs` before every signal keeps a pid recycled mid-wait from ever being touched.
+ */
+export function terminateProcesses(pids: Iterable<number>, stillOurs: (pid: number) => boolean, graceMs = 15_000): void {
+ const live = new Set(pids)
+
+ const signal = (sig: NodeJS.Signals) => {
+ for (const pid of live) {
+ try {
+ if (stillOurs(pid)) {
+ process.kill(pid, sig)
+ } else {
+ live.delete(pid)
+ }
+ } catch {
+ live.delete(pid)
+ }
+ }
+ }
+
+ signal('SIGTERM')
+
+ for (const deadline = Date.now() + graceMs; live.size && Date.now() < deadline; ) {
+ sleepSync(100)
+
+ for (const pid of [...live]) {
+ if (!stillOurs(pid)) {
+ live.delete(pid)
+ }
+ }
+ }
+
+ signal('SIGKILL')
+}
+
+function processHomeUnder(pid: number, root: string, hermesHome: string): boolean {
+ if (!procfsCensus) {
+ return isSandboxProcess(pid, hermesHome)
+ }
+
+ const info = readProc(pid)
+
+ return Boolean(info?.cmdline) && info!.environ.split('\0').some(entry => entry.startsWith(`HERMES_HOME=${root}`))
+}
+
+/**
+ * SIGTERM (then SIGKILL after 15 s) every daemon recorded in a sandbox `gateway.lock` (launch
+ * home and each profile home), while its environment still names this sandbox.
+ */
+export function stopSandboxGatewayDaemons(root: string, hermesHome: string): void {
+ const profilesDir = path.join(hermesHome, 'profiles')
+
+ const homes = [
+ hermesHome,
+ ...(fs.existsSync(profilesDir) ? fs.readdirSync(profilesDir).map(name => path.join(profilesDir, name)) : [])
+ ]
+
+ const pids = new Set<number>()
+
+ for (const home of homes) {
+ try {
+ const pid = Number(JSON.parse(fs.readFileSync(path.join(home, 'gateway.lock'), 'utf8'))?.pid)
+
+ if (Number.isInteger(pid) && pid > 0 && processHomeUnder(pid, root, hermesHome)) {
+ pids.add(pid)
+ }
+ } catch {
+ // No lock (remote-only spec, never booted) or an empty one: nothing to stop.
+ }
+ }
+
+ terminateProcesses(pids, pid => processHomeUnder(pid, root, hermesHome))
+}
+
+/** Kernel start time of `pid` (/proc/<pid>/stat field 22): with the pid, a process identity no recycle repeats. */
+function startTicks(pid: number): null | string {
+ try {
+ const stat = fs.readFileSync(`/proc/${pid}/stat`, 'utf8')
+
+ return stat.slice(stat.lastIndexOf(')') + 2).split(' ')[19] ?? null
+ } catch {
+ return null
+ }
+}
+
+/** Record the calling process as the runner that owns `root`. */
+export function recordRunOwner(root: string): void {
+ fs.writeFileSync(path.join(root, OWNER_FILE), JSON.stringify({ pid: process.pid, start: startTicks(process.pid) }))
+}
+
+/** False only when `root`'s recorded runner is provably gone (a root with no record counts as live). */
+function runnerAlive(root: string): boolean {
+ let owner: { pid?: unknown; start?: unknown }
+
+ try {
+ owner = JSON.parse(fs.readFileSync(path.join(root, OWNER_FILE), 'utf8'))
+ } catch {
+ return true
+ }
+
+ return typeof owner.pid === 'number' && owner.start !== null && startTicks(owner.pid) === owner.start
+}
+
+/** The run root a process belongs to: its HERMES_E2E_RUN_TMP, when its HERMES_HOME also lies under it. */
+function runRootOf(pid: number): null | string {
+ const env = readProc(pid)?.environ.split('\0') ?? []
+ const root = env.find(entry => entry.startsWith('HERMES_E2E_RUN_TMP='))?.slice('HERMES_E2E_RUN_TMP='.length)
+ const home = env.find(entry => entry.startsWith('HERMES_HOME='))?.slice('HERMES_HOME='.length)
+
+ return root && home?.startsWith(root + path.sep) ? root : null
+}
+
+/**
+ * Stop every process left under a run root `shouldReap` selects (Linux /proc only), then remove
+ * those roots. Returns the roots reaped.
+ */
+export function reapRunBackends(shouldReap: (root: string) => boolean): string[] {
+ if (!procfsCensus) {
+ return []
+ }
+
+ const verdicts = new Map<string, boolean>()
+ const victims = new Map<number, string>()
+
+ for (const entry of fs.readdirSync('/proc')) {
+ const pid = Number(entry)
+ const root = Number.isInteger(pid) && pid !== process.pid ? runRootOf(pid) : null
+
+ if (root) {
+ if (!verdicts.has(root)) {
+ verdicts.set(root, shouldReap(root))
+ }
+
+ if (verdicts.get(root)) {
+ victims.set(pid, root)
+ }
+ }
+ }
+
+ terminateProcesses(victims.keys(), pid => runRootOf(pid) === victims.get(pid))
+ const roots = [...new Set(victims.values())]
+
+ for (const root of roots) {
+ fs.rmSync(root, { recursive: true, force: true })
+ }
+
+ return roots
+}
+
+/** Reap the backends of every run whose runner is dead. */
+export function reapAbandonedRuns(): string[] {
+ return reapRunBackends(root => !runnerAlive(root))
+}
diff --git a/apps/desktop/e2e/run-tmp-reaper.unit.test.ts b/apps/desktop/e2e/run-tmp-reaper.unit.test.ts
new file mode 100644
index 00000000000..146ded5733a
--- /dev/null
+++ b/apps/desktop/e2e/run-tmp-reaper.unit.test.ts
@@ -0,0 +1,69 @@
+import { type ChildProcess, spawn } from 'node:child_process'
+import * as fs from 'node:fs'
+import * as os from 'node:os'
+import * as path from 'node:path'
+
+import { afterEach, describe, expect, it } from 'vitest'
+
+import { reapAbandonedRuns, recordRunOwner } from './run-tmp-reaper'
+
+const children: ChildProcess[] = []
+const dirs: string[] = []
+
+afterEach(() => {
+ for (const child of children.splice(0)) {
+ child.kill('SIGKILL')
+ }
+
+ for (const dir of dirs.splice(0)) {
+ fs.rmSync(dir, { recursive: true, force: true })
+ }
+})
+
+function exited(child: ChildProcess): Promise<NodeJS.Signals | null> {
+ return child.exitCode !== null || child.signalCode !== null
+ ? Promise.resolve(child.signalCode)
+ : new Promise(resolve => child.once('exit', (_code, signal) => resolve(signal)))
+}
+
+/** A stand-in backend: a real process whose environment names `root` the way a sandbox gateway's does. */
+function backendUnder(root: string): ChildProcess {
+ const child = spawn('sleep', ['300'], {
+ env: { PATH: process.env.PATH, HERMES_E2E_RUN_TMP: root, HERMES_HOME: path.join(root, 'sandbox', 'hermes-home') },
+ stdio: 'ignore'
+ })
+
+ children.push(child)
+
+ return child
+}
+
+describe.runIf(process.platform === 'linux')('reapAbandonedRuns', () => {
+ it("stops a killed runner's backends and removes its root; a live runner's backends are untouched", async () => {
+ const base = fs.realpathSync(fs.mkdtempSync(path.join(os.tmpdir(), 'reaper-')))
+ dirs.push(base)
+ const deadRoot = fs.mkdtempSync(path.join(base, 'pw-'))
+ const liveRoot = fs.mkdtempSync(path.join(base, 'pw-'))
+
+ // The killed run's record: the pid and kernel start time of a runner that no longer exists.
+ const runner = spawn('sleep', ['300'], { stdio: 'ignore' })
+ const stat = fs.readFileSync(`/proc/${runner.pid}/stat`, 'utf8')
+ const start = stat.slice(stat.lastIndexOf(')') + 2).split(' ')[19]
+ runner.kill('SIGKILL')
+ await exited(runner)
+ fs.writeFileSync(path.join(deadRoot, '.runner.json'), JSON.stringify({ pid: runner.pid, start }))
+ recordRunOwner(liveRoot)
+
+ const orphan = backendUnder(deadRoot)
+ const concurrent = backendUnder(liveRoot)
+
+ expect(reapAbandonedRuns()).toContain(deadRoot)
+ expect(await exited(orphan)).toBe('SIGTERM')
+ expect(fs.existsSync(deadRoot)).toBe(false)
+
+ expect(concurrent.exitCode).toBeNull()
+ expect(concurrent.signalCode).toBeNull()
+ expect(() => process.kill(concurrent.pid!, 0)).not.toThrow()
+ expect(fs.existsSync(liveRoot)).toBe(true)
+ })
+})
diff --git a/apps/desktop/e2e/run-tmp.ts b/apps/desktop/e2e/run-tmp.ts
index 2e963da87e8..56c12a0d60c 100644
--- a/apps/desktop/e2e/run-tmp.ts
+++ b/apps/desktop/e2e/run-tmp.ts
@@ -7,6 +7,11 @@
* `playwright-tracing-*` dirs). Workers and the apps they launch inherit
* TMPDIR, so everything lands here; a SIGKILLed run leaves one dir, not dozens.
*
+ * The root also names the run's backends (`HERMES_E2E_RUN_TMP` in their
+ * environment): the exit hook stops them before removing it, and the next
+ * runner stops the ones a killed run left (./run-tmp-reaper). Canonical, so a
+ * backend's resolved HERMES_HOME still lies under it behind a symlinked tmpdir.
+ *
* Config modules load in the runner and again in each worker; only the first
* (runner) load creates the root.
*/
@@ -14,11 +19,21 @@ import fs from 'node:fs'
import os from 'node:os'
import path from 'node:path'
+import { reapAbandonedRuns, reapRunBackends, recordRunOwner } from './run-tmp-reaper'
+
if (!process.env.HERMES_E2E_RUN_TMP) {
- const root = fs.mkdtempSync(path.join(os.tmpdir(), 'pw-'))
+ for (const abandoned of reapAbandonedRuns()) {
+ console.warn(`[e2e] stopped the backends a killed run left under ${abandoned}`)
+ }
+
+ const root = fs.realpathSync(fs.mkdtempSync(path.join(os.tmpdir(), 'pw-')))
+ recordRunOwner(root)
process.env.HERMES_E2E_RUN_TMP = root
process.env.TMPDIR = root
process.env.TEMP = root
process.env.TMP = root
- process.on('exit', () => fs.rmSync(root, { recursive: true, force: true }))
+ process.on('exit', () => {
+ reapRunBackends(candidate => candidate === root)
+ fs.rmSync(root, { recursive: true, force: true })
+ })
}