The agent that grows with you
Python aiai-agentai-agentsanthropicchatgptclaudeclaude-codecodexhermeshermes-agentllmnous-researchopenai
fix(desktop-e2e): stop the gateways Playwright runs leave behind, including a killed run's
Teknium29d13bd134c5
diff --git a/apps/desktop/e2e/core/harness.ts b/apps/desktop/e2e/core/harness.tsindex 82d447171d0..edae0ca9778 100644--- a/apps/desktop/e2e/core/harness.ts+++ b/apps/desktop/e2e/core/harness.ts@@ -18,8 +18,9 @@ import { DatabaseSync } from 'node:sqlite'import { _electron, type ElectronApplication, expect, type Page } from '@playwright/test'import { resolveElectronBinary } from '../electron-binary'+import { stopSandboxGatewayDaemons } from '../run-tmp-reaper'-import { isSandboxProcess, procfsCensus, type ProcInfo, readProc, sandboxProcessesOf } from './process-census'+import { type ProcInfo, sandboxProcessesOf } from './process-census'export type { ProcInfo } from './process-census'@@ -86,73 +87,6 @@ export function createCoreSandbox(label: string): CoreSandbox {}}-function sleepSync(ms: number): void {- Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, ms)-}--function processHomeUnder(pid: number, root: string, hermesHome: string): boolean {- if (!procfsCensus) {- return isSandboxProcess(pid, hermesHome)- }-- const info = readProc(pid)-- return Boolean(info?.cmdline) && info!.environ.split('\0').some(entry => entry.startsWith(`HERMES_HOME=${root}`))-}--/**- * SIGTERM (then SIGKILL after 15 s) every daemon recorded in a sandbox- * `gateway.lock` (launch home and each profile home). The pid is only- * signalled while its environment still names this sandbox, so a recycled pid- * is never touched.- */-function stopSandboxGatewayDaemons(root: string, hermesHome: string): void {- const profilesDir = path.join(hermesHome, 'profiles')-- const homes = [- hermesHome,- ...(fs.existsSync(profilesDir) ? fs.readdirSync(profilesDir).map(name => path.join(profilesDir, name)) : [])- ]-- const pids = new Set<number>()-- for (const home of homes) {- try {- const pid = Number(JSON.parse(fs.readFileSync(path.join(home, 'gateway.lock'), 'utf8'))?.pid)-- if (Number.isInteger(pid) && pid > 0 && processHomeUnder(pid, root, hermesHome)) {- pids.add(pid)- }- } catch {- // No lock (remote-only spec, never booted) or an empty one: nothing to stop.- }- }-- const signal = (sig: NodeJS.Signals) => {- for (const pid of pids) {- try {- process.kill(pid, sig)- } catch {- pids.delete(pid)- }- }- }-- signal('SIGTERM')-- for (const deadline = Date.now() + 15_000; pids.size && Date.now() < deadline; ) {- sleepSync(100)-- for (const pid of [...pids]) {- if (!processHomeUnder(pid, root, hermesHome)) {- pids.delete(pid)- }- }- }-- signal('SIGKILL')-}-/*** Sandbox config: only the scripted provider. The approval prompts under test* come from Hermes's own detector.diff --git a/apps/desktop/e2e/fixtures.ts b/apps/desktop/e2e/fixtures.tsindex 3ce674c2ba6..57003cbb29b 100644--- a/apps/desktop/e2e/fixtures.ts+++ b/apps/desktop/e2e/fixtures.ts@@ -31,6 +31,7 @@ import { writeEnvFile, writeMockProviderConfig } from '../../../tests-js/scriptsimport { type MockServerOptions, startMockServer } from '../../../tests-js/scripts/mock-server'import { resolveElectronBinary } from './electron-binary'+import { stopSandboxGatewayDaemons } from './run-tmp-reaper'import { installErrorBannerGuard } from './test'const DESKTOP_ROOT = path.resolve(import.meta.dirname, '..')@@ -150,6 +151,10 @@ export function createSandbox(prefix: string): Sandbox {hermesHome,userDataDir,cleanup: () => {+ // `app.close()` leaves the detached gateway the app dialled; alive, it+ // would also rewrite this home right after the rmSync below.+ stopSandboxGatewayDaemons(root, hermesHome)+try {fs.rmSync(root, { recursive: true, force: true })} catch {diff --git a/apps/desktop/e2e/run-tmp-reaper.ts b/apps/desktop/e2e/run-tmp-reaper.tsnew file mode 100644index 00000000000..1db0f43569b--- /dev/null+++ b/apps/desktop/e2e/run-tmp-reaper.ts@@ -0,0 +1,187 @@+/**+ * Stop the backends E2E runs start, at three points.+ *+ * Desktop dials its backend through `hermes gateway ensure`, which spawns a detached+ * `gateway run` that outlives the app on purpose (production attach mode). Closing the app+ * therefore never stops it, and an orphan rewrites its sandbox HERMES_HOME after the+ * fixture's rmSync, so the leaked run root reappears. Each layer below covers a teardown the+ * one before it cannot reach:+ *+ * - per sandbox, `stopSandboxGatewayDaemons`: the daemons a sandbox's `gateway.lock` files+ * record (every platform);+ * - per run, at runner exit: every process whose environment names this run's root (a spec+ * whose fixture never returned, profile gateways outside the lock homes);+ * - per host, at the next runner's start: every process left under a run root whose runner+ * is dead (SIGKILLed by a CI or agent timeout, so neither teardown nor the exit hook ran).+ *+ * The run-level layers read /proc (Linux only; elsewhere they are no-ops). They only signal a+ * process whose environment carries BOTH `HERMES_E2E_RUN_TMP=<root>` and a HERMES_HOME under+ * that root, so a developer's own gateway (no run root) is never a candidate, and a root whose+ * runner is alive (a concurrent run) or unrecorded (an older harness) is left alone.+ */++import * as fs from 'node:fs'+import * as path from 'node:path'++import { isSandboxProcess, procfsCensus, readProc } from './core/process-census'++const OWNER_FILE = '.runner.json'++function sleepSync(ms: number): void {+ Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, ms)+}++/**+ * SIGTERM `pids`, then SIGKILL whatever `stillOurs` still claims after `graceMs`. Re-checking+ * `stillOurs` before every signal keeps a pid recycled mid-wait from ever being touched.+ */+export function terminateProcesses(pids: Iterable<number>, stillOurs: (pid: number) => boolean, graceMs = 15_000): void {+ const live = new Set(pids)++ const signal = (sig: NodeJS.Signals) => {+ for (const pid of live) {+ try {+ if (stillOurs(pid)) {+ process.kill(pid, sig)+ } else {+ live.delete(pid)+ }+ } catch {+ live.delete(pid)+ }+ }+ }++ signal('SIGTERM')++ for (const deadline = Date.now() + graceMs; live.size && Date.now() < deadline; ) {+ sleepSync(100)++ for (const pid of [...live]) {+ if (!stillOurs(pid)) {+ live.delete(pid)+ }+ }+ }++ signal('SIGKILL')+}++function processHomeUnder(pid: number, root: string, hermesHome: string): boolean {+ if (!procfsCensus) {+ return isSandboxProcess(pid, hermesHome)+ }++ const info = readProc(pid)++ return Boolean(info?.cmdline) && info!.environ.split('\0').some(entry => entry.startsWith(`HERMES_HOME=${root}`))+}++/**+ * SIGTERM (then SIGKILL after 15 s) every daemon recorded in a sandbox `gateway.lock` (launch+ * home and each profile home), while its environment still names this sandbox.+ */+export function stopSandboxGatewayDaemons(root: string, hermesHome: string): void {+ const profilesDir = path.join(hermesHome, 'profiles')++ const homes = [+ hermesHome,+ ...(fs.existsSync(profilesDir) ? fs.readdirSync(profilesDir).map(name => path.join(profilesDir, name)) : [])+ ]++ const pids = new Set<number>()++ for (const home of homes) {+ try {+ const pid = Number(JSON.parse(fs.readFileSync(path.join(home, 'gateway.lock'), 'utf8'))?.pid)++ if (Number.isInteger(pid) && pid > 0 && processHomeUnder(pid, root, hermesHome)) {+ pids.add(pid)+ }+ } catch {+ // No lock (remote-only spec, never booted) or an empty one: nothing to stop.+ }+ }++ terminateProcesses(pids, pid => processHomeUnder(pid, root, hermesHome))+}++/** Kernel start time of `pid` (/proc/<pid>/stat field 22): with the pid, a process identity no recycle repeats. */+function startTicks(pid: number): null | string {+ try {+ const stat = fs.readFileSync(`/proc/${pid}/stat`, 'utf8')++ return stat.slice(stat.lastIndexOf(')') + 2).split(' ')[19] ?? null+ } catch {+ return null+ }+}++/** Record the calling process as the runner that owns `root`. */+export function recordRunOwner(root: string): void {+ fs.writeFileSync(path.join(root, OWNER_FILE), JSON.stringify({ pid: process.pid, start: startTicks(process.pid) }))+}++/** False only when `root`'s recorded runner is provably gone (a root with no record counts as live). */+function runnerAlive(root: string): boolean {+ let owner: { pid?: unknown; start?: unknown }++ try {+ owner = JSON.parse(fs.readFileSync(path.join(root, OWNER_FILE), 'utf8'))+ } catch {+ return true+ }++ return typeof owner.pid === 'number' && owner.start !== null && startTicks(owner.pid) === owner.start+}++/** The run root a process belongs to: its HERMES_E2E_RUN_TMP, when its HERMES_HOME also lies under it. */+function runRootOf(pid: number): null | string {+ const env = readProc(pid)?.environ.split('\0') ?? []+ const root = env.find(entry => entry.startsWith('HERMES_E2E_RUN_TMP='))?.slice('HERMES_E2E_RUN_TMP='.length)+ const home = env.find(entry => entry.startsWith('HERMES_HOME='))?.slice('HERMES_HOME='.length)++ return root && home?.startsWith(root + path.sep) ? root : null+}++/**+ * Stop every process left under a run root `shouldReap` selects (Linux /proc only), then remove+ * those roots. Returns the roots reaped.+ */+export function reapRunBackends(shouldReap: (root: string) => boolean): string[] {+ if (!procfsCensus) {+ return []+ }++ const verdicts = new Map<string, boolean>()+ const victims = new Map<number, string>()++ for (const entry of fs.readdirSync('/proc')) {+ const pid = Number(entry)+ const root = Number.isInteger(pid) && pid !== process.pid ? runRootOf(pid) : null++ if (root) {+ if (!verdicts.has(root)) {+ verdicts.set(root, shouldReap(root))+ }++ if (verdicts.get(root)) {+ victims.set(pid, root)+ }+ }+ }++ terminateProcesses(victims.keys(), pid => runRootOf(pid) === victims.get(pid))+ const roots = [...new Set(victims.values())]++ for (const root of roots) {+ fs.rmSync(root, { recursive: true, force: true })+ }++ return roots+}++/** Reap the backends of every run whose runner is dead. */+export function reapAbandonedRuns(): string[] {+ return reapRunBackends(root => !runnerAlive(root))+}diff --git a/apps/desktop/e2e/run-tmp-reaper.unit.test.ts b/apps/desktop/e2e/run-tmp-reaper.unit.test.tsnew file mode 100644index 00000000000..146ded5733a--- /dev/null+++ b/apps/desktop/e2e/run-tmp-reaper.unit.test.ts@@ -0,0 +1,69 @@+import { type ChildProcess, spawn } from 'node:child_process'+import * as fs from 'node:fs'+import * as os from 'node:os'+import * as path from 'node:path'++import { afterEach, describe, expect, it } from 'vitest'++import { reapAbandonedRuns, recordRunOwner } from './run-tmp-reaper'++const children: ChildProcess[] = []+const dirs: string[] = []++afterEach(() => {+ for (const child of children.splice(0)) {+ child.kill('SIGKILL')+ }++ for (const dir of dirs.splice(0)) {+ fs.rmSync(dir, { recursive: true, force: true })+ }+})++function exited(child: ChildProcess): Promise<NodeJS.Signals | null> {+ return child.exitCode !== null || child.signalCode !== null+ ? Promise.resolve(child.signalCode)+ : new Promise(resolve => child.once('exit', (_code, signal) => resolve(signal)))+}++/** A stand-in backend: a real process whose environment names `root` the way a sandbox gateway's does. */+function backendUnder(root: string): ChildProcess {+ const child = spawn('sleep', ['300'], {+ env: { PATH: process.env.PATH, HERMES_E2E_RUN_TMP: root, HERMES_HOME: path.join(root, 'sandbox', 'hermes-home') },+ stdio: 'ignore'+ })++ children.push(child)++ return child+}++describe.runIf(process.platform === 'linux')('reapAbandonedRuns', () => {+ it("stops a killed runner's backends and removes its root; a live runner's backends are untouched", async () => {+ const base = fs.realpathSync(fs.mkdtempSync(path.join(os.tmpdir(), 'reaper-')))+ dirs.push(base)+ const deadRoot = fs.mkdtempSync(path.join(base, 'pw-'))+ const liveRoot = fs.mkdtempSync(path.join(base, 'pw-'))++ // The killed run's record: the pid and kernel start time of a runner that no longer exists.+ const runner = spawn('sleep', ['300'], { stdio: 'ignore' })+ const stat = fs.readFileSync(`/proc/${runner.pid}/stat`, 'utf8')+ const start = stat.slice(stat.lastIndexOf(')') + 2).split(' ')[19]+ runner.kill('SIGKILL')+ await exited(runner)+ fs.writeFileSync(path.join(deadRoot, '.runner.json'), JSON.stringify({ pid: runner.pid, start }))+ recordRunOwner(liveRoot)++ const orphan = backendUnder(deadRoot)+ const concurrent = backendUnder(liveRoot)++ expect(reapAbandonedRuns()).toContain(deadRoot)+ expect(await exited(orphan)).toBe('SIGTERM')+ expect(fs.existsSync(deadRoot)).toBe(false)++ expect(concurrent.exitCode).toBeNull()+ expect(concurrent.signalCode).toBeNull()+ expect(() => process.kill(concurrent.pid!, 0)).not.toThrow()+ expect(fs.existsSync(liveRoot)).toBe(true)+ })+})diff --git a/apps/desktop/e2e/run-tmp.ts b/apps/desktop/e2e/run-tmp.tsindex 2e963da87e8..56c12a0d60c 100644--- a/apps/desktop/e2e/run-tmp.ts+++ b/apps/desktop/e2e/run-tmp.ts@@ -7,6 +7,11 @@* `playwright-tracing-*` dirs). Workers and the apps they launch inherit* TMPDIR, so everything lands here; a SIGKILLed run leaves one dir, not dozens.*+ * The root also names the run's backends (`HERMES_E2E_RUN_TMP` in their+ * environment): the exit hook stops them before removing it, and the next+ * runner stops the ones a killed run left (./run-tmp-reaper). Canonical, so a+ * backend's resolved HERMES_HOME still lies under it behind a symlinked tmpdir.+ ** Config modules load in the runner and again in each worker; only the first* (runner) load creates the root.*/@@ -14,11 +19,21 @@ import fs from 'node:fs'import os from 'node:os'import path from 'node:path'+import { reapAbandonedRuns, reapRunBackends, recordRunOwner } from './run-tmp-reaper'+if (!process.env.HERMES_E2E_RUN_TMP) {- const root = fs.mkdtempSync(path.join(os.tmpdir(), 'pw-'))+ for (const abandoned of reapAbandonedRuns()) {+ console.warn(`[e2e] stopped the backends a killed run left under ${abandoned}`)+ }++ const root = fs.realpathSync(fs.mkdtempSync(path.join(os.tmpdir(), 'pw-')))+ recordRunOwner(root)process.env.HERMES_E2E_RUN_TMP = rootprocess.env.TMPDIR = rootprocess.env.TEMP = rootprocess.env.TMP = root- process.on('exit', () => fs.rmSync(root, { recursive: true, force: true }))+ process.on('exit', () => {+ reapRunBackends(candidate => candidate === root)+ fs.rmSync(root, { recursive: true, force: true })+ })}