Skip to content

Beautiful, AI-native markdown IDE and LLM wiki

Fix nested owner-loss scratch cleanup (#5737)

Nick Gomezb29f00e961ee
diff --git a/packages/app/tests/integration/e2e-stop-rules.test.ts b/packages/app/tests/integration/e2e-stop-rules.test.ts
index 20dbc5b1c..609640d6f 100644
--- a/packages/app/tests/integration/e2e-stop-rules.test.ts
+++ b/packages/app/tests/integration/e2e-stop-rules.test.ts
@@ -108,6 +108,7 @@ const DEV_SERVER_SPAWN_SITES = [
] as const;
const NON_DEV_SERVER_SPAWN_SITES = [
'packages/app/tests/stress/_helpers/i18n-catalog-freshness.ts',
+ 'packages/app/tests/stress/_helpers/port-ownership/cleanup-order-control.test-helper.ts',
'packages/app/tests/stress/_helpers/port-ownership/lifetime-global-setup.ts',
'packages/app/tests/stress/_helpers/port-ownership/owner-loss.test-helper.ts',
'packages/app/tests/stress/_helpers/port-ownership/run-case.test-helper.ts',
diff --git a/packages/app/tests/stress/_helpers/port-ownership/cleanup-order-control.cjs b/packages/app/tests/stress/_helpers/port-ownership/cleanup-order-control.cjs
new file mode 100644
index 000000000..a8d921923
--- /dev/null
+++ b/packages/app/tests/stress/_helpers/port-ownership/cleanup-order-control.cjs
@@ -0,0 +1,20 @@
+const { meet } = require('./cleanup-order-gate.cjs');
+const net = require('node:net');
+
+const role = process.argv[2];
+const socket = net.connect(
+ {
+ port: Number(process.env.OK_PORT_OWNERSHIP_SCHEDULE_PORT),
+ host: '127.0.0.1',
+ },
+ () => socket.write(`${role}\t${process.pid}\n`),
+);
+socket.unref();
+process.on('message', (message) => {
+ if (message === 'finish') {
+ process.disconnect();
+ return;
+ }
+ meet(message);
+ if (process.connected) process.send(message);
+});
diff --git a/packages/app/tests/stress/_helpers/port-ownership/cleanup-order-control.test-helper.ts b/packages/app/tests/stress/_helpers/port-ownership/cleanup-order-control.test-helper.ts
new file mode 100644
index 000000000..93da5d0ac
--- /dev/null
+++ b/packages/app/tests/stress/_helpers/port-ownership/cleanup-order-control.test-helper.ts
@@ -0,0 +1,145 @@
+import { type ChildProcess, type StdioOptions, spawn } from 'node:child_process';
+import { fileURLToPath } from 'node:url';
+import { createCleanupOrder } from './cleanup-order.test-helper.ts';
+
+const CONTROL = fileURLToPath(new URL('./cleanup-order-control.cjs', import.meta.url));
+const GATE = fileURLToPath(new URL('./cleanup-order-gate.cjs', import.meta.url));
+const WITH_IPC: StdioOptions = ['ignore', 'pipe', 'pipe', 'ipc'];
+
+type Exit = [code: number | null, signal: NodeJS.Signals | null];
+
+interface Participant {
+ role: 'runner' | 'server' | 'remover';
+ child: ChildProcess;
+ output: string[];
+ closed: Promise<Exit>;
+}
+
+function report(failure: unknown, participants: Participant[]): Error {
+ const sections = participants.map(
+ ({ role, child, output }) =>
+ `--- ${role} (pid ${child.pid}, code ${child.exitCode}, signal ${child.signalCode})\n${output.join('')}`,
+ );
+ const message = failure instanceof Error ? failure.message : String(failure);
+ return new Error([message, ...sections].join('\n'), { cause: failure });
+}
+
+export async function runCleanupOrderControl(
+ outputDir: string,
+ progress: 'yield' | 'exit' | 'remover',
+) {
+ const ordering = await createCleanupOrder(outputDir, 'control');
+ const participants: Participant[] = [];
+ const dismissed = new Set<ChildProcess>();
+ let reject: (error: Error) => void = () => {};
+ const interrupted = new Promise<never>((_, fail) => {
+ reject = fail;
+ });
+ const interrupt = (child: ChildProcess, error: Error) => {
+ if (!dismissed.has(child)) reject(error);
+ };
+ const during = <T>(wait: Promise<T>) => Promise.race([wait, interrupted]);
+ const reply = ({ child }: Participant) =>
+ new Promise<void>((resolve) => child.once('message', () => resolve()));
+
+ function start(role: Participant['role'], args: string[], stdio: StdioOptions) {
+ const child = spawn(process.execPath, args, { env: ordering.env, stdio });
+ const closed = new Promise<Exit>((resolve) => {
+ child.once('close', (code: number | null, signal: NodeJS.Signals | null) => {
+ const acknowledged = ordering.events.some(
+ (event) => event.pid === child.pid && event.phase === 'ack:remove',
+ );
+ if (!acknowledged) {
+ interrupt(
+ child,
+ new Error(
+ `cleanup ordering ${role} exited during the control protocol (code ${code}, signal ${signal})`,
+ ),
+ );
+ }
+ resolve([code, signal]);
+ });
+ });
+ const participant: Participant = { role, child, output: [], closed };
+ participants.push(participant);
+ for (const stream of [child.stdout, child.stderr]) {
+ stream?.on('data', (chunk: Buffer) => participant.output.push(chunk.toString('utf8')));
+ }
+ child.on('error', (error) =>
+ interrupt(child, new Error(`cleanup ordering ${role} failed`, { cause: error })),
+ );
+ if (child.pid === undefined) throw new Error(`cleanup ordering ${role} did not spawn`);
+ return { ...participant, pid: child.pid };
+ }
+
+ function send({ role, child }: Participant, message: string) {
+ if (!child.connected)
+ throw new Error(`cleanup ordering ${role} disconnected before ${message}`);
+ child.send(message, (error) => {
+ if (error) {
+ interrupt(
+ child,
+ new Error(`cleanup ordering ${role} did not receive ${message}`, { cause: error }),
+ );
+ }
+ });
+ }
+
+ let failure: unknown;
+ try {
+ const runner = start('runner', [CONTROL, 'runner'], WITH_IPC);
+ const server = start('server', [CONTROL, 'server'], WITH_IPC);
+ await during(ordering.beforeOwnerExit(server.pid));
+ const recorded = reply(server);
+ send(server, 'before-record');
+ await during(ordering.waitForArrival('before-record', server.pid));
+ if (progress === 'remover') {
+ const remover = start('remover', [GATE, 'remove'], ['ignore', 'pipe', 'pipe']);
+ await during(ordering.waitForArrival('remove', remover.pid));
+ await during(recorded);
+ const completed = reply(server);
+ send(server, 'after-record');
+ await during(ordering.waitForArrival('after-record', server.pid));
+ send(runner, 'yield');
+ await during(completed);
+ const removerExit = await during(remover.closed);
+ return {
+ removerExit,
+ order: ordering.events.flatMap((event) => {
+ if (event.pid === remover.pid && event.phase === 'remove') return ['request'];
+ if (event.pid === server.pid && event.phase === 'after-record') return ['write-complete'];
+ if (event.pid === remover.pid && event.phase === 'ack:remove') return ['acknowledgment'];
+ return [];
+ }),
+ };
+ }
+ if (progress === 'exit') {
+ dismissed.add(runner.child);
+ send(runner, 'finish');
+ } else {
+ const yielded = reply(runner);
+ send(runner, 'yield');
+ await during(yielded);
+ }
+ await during(recorded);
+ return {
+ order: ordering.events.flatMap((event) => {
+ if (event.pid === server.pid && event.phase === 'before-record') return ['request'];
+ if (event.pid === runner.pid && event.phase === progress) return ['progress'];
+ if (event.pid === server.pid && event.phase === 'ack:before-record')
+ return ['acknowledgment'];
+ return [];
+ }),
+ };
+ } catch (error) {
+ failure = error;
+ } finally {
+ for (const { child } of participants) dismissed.add(child);
+ await ordering.close();
+ for (const participant of participants) {
+ if (participant.child.connected) send(participant, 'finish');
+ }
+ await Promise.all(participants.map(({ closed }) => closed));
+ }
+ throw report(failure, participants);
+}
diff --git a/packages/app/tests/stress/_helpers/port-ownership/cleanup-order-gate.cjs b/packages/app/tests/stress/_helpers/port-ownership/cleanup-order-gate.cjs
new file mode 100644
index 000000000..1ba2cb8ff
--- /dev/null
+++ b/packages/app/tests/stress/_helpers/port-ownership/cleanup-order-gate.cjs
@@ -0,0 +1,32 @@
+const { execFileSync } = require('node:child_process');
+const net = require('node:net');
+
+function meet(phase) {
+ const env = { ...process.env };
+ delete env.NODE_OPTIONS;
+ delete env.DYLD_INSERT_LIBRARIES;
+ delete env.LD_PRELOAD;
+ execFileSync(process.execPath, [__filename, phase, String(process.pid)], { env });
+}
+
+if (require.main === module) {
+ const phase = process.argv[2];
+ const pid = process.argv[3] ?? String(process.pid);
+ let acknowledged = false;
+ const socket = net.connect(
+ { port: Number(process.env.OK_PORT_OWNERSHIP_SCHEDULE_PORT), host: '127.0.0.1' },
+ () => socket.write(`${phase}\t${pid}\n`),
+ );
+ socket.once('data', (data) => {
+ acknowledged = data[0] === 33;
+ });
+ socket.once('error', (error) => {
+ process.stderr.write(`${error.message}\n`);
+ process.exitCode = 1;
+ });
+ socket.once('close', () => {
+ if (!acknowledged) process.exitCode = 1;
+ });
+}
+
+module.exports = { meet };
diff --git a/packages/app/tests/stress/_helpers/port-ownership/cleanup-order-preload.cjs b/packages/app/tests/stress/_helpers/port-ownership/cleanup-order-preload.cjs
new file mode 100644
index 000000000..8b9dbd1ea
--- /dev/null
+++ b/packages/app/tests/stress/_helpers/port-ownership/cleanup-order-preload.cjs
@@ -0,0 +1,54 @@
+const fs = require('node:fs');
+const net = require('node:net');
+const { basename } = require('node:path');
+const { meet } = require('./cleanup-order-gate.cjs');
+const { syncBuiltinESMExports } = require('node:module');
+
+const library = process.env.OK_PORT_OWNERSHIP_SCHEDULE_LIBRARY;
+if (library) {
+ process.env[process.platform === 'darwin' ? 'DYLD_INSERT_LIBRARIES' : 'LD_PRELOAD'] = library;
+}
+
+if (process.env.OK_PORT_OWNERSHIP_CALLER === 'lifetime-owner-loss') {
+ const port = Number(process.env.OK_PORT_OWNERSHIP_SCHEDULE_PORT);
+ const role = process.argv.some((value) => basename(value) === 'lifetime-server.cjs')
+ ? 'server'
+ : 'runner';
+ const lifetime = net.connect({ port, host: '127.0.0.1' }, () => {
+ lifetime.write(`${role}\t${process.pid}\n`);
+ });
+ lifetime.unref();
+ const connect = net.connect;
+ net.connect = function (...args) {
+ const socket = connect.apply(this, args);
+ if (args[0]?.port === Number(process.env.OK_PORT_OWNERSHIP_CONTROL_PORT)) {
+ let observed = false;
+ const yielding = () => {
+ if (observed) return;
+ observed = true;
+ setImmediate(() => meet('yield'));
+ };
+ socket.once('end', yielding);
+ socket.once('error', yielding);
+ socket.once('close', yielding);
+ }
+ return socket;
+ };
+ const append = fs.appendFileSync;
+ fs.appendFileSync = function (...args) {
+ const data = String(args[1]);
+ const phase =
+ role === 'server' && data.includes('"event":"owner-control-closed"')
+ ? 'record'
+ : data.includes('"event":"owner-cleanup-error"')
+ ? 'error'
+ : undefined;
+ if (phase) meet(`before-${phase}`);
+ try {
+ return append.apply(this, args);
+ } finally {
+ if (phase) meet(`after-${phase}`);
+ }
+ };
+ syncBuiltinESMExports();
+}
diff --git a/packages/app/tests/stress/_helpers/port-ownership/cleanup-order.c b/packages/app/tests/stress/_helpers/port-ownership/cleanup-order.c
new file mode 100644
index 000000000..548f6f826
--- /dev/null
+++ b/packages/app/tests/stress/_helpers/port-ownership/cleanup-order.c
@@ -0,0 +1,122 @@
+#define _GNU_SOURCE
+#include <arpa/inet.h>
+#include <dirent.h>
+#include <dlfcn.h>
+#include <errno.h>
+#include <fcntl.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <sys/socket.h>
+#include <unistd.h>
+
+__attribute__((noreturn)) static void fail(const char *step, long result, int error, int pid,
+ int code) {
+ char message[160];
+ int length = snprintf(message, sizeof(message),
+ "cleanup-order rendezvous %s failed: result=%ld errno=%d pid=%d\n", step,
+ result, error, pid);
+ if (length > 0) {
+ size_t size = (size_t)length < sizeof(message) ? (size_t)length : sizeof(message) - 1;
+ ssize_t written = write(STDERR_FILENO, message, size);
+ (void)written;
+ }
+ _exit(code);
+}
+
+static void rendezvous(const char *phase, int pid) {
+ const char *port = getenv("OK_PORT_OWNERSHIP_SCHEDULE_PORT");
+ if (!port) return;
+ int saved_errno = errno;
+ int fd = socket(AF_INET, SOCK_STREAM, 0);
+ if (fd < 0) fail("socket", fd, errno, pid, 90);
+ struct sockaddr_in address = {0};
+ address.sin_family = AF_INET;
+ address.sin_port = htons((unsigned short)atoi(port));
+ address.sin_addr.s_addr = htonl(INADDR_LOOPBACK);
+ if (connect(fd, (struct sockaddr *)&address, sizeof(address)) != 0) {
+ fail("connect", -1, errno, pid, 91);
+ }
+ char line[256];
+ int length = snprintf(line, sizeof(line), "%s\t%d\n", phase, pid);
+ ssize_t sent = write(fd, line, length);
+ if (sent != length) fail("write", sent, sent < 0 ? errno : 0, pid, 92);
+ char reply;
+ ssize_t received = read(fd, &reply, 1);
+ if (received != 1) fail("read", received, received < 0 ? errno : 0, pid, 93);
+ close(fd);
+ errno = saved_errno;
+}
+
+static void before_remove(const char *target) {
+ const char *run_dir = getenv("OK_PORT_OWNERSHIP_SCHEDULE_RUN_DIR");
+ if (!run_dir || strcmp(target, run_dir)) return;
+ DIR *directory = opendir(target);
+ if (!directory) return;
+ int empty = 1;
+ struct dirent *entry;
+ while ((entry = readdir(directory))) {
+ if (strcmp(entry->d_name, ".") && strcmp(entry->d_name, "..")) {
+ empty = 0;
+ break;
+ }
+ }
+ closedir(directory);
+ if (empty) rendezvous("remove", getpid());
+}
+#ifdef __APPLE__
+#define REPLACEMENT(name) observed_##name
+#define ORIGINAL(name) name
+#else
+#define REPLACEMENT(name) name
+#define ORIGINAL(name) original_##name
+#endif
+int REPLACEMENT(rmdir)(const char *target) {
+#ifndef __APPLE__
+ int (*original_rmdir)(const char *) = dlsym(RTLD_NEXT, "rmdir");
+#endif
+ before_remove(target);
+ return ORIGINAL(rmdir)(target);
+}
+int REPLACEMENT(remove)(const char *target) {
+#ifndef __APPLE__
+ int (*original_remove)(const char *) = dlsym(RTLD_NEXT, "remove");
+#endif
+ before_remove(target);
+ return ORIGINAL(remove)(target);
+}
+int REPLACEMENT(unlinkat)(int dirfd, const char *target, int flags) {
+#ifndef __APPLE__
+ int (*original_unlinkat)(int, const char *, int) = dlsym(RTLD_NEXT, "unlinkat");
+#endif
+ char resolved[4096];
+ int resolved_fd = 0;
+#ifdef __APPLE__
+ resolved_fd = fcntl(dirfd, F_GETPATH, resolved) == 0;
+#else
+ char descriptor[64];
+ snprintf(descriptor, sizeof(descriptor), "/proc/self/fd/%d", dirfd);
+ ssize_t length = readlink(descriptor, resolved, sizeof(resolved) - 1);
+ if (length >= 0) {
+ resolved[length] = '\0';
+ resolved_fd = 1;
+ }
+#endif
+ if (target[0] == '/') {
+ before_remove(target);
+ } else if (resolved_fd) {
+ size_t length = strlen(resolved);
+ snprintf(resolved + length, sizeof(resolved) - length, "/%s", target);
+ before_remove(resolved);
+ }
+ return ORIGINAL(unlinkat)(dirfd, target, flags);
+}
+#ifdef __APPLE__
+#define INTERPOSE(replacement, original) \
+ __attribute__((used)) static struct { const void *new_function; const void *old_function; } \
+ interpose_##original __attribute__((section("__DATA,__interpose"))) = \
+ { (const void *)(unsigned long)&replacement, (const void *)(unsigned long)&original };
+INTERPOSE(observed_rmdir, rmdir)
+INTERPOSE(observed_remove, remove)
+INTERPOSE(observed_unlinkat, unlinkat)
+#endif
diff --git a/packages/app/tests/stress/_helpers/port-ownership/cleanup-order.test-helper.ts b/packages/app/tests/stress/_helpers/port-ownership/cleanup-order.test-helper.ts
new file mode 100644
index 000000000..113a9bce5
--- /dev/null
+++ b/packages/app/tests/stress/_helpers/port-ownership/cleanup-order.test-helper.ts
@@ -0,0 +1,208 @@
+import { execFile } from 'node:child_process';
+import { mkdirSync, writeFileSync } from 'node:fs';
+import { createServer, type Socket } from 'node:net';
+import { join } from 'node:path';
+import { fileURLToPath } from 'node:url';
+import { promisify } from 'node:util';
+
+const execute = promisify(execFile);
+const SOURCE = fileURLToPath(new URL('./cleanup-order.c', import.meta.url));
+const PRELOAD = fileURLToPath(new URL('./cleanup-order-preload.cjs', import.meta.url));
+
+export async function createCleanupOrder(
+ outputDir: string,
+ purpose: 'owner-loss' | 'control' = 'owner-loss',
+) {
+ const nativeOrdering =
+ purpose === 'owner-loss' && (process.platform === 'darwin' || process.platform === 'linux');
+ mkdirSync(outputDir, { recursive: true });
+ writeFileSync(
+ join(outputDir, 'capabilities.json'),
+ JSON.stringify({ platform: process.platform, nativeOrdering }),
+ );
+ const library = join(outputDir, process.platform === 'darwin' ? 'order.dylib' : 'order.so');
+ if (nativeOrdering) {
+ try {
+ await execute('cc', [
+ '-Wall',
+ '-Wextra',
+ '-Werror=implicit-function-declaration',
+ '-Werror=implicit-int',
+ '-Werror=int-conversion',
+ '-Werror=incompatible-pointer-types',
+ '-Werror=format',
+ '-Werror=return-type',
+ ...(process.platform === 'darwin' ? ['-dynamiclib'] : ['-shared', '-fPIC']),
+ SOURCE,
+ '-o',
+ library,
+ ...(process.platform === 'linux' ? ['-ldl'] : []),
+ ]);
+ } catch (error) {
+ throw new Error(
+ `native cleanup ordering needs a C compiler on PATH as cc (Xcode Command Line Tools on macOS, gcc on Linux) to build ${SOURCE}`,
+ { cause: error },
+ );
+ }
+ }
+
+ const clients = new Set<Socket>();
+ const held = new Map<string, Socket>();
+ const seen = new Set<string>();
+ const ended = new Set<number>();
+ const yielded = new Set<number>();
+ const events: Array<{ phase: string; pid: number }> = [];
+ const arrivals = new Map<string, () => void>();
+ const removers = new Set<number>();
+ let runner: number | undefined;
+ let server: number | undefined;
+ let readyResolve: ((error?: Error) => void) | undefined;
+ const ready = new Promise<Error | undefined>((resolve) => {
+ readyResolve = resolve;
+ });
+ const key = (phase: string, pid: number | undefined) => `${phase}:${pid}`;
+ const has = (phase: string, pid: number | undefined) => seen.has(key(phase, pid));
+ const advanced = (pid: number | undefined) =>
+ pid !== undefined && (ended.has(pid) || yielded.has(pid));
+ function release(phase: string, pid: number | undefined) {
+ const id = key(phase, pid);
+ const socket = held.get(id);
+ if (!socket) return;
+ held.delete(id);
+ socket.end('!');
+ events.push({ phase: `ack:${phase}`, pid: Number(pid) });
+ }
+ function advance() {
+ if (runner !== undefined && server !== undefined) readyResolve?.();
+ if (removers.size > 0 || advanced(runner)) release('before-record', server);
+ if ([...removers].some((pid) => has('before-error', pid)) || advanced(runner)) {
+ release('after-record', server);
+ }
+ for (const pid of removers) {
+ if (pid === server) {
+ if (
+ [...removers].some((other) => other !== server && has('after-error', other)) ||
+ advanced(runner)
+ ) {
+ release('remove', pid);
+ }
+ } else {
+ if (has('after-record', server) || has('remove', server) || advanced(server))
+ release('remove', pid);
+ if (has('remove', server) || advanced(server)) release('before-error', pid);
+ }
+ }
+ }
+ const controller = createServer((socket) => {
+ clients.add(socket);
+ let buffer = '';
+ let actor: number | undefined;
+ socket.on('data', (chunk) => {
+ buffer += chunk.toString('utf8');
+ if (!buffer.includes('\n')) return;
+ const [phase, rawPid] = buffer.trim().split('\t');
+ const pid = Number(rawPid);
+ if (!phase || !Number.isInteger(pid) || pid <= 1) {
+ readyResolve?.(new Error('invalid cleanup ordering participant'));
+ socket.destroy();
+ return;
+ }
+ events.push({ phase, pid });
+ if (phase === 'runner' || phase === 'server') {
+ actor = pid;
+ if (phase === 'runner') runner = pid;
+ else server = pid;
+ } else {
+ const id = key(phase, pid);
+ seen.add(id);
+ held.set(id, socket);
+ arrivals.get(id)?.();
+ arrivals.delete(id);
+ if (phase === 'remove' && runner !== undefined && server !== undefined) removers.add(pid);
+ if (phase === 'yield') yielded.add(pid);
+ const constrained =
+ (pid === server && ['before-record', 'after-record'].includes(phase)) ||
+ (phase === 'remove' && removers.has(pid)) ||
+ (phase === 'before-error' && pid !== server && removers.has(pid));
+ if (!constrained) release(phase, pid);
+ }
+ advance();
+ });
+ socket.once('close', () => {
+ clients.delete(socket);
+ if (actor !== undefined) {
+ ended.add(actor);
+ events.push({ phase: 'exit', pid: actor });
+ if (runner === undefined || server === undefined) {
+ readyResolve?.(
+ new Error('cleanup ordering participant exited before registration completed'),
+ );
+ }
+ }
+ advance();
+ });
+ });
+ await new Promise<void>((resolve, reject) => {
+ controller.once('error', reject);
+ controller.listen(0, '127.0.0.1', resolve);
+ });
+ const address = controller.address();
+ if (!address || typeof address === 'string') throw new Error('cleanup ordering has no address');
+ const env: NodeJS.ProcessEnv = {
+ ...process.env,
+ OK_PORT_OWNERSHIP_SCHEDULE_PORT: String(address.port),
+ ...(nativeOrdering
+ ? {
+ OK_PORT_OWNERSHIP_SCHEDULE_LIBRARY: library,
+ NODE_OPTIONS: `${process.env.NODE_OPTIONS ?? ''} --require=${PRELOAD}`.trim(),
+ [process.platform === 'darwin' ? 'DYLD_INSERT_LIBRARIES' : 'LD_PRELOAD']: library,
+ }
+ : {}),
+ };
+ async function close() {
+ writeFileSync(join(outputDir, 'events.json'), JSON.stringify(events, null, 2));
+ for (const socket of held.values()) socket.end('!');
+ held.clear();
+ for (const socket of clients) socket.destroy();
+ await new Promise<void>((resolve) => controller.close(() => resolve()));
+ }
+ try {
+ if (nativeOrdering) {
+ const canary = join(outputDir, 'canary');
+ mkdirSync(canary);
+ await execute(
+ process.execPath,
+ [
+ '-e',
+ 'require("node:fs").rmSync(process.argv[1], { recursive: true, force: true })',
+ canary,
+ ],
+ { env: { ...env, OK_PORT_OWNERSHIP_SCHEDULE_RUN_DIR: canary } },
+ );
+ if (!events.some((event) => event.phase === 'remove')) {
+ throw new Error('native cleanup ordering did not observe the real removal canary');
+ }
+ }
+ return {
+ env,
+ nativeOrdering,
+ waitForArrival: (phase: string, pid: number) => {
+ const id = key(phase, pid);
+ return seen.has(id)
+ ? Promise.resolve()
+ : new Promise<void>((resolve) => arrivals.set(id, resolve));
+ },
+ beforeOwnerExit: async (pid: number) => {
+ const error = await ready;
+ if (error) throw error;
+ if (server !== pid)
+ throw new Error('cleanup ordering did not register the lifetime server');
+ },
+ events,
+ close,
+ };
+ } catch (error) {
+ await close();
+ throw error;
+ }
+}
diff --git a/packages/app/tests/stress/_helpers/port-ownership/expect-owned-server-run.test-helper.ts b/packages/app/tests/stress/_helpers/port-ownership/expect-owned-server-run.test-helper.ts
index a970b926d..853ced1e0 100644
--- a/packages/app/tests/stress/_helpers/port-ownership/expect-owned-server-run.test-helper.ts
+++ b/packages/app/tests/stress/_helpers/port-ownership/expect-owned-server-run.test-helper.ts
@@ -3,7 +3,7 @@ import { expect, test } from '@playwright/test';
import { type OwnershipCase, runOwnershipCase } from './run-case.test-helper.ts';
export async function expectOwnedServerRun(ownershipCase: OwnershipCase): Promise<void> {
- const run = await runOwnershipCase(ownershipCase);
+ const { run, scratchReleased } = await runOwnershipCase(ownershipCase);
const testInfo = test.info();
expect(run.report, run.transcript).toBeDefined();
const reportPath = testInfo.outputPath('inner-report.json');
@@ -46,4 +46,13 @@ export async function expectOwnedServerRun(ownershipCase: OwnershipCase): Promis
`${run.transcript}\n${JSON.stringify(run.events)}`,
).toBe('passed');
expect(run.exitCode, run.transcript).toBe(0);
+ await expectScratchReleased(scratchReleased);
+}
+
+export async function expectScratchReleased(
+ scratchReleased: Promise<Error | undefined>,
+): Promise<void> {
+ await test.step('scratch lease released', async () => {
+ expect(await scratchReleased).toBeUndefined();
+ });
}
diff --git a/packages/app/tests/stress/_helpers/port-ownership/lease-inheritance-probe.cjs b/packages/app/tests/stress/_helpers/port-ownership/lease-inheritance-probe.cjs
new file mode 100644
index 000000000..1d96d9603
--- /dev/null
+++ b/packages/app/tests/stress/_helpers/port-ownership/lease-inheritance-probe.cjs
@@ -0,0 +1,93 @@
+const childProcess = require('node:child_process');
+const { once } = require('node:events');
+const { writeSync } = require('node:fs');
+
+const key = 'OK_PORT_OWNERSHIP_LEASE_FD';
+const [role, label] = process.argv.slice(2);
+const writeLabel = (name) => writeSync(Number(process.env[key]), `${name}\n`);
+const quote = (value) => `'${value.replaceAll("'", "'\\''")}'`;
+
+async function observe(name, child) {
+ let stdout = '';
+ let message;
+ child.stdout.on('data', (chunk) => {
+ stdout += chunk;
+ });
+ child.stderr.pipe(process.stderr, { end: false });
+ child.on('message', (value) => {
+ message = value;
+ });
+ try {
+ const [code, signal] = await once(child, 'close');
+ return { label: name, code, signal, stdioLength: child.stdio.length, stdout, message };
+ } catch (error) {
+ return { label: name, error: String(error) };
+ }
+}
+
+function observeSync(name, run) {
+ try {
+ return { label: name, code: 0, signal: null, stdout: run() };
+ } catch (error) {
+ return {
+ label: name,
+ code: error.status ?? null,
+ signal: error.signal ?? null,
+ stdout: String(error.stdout ?? ''),
+ error: String(error),
+ };
+ }
+}
+
+async function drive() {
+ const env = { PATH: process.env.PATH, NODE_OPTIONS: process.env.NODE_OPTIONS };
+ const actor = (name) => [__filename, 'actor', name];
+ const command = (name) => [process.execPath, ...actor(name)].map(quote).join(' ');
+ const routes = [
+ await observe('spawn', childProcess.spawn(process.execPath, actor('spawn'), { env })),
+ await observe('fork', childProcess.fork(__filename, ['actor', 'fork'], { env, silent: true })),
+ await observe('execFile', childProcess.execFile(process.execPath, actor('execFile'), { env })),
+ await observe('exec', childProcess.exec(command('exec'), { env })),
+ ];
+ const sync = childProcess.spawnSync(process.execPath, actor('spawnSync'), {
+ env,
+ encoding: 'utf8',
+ });
+ process.stderr.write(sync.stderr ?? '');
+ routes.push(
+ {
+ label: 'spawnSync',
+ code: sync.status,
+ signal: sync.signal,
+ stdioLength: sync.output?.length,
+ stdout: sync.stdout,
+ error: sync.error ? String(sync.error) : undefined,
+ },
+ observeSync('execFileSync', () =>
+ childProcess.execFileSync(process.execPath, actor('execFileSync'), {
+ env,
+ encoding: 'utf8',
+ }),
+ ),
+ observeSync('execSync', () =>
+ childProcess.execSync(command('execSync'), { env, encoding: 'utf8' }),
+ ),
+ );
+ process.once('message', () => {
+ writeLabel('after-owner-release');
+ process.disconnect();
+ });
+ process.send({ routes });
+}
+
+if (role === 'actor') {
+ writeLabel(label);
+ process.stdout.write(`${label}\n`);
+ if (process.send) process.send(label, () => process.disconnect());
+} else if (role === 'driver') {
+ void drive();
+} else if (role === 'reader') {
+ process.stdin.pipe(process.stdout);
+} else {
+ throw new Error(`unknown lease probe role: ${role}`);
+}
diff --git a/packages/app/tests/stress/_helpers/port-ownership/occupy-selected-port.cjs b/packages/app/tests/stress/_helpers/port-ownership/occupy-selected-port.cjs
index 91e5a775b..d20a4293b 100644
--- a/packages/app/tests/stress/_helpers/port-ownership/occupy-selected-port.cjs
+++ b/packages/app/tests/stress/_helpers/port-ownership/occupy-selected-port.cjs
@@ -1,9 +1,9 @@
+require('./owner-scratch-lease.cjs');
const net = require('node:net');
const childProcess = require('node:child_process');
const fs = require('node:fs');
const http = require('node:http');
const { syncBuiltinESMExports } = require('node:module');
-const path = require('node:path');
const { createOwnerLossHandler } = require('./owner-lifetime.cjs');
const createServer = net.createServer;
@@ -27,25 +27,7 @@ if (Number.isInteger(controlPort) && controlPort > 0) {
exit: (code) => process.exit(code),
schedule: (callback, ms) => setTimeout(callback, ms),
record,
- cleanup: () => {
- const runDir = process.env.OK_PORT_OWNERSHIP_RUN_DIR;
- if (!runDir || fs.existsSync(path.join(runDir, 'owner-release'))) return;
- const tempRoot = fs.realpathSync(process.env.OK_PORT_OWNERSHIP_SYSTEM_TEMP_ROOT);
- const resolved = fs.realpathSync(runDir);
- if (
- path.dirname(resolved) === tempRoot &&
- path.basename(resolved).startsWith('ok-port-ownership-') &&
- !fs.lstatSync(runDir).isSymbolicLink()
- ) {
- try {
- fs.mkdirSync(path.join(runDir, '.owner-cleanup-lock'));
- } catch (error) {
- if (error?.code === 'EEXIST') return;
- throw error;
- }
- fs.rmSync(runDir, { recursive: true, force: true });
- }
- },
+ cleanup: () => {},
});
control.once('error', leave);
control.once('end', leave);
diff --git a/packages/app/tests/stress/_helpers/port-ownership/owner-loss.test-helper.ts b/packages/app/tests/stress/_helpers/port-ownership/owner-loss.test-helper.ts
index d084b0429..8b79d8911 100644
--- a/packages/app/tests/stress/_helpers/port-ownership/owner-loss.test-helper.ts
+++ b/packages/app/tests/stress/_helpers/port-ownership/owner-loss.test-helper.ts
@@ -5,6 +5,7 @@ import { connect } from 'node:net';
import { tmpdir } from 'node:os';
import { join } from 'node:path';
import { fileURLToPath } from 'node:url';
+import { createCleanupOrder } from './cleanup-order.test-helper.ts';
import {
type OwnedEndpointRelease,
probeOwnedEndpoint,
@@ -21,6 +22,11 @@ interface Receipt {
pid: number;
}
+interface OrderingEvent {
+ phase: string;
+ pid: number;
+}
+
export interface OwnerLossRun extends OwnedEndpointRelease {
driverPid: number | undefined;
driverExit: number | null;
@@ -30,9 +36,25 @@ export interface OwnerLossRun extends OwnedEndpointRelease {
scratchGone: boolean;
events: unknown[];
servingAfterOwnerExit: boolean;
+ ordering: { native: boolean; order: string[]; events: OrderingEvent[] };
transcript: string;
}
+const UNORDERED_PHASES = new Set(['runner', 'server', 'exit', 'yield', 'ack:yield']);
+
+function cleanupOrderFrom(events: OrderingEvent[], serverPid: number): string[] {
+ const roles = new Map<number, string>();
+ for (const { phase, pid } of events) {
+ if (phase === 'runner' || phase === 'server') roles.set(pid, phase);
+ }
+ let started = false;
+ return events.flatMap(({ phase, pid }) => {
+ if (phase === 'server' && pid === serverPid) started = true;
+ if (!started || UNORDERED_PHASES.has(phase)) return [];
+ return [`${roles.get(pid) ?? 'unregistered'}:${phase}`];
+ });
+}
+
async function awaitReceipt(runDir: string, child: ChildProcess): Promise<Receipt> {
const receiptPath = join(runDir, 'lifetime-receipt.json');
return new Promise<Receipt>((resolve, reject) => {
@@ -108,11 +130,12 @@ function watchScratchGone(runDir: string): { result: Promise<boolean>; stop: ()
return { result, stop: () => finish(false) };
}
-export async function runOwnerLossControl(): Promise<OwnerLossRun> {
+export async function runOwnerLossControl(outputDir: string): Promise<OwnerLossRun> {
+ const ordering = await createCleanupOrder(outputDir);
const runDir = mkdtempSync(join(tmpdir(), 'ok-port-ownership-'));
const child = spawn(process.execPath, ['--import', 'tsx', DRIVER, runDir], {
cwd: APP_ROOT,
- env: process.env,
+ env: { ...ordering.env, OK_PORT_OWNERSHIP_SCHEDULE_RUN_DIR: runDir },
stdio: ['ignore', 'pipe', 'pipe', 'ipc'] as const,
});
let transcript = '';
@@ -124,6 +147,7 @@ export async function runOwnerLossControl(): Promise<OwnerLossRun> {
transcript += chunk.toString('utf8');
});
+ let failure: unknown;
try {
const receipt = await awaitReceipt(runDir, child);
const witness = connect(receipt.port, '127.0.0.1');
@@ -131,6 +155,14 @@ export async function runOwnerLossControl(): Promise<OwnerLossRun> {
scratchWatcher = watchScratchGone(runDir);
const closed = once(witness, 'close');
const exited = once(child, 'exit');
+ if (ordering.nativeOrdering) {
+ await Promise.race([
+ ordering.beforeOwnerExit(receipt.pid),
+ exited.then(() => {
+ throw new Error('owner exited before cleanup ordering was ready');
+ }),
+ ]);
+ }
child.send('exit');
const [driverExit, driverSignal] = (await exited) as [number | null, NodeJS.Signals | null];
let closeBound: ReturnType<typeof setTimeout> | undefined;
@@ -142,6 +174,7 @@ export async function runOwnerLossControl(): Promise<OwnerLossRun> {
]).finally(() => clearTimeout(closeBound));
if (!witnessClosed) witness.destroy();
const scratchGone = await scratchWatcher.result;
+ const orderingEvents = [...ordering.events];
const eventsPath = join(runDir, 'events.jsonl');
const events = existsSync(eventsPath) ? readOwnershipRecords(eventsPath) : [];
const servingAfterOwnerExit = await probeOwnedEndpoint(receipt);
@@ -156,8 +189,15 @@ export async function runOwnerLossControl(): Promise<OwnerLossRun> {
events,
servingAfterOwnerExit,
...release,
+ ordering: {
+ native: ordering.nativeOrdering,
+ order: cleanupOrderFrom(orderingEvents, receipt.pid),
+ events: orderingEvents,
+ },
transcript,
};
+ } catch (error) {
+ failure = error;
} finally {
scratchWatcher?.stop();
if (child.exitCode === null && child.signalCode === null && child.connected) {
@@ -165,6 +205,9 @@ export async function runOwnerLossControl(): Promise<OwnerLossRun> {
child.send('exit');
await exited.catch(() => {});
}
+ await ordering.close();
if (existsSync(runDir)) rmSync(runDir, { recursive: true, force: true });
}
+ const message = failure instanceof Error ? failure.message : String(failure);
+ throw new Error(`${message}\n--- owner-loss driver output\n${transcript}`, { cause: failure });
}
diff --git a/packages/app/tests/stress/_helpers/port-ownership/owner-scratch-janitor.cjs b/packages/app/tests/stress/_helpers/port-ownership/owner-scratch-janitor.cjs
new file mode 100644
index 000000000..a628e84e6
--- /dev/null
+++ b/packages/app/tests/stress/_helpers/port-ownership/owner-scratch-janitor.cjs
@@ -0,0 +1,24 @@
+const fs = require('node:fs');
+const path = require('node:path');
+
+const [runDir, systemTempRoot] = process.argv.slice(2);
+const resolved = fs.realpathSync(runDir);
+if (
+ path.dirname(resolved) !== fs.realpathSync(systemTempRoot) ||
+ !path.basename(resolved).startsWith('ok-port-ownership-') ||
+ fs.lstatSync(runDir).isSymbolicLink()
+) {
+ throw new Error('invalid owner scratch directory');
+}
+
+process.stdin.once('end', () => {
+ try {
+ fs.rmSync(runDir, { recursive: true, force: true });
+ } catch (error) {
+ console.error(error);
+ process.exitCode = 1;
+ }
+});
+process.stdin.resume();
+process.send('ready');
+process.disconnect();
diff --git a/packages/app/tests/stress/_helpers/port-ownership/owner-scratch-lease.cjs b/packages/app/tests/stress/_helpers/port-ownership/owner-scratch-lease.cjs
new file mode 100644
index 000000000..a6894f8c0
--- /dev/null
+++ b/packages/app/tests/stress/_helpers/port-ownership/owner-scratch-lease.cjs
@@ -0,0 +1,50 @@
+const { ChildProcess } = require('node:child_process');
+const { fstatSync } = require('node:fs');
+
+const key = 'OK_PORT_OWNERSHIP_LEASE_FD';
+const inherited = process.env[key];
+
+if (inherited !== undefined) {
+ const fd = Number(inherited);
+ if (!Number.isInteger(fd) || fd < 3) throw new Error('invalid owner scratch lease descriptor');
+ fstatSync(fd);
+ const spawn = ChildProcess.prototype.spawn;
+ const spawnSyncBinding = process.binding('spawn_sync');
+ const spawnSync = spawnSyncBinding.spawn;
+ const withLeaseEnv = (options, index) => ({
+ ...options,
+ envPairs: [
+ ...(options.envPairs ?? []).filter((pair) => !pair.startsWith(`${key}=`)),
+ `${key}=${index}`,
+ ],
+ });
+
+ ChildProcess.prototype.spawn = function (options) {
+ const requested = options.stdio ?? 'pipe';
+ let stdio;
+ if (Array.isArray(requested)) {
+ stdio = [...requested];
+ while (stdio.length < 3) stdio.push(undefined);
+ } else if (requested === 'inherit') {
+ stdio = [0, 1, 2];
+ } else if (['pipe', 'ignore', 'overlapped'].includes(requested)) {
+ stdio = [requested, requested, requested];
+ } else {
+ return spawn.call(this, options);
+ }
+ const index = stdio.length;
+ const result = spawn.call(this, { ...withLeaseEnv(options, index), stdio: [...stdio, fd] });
+ if (this.stdio) this.stdio.length = index;
+ return result;
+ };
+
+ spawnSyncBinding.spawn = function (options) {
+ const index = options.stdio.length;
+ const result = spawnSync.call(this, {
+ ...withLeaseEnv(options, index),
+ stdio: [...options.stdio, { type: 'fd', fd }],
+ });
+ if (result.output) result.output.length = index;
+ return result;
+ };
+}
diff --git a/packages/app/tests/stress/_helpers/port-ownership/run-case.test-helper.ts b/packages/app/tests/stress/_helpers/port-ownership/run-case.test-helper.ts
index 9e1a1da38..51fa4581e 100644
--- a/packages/app/tests/stress/_helpers/port-ownership/run-case.test-helper.ts
+++ b/packages/app/tests/stress/_helpers/port-ownership/run-case.test-helper.ts
@@ -1,13 +1,6 @@
import { spawn } from 'node:child_process';
-import {
- existsSync,
- mkdirSync,
- mkdtempSync,
- readFileSync,
- realpathSync,
- rmSync,
- writeFileSync,
-} from 'node:fs';
+import { once } from 'node:events';
+import { existsSync, mkdirSync, mkdtempSync, readFileSync, realpathSync, rmSync } from 'node:fs';
import { createRequire } from 'node:module';
import { createServer, type Socket } from 'node:net';
import { tmpdir } from 'node:os';
@@ -20,9 +13,33 @@ const APP_ROOT = fileURLToPath(new URL('../../../..', import.meta.url));
const SUBTREE_ROOT = fileURLToPath(new URL('../../../../../..', import.meta.url));
const RUN_CONFIG = fileURLToPath(new URL('./ownership-run.config.ts', import.meta.url));
const PRELOAD = fileURLToPath(new URL('./occupy-selected-port.cjs', import.meta.url));
+const JANITOR = fileURLToPath(new URL('./owner-scratch-janitor.cjs', import.meta.url));
+const LEASE_PROBE = fileURLToPath(new URL('./lease-inheritance-probe.cjs', import.meta.url));
const PLAYWRIGHT_CLI = createRequire(join(APP_ROOT, 'package.json')).resolve(
'@playwright/test/cli',
);
+const RUNNER_STDIO = ['ignore', 'pipe', 'pipe'] as const;
+const LEASE_PROBE_STDIO = [...RUNNER_STDIO, 'ipc'] as const;
+
+type ProcessExit = [code: number | null, signal: NodeJS.Signals | null];
+
+export interface LeaseRoute {
+ label: string;
+ code?: number | null;
+ signal?: NodeJS.Signals | null;
+ stdioLength?: number;
+ stdout?: string;
+ message?: string;
+ error?: string;
+}
+
+export interface LeaseInheritanceRun {
+ labels: string[];
+ routes: LeaseRoute[] | undefined;
+ driverExit: ProcessExit;
+ readerExit: ProcessExit;
+ transcript: string;
+}
export interface OwnershipCase {
file: string;
@@ -55,6 +72,11 @@ export interface OwnershipRun {
| undefined;
}
+export interface OwnershipCaseResult {
+ run: OwnershipRun;
+ scratchReleased: Promise<Error | undefined>;
+}
+
async function openOwnerControl(): Promise<{ port: number; close: () => Promise<void> }> {
const clients = new Set<Socket>();
const server = createServer((socket) => {
@@ -81,6 +103,41 @@ async function openOwnerControl(): Promise<{ port: number; close: () => Promise<
};
}
+async function startScratchJanitor(runDir: string) {
+ const janitor = spawn(process.execPath, [JANITOR, runDir, realpathSync(tmpdir())], {
+ stdio: ['pipe', 'ignore', 'inherit', 'ipc'],
+ });
+ const completed = new Promise<Error | undefined>((resolve) => {
+ janitor.once('error', resolve);
+ janitor.once('exit', (code, signal) => {
+ resolve(
+ code === 0
+ ? undefined
+ : new Error(`owner scratch cleanup failed: code=${code} signal=${signal}`),
+ );
+ });
+ });
+ const lease = janitor.stdin;
+ if (!lease) {
+ const error = await completed;
+ throw error ?? new Error('owner scratch janitor has no lease pipe');
+ }
+ try {
+ const [message] = await Promise.race([
+ once(janitor, 'message'),
+ completed.then((error) => {
+ throw error ?? new Error('owner scratch janitor exited before readiness');
+ }),
+ ]);
+ if (message !== 'ready') throw new Error('invalid owner scratch janitor readiness');
+ } catch (error) {
+ lease.destroy();
+ await completed;
+ throw error;
+ }
+ return { lease, completed };
+}
+
function specsOf(suites: JSONReportSuite[]): JSONReportSpec[] {
return suites.flatMap((suite) => [...suite.specs, ...specsOf(suite.suites ?? [])]);
}
@@ -138,7 +195,7 @@ export function readOwnershipRecords(path: string): unknown[] {
.map((line) => JSON.parse(line) as unknown);
}
-export async function runOwnershipCase(ownershipCase: OwnershipCase): Promise<OwnershipRun> {
+export async function runOwnershipCase(ownershipCase: OwnershipCase): Promise<OwnershipCaseResult> {
const runDir = ownershipCase.runDir ?? mkdtempSync(join(tmpdir(), 'ok-port-ownership-'));
const runtimeTmp = join(runDir, 'tmp');
mkdirSync(runtimeTmp);
@@ -146,12 +203,17 @@ export async function runOwnershipCase(ownershipCase: OwnershipCase): Promise<Ow
rmSync(runDir, { recursive: true, force: true });
throw error;
});
+ const janitor = await startScratchJanitor(runDir).catch(async (error: unknown) => {
+ await ownerControl.close();
+ rmSync(runDir, { recursive: true, force: true });
+ throw error;
+ });
const env = createNestedPlaywrightEnv({
NODE_OPTIONS: `${process.env.NODE_OPTIONS ?? ''} --require=${PRELOAD}`.trim(),
OK_PORT_OWNERSHIP_CALLER: ownershipCase.caller,
OK_PORT_OWNERSHIP_RECORDS: join(runDir, 'events.jsonl'),
OK_PORT_OWNERSHIP_RUN_DIR: runDir,
- OK_PORT_OWNERSHIP_SYSTEM_TEMP_ROOT: realpathSync(tmpdir()),
+ OK_PORT_OWNERSHIP_LEASE_FD: String(RUNNER_STDIO.length),
OK_PORT_OWNERSHIP_CONTROL_PORT: String(ownerControl.port),
...(ownershipCase.caller === 'global-warm-cache.ts'
? { OK_TEST_VITE_SEED_DIR: join(runDir, 'seed') }
@@ -175,13 +237,15 @@ export async function runOwnershipCase(ownershipCase: OwnershipCase): Promise<Ow
'--workers=2',
'--retries=0',
],
- { cwd: APP_ROOT, env, stdio: ['ignore', 'pipe', 'pipe'] },
+ { cwd: APP_ROOT, env, stdio: [...RUNNER_STDIO, janitor.lease] as const },
);
+ const { stdout, stderr } = child;
+ if (!stdout || !stderr) throw new Error('nested ownership runner has no output pipes');
let transcript = '';
- child.stdout.on('data', (chunk: Buffer) => {
+ stdout.on('data', (chunk: Buffer) => {
transcript += chunk.toString('utf8');
});
- child.stderr.on('data', (chunk: Buffer) => {
+ stderr.on('data', (chunk: Buffer) => {
transcript += chunk.toString('utf8');
});
const closed = new Promise<true>((resolve) => {
@@ -193,13 +257,6 @@ export async function runOwnershipCase(ownershipCase: OwnershipCase): Promise<Ow
}>((resolve, reject) => {
child.once('error', reject);
child.once('exit', (code, exitSignal) => {
- try {
- writeFileSync(join(runDir, 'owner-release'), 'child-exit');
- } catch (error) {
- reject(error);
- void ownerControl.close();
- return;
- }
void ownerControl
.close()
.then(() => resolve({ exitCode: code, signal: exitSignal }), reject);
@@ -213,8 +270,8 @@ export async function runOwnershipCase(ownershipCase: OwnershipCase): Promise<Ow
}),
]).finally(() => clearTimeout(closeBound));
if (!stdioClosed) {
- child.stdout.destroy();
- child.stderr.destroy();
+ stdout.destroy();
+ stderr.destroy();
}
const report = existsSync(join(runDir, 'results.json'))
? (JSON.parse(readFileSync(join(runDir, 'results.json'), 'utf8')) as JSONReport)
@@ -242,21 +299,69 @@ export async function runOwnershipCase(ownershipCase: OwnershipCase): Promise<Ow
}
}
return {
- exitCode,
- signal,
- stdioClosed,
- transcript,
- events,
- eventFilePresent,
- report,
- matchingSpecs: specsOf(report?.suites ?? []).filter(
- (spec) => spec.title === ownershipCase.name,
- ),
- errors: report?.errors,
- lifetime,
+ run: {
+ exitCode,
+ signal,
+ stdioClosed,
+ transcript,
+ events,
+ eventFilePresent,
+ report,
+ matchingSpecs: specsOf(report?.suites ?? []).filter(
+ (spec) => spec.title === ownershipCase.name,
+ ),
+ errors: report?.errors,
+ lifetime,
+ },
+ scratchReleased: janitor.completed,
};
} finally {
await ownerControl.close();
- rmSync(runDir, { recursive: true, force: true });
+ janitor.lease.destroy();
+ }
+}
+
+export async function runLeaseInheritanceProbe(): Promise<LeaseInheritanceRun> {
+ const reader = spawn(process.execPath, [LEASE_PROBE, 'reader'], {
+ stdio: ['pipe', 'pipe', 'pipe'],
+ });
+ const readerClosed = once(reader, 'close') as Promise<ProcessExit>;
+ let received = '';
+ let transcript = '';
+ reader.stdout.on('data', (chunk: Buffer) => {
+ received += chunk.toString('utf8');
+ });
+ reader.stderr.on('data', (chunk: Buffer) => {
+ transcript += chunk.toString('utf8');
+ });
+ const lease = reader.stdin;
+ const driver = spawn(process.execPath, [LEASE_PROBE, 'driver'], {
+ env: {
+ ...process.env,
+ NODE_OPTIONS: `${process.env.NODE_OPTIONS ?? ''} --require=${PRELOAD}`.trim(),
+ OK_PORT_OWNERSHIP_LEASE_FD: String(LEASE_PROBE_STDIO.length),
+ },
+ stdio: [...LEASE_PROBE_STDIO, lease] as const,
+ });
+ for (const stream of [driver.stdout, driver.stderr]) {
+ stream?.on('data', (chunk: Buffer) => {
+ transcript += chunk.toString('utf8');
+ });
}
+ const driverClosed = once(driver, 'close') as Promise<ProcessExit>;
+ const ready = await Promise.race([
+ once(driver, 'message').then(([message]) => message as { routes: LeaseRoute[] }),
+ driverClosed.then(() => undefined),
+ ]);
+ const released = once(lease, 'close');
+ lease.destroy();
+ await released;
+ if (ready && driver.connected) driver.send('write');
+ return {
+ routes: ready?.routes,
+ driverExit: await driverClosed,
+ readerExit: await readerClosed,
+ labels: received.split('\n').filter(Boolean),
+ transcript,
+ };
}
diff --git a/packages/app/tests/stress/harness-port-ownership.e2e.ts b/packages/app/tests/stress/harness-port-ownership.e2e.ts
index 88095b142..1f8a99c78 100644
--- a/packages/app/tests/stress/harness-port-ownership.e2e.ts
+++ b/packages/app/tests/stress/harness-port-ownership.e2e.ts
@@ -1,9 +1,15 @@
import { writeFileSync } from 'node:fs';
import { expect, test } from '@playwright/test';
-import { expectOwnedServerRun } from './_helpers/port-ownership/expect-owned-server-run.test-helper.ts';
+import { runCleanupOrderControl } from './_helpers/port-ownership/cleanup-order-control.test-helper.ts';
+import {
+ expectOwnedServerRun,
+ expectScratchReleased,
+} from './_helpers/port-ownership/expect-owned-server-run.test-helper.ts';
import { runOwnerLossControl } from './_helpers/port-ownership/owner-loss.test-helper.ts';
import {
+ type LeaseRoute,
type OwnershipRun,
+ runLeaseInheritanceProbe,
runOwnershipCase,
} from './_helpers/port-ownership/run-case.test-helper.ts';
@@ -48,7 +54,7 @@ async function attachLifetimeRun(run: OwnershipRun): Promise<void> {
}
test('nested runner releases owned setup resources on timeout', async () => {
- const run = await runOwnershipCase({
+ const { run, scratchReleased } = await runOwnershipCase({
file: 'tests/stress/_helpers/port-ownership/lifetime.ownership-case.ts',
name: 'nested lifetime control reaches its test body',
caller: 'lifetime-timeout',
@@ -59,10 +65,11 @@ test('nested runner releases owned setup resources on timeout', async () => {
expect(run.exitCode, run.transcript).toBe(1);
expect(run.stdioClosed, run.transcript).toBe(true);
expect(run.lifetime?.servingAfterExit, JSON.stringify(run.lifetime)).toBe(false);
+ await expectScratchReleased(scratchReleased);
});
test('nested runner releases owned setup resources on normal completion', async () => {
- const run = await runOwnershipCase({
+ const { run, scratchReleased } = await runOwnershipCase({
file: 'tests/stress/_helpers/port-ownership/lifetime.ownership-case.ts',
name: 'nested lifetime control reaches its test body',
caller: 'lifetime-normal',
@@ -74,10 +81,52 @@ test('nested runner releases owned setup resources on normal completion', async
expect(run.exitCode, run.transcript).toBe(0);
expect(run.stdioClosed, run.transcript).toBe(true);
expect(run.lifetime?.servingAfterExit, run.transcript).toBe(false);
+ await expectScratchReleased(scratchReleased);
+});
+
+for (const progress of ['yield', 'exit'] as const) {
+ test(`cleanup ordering releases a delayed writer after its peer ${progress}s`, async () => {
+ const observed = await runCleanupOrderControl(
+ test.info().outputPath('cleanup-order'),
+ progress,
+ );
+ expect(observed.order).toEqual(['request', 'progress', 'acknowledgment']);
+ });
+}
+
+test('cleanup ordering holds an unregistered remover until the pending write completes', async () => {
+ const observed = await runCleanupOrderControl(test.info().outputPath('cleanup-order'), 'remover');
+ expect(observed.removerExit).toEqual([0, null]);
+ expect(observed.order).toEqual(['request', 'write-complete', 'acknowledgment']);
+});
+
+const LEASE_ROUTES: LeaseRoute[] = [
+ { label: 'spawn', code: 0, signal: null, stdioLength: 3, stdout: 'spawn\n' },
+ { label: 'fork', code: 0, signal: null, stdioLength: 4, stdout: 'fork\n', message: 'fork' },
+ { label: 'execFile', code: 0, signal: null, stdioLength: 3, stdout: 'execFile\n' },
+ { label: 'exec', code: 0, signal: null, stdioLength: 3, stdout: 'exec\n' },
+ { label: 'spawnSync', code: 0, signal: null, stdioLength: 3, stdout: 'spawnSync\n' },
+ { label: 'execFileSync', code: 0, signal: null, stdout: 'execFileSync\n' },
+ { label: 'execSync', code: 0, signal: null, stdout: 'execSync\n' },
+];
+
+test('scratch lease reaches descendants through every spawn entry point', async () => {
+ const probe = await runLeaseInheritanceProbe();
+ const diagnostics = JSON.stringify(probe);
+ expect([...probe.labels].sort(), diagnostics).toEqual(
+ [...LEASE_ROUTES.map(({ label }) => label), 'after-owner-release'].sort(),
+ );
+ expect(probe.routes, diagnostics).toEqual(LEASE_ROUTES);
+ expect(probe.driverExit, diagnostics).toEqual([0, null]);
+ expect(probe.readerExit, diagnostics).toEqual([0, null]);
});
test('nested runner releases owned setup resources when its owner exits', async () => {
- const result = await runOwnerLossControl();
+ const result = await runOwnerLossControl(test.info().outputPath('cleanup-order'));
+ await test.info().attach('cleanup-ordering', {
+ path: test.info().outputPath('cleanup-order/capabilities.json'),
+ contentType: 'application/json',
+ });
const path = test.info().outputPath('owner-loss-run.json');
writeFileSync(path, JSON.stringify(result));
await test.info().attach('owner-loss-run', { path, contentType: 'application/json' });
@@ -87,6 +136,21 @@ test('nested runner releases owned setup resources when its owner exits', async
expect(result.witnessClosed, JSON.stringify(result)).toBe(true);
expect(result.servingAfterOwnerExit, JSON.stringify(result)).toBe(false);
expect(result.scratchGone, JSON.stringify(result)).toBe(true);
+ if (result.ordering.native) {
+ expect(result.ordering.order, JSON.stringify(result.ordering.events)).toEqual([
+ 'server:before-record',
+ 'server:ack:before-record',
+ 'server:after-record',
+ 'server:ack:after-record',
+ 'unregistered:remove',
+ 'unregistered:ack:remove',
+ ]);
+ } else {
+ test.info().annotations.push({
+ type: 'cleanup-ordering',
+ description: `native cleanup ordering is unavailable on ${process.platform}; this run used natural timing`,
+ });
+ }
});
test('automatically selected worker server owns its endpoint', async () => {