Skip to content

Beautiful, AI-native markdown IDE and LLM wiki

Add the OK E2E server reuse and prebuilt serving half (#5894)

Nick Gomezd2ac690d05cf
diff --git a/knip.config.ts b/knip.config.ts
index 69344114d..e43118957 100644
--- a/knip.config.ts
+++ b/knip.config.ts
@@ -117,6 +117,8 @@ export default {
'tests/**/*.{test,e2e}.ts',
'tests/perf/lib/*.ts',
'tests/dom/**/*.ts',
+ 'vite.prebuilt-dev.ts',
+ 'tests/stress/_helpers/reuse/server-supervisor.ts',
],
project: 'src/**',
ignoreDependencies: [
@@ -147,7 +149,11 @@ export default {
entry: ['src/**/*.test.{ts,tsx}'],
},
'packages/server': {
- entry: ['src/**/*.test.ts', 'src/parse-worker.ts'],
+ entry: [
+ 'src/**/*.test.ts',
+ 'src/parse-worker.ts',
+ 'src/shadow-fixture-launch.test-helper.mjs',
+ ],
project: 'src/**',
ignoreDependencies: ['@types/shell-quote'],
},
diff --git a/packages/app/package.json b/packages/app/package.json
index 1e060b07b..fbd4dd3b7 100644
--- a/packages/app/package.json
+++ b/packages/app/package.json
@@ -32,6 +32,7 @@
"test:integration:shard1": "vitest run --config vitest.integration.config.ts --shard=1/3",
"test:integration:shard2": "vitest run --config vitest.integration.config.ts --shard=2/3",
"test:integration:shard3": "vitest run --config vitest.integration.config.ts --shard=3/3",
+ "test:worker-lifetime": "vitest run --config vitest.worker-lifetime.config.ts",
"test:perf:sessions": "vitest run --config vitest.config.ts tests/integration/per-session-um-perf.test.ts --testTimeout 60000",
"test:conversion": "vitest run --config vitest.config.ts tests/conversion/",
"test:stress:api": "node --import tsx tests/stress/stress-api.ts",
@@ -39,7 +40,7 @@
"measure:stress": "bash scripts/measure-stress.sh",
"measure:sweep": "bash scripts/measure-sweep.sh",
"perf:compare": "bash scripts/perf-compare.sh",
- "test:e2e": "playwright test tests/stress/empty-bullet-before-link.e2e.ts tests/stress/list-item-drag.e2e.ts tests/stress/ux-interactions.e2e.ts tests/stress/file-tree-create.e2e.ts tests/stress/file-tree-drag-to-root.e2e.ts tests/stress/file-tree-deselect-to-root.e2e.ts tests/stress/file-tree-collapse-spaced-folder.e2e.ts tests/stress/sidebar-file-list-fills.e2e.ts tests/stress/file-tree-sticky-collapse.e2e.ts tests/stress/file-tree-collapse-persistence.e2e.ts tests/stress/file-tree-fractional-zoom.e2e.ts tests/stress/create-then-rename-editable.e2e.ts tests/stress/find-replace.e2e.ts tests/stress/external-link-cue.e2e.ts tests/stress/editor-tabs.e2e.ts tests/stress/crdt-stress.e2e.ts tests/stress/slash-command.e2e.ts tests/stress/paste-fidelity.e2e.ts tests/stress/fr-7a-disconnect-source-mode.e2e.ts tests/stress/docs-open.e2e.ts tests/stress/frozen-table-headers.e2e.ts tests/stress/asset-embed.e2e.ts tests/stress/asset-embed-advanced.e2e.ts tests/stress/asset-click-dispatch.e2e.ts tests/stress/handoff.e2e.ts tests/stress/multi-agent-presence.e2e.ts tests/stress/editor-mode-persistence.e2e.ts tests/stress/lint-config-toggle.e2e.ts tests/stress/plugin-enable-notice.e2e.ts tests/stress/schema-edit-fields-view.e2e.ts tests/stress/sidebar-search-pill.e2e.ts tests/stress/navigation-history.e2e.ts tests/stress/command-palette-semantic.e2e.ts tests/stress/agent-activity-panel.e2e.ts tests/stress/drop-pipeline-auto-open.e2e.ts tests/stress/command-palette-flicker.e2e.ts tests/stress/cm6-list-hanging-indent.e2e.ts tests/stress/ng7-rapid-nav-coherence.e2e.ts tests/stress/reveal-on-activate.e2e.ts tests/stress/selection-indicator.e2e.ts tests/stress/new-file-cross-doc-bleed.e2e.ts tests/stress/new-item-shortcut-fast-path.e2e.ts tests/stress/editor-mode-flip-cross-doc-bleed.e2e.ts tests/stress/editor-area-viewport-resize.e2e.ts tests/stress/qa-sidebar-responsive.e2e.ts tests/stress/prd-6955-reassertion-repro.e2e.ts tests/stress/prd-6955-reassertion-wedge.e2e.ts tests/stress/prd-6914-repro.e2e.ts tests/stress/showall-lazy-tree.e2e.ts tests/stress/tabs-component-strip.e2e.ts tests/stress/source-find-scroll.e2e.ts tests/stress/harness-app-warmth.e2e.ts tests/stress/qa-canary-authoring-both-modes.e2e.ts tests/stress/qa-canary-live-typing.e2e.ts tests/stress/keystroke-cadence-danger-space.e2e.ts tests/stress/jsx-unregistered-backspace-delete.e2e.ts tests/stress/jsx-unregistered-ime-concurrent.e2e.ts tests/stress/jsx-backspace-delete.e2e.ts tests/stress/link-authoring-bytes.e2e.ts tests/stress/link-authoring-apex.e2e.ts tests/stress/qa019-suffix-precedence.e2e.ts tests/stress/command-palette-parity.e2e.ts tests/stress/settings-search.e2e.ts tests/stress/settings-close-button.e2e.ts tests/stress/okf-generated-index-settings.e2e.ts tests/stress/okf-recommended-skill-install.e2e.ts tests/stress/chip-popover-positioning.e2e.ts tests/stress/docs-open-scroll-restore.e2e.ts tests/stress/file-tree-decoration-chip-strip-repro.e2e.ts tests/stress/grip-click-nodeselect.e2e.ts tests/stress/jsx-halo-semantics.e2e.ts tests/stress/jsx-image-body-click.e2e.ts tests/stress/jsx-prop-panel-placeholder.e2e.ts tests/stress/jsx-range-encompass-halo.e2e.ts tests/stress/jsx-wildcard-convert.e2e.ts tests/stress/markdown-lint.e2e.ts tests/stress/frontmatter-schema-badge.e2e.ts tests/stress/frontmatter-glob-picker.e2e.ts tests/stress/unified-problems.e2e.ts tests/stress/mid-type-recovery.e2e.ts tests/stress/observer-a-multi-client.e2e.ts tests/stress/outline-toolbar-occlusion.e2e.ts tests/stress/selection-surface-pane-clip.e2e.ts tests/stress/quiet-tree-hidden-doc.e2e.ts tests/stress/raw-mdx-fallback-onblur-upgrade.e2e.ts tests/stress/rename-noext-probe.e2e.ts tests/stress/show-ok-folders.e2e.ts tests/stress/source-polish.e2e.ts tests/stress/warm-skeleton-scroll-restore.e2e.ts tests/stress/props-toggle-scroll-preserve.e2e.ts tests/stress/yjs-no-dual-import.e2e.ts tests/stress/graph-panel-surfaces.e2e.ts tests/stress/asset-embed-real-fidelity.e2e.ts tests/stress/image-invalid-placeholder.e2e.ts tests/stress/local-target-audit.e2e.ts tests/stress/rename-content-preservation.e2e.ts tests/stress/rename-consolidation.e2e.ts tests/stress/outline-active-heading.e2e.ts tests/stress/outline-navigation.e2e.ts tests/stress/prd-7627-default-swatch.e2e.ts tests/stress/clipboard-dollar-math-source-fallback.e2e.ts tests/stress/source-mode-undo-keymap.e2e.ts tests/stress/source-undo-mode-flip.e2e.ts tests/stress/timeline-recovered-restore.e2e.ts tests/stress/blank-line-preservation.e2e.ts tests/stress/landing-helpers.e2e.ts tests/stress/agent-follow-scroll.e2e.ts tests/stress/mode-switch-landing.e2e.ts tests/stress/mode-switch-ordinal-divergence.e2e.ts tests/stress/comment-annotation-visibility.e2e.ts tests/stress/comment-opens-in-panel.e2e.ts tests/stress/qa-miles-edge-blanks.e2e.ts tests/stress/prop-upload.e2e.ts tests/stress/slash-command-auto-open.e2e.ts tests/stress/clipboard-relative-url-source-fallback.e2e.ts tests/stress/editor-split-view.e2e.ts tests/stress/language-load-failure.e2e.ts tests/stress/language-first-paint.e2e.ts tests/stress/user-text-direction.e2e.ts tests/stress/pseudolocale.e2e.ts tests/stress/language-picker.e2e.ts tests/stress/zh-hans-coverage-sweep.e2e.ts tests/stress/preview-tab-promotion.e2e.ts tests/stress/cv-paint-lock-click.e2e.ts tests/stress/cv-auto-paint-lock-click.e2e.ts tests/stress/harness-port-ownership.e2e.ts tests/stress/history-traversal-preview-tab.e2e.ts tests/stress/tab-close-scroll-restore.e2e.ts tests/stress/skills-sidebar-open.e2e.ts tests/stress/trailing-affordance.e2e.ts tests/stress/text-doc-composer-inset.e2e.ts tests/stress/composer-control-alignment.e2e.ts tests/stress/agents-settings-accessible-names.e2e.ts tests/stress/composer-growth-eof-reveal.e2e.ts tests/stress/theme-fade.e2e.ts tests/stress/theme-color-conversion.e2e.ts tests/stress/composer-open-caret-reveal.e2e.ts tests/stress/spellcheck-browser-absence.e2e.ts tests/stress/agents-panel-reload-visibility.e2e.ts tests/stress/rail-flex-distribution-law.e2e.ts tests/stress/audit-409-console-record.e2e.ts tests/stress/pagelist-refresh-lifetime.e2e.ts tests/stress/list-keymap.e2e.ts tests/stress/okignore-settings.e2e.ts tests/stress/template-admission-detail.e2e.ts",
+ "test:e2e": "playwright test tests/stress/empty-bullet-before-link.e2e.ts tests/stress/list-item-drag.e2e.ts tests/stress/ux-interactions.e2e.ts tests/stress/file-tree-create.e2e.ts tests/stress/file-tree-drag-to-root.e2e.ts tests/stress/file-tree-deselect-to-root.e2e.ts tests/stress/file-tree-collapse-spaced-folder.e2e.ts tests/stress/sidebar-file-list-fills.e2e.ts tests/stress/file-tree-sticky-collapse.e2e.ts tests/stress/file-tree-collapse-persistence.e2e.ts tests/stress/file-tree-fractional-zoom.e2e.ts tests/stress/create-then-rename-editable.e2e.ts tests/stress/find-replace.e2e.ts tests/stress/external-link-cue.e2e.ts tests/stress/editor-tabs.e2e.ts tests/stress/crdt-stress.e2e.ts tests/stress/slash-command.e2e.ts tests/stress/paste-fidelity.e2e.ts tests/stress/fr-7a-disconnect-source-mode.e2e.ts tests/stress/docs-open.e2e.ts tests/stress/frozen-table-headers.e2e.ts tests/stress/asset-embed.e2e.ts tests/stress/asset-embed-advanced.e2e.ts tests/stress/asset-click-dispatch.e2e.ts tests/stress/handoff.e2e.ts tests/stress/multi-agent-presence.e2e.ts tests/stress/editor-mode-persistence.e2e.ts tests/stress/lint-config-toggle.e2e.ts tests/stress/plugin-enable-notice.e2e.ts tests/stress/schema-edit-fields-view.e2e.ts tests/stress/sidebar-search-pill.e2e.ts tests/stress/navigation-history.e2e.ts tests/stress/command-palette-semantic.e2e.ts tests/stress/agent-activity-panel.e2e.ts tests/stress/drop-pipeline-auto-open.e2e.ts tests/stress/command-palette-flicker.e2e.ts tests/stress/cm6-list-hanging-indent.e2e.ts tests/stress/ng7-rapid-nav-coherence.e2e.ts tests/stress/reveal-on-activate.e2e.ts tests/stress/selection-indicator.e2e.ts tests/stress/new-file-cross-doc-bleed.e2e.ts tests/stress/new-item-shortcut-fast-path.e2e.ts tests/stress/editor-mode-flip-cross-doc-bleed.e2e.ts tests/stress/editor-area-viewport-resize.e2e.ts tests/stress/qa-sidebar-responsive.e2e.ts tests/stress/prd-6955-reassertion-repro.e2e.ts tests/stress/prd-6955-reassertion-wedge.e2e.ts tests/stress/prd-6914-repro.e2e.ts tests/stress/showall-lazy-tree.e2e.ts tests/stress/tabs-component-strip.e2e.ts tests/stress/source-find-scroll.e2e.ts tests/stress/harness-app-warmth.e2e.ts tests/stress/qa-canary-authoring-both-modes.e2e.ts tests/stress/qa-canary-live-typing.e2e.ts tests/stress/keystroke-cadence-danger-space.e2e.ts tests/stress/jsx-unregistered-backspace-delete.e2e.ts tests/stress/jsx-unregistered-ime-concurrent.e2e.ts tests/stress/jsx-backspace-delete.e2e.ts tests/stress/link-authoring-bytes.e2e.ts tests/stress/link-authoring-apex.e2e.ts tests/stress/qa019-suffix-precedence.e2e.ts tests/stress/command-palette-parity.e2e.ts tests/stress/settings-search.e2e.ts tests/stress/settings-close-button.e2e.ts tests/stress/okf-generated-index-settings.e2e.ts tests/stress/okf-recommended-skill-install.e2e.ts tests/stress/chip-popover-positioning.e2e.ts tests/stress/docs-open-scroll-restore.e2e.ts tests/stress/file-tree-decoration-chip-strip-repro.e2e.ts tests/stress/grip-click-nodeselect.e2e.ts tests/stress/jsx-halo-semantics.e2e.ts tests/stress/jsx-image-body-click.e2e.ts tests/stress/jsx-prop-panel-placeholder.e2e.ts tests/stress/jsx-range-encompass-halo.e2e.ts tests/stress/jsx-wildcard-convert.e2e.ts tests/stress/markdown-lint.e2e.ts tests/stress/frontmatter-schema-badge.e2e.ts tests/stress/frontmatter-glob-picker.e2e.ts tests/stress/unified-problems.e2e.ts tests/stress/mid-type-recovery.e2e.ts tests/stress/observer-a-multi-client.e2e.ts tests/stress/outline-toolbar-occlusion.e2e.ts tests/stress/selection-surface-pane-clip.e2e.ts tests/stress/quiet-tree-hidden-doc.e2e.ts tests/stress/raw-mdx-fallback-onblur-upgrade.e2e.ts tests/stress/rename-noext-probe.e2e.ts tests/stress/show-ok-folders.e2e.ts tests/stress/source-polish.e2e.ts tests/stress/warm-skeleton-scroll-restore.e2e.ts tests/stress/props-toggle-scroll-preserve.e2e.ts tests/stress/yjs-no-dual-import.e2e.ts tests/stress/graph-panel-surfaces.e2e.ts tests/stress/asset-embed-real-fidelity.e2e.ts tests/stress/image-invalid-placeholder.e2e.ts tests/stress/local-target-audit.e2e.ts tests/stress/rename-content-preservation.e2e.ts tests/stress/rename-consolidation.e2e.ts tests/stress/outline-active-heading.e2e.ts tests/stress/outline-navigation.e2e.ts tests/stress/prd-7627-default-swatch.e2e.ts tests/stress/clipboard-dollar-math-source-fallback.e2e.ts tests/stress/source-mode-undo-keymap.e2e.ts tests/stress/source-undo-mode-flip.e2e.ts tests/stress/timeline-recovered-restore.e2e.ts tests/stress/blank-line-preservation.e2e.ts tests/stress/landing-helpers.e2e.ts tests/stress/agent-follow-scroll.e2e.ts tests/stress/mode-switch-landing.e2e.ts tests/stress/mode-switch-ordinal-divergence.e2e.ts tests/stress/comment-annotation-visibility.e2e.ts tests/stress/comment-opens-in-panel.e2e.ts tests/stress/qa-miles-edge-blanks.e2e.ts tests/stress/prop-upload.e2e.ts tests/stress/slash-command-auto-open.e2e.ts tests/stress/clipboard-relative-url-source-fallback.e2e.ts tests/stress/editor-split-view.e2e.ts tests/stress/language-load-failure.e2e.ts tests/stress/language-first-paint.e2e.ts tests/stress/user-text-direction.e2e.ts tests/stress/pseudolocale.e2e.ts tests/stress/language-picker.e2e.ts tests/stress/zh-hans-coverage-sweep.e2e.ts tests/stress/preview-tab-promotion.e2e.ts tests/stress/cv-paint-lock-click.e2e.ts tests/stress/cv-auto-paint-lock-click.e2e.ts tests/stress/harness-port-ownership.e2e.ts tests/stress/server-reuse-admission.e2e.ts tests/stress/server-reuse-reset.e2e.ts tests/stress/prebuilt-dev-serving.e2e.ts tests/stress/history-traversal-preview-tab.e2e.ts tests/stress/tab-close-scroll-restore.e2e.ts tests/stress/skills-sidebar-open.e2e.ts tests/stress/trailing-affordance.e2e.ts tests/stress/text-doc-composer-inset.e2e.ts tests/stress/composer-control-alignment.e2e.ts tests/stress/agents-settings-accessible-names.e2e.ts tests/stress/composer-growth-eof-reveal.e2e.ts tests/stress/theme-fade.e2e.ts tests/stress/theme-color-conversion.e2e.ts tests/stress/composer-open-caret-reveal.e2e.ts tests/stress/spellcheck-browser-absence.e2e.ts tests/stress/agents-panel-reload-visibility.e2e.ts tests/stress/rail-flex-distribution-law.e2e.ts tests/stress/audit-409-console-record.e2e.ts tests/stress/pagelist-refresh-lifetime.e2e.ts tests/stress/list-keymap.e2e.ts tests/stress/okignore-settings.e2e.ts tests/stress/assertion-timeout.e2e.ts tests/stress/loop-markers.e2e.ts tests/stress/template-admission-detail.e2e.ts",
"test:e2e:install-browsers": "playwright install chromium webkit firefox",
"test:visual": "playwright test --config playwright.visual.config.ts",
"test:visual:update": "playwright test --config playwright.visual.config.ts --update-snapshots",
diff --git a/packages/app/playwright.a11y.config.ts b/packages/app/playwright.a11y.config.ts
index a05e765c2..d2a4533f6 100644
--- a/packages/app/playwright.a11y.config.ts
+++ b/packages/app/playwright.a11y.config.ts
@@ -41,7 +41,10 @@ const isCI = !!process.env.CI;
export default defineConfig({
testDir: './tests/a11y',
- globalSetup: './tests/stress/_helpers/i18n-catalog-freshness.ts',
+ globalSetup: [
+ './tests/stress/_helpers/i18n-catalog-freshness.ts',
+ './tests/stress/_helpers/prebuilt/prebuilt-dev-setup.ts',
+ ],
testMatch: /.*\.e2e\.ts$/,
timeout: 120_000,
expect: { timeout: EXPECT_TIMEOUT_MS },
diff --git a/packages/app/playwright.config.ts b/packages/app/playwright.config.ts
index 68cdcb7d2..17c82f573 100644
--- a/packages/app/playwright.config.ts
+++ b/packages/app/playwright.config.ts
@@ -70,7 +70,7 @@ export function resolveWorkerCount(logicalCpus: number): number {
* run on the same per-worker fixture, so they share one convergence band.
* Rationale for the value itself is on `expect` below.
*/
-export const EXPECT_TIMEOUT_MS = isCI ? 15_000 : 5_000;
+export const EXPECT_TIMEOUT_MS = 15_000;
export default defineConfig({
testDir: './tests/stress',
@@ -84,6 +84,8 @@ export default defineConfig({
// mid-test reload storms were the suite's dominant cross-cutting flake
// class. Fail-open: a failed warm build logs and falls back to cold boots.
'./tests/stress/_helpers/global-warm-cache.ts',
+ './tests/stress/_helpers/reuse/reuse-run.ts',
+ './tests/stress/_helpers/prebuilt/prebuilt-dev-setup.ts',
],
timeout: 120_000,
// Web-first assertions poll until this budget, not Playwright's 5s default.
diff --git a/packages/app/playwright.worker-lifetime.config.ts b/packages/app/playwright.worker-lifetime.config.ts
new file mode 100644
index 000000000..f59bb2b23
--- /dev/null
+++ b/packages/app/playwright.worker-lifetime.config.ts
@@ -0,0 +1,12 @@
+import { defineConfig } from '@playwright/test';
+import ordinary from './playwright.config.ts';
+
+const directory = process.env.OK_TEST_LIFETIME_DIR;
+if (directory === undefined) throw new Error('OK_TEST_LIFETIME_DIR is required');
+
+export default defineConfig({
+ ...ordinary,
+ testDir: `${directory}/cases`,
+ workers: 2,
+ retries: 0,
+});
diff --git a/packages/app/src/build/chrome-tokens-vite-plugin.test.ts b/packages/app/src/build/chrome-tokens-vite-plugin.test.ts
index b21796750..03e1e5d01 100644
--- a/packages/app/src/build/chrome-tokens-vite-plugin.test.ts
+++ b/packages/app/src/build/chrome-tokens-vite-plugin.test.ts
@@ -1,4 +1,6 @@
-import { dirname, resolve } from 'node:path';
+import { mkdtempSync, rmSync, writeFileSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { dirname, join, resolve } from 'node:path';
import { fileURLToPath } from 'node:url';
import { describe, expect, test } from 'vitest';
import { CHROME_BG_DARK, CHROME_BG_LIGHT } from '../../../core/src/constants/chrome.ts';
@@ -47,3 +49,30 @@ describe('chromeTokensVitePlugin', () => {
expect(transform.handler(html)).toBe(`${CHROME_BG_LIGHT}/${CHROME_BG_LIGHT}/${CHROME_BG_DARK}`);
});
});
+
+function sidebarCss(light: string, dark: string): string {
+ return `:root {\n --sidebar: ${light};\n}\n.dark {\n --sidebar: ${dark};\n}\n`;
+}
+
+test('a running dev server serves the --sidebar token the stylesheet holds now, not the one it held at start', () => {
+ const dir = mkdtempSync(join(tmpdir(), 'ok-chrome-tokens-'));
+ try {
+ const cssPath = join(dir, 'globals.css');
+ writeFileSync(cssPath, sidebarCss('oklch(0.985 0 0)', 'oklch(0.205 0 0)'));
+ const plugin = chromeTokensVitePlugin({ globalsCssPath: cssPath });
+ const buildStart = plugin.buildStart as () => void;
+ const transform = plugin.transformIndexHtml as { handler: (html: string) => string };
+ buildStart();
+ expect(transform.handler('__OK_CHROME_BG_LIGHT__ __OK_CHROME_BG_DARK__')).toBe(
+ '#fafafa #171717',
+ );
+
+ writeFileSync(cssPath, sidebarCss('oklch(0.9 0 0)', 'oklch(0.205 0 0)'));
+
+ expect(transform.handler('__OK_CHROME_BG_LIGHT__ __OK_CHROME_BG_DARK__')).toBe(
+ '#dedede #171717',
+ );
+ } finally {
+ rmSync(dir, { recursive: true, force: true });
+ }
+});
diff --git a/packages/app/src/build/chrome-tokens-vite-plugin.ts b/packages/app/src/build/chrome-tokens-vite-plugin.ts
index e279a4bc4..c078d58c2 100644
--- a/packages/app/src/build/chrome-tokens-vite-plugin.ts
+++ b/packages/app/src/build/chrome-tokens-vite-plugin.ts
@@ -12,24 +12,17 @@ interface ChromeTokensPluginOptions {
export function chromeTokensVitePlugin(options: ChromeTokensPluginOptions = {}): Plugin {
const HERE = import.meta.dirname ?? new URL('.', import.meta.url).pathname;
const cssPath = options.globalsCssPath ?? resolve(HERE, '../globals.css');
- let resolved: { light: string; dark: string } | null = null;
-
- function resolveOnce(): { light: string; dark: string } {
- if (resolved !== null) return resolved;
- resolved = resolveChromeTokensFromCss(cssPath);
- return resolved;
- }
return {
name: 'ok:chrome-tokens',
enforce: 'pre',
buildStart() {
- resolveOnce();
+ resolveChromeTokensFromCss(cssPath);
},
transformIndexHtml: {
order: 'pre',
handler(html: string): string {
- const tokens = resolveOnce();
+ const tokens = resolveChromeTokensFromCss(cssPath);
return html
.replaceAll(LIGHT_PLACEHOLDER, tokens.light)
.replaceAll(DARK_PLACEHOLDER, tokens.dark);
diff --git a/packages/app/src/build/prebuilt-dev-contract.ts b/packages/app/src/build/prebuilt-dev-contract.ts
new file mode 100644
index 000000000..9cf67b046
--- /dev/null
+++ b/packages/app/src/build/prebuilt-dev-contract.ts
@@ -0,0 +1,15 @@
+import type { Rolldown } from 'vite';
+
+export const PREBUILT_OUT_DIR_ENV = 'OK_TEST_PREBUILT_OUT_DIR';
+
+export const PREBUILT_MODULE_MANIFEST = '.vite/manifest.json';
+
+export const E2E_PAGE_MODULES_CHUNK = 'e2e-page-modules';
+
+export const E2E_PAGE_MODULES_SOURCE = 'src/test-utils/e2e-page-modules.ts';
+
+export const PREBUILT_MODULE_CHUNKS = '.vite/module-chunks.json';
+
+export const PREBUILT_OUTPUT_OPTIONS = {
+ keepNames: true,
+} as const satisfies Rolldown.OutputOptions;
diff --git a/packages/app/src/build/prebuilt-module-chunks-plugin.ts b/packages/app/src/build/prebuilt-module-chunks-plugin.ts
new file mode 100644
index 000000000..56ff35585
--- /dev/null
+++ b/packages/app/src/build/prebuilt-module-chunks-plugin.ts
@@ -0,0 +1,32 @@
+import { isAbsolute, relative, sep } from 'node:path';
+import type { Plugin } from 'vite';
+import { PREBUILT_MODULE_CHUNKS } from './prebuilt-dev-contract';
+
+export function prebuiltModuleChunksPlugin(appRoot: string): Plugin {
+ return {
+ name: 'ok:prebuilt-module-chunks',
+ apply: 'build',
+ generateBundle(_options, bundle) {
+ const chunks = new Map<string, string>();
+ for (const output of Object.values(bundle)) {
+ if (output.type !== 'chunk') continue;
+ for (const id of output.moduleIds) {
+ const path = id.split('?')[0] ?? '';
+ if (!isAbsolute(path)) continue;
+ const source = relative(appRoot, path).split(sep).join('/');
+ if (source.startsWith('../')) continue;
+ const carrier = chunks.get(source);
+ if (carrier !== undefined && carrier !== output.fileName) {
+ this.error(`${source} is bundled into both ${carrier} and ${output.fileName}`);
+ }
+ chunks.set(source, output.fileName);
+ }
+ }
+ this.emitFile({
+ type: 'asset',
+ fileName: PREBUILT_MODULE_CHUNKS,
+ source: `${JSON.stringify(Object.fromEntries([...chunks].sort()), null, 1)}\n`,
+ });
+ },
+ };
+}
diff --git a/packages/app/src/build/test-loop-marker-contract.ts b/packages/app/src/build/test-loop-marker-contract.ts
new file mode 100644
index 000000000..e38d34810
--- /dev/null
+++ b/packages/app/src/build/test-loop-marker-contract.ts
@@ -0,0 +1 @@
+export const TEST_LOOP_MARKERS_PATH = '/__ok-test-loop-markers';
diff --git a/packages/app/src/build/test-loop-marker-plugin.ts b/packages/app/src/build/test-loop-marker-plugin.ts
new file mode 100644
index 000000000..58ded9632
--- /dev/null
+++ b/packages/app/src/build/test-loop-marker-plugin.ts
@@ -0,0 +1,31 @@
+import type { ServerResponse } from 'node:http';
+import { TEST_LOOP_CORE_MARKER, TEST_LOOP_SERVER_MARKER } from '@inkeep/open-knowledge-server';
+import type { Connect, Plugin } from 'vite';
+import { TEST_LOOP_MARKERS_PATH } from './test-loop-marker-contract';
+
+function serveLoopMarkers(
+ request: Connect.IncomingMessage,
+ response: ServerResponse,
+ next: Connect.NextFunction,
+): void {
+ if (request.method !== 'GET' || request.url !== TEST_LOOP_MARKERS_PATH) {
+ next();
+ return;
+ }
+ response.statusCode = 200;
+ response.setHeader('content-type', 'application/json');
+ response.setHeader('cache-control', 'no-store');
+ response.end(JSON.stringify({ server: TEST_LOOP_SERVER_MARKER, core: TEST_LOOP_CORE_MARKER }));
+}
+
+export function testLoopMarkerPlugin(): Plugin {
+ return {
+ name: 'ok-test-loop-markers',
+ configureServer(server) {
+ server.middlewares.use(serveLoopMarkers);
+ },
+ configurePreviewServer(server) {
+ server.middlewares.use(serveLoopMarkers);
+ },
+ };
+}
diff --git a/packages/app/src/build/test-server-loaded-inputs.ts b/packages/app/src/build/test-server-loaded-inputs.ts
new file mode 100644
index 000000000..29302b777
--- /dev/null
+++ b/packages/app/src/build/test-server-loaded-inputs.ts
@@ -0,0 +1,59 @@
+import { createHash } from 'node:crypto';
+import { readFileSync } from 'node:fs';
+import { isAbsolute, join } from 'node:path';
+import type { ResolvedConfig } from 'vite';
+
+export type TestServerLoadedInputs = Readonly<Record<string, string | null>>;
+
+type RestartInputConfig = Pick<
+ ResolvedConfig,
+ 'configFile' | 'configFileDependencies' | 'envDir' | 'mode'
+>;
+
+export function viteRestartInputs({
+ configFile,
+ configFileDependencies,
+ envDir,
+ mode,
+}: RestartInputConfig): string[] {
+ // UPSTREAM([email protected]): Vite restarts on the env files getEnvFilesForMode lists, which it does not export; keep these names equal to it
+ const envFiles =
+ envDir === false
+ ? []
+ : ['.env', '.env.local', `.env.${mode}`, `.env.${mode}.local`].map((name) =>
+ join(envDir, name),
+ );
+ const config = configFile === undefined ? [] : [configFile];
+ return [...new Set([...config, ...configFileDependencies, ...envFiles])].sort();
+}
+
+function contentDigest(path: string): string | null {
+ try {
+ return createHash('sha256').update(readFileSync(path)).digest('hex');
+ } catch (error) {
+ const code = (error as NodeJS.ErrnoException).code;
+ if (code === 'ENOENT' || code === 'ENOTDIR') return null;
+ return `unreadable (${code ?? String(error)})`;
+ }
+}
+
+export function digestLoadedInputs(paths: Iterable<string>): TestServerLoadedInputs {
+ return Object.fromEntries([...paths].sort().map((path) => [path, contentDigest(path)]));
+}
+
+export function parseTestServerLoadedInputs(value: unknown): TestServerLoadedInputs | undefined {
+ if (typeof value !== 'object' || value === null || Array.isArray(value)) return undefined;
+ const entries = Object.entries(value);
+ if (entries.length === 0) return undefined;
+ const wellFormed = entries.every(
+ ([path, digest]) => isAbsolute(path) && (digest === null || typeof digest === 'string'),
+ );
+ return wellFormed ? (Object.fromEntries(entries) as TestServerLoadedInputs) : undefined;
+}
+
+export function changedLoadedInputs(loaded: TestServerLoadedInputs): string[] {
+ const current = digestLoadedInputs(Object.keys(loaded));
+ return Object.keys(loaded)
+ .filter((path) => loaded[path] !== current[path])
+ .sort();
+}
diff --git a/packages/app/src/build/test-server-startup-contract.ts b/packages/app/src/build/test-server-startup-contract.ts
index 4ef65f135..186d7896c 100644
--- a/packages/app/src/build/test-server-startup-contract.ts
+++ b/packages/app/src/build/test-server-startup-contract.ts
@@ -1,5 +1,6 @@
export const TEST_SERVER_STARTUP_ENV = 'OK_TEST_VITE_START_REQUEST';
export const TEST_SERVER_IDENTITY_PATH = '/__ok-test-server-identity';
+export const TEST_SERVER_LOADED_INPUTS_PATH = '/__ok-test-server-loaded-inputs';
export const TEST_SERVER_CANDIDATE_PORTS = { start: 49152, end: 57344 } as const;
export const TEST_SERVER_HOST_FAMILY = {
'127.0.0.1': 'IPv4',
diff --git a/packages/app/src/build/test-server-startup-plugin.ts b/packages/app/src/build/test-server-startup-plugin.ts
index 499529e66..f3bd2bdaf 100644
--- a/packages/app/src/build/test-server-startup-plugin.ts
+++ b/packages/app/src/build/test-server-startup-plugin.ts
@@ -2,13 +2,18 @@ import { randomUUID } from 'node:crypto';
import { existsSync, renameSync, rmSync, writeFileSync } from 'node:fs';
import type { AddressInfo } from 'node:net';
import { isAbsolute } from 'node:path';
-import type { Plugin } from 'vite';
-
+import type { Plugin, ViteDevServer } from 'vite';
+import {
+ digestLoadedInputs,
+ type TestServerLoadedInputs,
+ viteRestartInputs,
+} from './test-server-loaded-inputs';
import {
isTestServerHost,
TEST_SERVER_CANDIDATE_PORTS,
TEST_SERVER_HOST_FAMILY,
TEST_SERVER_IDENTITY_PATH,
+ TEST_SERVER_LOADED_INPUTS_PATH,
TEST_SERVER_STARTUP_ENV,
type TestServerStartupReceipt,
type TestServerStartupRequest,
@@ -80,37 +85,85 @@ function boundReceipt(
};
}
+type StartupHostServer = Pick<ViteDevServer, 'httpServer' | 'middlewares'>;
+
+const RECEIPTS_WRITTEN_BY_THIS_PROCESS = Symbol.for('ok:test-server-startup:receipts-written');
+
+function receiptsWrittenByThisProcess(): Map<string, string> {
+ const processScope = globalThis as { [RECEIPTS_WRITTEN_BY_THIS_PROCESS]?: Map<string, string> };
+ processScope[RECEIPTS_WRITTEN_BY_THIS_PROCESS] ??= new Map();
+ return processScope[RECEIPTS_WRITTEN_BY_THIS_PROCESS];
+}
+
+function publishReceipt(request: TestServerStartupRequest, receipt: string): void {
+ const written = receiptsWrittenByThisProcess();
+ const earlier = written.get(request.receiptPath);
+ if (earlier !== undefined) {
+ if (earlier !== receipt) {
+ throw new Error(
+ `automatic Vite startup restarted in-process and bound ${receipt}, not the endpoint its receipt records (${earlier})`,
+ );
+ }
+ return;
+ }
+ if (existsSync(request.receiptPath)) {
+ throw new Error('automatic Vite startup receipt already exists');
+ }
+ const temporaryPath = `${request.receiptPath}.${process.pid}.${randomUUID()}`;
+ try {
+ writeFileSync(temporaryPath, receipt, { flag: 'wx', mode: 0o600 });
+ renameSync(temporaryPath, request.receiptPath);
+ } finally {
+ rmSync(temporaryPath, { force: true });
+ }
+ written.set(request.receiptPath, receipt);
+}
+
+function installStartupIdentity(
+ request: TestServerStartupRequest,
+ server: StartupHostServer,
+ configuredHost: string | boolean | undefined,
+ loadedInputs: TestServerLoadedInputs | undefined,
+): void {
+ const httpServer = server.httpServer;
+ if (httpServer === null) {
+ throw new Error('automatic Vite startup needs an HTTP server');
+ }
+ if (configuredHost !== request.host) {
+ throw new Error('automatic Vite startup host differs from its request');
+ }
+ if (loadedInputs === undefined) {
+ throw new Error('automatic Vite startup has no record of the configuration it loaded');
+ }
+ server.middlewares.use(TEST_SERVER_IDENTITY_PATH, (_req, res) => {
+ const receipt = boundReceipt(request, httpServer.address());
+ res.setHeader('Content-Type', 'application/json');
+ res.setHeader('Cache-Control', 'no-store');
+ res.end(JSON.stringify(receipt));
+ });
+ server.middlewares.use(TEST_SERVER_LOADED_INPUTS_PATH, (_req, res) => {
+ res.setHeader('Content-Type', 'application/json');
+ res.setHeader('Cache-Control', 'no-store');
+ res.end(JSON.stringify(loadedInputs));
+ });
+ httpServer.once('listening', () => {
+ publishReceipt(request, JSON.stringify(boundReceipt(request, httpServer.address())));
+ });
+}
+
export function testServerStartupPlugin(request: TestServerStartupRequest): Plugin {
+ let loadedInputs: TestServerLoadedInputs | undefined;
return {
name: 'ok:test-server-startup',
apply: 'serve',
+ configResolved(config) {
+ loadedInputs = digestLoadedInputs(viteRestartInputs(config));
+ },
configureServer(server) {
- const httpServer = server.httpServer;
- if (httpServer === null) {
- throw new Error('automatic Vite startup needs an HTTP server');
- }
- if (server.config.server.host !== request.host) {
- throw new Error('automatic Vite startup host differs from its request');
- }
- server.middlewares.use(TEST_SERVER_IDENTITY_PATH, (_req, res) => {
- const receipt = boundReceipt(request, httpServer.address());
- res.setHeader('Content-Type', 'application/json');
- res.setHeader('Cache-Control', 'no-store');
- res.end(JSON.stringify(receipt));
- });
- httpServer.once('listening', () => {
- const receipt = boundReceipt(request, httpServer.address());
- if (existsSync(request.receiptPath)) {
- throw new Error('automatic Vite startup receipt already exists');
- }
- const temporaryPath = `${request.receiptPath}.${process.pid}.${randomUUID()}`;
- try {
- writeFileSync(temporaryPath, JSON.stringify(receipt), { flag: 'wx', mode: 0o600 });
- renameSync(temporaryPath, request.receiptPath);
- } finally {
- rmSync(temporaryPath, { force: true });
- }
- });
+ installStartupIdentity(request, server, server.config.server.host, loadedInputs);
+ },
+ configurePreviewServer(server) {
+ installStartupIdentity(request, server, server.config.preview.host, loadedInputs);
},
};
}
diff --git a/packages/app/src/server/asset-routing.ts b/packages/app/src/server/asset-routing.ts
new file mode 100644
index 000000000..c9b2038c1
--- /dev/null
+++ b/packages/app/src/server/asset-routing.ts
@@ -0,0 +1,51 @@
+import sirv from 'sirv';
+import type { Connect } from 'vite';
+
+export type AssetRouting =
+ | { readonly kind: 'dev-server' }
+ | { readonly kind: 'prebuilt'; readonly bundleDir: string };
+
+function servedByViteDevServer(url: string): boolean {
+ const path = url.split('?')[0] ?? '';
+ const queryStart = url.indexOf('?');
+ const query = queryStart >= 0 ? url.slice(queryStart) : '';
+ const params = query ? new URLSearchParams(query) : null;
+ return (
+ path.startsWith('/@vite/') ||
+ path.startsWith('/@fs/') ||
+ path.startsWith('/@id/') ||
+ path === '/@react-refresh' ||
+ path.startsWith('/node_modules/') ||
+ path.startsWith('/src/') ||
+ path === '/favicon.svg' ||
+ params?.has('import') === true ||
+ params?.has('html-proxy') === true
+ );
+}
+
+// WARN: these are the bundle prefixes packages/server/src/http/content-dispatch.ts serves shell-first; keep them in step.
+const BUNDLE_FIRST_PREFIXES = ['/assets/', '/excalidraw-assets/'] as const;
+
+export function routeAssets(
+ routing: AssetRouting,
+ assetMiddleware: Connect.NextHandleFunction,
+): Connect.NextHandleFunction {
+ if (routing.kind === 'dev-server') {
+ return (req, res, next) =>
+ servedByViteDevServer(req.url ?? '') ? next() : assetMiddleware(req, res, next);
+ }
+ const bundle = sirv(routing.bundleDir, { dev: true, etag: true });
+ return (req, res, next) => {
+ const path = req.url?.split('?')[0] ?? '';
+ if (path === '/favicon.svg') return next();
+ if (!BUNDLE_FIRST_PREFIXES.some((prefix) => path.startsWith(prefix))) {
+ return assetMiddleware(req, res, next);
+ }
+ bundle(req, res, () =>
+ assetMiddleware(req, res, () => {
+ res.statusCode = 404;
+ res.end();
+ }),
+ );
+ };
+}
diff --git a/packages/app/src/server/hocuspocus-plugin.ts b/packages/app/src/server/hocuspocus-plugin.ts
index 62d787a25..cea0e1d7c 100644
--- a/packages/app/src/server/hocuspocus-plugin.ts
+++ b/packages/app/src/server/hocuspocus-plugin.ts
@@ -20,10 +20,11 @@ import {
updateServerLockPort,
} from '@inkeep/open-knowledge-server';
import sirv from 'sirv';
-import type { Plugin } from 'vite';
+import type { Plugin, ViteDevServer } from 'vite';
import type { WebSocketServer } from 'ws';
import { parse as parseYaml } from 'yaml';
import { computeDevApiConfigResponse } from './api-config-handler.ts';
+import { type AssetRouting, routeAssets } from './asset-routing.ts';
let configureServerInvocations = 0;
@@ -118,8 +119,12 @@ function closeWebSocketServer(wss: WebSocketServer): Promise<void> {
});
}
+type OkServerHost = Pick<ViteDevServer, 'httpServer' | 'middlewares'>;
+
+type RuntimeTeardowns = Set<() => Promise<void>>;
+
export function hocuspocusPlugin(): Plugin {
- const activeRuntimeTeardowns = new Set<() => Promise<void>>();
+ const activeRuntimeTeardowns: RuntimeTeardowns = new Set();
return {
name: 'hocuspocus',
async buildEnd() {
@@ -132,199 +137,202 @@ export function hocuspocusPlugin(): Plugin {
if (errors.length > 0) throw new AggregateError(errors, 'Hocuspocus Vite teardown failed');
},
async configureServer(server) {
- configureServerInvocations += 1;
- if (configureServerInvocations > 1) {
- console.warn(
- `[collab] configureServer invoked ${configureServerInvocations}× — Vite restarted; spinning up a fresh ServerInstance. The previous srv will be destroyed by its httpServer close handler.`,
- );
- } else {
- console.info(`[collab] configureServer invocation=1 pid=${process.pid}`);
- }
+ await attachOkServer(server, { kind: 'dev-server' }, activeRuntimeTeardowns);
+ },
+ };
+}
- const currentSrv = createServer({
- contentDir: CONTENT_DIR,
- projectDir: TEST_PROJECT_DIR ?? (isTestIsolated ? CONTENT_DIR : PROJECT_ROOT),
- contentRoot: isTestIsolated ? '' : CONTENT_ROOT,
- ...(isTestIsolated ? { configHomedirOverride: CONTENT_DIR } : {}),
- gitEnabled: isEphemeralTest ? false : !isTestIsolated || gitEnabledForTest,
- enableTestRoutes: true,
- quiet: true,
- embeddingsKeyStore: makeLazyEmbeddingsKeyStore(),
- capabilities: ['http', 'ws', 'ui'],
- ...(isEphemeralTest ? { ephemeral: true, singleDocRelPath: SINGLE_DOC_REL_PATH } : {}),
- });
+export function prebuiltHocuspocusPlugin(): Plugin {
+ const activeRuntimeTeardowns: RuntimeTeardowns = new Set();
+ return {
+ name: 'hocuspocus:prebuilt',
+ apply: 'serve',
+ async configurePreviewServer(server) {
+ const teardown = await attachOkServer(
+ server,
+ { kind: 'prebuilt', bundleDir: resolve(server.config.root, server.config.build.outDir) },
+ activeRuntimeTeardowns,
+ );
+ const closePreview = server.close.bind(server);
+ server.close = async () => {
+ const results = await Promise.allSettled([closePreview(), teardown()]);
+ const errors = results
+ .filter((result): result is PromiseRejectedResult => result.status === 'rejected')
+ .map((result) => result.reason);
+ if (errors.length > 0) throw new AggregateError(errors, 'Prebuilt preview close failed');
+ };
+ },
+ };
+}
- latestLockDir = currentSrv.lockDir;
- if (!exitHandlerRegistered) {
- exitHandlerRegistered = true;
- process.once('exit', () => {
- if (latestLockDir === null) return;
- try {
- releaseServerLock(latestLockDir);
- } catch {}
- });
- }
+async function attachOkServer(
+ server: OkServerHost,
+ assetRouting: AssetRouting,
+ activeRuntimeTeardowns: RuntimeTeardowns,
+): Promise<() => Promise<void>> {
+ configureServerInvocations += 1;
+ if (configureServerInvocations > 1) {
+ console.warn(
+ `[collab] configureServer invoked ${configureServerInvocations}× — Vite restarted; spinning up a fresh ServerInstance. The previous srv will be destroyed by its httpServer close handler.`,
+ );
+ } else {
+ console.info(`[collab] configureServer invocation=1 pid=${process.pid}`);
+ }
- if (configureServerInvocations === 1) {
- getLogger('hocuspocus').info({ contentDir: CONTENT_DIR }, 'content dir');
- }
+ const currentSrv = createServer({
+ contentDir: CONTENT_DIR,
+ projectDir: TEST_PROJECT_DIR ?? (isTestIsolated ? CONTENT_DIR : PROJECT_ROOT),
+ contentRoot: isTestIsolated ? '' : CONTENT_ROOT,
+ ...(isTestIsolated ? { configHomedirOverride: CONTENT_DIR } : {}),
+ gitEnabled: isEphemeralTest ? false : !isTestIsolated || gitEnabledForTest,
+ enableTestRoutes: true,
+ quiet: true,
+ embeddingsKeyStore: makeLazyEmbeddingsKeyStore(),
+ capabilities: ['http', 'ws', 'ui'],
+ ...(isEphemeralTest ? { ephemeral: true, singleDocRelPath: SINGLE_DOC_REL_PATH } : {}),
+ });
- server.httpServer?.once('listening', () => {
- const addr = server.httpServer?.address();
- if (typeof addr === 'object' && addr !== null) {
- updateServerLockPort(currentSrv.lockDir, addr.port, `http://localhost:${addr.port}`);
- }
- });
+ latestLockDir = currentSrv.lockDir;
+ if (!exitHandlerRegistered) {
+ exitHandlerRegistered = true;
+ process.once('exit', () => {
+ if (latestLockDir === null) return;
+ try {
+ releaseServerLock(latestLockDir);
+ } catch {}
+ });
+ }
- const {
- hocuspocus,
- nativeApi,
- sessionManager,
- agentFocusBroadcaster,
- agentPresenceBroadcaster,
- } = currentSrv;
+ if (configureServerInvocations === 1) {
+ getLogger('hocuspocus').info({ contentDir: CONTENT_DIR }, 'content dir');
+ }
- const acpThreadManager = isEphemeralTest
- ? null
- : new AcpThreadManager({
- contentDir: CONTENT_DIR,
- localDir: currentSrv.lockDir,
- globalDir: okUserHomeDir(),
- registry: currentSrv.acpRegistry,
- permissions: currentSrv.acpPermissions,
- sessionManager,
- agentPresenceBroadcaster,
- resolveEmbed: currentSrv.resolveEmbed,
- isExcludedPath: (rel) => currentSrv.contentFilter.isExcluded(rel),
- isIgnoredPath: (rel) => currentSrv.contentFilter.isPathIgnored(rel),
- getLoadedDocText: (docName) =>
- hocuspocus.documents.get(docName)?.getText('source').toString() ?? null,
- getServerUrl: () => {
- const addr = server.httpServer?.address();
- const port = typeof addr === 'object' && addr !== null ? addr.port : 5173;
- return `http://localhost:${port}`;
- },
- getMcpStdioCommand: () => {
- const addr = server.httpServer?.address();
- const port = typeof addr === 'object' && addr !== null ? addr.port : 5173;
- return buildOkMcpStdioCommand(undefined, port, { log: getLogger('acp-threads') });
- },
- agentBrowserTools: () =>
- !isTestIsolated &&
- readAgentBrowserTools(PROJECT_ROOT, undefined, getLogger('acp-permissions')),
- log: getLogger('acp-threads'),
- });
- await acpThreadManager?.init();
+ server.httpServer?.once('listening', () => {
+ const addr = server.httpServer?.address();
+ if (typeof addr === 'object' && addr !== null) {
+ updateServerLockPort(currentSrv.lockDir, addr.port, `http://localhost:${addr.port}`);
+ }
+ });
- const collaborationHost = createCollaborationHost({
- hocuspocus,
- log: getLogger('hocuspocus'),
+ const { hocuspocus, nativeApi, sessionManager, agentFocusBroadcaster, agentPresenceBroadcaster } =
+ currentSrv;
+
+ const acpThreadManager = isEphemeralTest
+ ? null
+ : new AcpThreadManager({
+ contentDir: CONTENT_DIR,
+ localDir: currentSrv.lockDir,
+ globalDir: okUserHomeDir(),
+ registry: currentSrv.acpRegistry,
+ permissions: currentSrv.acpPermissions,
sessionManager,
- agentFocusBroadcaster,
agentPresenceBroadcaster,
- maintenanceCoordinator: currentSrv.maintenanceCoordinator,
- acpThreadManager,
+ resolveEmbed: currentSrv.resolveEmbed,
+ isExcludedPath: (rel) => currentSrv.contentFilter.isExcluded(rel),
+ isIgnoredPath: (rel) => currentSrv.contentFilter.isPathIgnored(rel),
+ getLoadedDocText: (docName) =>
+ hocuspocus.documents.get(docName)?.getText('source').toString() ?? null,
+ getServerUrl: () => {
+ const addr = server.httpServer?.address();
+ const port = typeof addr === 'object' && addr !== null ? addr.port : 5173;
+ return `http://localhost:${port}`;
+ },
+ getMcpStdioCommand: () => {
+ const addr = server.httpServer?.address();
+ const port = typeof addr === 'object' && addr !== null ? addr.port : 5173;
+ return buildOkMcpStdioCommand(undefined, port, { log: getLogger('acp-threads') });
+ },
+ agentBrowserTools: () =>
+ !isTestIsolated &&
+ readAgentBrowserTools(PROJECT_ROOT, undefined, getLogger('acp-permissions')),
+ log: getLogger('acp-threads'),
});
- server.httpServer?.prependListener('upgrade', collaborationHost.handleUpgrade);
+ await acpThreadManager?.init();
- const assetMiddleware = createAssetServeMiddleware({
- contentDir: CONTENT_DIR,
- contentFilter: currentSrv.contentFilter,
- contentSirv: sirv(CONTENT_DIR, { dev: true, dotfiles: false }),
- inlineExtensions: INLINE_RENDERABLE_EXTENSIONS,
- assetExtensions: ASSET_EXTENSIONS,
- blocklistExtensions: EXECUTABLE_BLOCKLIST_EXTENSIONS,
- ingressPolicy: buildIngressPolicy({}),
- resolveTrackedFile: currentSrv.resolveTrackedFile,
- });
- server.middlewares.use((req, res, next) => {
- const url = req.url ?? '';
- const path = url.split('?')[0] ?? '';
- const queryStart = url.indexOf('?');
- const query = queryStart >= 0 ? url.slice(queryStart) : '';
- const params = query ? new URLSearchParams(query) : null;
- if (
- path.startsWith('/@vite/') ||
- path.startsWith('/@fs/') ||
- path.startsWith('/@id/') ||
- path === '/@react-refresh' ||
- path.startsWith('/node_modules/') ||
- path.startsWith('/src/') ||
- path === '/favicon.svg' ||
- params?.has('import') ||
- params?.has('html-proxy')
- ) {
- return next();
- }
- return assetMiddleware(req, res, next);
- });
+ const collaborationHost = createCollaborationHost({
+ hocuspocus,
+ log: getLogger('hocuspocus'),
+ sessionManager,
+ agentFocusBroadcaster,
+ agentPresenceBroadcaster,
+ maintenanceCoordinator: currentSrv.maintenanceCoordinator,
+ acpThreadManager,
+ });
+ server.httpServer?.prependListener('upgrade', collaborationHost.handleUpgrade);
+
+ const assetMiddleware = createAssetServeMiddleware({
+ contentDir: CONTENT_DIR,
+ contentFilter: currentSrv.contentFilter,
+ contentSirv: sirv(CONTENT_DIR, { dev: true, dotfiles: false }),
+ inlineExtensions: INLINE_RENDERABLE_EXTENSIONS,
+ assetExtensions: ASSET_EXTENSIONS,
+ blocklistExtensions: EXECUTABLE_BLOCKLIST_EXTENSIONS,
+ ingressPolicy: buildIngressPolicy({}),
+ resolveTrackedFile: currentSrv.resolveTrackedFile,
+ });
+ server.middlewares.use(routeAssets(assetRouting, assetMiddleware));
- server.middlewares.use(async (req, res, next) => {
- const url = req.url?.split('?')[0];
- if (url?.startsWith('/api/')) {
- if (url === '/api/config') {
- const addr = server.httpServer?.address();
- const port = typeof addr === 'object' && addr !== null ? addr.port : 0;
- const response = computeDevApiConfigResponse(req.method, port, isEphemeralTest);
- if (response) {
- for (const [name, value] of Object.entries(response.headers)) {
- res.setHeader(name, value);
- }
- res.statusCode = response.status;
- if (response.omitBody) {
- res.end();
- } else {
- res.end(response.body);
- }
- return;
- }
+ server.middlewares.use(async (req, res, next) => {
+ const url = req.url?.split('?')[0];
+ if (url?.startsWith('/api/')) {
+ if (url === '/api/config') {
+ const addr = server.httpServer?.address();
+ const port = typeof addr === 'object' && addr !== null ? addr.port : 0;
+ const response = computeDevApiConfigResponse(req.method, port, isEphemeralTest);
+ if (response) {
+ for (const [name, value] of Object.entries(response.headers)) {
+ res.setHeader(name, value);
+ }
+ res.statusCode = response.status;
+ if (response.omitBody) {
+ res.end();
+ } else {
+ res.end(response.body);
}
- if (await nativeApi.dispatch(req, res)) return;
- // biome-ignore lint/suspicious/noExplicitAny: Hocuspocus `hooks()` has no exported payload type for onRequest
- await hocuspocus.hooks('onRequest', { request: req, response: res } as any);
- if (res.writableEnded || res.headersSent) return;
- res.statusCode = 404;
- res.setHeader('Content-Type', 'application/json');
- res.end(JSON.stringify({ error: 'API route not found', path: url }));
return;
}
- next();
- });
+ }
+ if (await nativeApi.dispatch(req, res)) return;
+ // biome-ignore lint/suspicious/noExplicitAny: Hocuspocus `hooks()` has no exported payload type for onRequest
+ await hocuspocus.hooks('onRequest', { request: req, response: res } as any);
+ if (res.writableEnded || res.headersSent) return;
+ res.statusCode = 404;
+ res.setHeader('Content-Type', 'application/json');
+ res.end(JSON.stringify({ error: 'API route not found', path: url }));
+ return;
+ }
+ next();
+ });
- let teardownPromise: Promise<void> | undefined;
- const teardown = (): Promise<void> => {
- teardownPromise ??= (async () => {
- const errors: unknown[] = [];
- const run = async (label: string, action: () => Promise<void>): Promise<void> => {
- try {
- await action();
- } catch (err) {
- errors.push(err);
- getLogger('hocuspocus').error(
- { err },
- `Vite collaboration teardown failed: ${label}`,
- );
- }
- };
- server.httpServer?.off('upgrade', collaborationHost.handleUpgrade);
- await run('ACP thread manager', async () => acpThreadManager?.destroy());
- await run('collaboration host', () => collaborationHost.shutdown());
- await run('WebSocket server', () => closeWebSocketServer(collaborationHost.wss));
- await run('server instance', () => currentSrv.destroy());
- if (errors.length > 0)
- throw new AggregateError(errors, 'Vite collaboration teardown failed');
- })();
- void teardownPromise.finally(() => activeRuntimeTeardowns.delete(teardown)).catch(() => {});
- return teardownPromise;
+ let teardownPromise: Promise<void> | undefined;
+ const teardown = (): Promise<void> => {
+ teardownPromise ??= (async () => {
+ const errors: unknown[] = [];
+ const run = async (label: string, action: () => Promise<void>): Promise<void> => {
+ try {
+ await action();
+ } catch (err) {
+ errors.push(err);
+ getLogger('hocuspocus').error({ err }, `Vite collaboration teardown failed: ${label}`);
+ }
};
- activeRuntimeTeardowns.add(teardown);
- server.httpServer?.on('close', () => {
- void teardown().catch((err) =>
- getLogger('hocuspocus').error({ err }, 'HTTP close fallback teardown failed'),
- );
- });
-
- getLogger('hocuspocus').info({}, 'WebSocket server ready on /collab');
- },
+ server.httpServer?.off('upgrade', collaborationHost.handleUpgrade);
+ await run('ACP thread manager', async () => acpThreadManager?.destroy());
+ await run('collaboration host', () => collaborationHost.shutdown());
+ await run('WebSocket server', () => closeWebSocketServer(collaborationHost.wss));
+ await run('server instance', () => currentSrv.destroy());
+ if (errors.length > 0) throw new AggregateError(errors, 'Vite collaboration teardown failed');
+ })();
+ void teardownPromise.finally(() => activeRuntimeTeardowns.delete(teardown)).catch(() => {});
+ return teardownPromise;
};
+ activeRuntimeTeardowns.add(teardown);
+ server.httpServer?.on('close', () => {
+ void teardown().catch((err) =>
+ getLogger('hocuspocus').error({ err }, 'HTTP close fallback teardown failed'),
+ );
+ });
+
+ getLogger('hocuspocus').info({}, 'WebSocket server ready on /collab');
+ return teardown;
}
diff --git a/packages/app/src/test-utils/e2e-page-modules.ts b/packages/app/src/test-utils/e2e-page-modules.ts
new file mode 100644
index 000000000..d6219a33e
--- /dev/null
+++ b/packages/app/src/test-utils/e2e-page-modules.ts
@@ -0,0 +1,5 @@
+export { buildAnchorSegments, commentColor } from '@/comments/anchor-layers';
+export { computeLiveXtermTheme } from '@/components/terminal-theme';
+export { cssColorToHex } from '@/lib/css-color-to-hex';
+export { THEME_COLOR_PROPERTIES } from '@/lib/theme-color-properties';
+export { TEST_LOOP_APP_MARKER } from '@/test-utils/test-loop-marker';
diff --git a/packages/app/src/test-utils/test-loop-marker.ts b/packages/app/src/test-utils/test-loop-marker.ts
new file mode 100644
index 000000000..54c3872dc
--- /dev/null
+++ b/packages/app/src/test-utils/test-loop-marker.ts
@@ -0,0 +1 @@
+export const TEST_LOOP_APP_MARKER = 'app-base';
diff --git a/packages/app/tests/foundation/contract-dev-server.test-helper.ts b/packages/app/tests/foundation/contract-dev-server.test-helper.ts
new file mode 100644
index 000000000..a14a918cb
--- /dev/null
+++ b/packages/app/tests/foundation/contract-dev-server.test-helper.ts
@@ -0,0 +1,160 @@
+import { spawn } from 'node:child_process';
+import { mkdtempSync, realpathSync } from 'node:fs';
+import { createRequire } from 'node:module';
+import { tmpdir } from 'node:os';
+import { dirname, join } from 'node:path';
+import { TEST_SERVER_HOST_FAMILY } from '../../src/build/test-server-startup-contract.ts';
+import type { LaunchOwnedServer, OwnedServer } from '../stress/_helpers/reuse/server-supervisor.ts';
+import {
+ APP_PACKAGE_ROOT,
+ beginViteStartup,
+ createViteStartupRequest,
+ killGracefully,
+ waitForBoundViteEndpoint,
+} from '../stress/_helpers/server-process.ts';
+import { removeAllDuringTeardown } from '../stress/_helpers/teardown-fs.ts';
+import {
+ CONTRACT_SERVER_EXIT_PATH,
+ CONTRACT_SERVER_FAIL_READINESS_PATH,
+ CONTRACT_SERVER_FORGE_PATH,
+ CONTRACT_SERVER_GENERATION_PATH,
+ CONTRACT_SERVER_RESTART_PATH,
+ type ContractServerReadinessCheck,
+} from './fixtures/contract-dev-server-routes.ts';
+
+const VITE_CLI = join(
+ dirname(createRequire(join(APP_PACKAGE_ROOT, 'package.json')).resolve('vite/package.json')),
+ 'bin',
+ 'vite.js',
+);
+export const CONTRACT_SERVER_CONFIG = join(
+ import.meta.dirname,
+ 'fixtures',
+ 'contract-dev-server.ts',
+);
+const CONTRACT_SERVER_READY_MS = 15_000;
+
+export interface ContractServerLauncher {
+ readonly launch: LaunchOwnedServer;
+ readonly launched: OwnedServer[];
+ readonly environments: Array<Readonly<Record<string, string>>>;
+ stderrOf(server: OwnedServer): string;
+}
+
+export function contractServerLauncher({
+ configFile = CONTRACT_SERVER_CONFIG,
+}: {
+ readonly configFile?: string;
+} = {}): ContractServerLauncher {
+ const launched: OwnedServer[] = [];
+ const environments: Array<Readonly<Record<string, string>>> = [];
+ const stderrs = new WeakMap<OwnedServer, () => string>();
+ const launch: LaunchOwnedServer = async (slot, workerServerEnv) => {
+ environments.push(workerServerEnv);
+ const request = createViteStartupRequest('127.0.0.1');
+ const contentDir = realpathSync(mkdtempSync(join(tmpdir(), `ok-contract-s${slot}-`)));
+ let stderr = '';
+ const proc = spawn(
+ process.execPath,
+ [VITE_CLI, '--config', configFile, '--host', request.host],
+ {
+ cwd: contentDir,
+ detached: true,
+ env: { ...process.env, ...workerServerEnv, ...request.environment },
+ stdio: ['ignore', 'ignore', 'pipe'],
+ },
+ );
+ proc.stderr?.on('data', (chunk: Buffer) => {
+ stderr += chunk.toString('utf8');
+ });
+ let releasing: Promise<void> | undefined;
+ const release = () => {
+ releasing ??= killGracefully(proc).finally(() =>
+ removeAllDuringTeardown(contentDir, request.receiptDir),
+ );
+ return releasing;
+ };
+ try {
+ const endpoint = await waitForBoundViteEndpoint(
+ beginViteStartup(request, proc),
+ CONTRACT_SERVER_READY_MS,
+ );
+ const server: OwnedServer = {
+ proc,
+ identity: {
+ nonce: request.nonce,
+ address: request.host,
+ family: TEST_SERVER_HOST_FAMILY[request.host],
+ port: endpoint.port,
+ },
+ baseURL: endpoint.baseURL,
+ contentDir,
+ release,
+ };
+ launched.push(server);
+ stderrs.set(server, () => stderr);
+ return server;
+ } catch (error) {
+ let cleanupFailure = '';
+ try {
+ await release();
+ } catch (cleanupError) {
+ cleanupFailure = `\n--- cleanup after this failure did not complete: ${cleanupError instanceof Error ? cleanupError.message : String(cleanupError)} ---`;
+ }
+ throw new Error(
+ `the contract dev server did not start: ${String(error)}\n${stderr}${cleanupFailure}`,
+ );
+ }
+ };
+ const stderrOf = (server: OwnedServer): string => {
+ const read = stderrs.get(server);
+ if (read === undefined) throw new Error(`${server.baseURL} was not launched by this launcher`);
+ return read();
+ };
+ return { launch, launched, environments, stderrOf };
+}
+
+async function control(server: OwnedServer, path: string): Promise<void> {
+ const response = await fetch(`${server.baseURL}${path}`, { signal: AbortSignal.timeout(5_000) });
+ const body = await response.text();
+ if (!response.ok) throw new Error(`${path} returned status ${response.status}: ${body}`);
+}
+
+export async function exitContractServer(server: OwnedServer): Promise<void> {
+ const exited = new Promise<void>((resolve) => {
+ if (server.proc.exitCode !== null || server.proc.signalCode !== null) resolve();
+ else server.proc.once('exit', () => resolve());
+ });
+ await control(server, CONTRACT_SERVER_EXIT_PATH);
+ await exited;
+}
+
+export function forgeContractServerIdentity(server: OwnedServer): Promise<void> {
+ return control(server, CONTRACT_SERVER_FORGE_PATH);
+}
+
+export function failContractServerReadiness(
+ server: OwnedServer,
+ check: ContractServerReadinessCheck,
+): Promise<void> {
+ return control(server, `${CONTRACT_SERVER_FAIL_READINESS_PATH}/${check}`);
+}
+
+export function restartContractServer(server: OwnedServer, port?: number): Promise<void> {
+ return control(
+ server,
+ port === undefined
+ ? CONTRACT_SERVER_RESTART_PATH
+ : `${CONTRACT_SERVER_RESTART_PATH}?port=${port}`,
+ );
+}
+
+export async function contractServerGeneration(server: OwnedServer): Promise<number> {
+ const response = await fetch(`${server.baseURL}${CONTRACT_SERVER_GENERATION_PATH}`, {
+ signal: AbortSignal.timeout(5_000),
+ });
+ if (!response.ok) {
+ throw new Error(`${CONTRACT_SERVER_GENERATION_PATH} returned status ${response.status}`);
+ }
+ return Number(await response.text());
+}
diff --git a/packages/app/tests/foundation/fixtures/contract-dev-server-routes.ts b/packages/app/tests/foundation/fixtures/contract-dev-server-routes.ts
new file mode 100644
index 000000000..af22bd76f
--- /dev/null
+++ b/packages/app/tests/foundation/fixtures/contract-dev-server-routes.ts
@@ -0,0 +1,16 @@
+export const CONTRACT_SERVER_EXIT_PATH = '/__contract-server/exit';
+export const CONTRACT_SERVER_FORGE_PATH = '/__contract-server/forge-identity';
+export const CONTRACT_SERVER_FAIL_READINESS_PATH = '/__contract-server/fail-readiness';
+export const CONTRACT_SERVER_RESTART_PATH = '/__contract-server/restart';
+export const CONTRACT_SERVER_GENERATION_PATH = '/__contract-server/generation';
+export const CONTRACT_SERVER_EXCLUSIVE_RESOURCE_ENV = 'OK_CONTRACT_SERVER_EXCLUSIVE_RESOURCE';
+
+export const CONTRACT_SERVER_READINESS_CHECKS = ['api-config', 'collab'] as const;
+
+export type ContractServerReadinessCheck = (typeof CONTRACT_SERVER_READINESS_CHECKS)[number];
+
+export function isContractServerReadinessCheck(
+ value: string,
+): value is ContractServerReadinessCheck {
+ return (CONTRACT_SERVER_READINESS_CHECKS as readonly string[]).includes(value);
+}
diff --git a/packages/app/tests/foundation/fixtures/contract-dev-server.ts b/packages/app/tests/foundation/fixtures/contract-dev-server.ts
new file mode 100644
index 000000000..bf8b16b10
--- /dev/null
+++ b/packages/app/tests/foundation/fixtures/contract-dev-server.ts
@@ -0,0 +1,179 @@
+import { Hocuspocus } from '@hocuspocus/server';
+import { defineConfig, type Plugin } from 'vite';
+import { WebSocketServer } from 'ws';
+import {
+ TEST_SERVER_IDENTITY_PATH,
+ TEST_SERVER_STARTUP_ENV,
+} from '../../../src/build/test-server-startup-contract';
+import {
+ parseTestServerStartupRequest,
+ testServerStartupPlugin,
+} from '../../../src/build/test-server-startup-plugin';
+import {
+ CONTRACT_SERVER_EXCLUSIVE_RESOURCE_ENV,
+ CONTRACT_SERVER_EXIT_PATH,
+ CONTRACT_SERVER_FAIL_READINESS_PATH,
+ CONTRACT_SERVER_FORGE_PATH,
+ CONTRACT_SERVER_GENERATION_PATH,
+ CONTRACT_SERVER_RESTART_PATH,
+ type ContractServerReadinessCheck,
+ isContractServerReadinessCheck,
+} from './contract-dev-server-routes';
+
+const request = parseTestServerStartupRequest(process.env[TEST_SERVER_STARTUP_ENV]);
+if (request === undefined) {
+ throw new Error(`the contract dev server needs ${TEST_SERVER_STARTUP_ENV}`);
+}
+
+const RESTART_PORT = Symbol.for('ok:contract-server:restart-port');
+const GENERATION = Symbol.for('ok:contract-server:generation');
+const EXCLUSIVE_RESOURCE_HELD = Symbol.for('ok:contract-server:exclusive-resource-held');
+const contractProcess = globalThis as {
+ [RESTART_PORT]?: number;
+ [GENERATION]?: number;
+ [EXCLUSIVE_RESOURCE_HELD]?: boolean;
+};
+
+const EMPTY_SERVER_RESET_RESIDUE = {
+ loadedDocuments: [],
+ clientConnections: 0,
+ agentSessions: 0,
+ agentPresence: [],
+ staleExternalWrites: [],
+ removedDocuments: [],
+};
+
+function contractControl(): Plugin {
+ let forged = false;
+ return {
+ name: 'ok:contract-server-control',
+ apply: 'serve',
+ configureServer(server) {
+ const generation = (contractProcess[GENERATION] ?? 0) + 1;
+ contractProcess[GENERATION] = generation;
+ server.middlewares.use(CONTRACT_SERVER_GENERATION_PATH, (_req, res) => {
+ res.end(String(generation));
+ });
+ server.middlewares.use(CONTRACT_SERVER_RESTART_PATH, (req, res) => {
+ const port = new URL(req.url ?? '/', 'http://contract').searchParams.get('port');
+ if (port !== null) contractProcess[RESTART_PORT] = Number(port);
+ res.end('restarting');
+ res.once('finish', () => {
+ server.restart().catch((error: unknown) => {
+ process.stderr.write(`contract dev server restart failed: ${String(error)}\n`);
+ });
+ });
+ });
+ server.middlewares.use(CONTRACT_SERVER_EXIT_PATH, (_req, res) => {
+ res.end('exiting');
+ res.once('finish', () => process.exit(0));
+ });
+ server.middlewares.use(CONTRACT_SERVER_FORGE_PATH, (_req, res) => {
+ forged = true;
+ res.end('forged');
+ });
+ server.middlewares.use(TEST_SERVER_IDENTITY_PATH, (_req, res, next) => {
+ if (!forged) {
+ next();
+ return;
+ }
+ res.setHeader('Content-Type', 'application/json');
+ res.end(JSON.stringify({ nonce: 'forged-identity' }));
+ });
+ },
+ };
+}
+
+function contractReadiness(): Plugin {
+ const failing = new Set<ContractServerReadinessCheck>();
+ const hocuspocus = new Hocuspocus({ quiet: true });
+ const sockets = new WebSocketServer({ noServer: true });
+ return {
+ name: 'ok:contract-server-readiness',
+ apply: 'serve',
+ configureServer(server) {
+ server.middlewares.use(CONTRACT_SERVER_FAIL_READINESS_PATH, (req, res) => {
+ const check = (req.url ?? '').slice(1);
+ if (!isContractServerReadinessCheck(check)) {
+ res.statusCode = 404;
+ res.end(`no readiness check ${check}`);
+ return;
+ }
+ failing.add(check);
+ res.end(`failing ${check}`);
+ });
+ server.middlewares.use('/api/config', (_req, res) => {
+ const address = server.httpServer?.address();
+ if (failing.has('api-config') || typeof address !== 'object' || address === null) {
+ res.statusCode = 503;
+ res.end();
+ return;
+ }
+ res.setHeader('Content-Type', 'application/json');
+ res.end(JSON.stringify({ port: address.port, collabUrl: null }));
+ });
+ server.middlewares.use('/api/test-reset', (req, res) => {
+ const scope = new URL(req.url ?? '/', 'http://contract').searchParams.get('scope');
+ res.setHeader('Content-Type', 'application/json');
+ res.end(
+ JSON.stringify(
+ scope === 'server'
+ ? { scope: 'server', residue: EMPTY_SERVER_RESET_RESIDUE }
+ : { scope: 'document' },
+ ),
+ );
+ });
+ server.httpServer?.on('upgrade', (req, socket, head) => {
+ if (!req.url?.startsWith('/collab')) return;
+ if (failing.has('collab')) {
+ socket.destroy();
+ return;
+ }
+ sockets.handleUpgrade(req, socket, head, (ws) => {
+ const connection = hocuspocus.handleConnection(ws, req as unknown as Request);
+ ws.on('message', (data: Buffer) => connection.handleMessage(new Uint8Array(data)));
+ ws.on('close', (code: number, reason: Buffer) =>
+ connection.handleClose({ code, reason: reason.toString() }),
+ );
+ });
+ });
+ },
+ };
+}
+
+function contractExclusiveResource(): Plugin {
+ return {
+ name: 'ok:contract-server-exclusive-resource',
+ apply: 'serve',
+ configureServer(server) {
+ if (contractProcess[EXCLUSIVE_RESOURCE_HELD] === true) {
+ throw new Error('the contract exclusive resource is held by a running server');
+ }
+ contractProcess[EXCLUSIVE_RESOURCE_HELD] = true;
+ server.httpServer?.once('close', () => {
+ contractProcess[EXCLUSIVE_RESOURCE_HELD] = false;
+ });
+ },
+ };
+}
+
+export default defineConfig({
+ root: process.cwd(),
+ appType: 'custom',
+ clearScreen: false,
+ logLevel: 'error',
+ optimizeDeps: { noDiscovery: true, include: [] },
+ plugins: [
+ contractControl(),
+ contractReadiness(),
+ ...(process.env[CONTRACT_SERVER_EXCLUSIVE_RESOURCE_ENV] === '1'
+ ? [contractExclusiveResource()]
+ : []),
+ testServerStartupPlugin(request),
+ ],
+ server: {
+ port: contractProcess[RESTART_PORT] ?? request.candidatePort,
+ strictPort: false,
+ watch: null,
+ },
+});
diff --git a/packages/app/tests/foundation/fresh-slot.node.test.ts b/packages/app/tests/foundation/fresh-slot.node.test.ts
new file mode 100644
index 000000000..9aaf3d1e2
--- /dev/null
+++ b/packages/app/tests/foundation/fresh-slot.node.test.ts
@@ -0,0 +1,179 @@
+import { mkdirSync, mkdtempSync, realpathSync, rmSync, symlinkSync, writeFileSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { dirname, join } from 'node:path';
+import { afterEach, describe, expect, test } from 'vitest';
+import {
+ freshSlotResidue,
+ parseServerResetResidue,
+ serverResetResidueProblems,
+} from '../stress/_helpers/reuse/fresh-slot.ts';
+
+const FIXTURE_DOCS = ['test-doc', 'sidebar-folder/nested-doc'] as const;
+
+const roots: string[] = [];
+
+afterEach(() => {
+ for (const root of roots.splice(0)) rmSync(root, { recursive: true, force: true });
+});
+
+function write(root: string, path: string, text = ''): void {
+ mkdirSync(dirname(join(root, path)), { recursive: true });
+ writeFileSync(join(root, path), text, 'utf-8');
+}
+
+function freshSlot(): string {
+ const root = realpathSync(mkdtempSync(join(tmpdir(), 'ok-fresh-slot-')));
+ roots.push(root);
+ write(root, 'test-doc.md');
+ write(root, 'sidebar-folder/nested-doc.md');
+ mkdirSync(join(root, '.claude', 'skills'), { recursive: true });
+ mkdirSync(join(root, '.ok', 'skills'), { recursive: true });
+ write(root, '.ok/local/cache/main/backlinks.json', '{}');
+ write(root, '.ok/local/cache/tags.json', '{}');
+ write(root, '.ok/local/principal.json', '{}');
+ write(root, '.ok/local/server.lock', '{}');
+ write(root, '.ok/local/state.json', '{}');
+ write(root, '.ok/local/acp-registry-cache.json', '{}');
+ write(root, '.ok/local/stale-external-writes.json', '{}');
+ write(root, '.ok/local/removed-docs.json', '{}');
+ write(root, '.git/HEAD', 'ref: refs/heads/main\n');
+ return root;
+}
+
+describe('a reset slot is compared with what a freshly booted server holds', () => {
+ test('the tree a freshly booted test server writes has no residue', () => {
+ expect(freshSlotResidue(freshSlot(), FIXTURE_DOCS)).toEqual([]);
+ });
+
+ test('saved settings of every scope are residue', () => {
+ const root = freshSlot();
+ write(root, '.ok/config.yml', 'content: {}\n');
+ write(root, '.ok/local/config.yml', 'search: {}\n');
+ write(root, '.ok/global.yml', 'appearance:\n theme: dark\n');
+ write(root, '.ok/themes/mine.yaml', 'scheme: mine\n');
+
+ expect(freshSlotResidue(root, FIXTURE_DOCS)).toEqual([
+ '.ok/config.yml is not in a freshly booted slot',
+ '.ok/global.yml is not in a freshly booted slot',
+ '.ok/local/config.yml is not in a freshly booted slot',
+ '.ok/themes is not in a freshly booted slot',
+ ]);
+ });
+
+ test('created content, edited fixture documents and missing fixtures are residue', () => {
+ const root = freshSlot();
+ write(root, 'created.md', '# made by a test\n');
+ write(root, 'test-doc.md', 'edited\n');
+ write(root, 'sidebar-folder/extra.md', '');
+ rmSync(join(root, 'sidebar-folder', 'nested-doc.md'));
+ rmSync(join(root, '.claude', 'skills'), { recursive: true });
+
+ expect(freshSlotResidue(root, FIXTURE_DOCS)).toEqual([
+ '.claude/skills is missing',
+ 'created.md is not in a freshly booted slot',
+ 'sidebar-folder/extra.md is not in a freshly booted slot',
+ 'sidebar-folder/nested-doc.md is missing',
+ 'test-doc.md is not empty',
+ ]);
+ });
+
+ test('an installed skill and a non-empty ignore file are residue while an empty ignore file is not', () => {
+ const root = freshSlot();
+ write(root, '.okignore', '');
+ expect(freshSlotResidue(root, FIXTURE_DOCS)).toEqual([]);
+
+ write(root, '.okignore', 'drafts/\n');
+ write(root, '.claude/skills/helper/SKILL.md', '---\nname: helper\n---\n');
+ expect(freshSlotResidue(root, FIXTURE_DOCS)).toEqual([
+ '.claude/skills/helper is not in a freshly booted slot',
+ '.okignore is not empty',
+ ]);
+ });
+
+ test('a symlink where the slot expects a directory or file is residue', () => {
+ const root = freshSlot();
+ const elsewhere = realpathSync(mkdtempSync(join(tmpdir(), 'ok-fresh-slot-elsewhere-')));
+ roots.push(elsewhere);
+ rmSync(join(root, 'sidebar-folder'), { recursive: true });
+ symlinkSync(elsewhere, join(root, 'sidebar-folder'));
+ rmSync(join(root, 'test-doc.md'));
+ symlinkSync(join(elsewhere, 'missing.md'), join(root, 'test-doc.md'));
+
+ expect(freshSlotResidue(root, FIXTURE_DOCS)).toEqual([
+ 'sidebar-folder is not a directory',
+ 'test-doc.md is not a file',
+ ]);
+ });
+});
+
+describe('the server reset report', () => {
+ test('a report naming nothing left behind has no problems', () => {
+ const residue = parseServerResetResidue({
+ scope: 'server',
+ residue: {
+ loadedDocuments: [],
+ clientConnections: 0,
+ agentSessions: 0,
+ agentPresence: [],
+ staleExternalWrites: [],
+ removedDocuments: [],
+ },
+ });
+
+ expect(serverResetResidueProblems(residue)).toEqual([]);
+ });
+
+ test('documents, connections, sessions and presence the server kept are each named', () => {
+ const residue = parseServerResetResidue({
+ scope: 'server',
+ residue: {
+ loadedDocuments: ['notes'],
+ clientConnections: 2,
+ agentSessions: 1,
+ agentPresence: ['agent-a'],
+ staleExternalWrites: ['drafts/plan'],
+ removedDocuments: ['old-notes'],
+ },
+ });
+
+ expect(serverResetResidueProblems(residue)).toEqual([
+ 'loaded documents notes',
+ '2 client connections',
+ '1 agent sessions',
+ 'agent presence for agent-a',
+ 'stale external writes for drafts/plan',
+ 'removal records for old-notes',
+ ]);
+ });
+
+ test('a residue field this check does not inspect is rejected, so leftover state of a new kind discards the slot', () => {
+ expect(() =>
+ parseServerResetResidue({
+ scope: 'server',
+ residue: {
+ loadedDocuments: [],
+ clientConnections: 0,
+ agentSessions: 0,
+ agentPresence: [],
+ staleExternalWrites: [],
+ removedDocuments: [],
+ watchedFolders: ['drafts'],
+ },
+ }),
+ ).toThrow(
+ 'the test-reset response residue reports watchedFolders, which this check does not inspect',
+ );
+ });
+
+ test('a body that is not a server-scope report is rejected', () => {
+ expect(() => parseServerResetResidue({})).toThrow(
+ 'the test-reset response is not a server-scope report',
+ );
+ expect(() =>
+ parseServerResetResidue({
+ scope: 'server',
+ residue: { loadedDocuments: 'notes', clientConnections: 0, agentSessions: 0 },
+ }),
+ ).toThrow('the test-reset response residue.loadedDocuments is not a list of strings');
+ });
+});
diff --git a/packages/app/tests/foundation/owned-server-launch-failure.node.test.ts b/packages/app/tests/foundation/owned-server-launch-failure.node.test.ts
new file mode 100644
index 000000000..bb01acb3c
--- /dev/null
+++ b/packages/app/tests/foundation/owned-server-launch-failure.node.test.ts
@@ -0,0 +1,198 @@
+import { existsSync, mkdirSync, mkdtempSync, readdirSync, realpathSync, rmSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { basename, join } from 'node:path';
+import type { Browser, WorkerInfo } from '@playwright/test';
+import { afterEach, describe, expect, test, vi } from 'vitest';
+import { E2E_SERVING_ENV } from '../stress/_helpers/prebuilt/prebuilt-serving.ts';
+import { leaseWorkerServer, type ServerLease } from '../stress/_helpers/reuse/lease-fixture.ts';
+import { REUSE_RUN_ENV, REUSE_SOCKET_ENV } from '../stress/_helpers/reuse/lease-protocol.ts';
+import { launchOwnedWorkerServer } from '../stress/_helpers/reuse/owned-dev-server.ts';
+import { WorkerServerSupervisor } from '../stress/_helpers/reuse/server-supervisor.ts';
+import {
+ type SupervisorEndpoint,
+ serveSupervisor,
+} from '../stress/_helpers/reuse/supervisor-socket.ts';
+import { APP_PACKAGE_ROOT } from '../stress/_helpers/seed-key.ts';
+import { contractServerLauncher } from './contract-dev-server.test-helper.ts';
+
+const faults = vi.hoisted(() => ({
+ removal: new Set<string>(),
+ open: new Set<string>(),
+}));
+
+vi.mock('node:fs', async (importOriginal) => {
+ const fs = await importOriginal<typeof import('node:fs')>();
+ const faulted = (prefixes: ReadonlySet<string>, target: unknown) =>
+ [...prefixes].some((prefix) => basename(String(target)).startsWith(prefix));
+ const errnoError = (code: string, description: string, syscall: string, target: unknown) =>
+ Object.assign(new Error(`${code}: ${description}, ${syscall} '${String(target)}'`), {
+ code,
+ syscall,
+ path: String(target),
+ });
+ return {
+ ...fs,
+ rmSync: (
+ target: Parameters<typeof fs.rmSync>[0],
+ options?: Parameters<typeof fs.rmSync>[1],
+ ) => {
+ if (faulted(faults.removal, target)) {
+ throw errnoError('EACCES', 'permission denied', 'rmdir', target);
+ }
+ fs.rmSync(target, options);
+ },
+ openSync: (...args: Parameters<typeof fs.openSync>) => {
+ if (faulted(faults.open, args[0])) {
+ throw errnoError('ENOSPC', 'no space left on device', 'open', args[0]);
+ }
+ return fs.openSync(...args);
+ },
+ };
+});
+
+const WORKTREE = realpathSync(join(APP_PACKAGE_ROOT, '..', '..'));
+const ENV_NAMES = ['TMPDIR', REUSE_SOCKET_ENV, REUSE_RUN_ENV, E2E_SERVING_ENV] as const;
+const LABEL = 'launch-failure-s0';
+
+const WORKER_INFO = {
+ parallelIndex: 0,
+ workerIndex: 0,
+ config: { configFile: '/worktree/packages/app/playwright.config.ts' },
+ project: { name: 'chromium' },
+} as WorkerInfo;
+
+const BROWSER_THE_REFUSED_LEASE_NEVER_OPENS = {} as Browser;
+
+const savedEnv = new Map(ENV_NAMES.map((name) => [name, process.env[name]]));
+const openEndpoints: SupervisorEndpoint[] = [];
+const scratchDirs: string[] = [];
+
+afterEach(async () => {
+ faults.removal.clear();
+ faults.open.clear();
+ for (const [name, value] of savedEnv) {
+ if (value === undefined) delete process.env[name];
+ else process.env[name] = value;
+ }
+ for (const endpoint of openEndpoints.splice(0)) await endpoint.close();
+ for (const dir of scratchDirs.splice(0)) rmSync(dir, { recursive: true, force: true });
+});
+
+interface Scratch {
+ readonly tmp: string;
+ readonly missingBuild: string;
+ readonly socketPath: string;
+}
+
+function scratchWithOwnTmpdir(): Scratch {
+ const root = realpathSync(mkdtempSync(join(tmpdir(), 'ok-owned-launch-scratch-')));
+ scratchDirs.push(root);
+ const tmp = join(root, 'tmp');
+ mkdirSync(tmp);
+ process.env.TMPDIR = tmp;
+ return {
+ tmp,
+ missingBuild: join(root, 'no-prebuilt-build'),
+ socketPath: join(root, 'supervisor.sock'),
+ };
+}
+
+function viteCacheDirsOf(label: string): string[] {
+ return readdirSync(join(APP_PACKAGE_ROOT, 'node_modules')).filter((name) =>
+ name.startsWith(`.vite-${label}-`),
+ );
+}
+
+describe('an owned server whose launch fails', () => {
+ test('a worker whose owned server fails to start, and whose cleanup then fails too, is told both failures and the dev server log tail', async () => {
+ const scratch = scratchWithOwnTmpdir();
+ faults.removal.add(`ok-${LABEL}-`);
+ const supervisor = new WorkerServerSupervisor({
+ okRoot: WORKTREE,
+ launch: (_slot, workerServerEnv) =>
+ launchOwnedWorkerServer({
+ label: LABEL,
+ workerServerEnv,
+ serving: { kind: 'prebuilt', outputDir: scratch.missingBuild },
+ }),
+ identityTimeoutMs: 2_000,
+ });
+ const endpoint = await serveSupervisor(supervisor, scratch.socketPath);
+ openEndpoints.push(endpoint);
+ process.env[REUSE_SOCKET_ENV] = scratch.socketPath;
+ process.env[REUSE_RUN_ENV] = 'run-a';
+ delete process.env[E2E_SERVING_ENV];
+ const offered: Array<ServerLease | undefined> = [];
+
+ await leaseWorkerServer(
+ { workerServerEnv: {}, browser: BROWSER_THE_REFUSED_LEASE_NEVER_OPENS },
+ async (lease) => {
+ offered.push(lease);
+ },
+ WORKER_INFO,
+ );
+
+ const leftBehind = readdirSync(scratch.tmp);
+ expect(leftBehind).toEqual([expect.stringMatching(new RegExp(`^ok-${LABEL}-`))]);
+ const contentDir = join(scratch.tmp, leftBehind[0] ?? '');
+ expect(viteCacheDirsOf(LABEL)).toEqual([]);
+ expect(offered).toEqual([
+ {
+ kind: 'fresh',
+ note: `slot 0 was refused (launch-failed: slot 0 could not start an owned server: the prebuilt DEV app at ${scratch.missingBuild} has no build manifest, so it is not served\n--- dev server log tail ---\n\n--- cleanup after this failure did not complete: EACCES: permission denied, rmdir '${contentDir}' ---), so this worker starts its own server`,
+ },
+ ]);
+ expect(existsSync(contentDir)).toBe(true);
+ }, 60_000);
+
+ test('a launch whose server log cannot be opened releases its startup request and receipt directory, and says no log was open', async () => {
+ const scratch = scratchWithOwnTmpdir();
+ faults.open.add(`ok-e2e-${LABEL}-`);
+
+ const failure = await launchOwnedWorkerServer({
+ label: LABEL,
+ workerServerEnv: {},
+ serving: { kind: 'dev-server' },
+ }).then(
+ () => undefined,
+ (error: unknown) => error,
+ );
+
+ expect(readdirSync(scratch.tmp)).toEqual([]);
+ expect(failure).toBeInstanceOf(Error);
+ expect((failure as Error).message).toMatch(
+ new RegExp(
+ `^ENOSPC: no space left on device, open '${scratch.tmp}/ok-e2e-${LABEL}-\\d+-[a-z0-9]+\\.log'\\n--- no dev server log was open when this failure ended the launch ---$`,
+ ),
+ );
+ });
+
+ test('a contract dev server that exits before it is ready, and whose cleanup then fails too, reports its exit, its stderr and the cleanup failure', async () => {
+ const scratch = scratchWithOwnTmpdir();
+ faults.removal.add('ok-contract-s0-');
+ const launcher = contractServerLauncher();
+
+ const failure = await launcher
+ .launch(0, { NODE_OPTIONS: `--require=${scratch.missingBuild}/preload.cjs` })
+ .then(
+ () => undefined,
+ (error: unknown) => error,
+ );
+
+ const leftBehind = readdirSync(scratch.tmp);
+ expect(leftBehind).toEqual([expect.stringMatching(/^ok-contract-s0-/)]);
+ const contentDir = join(scratch.tmp, leftBehind[0] ?? '');
+ expect(failure).toBeInstanceOf(Error);
+ const message = (failure as Error).message;
+ expect(message).toMatch(
+ /^the contract dev server did not start: Error: automatic Vite startup command exited with code 1 without a receipt\n/,
+ );
+ expect(message).toContain(`Cannot find module '${scratch.missingBuild}/preload.cjs'`);
+ expect(message).toMatch(
+ new RegExp(
+ `\\n--- cleanup after this failure did not complete: EACCES: permission denied, rmdir '${contentDir}' ---$`,
+ ),
+ );
+ expect(launcher.launched).toEqual([]);
+ });
+});
diff --git a/packages/app/tests/foundation/owned-server-lifetime-loss.node.test.ts b/packages/app/tests/foundation/owned-server-lifetime-loss.node.test.ts
new file mode 100644
index 000000000..0c254b53e
--- /dev/null
+++ b/packages/app/tests/foundation/owned-server-lifetime-loss.node.test.ts
@@ -0,0 +1,93 @@
+import { EventEmitter } from 'node:events';
+import { mkdirSync, mkdtempSync, readdirSync, realpathSync, rmSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { afterEach, describe, expect, test, vi } from 'vitest';
+import { launchOwnedWorkerServer } from '../stress/_helpers/reuse/owned-dev-server.ts';
+
+const harness = vi.hoisted(() => ({
+ proc: undefined as (import('node:events').EventEmitter & Record<string, unknown>) | undefined,
+ readiness: undefined as
+ | { promise: Promise<unknown>; resolve: (value: unknown) => void }
+ | undefined,
+ releases: [] as string[],
+}));
+
+vi.mock('../stress/_helpers/fixtures.ts', async (importOriginal) => {
+ const fixtures = await importOriginal<typeof import('../stress/_helpers/fixtures.ts')>();
+ return {
+ ...fixtures,
+ spawnWorkerDevServer: () => harness.proc,
+ waitForServerReady: () => harness.readiness?.promise,
+ };
+});
+
+vi.mock('../stress/_helpers/worker-server-lifetime.ts', async (importOriginal) => {
+ const lifetime =
+ await importOriginal<typeof import('../stress/_helpers/worker-server-lifetime.ts')>();
+ return {
+ ...lifetime,
+ killGracefully: async () => {
+ harness.releases.push('stop');
+ },
+ completeWorkerServerRelease: async () => {
+ harness.releases.push('release');
+ },
+ };
+});
+
+const LIFETIME_LOSS =
+ 'worker server lifetime closed without completed release (code=1, signal=null)';
+const LABEL = 'lifetime-loss-s0';
+const savedTmpdir = process.env.TMPDIR;
+const scratchDirs: string[] = [];
+
+afterEach(() => {
+ harness.proc = undefined;
+ harness.readiness = undefined;
+ harness.releases.splice(0);
+ if (savedTmpdir === undefined) delete process.env.TMPDIR;
+ else process.env.TMPDIR = savedTmpdir;
+ for (const dir of scratchDirs.splice(0)) rmSync(dir, { recursive: true, force: true });
+});
+
+function scratchTmpdir(): string {
+ const root = realpathSync(mkdtempSync(join(tmpdir(), 'ok-owned-lifetime-scratch-')));
+ scratchDirs.push(root);
+ const tmp = join(root, 'tmp');
+ mkdirSync(tmp);
+ process.env.TMPDIR = tmp;
+ return tmp;
+}
+
+describe('an owned server whose lifetime is lost while it launches', () => {
+ test('a lifetime lost before the server answers fails the launch with that loss and releases the server, even when the server answers afterwards', async () => {
+ const tmp = scratchTmpdir();
+ harness.proc = Object.assign(new EventEmitter(), { exitCode: null, signalCode: null });
+ let resolveReadiness!: (value: unknown) => void;
+ harness.readiness = {
+ promise: new Promise((resolve) => {
+ resolveReadiness = resolve;
+ }),
+ resolve: (value) => resolveReadiness(value),
+ };
+
+ const outcome = launchOwnedWorkerServer({
+ label: LABEL,
+ workerServerEnv: {},
+ serving: { kind: 'dev-server' },
+ }).then(
+ () => 'granted',
+ (error: unknown) => error,
+ );
+ await vi.waitFor(() => expect(harness.proc?.listenerCount('error')).toBeGreaterThan(0));
+ harness.proc.emit('error', new Error(LIFETIME_LOSS));
+ harness.readiness.resolve({ baseURL: 'http://127.0.0.1:5999', port: 5999 });
+
+ const failure = await outcome;
+ expect(failure).toBeInstanceOf(Error);
+ expect((failure as Error).message.split('\n')[0]).toBe(LIFETIME_LOSS);
+ expect(harness.releases).toEqual(['stop', 'release']);
+ expect(readdirSync(tmp)).toEqual([]);
+ });
+});
diff --git a/packages/app/tests/foundation/prebuilt-build-env.node.test.ts b/packages/app/tests/foundation/prebuilt-build-env.node.test.ts
new file mode 100644
index 000000000..ea4dceebe
--- /dev/null
+++ b/packages/app/tests/foundation/prebuilt-build-env.node.test.ts
@@ -0,0 +1,46 @@
+import { mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { afterEach, describe, expect, test, vi } from 'vitest';
+import { PREBUILT_OUT_DIR_ENV } from '../../src/build/prebuilt-dev-contract';
+import { prebuiltBuildStep, runBuildStep } from '../stress/_helpers/prebuilt/prebuilt-dev-build';
+
+const RECORD_ENV = `
+import { writeFileSync } from 'node:fs';
+writeFileSync(process.argv[2], JSON.stringify(process.env));
+`;
+
+const roots: string[] = [];
+
+afterEach(() => {
+ vi.unstubAllEnvs();
+ for (const root of roots.splice(0)) rmSync(root, { recursive: true, force: true });
+});
+
+describe('the environment of the prebuilt DEV app build', () => {
+ test('no VITE_ variable of the shell reaches the build, while the rest of its environment does', async () => {
+ const root = mkdtempSync(join(tmpdir(), 'ok-prebuilt-env-'));
+ roots.push(root);
+ const script = join(root, 'record-env.mjs');
+ const recorded = join(root, 'env.json');
+ writeFileSync(script, RECORD_ENV);
+ vi.stubEnv('VITE_OTEL_ENABLED', 'true');
+ vi.stubEnv('VITE_OK_PERF_INSTRUMENT', '1');
+ vi.stubEnv('OK_PREBUILT_ENV_PROBE', 'kept');
+ const distDir = join(root, 'dist');
+
+ await runBuildStep({
+ ...prebuiltBuildStep(distDir),
+ command: process.execPath,
+ args: [script, recorded],
+ });
+
+ const env = JSON.parse(readFileSync(recorded, 'utf8')) as Record<string, string>;
+ expect(Object.keys(env).filter((name) => name.startsWith('VITE_'))).toEqual([]);
+ expect(env).toMatchObject({
+ OK_PREBUILT_ENV_PROBE: 'kept',
+ NODE_ENV: 'development',
+ [PREBUILT_OUT_DIR_ENV]: distDir,
+ });
+ });
+});
diff --git a/packages/app/tests/foundation/prebuilt-build-silence.node.test.ts b/packages/app/tests/foundation/prebuilt-build-silence.node.test.ts
new file mode 100644
index 000000000..0c46ad69c
--- /dev/null
+++ b/packages/app/tests/foundation/prebuilt-build-silence.node.test.ts
@@ -0,0 +1,174 @@
+import { existsSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { DatabaseSync } from 'node:sqlite';
+import { setImmediate as nextTurn, setTimeout as wait } from 'node:timers/promises';
+import { afterEach, describe, expect, test, vi } from 'vitest';
+import {
+ BUILD_STEP_SILENCE_MS,
+ runBuildStep,
+} from '../stress/_helpers/prebuilt/prebuilt-dev-build';
+
+const HOLDER = `
+import { DatabaseSync } from 'node:sqlite';
+const lock = new DatabaseSync(process.argv[2]);
+lock.exec('BEGIN');
+lock.prepare('SELECT count(*) FROM sqlite_schema').get();
+setInterval(() => {}, 1 << 30);
+`;
+
+const STALLED_BUILD = `
+import { spawn } from 'node:child_process';
+process.stdout.write('transforming...\\n');
+spawn(process.execPath, [process.argv[2], process.argv[3]], { stdio: 'inherit' });
+setInterval(() => {}, 1 << 30);
+`;
+
+const PROGRESSING_BUILD = `
+import { existsSync, writeFileSync } from 'node:fs';
+import { join } from 'node:path';
+const dir = process.argv[2];
+const after = (name, then) => {
+ const poll = setInterval(() => {
+ if (!existsSync(join(dir, name))) return;
+ clearInterval(poll);
+ then();
+ }, 10);
+};
+process.stdout.write('transforming...\\n', () => writeFileSync(join(dir, 'first-printed'), ''));
+after('print-again', () => {
+ process.stdout.write('rendering chunks...\\n', () => writeFileSync(join(dir, 'second-printed'), ''));
+ after('finish', () => process.exit(0));
+});
+`;
+
+const roots: string[] = [];
+
+function scratch(): string {
+ const root = mkdtempSync(join(tmpdir(), 'ok-prebuilt-silence-'));
+ roots.push(root);
+ return root;
+}
+
+function fixture(dir: string, name: string, source: string): string {
+ const path = join(dir, name);
+ writeFileSync(path, source);
+ return path;
+}
+
+function lockHeld(path: string): boolean {
+ if (!existsSync(path)) return false;
+ const probe = new DatabaseSync(path, { timeout: 0 });
+ try {
+ probe.exec('BEGIN EXCLUSIVE');
+ probe.exec('ROLLBACK');
+ return false;
+ } catch {
+ return true;
+ } finally {
+ probe.close();
+ }
+}
+
+async function until(condition: () => boolean, what: string): Promise<void> {
+ const deadline = Date.now() + 20_000;
+ while (!condition()) {
+ if (Date.now() > deadline) throw new Error(`${what} did not happen within 20 s`);
+ await wait(10);
+ }
+ await nextTurn();
+}
+
+function handled(run: Promise<void>): void {
+ run.catch(() => undefined);
+}
+
+afterEach(() => {
+ vi.useRealTimers();
+ for (const root of roots.splice(0)) rmSync(root, { recursive: true, force: true });
+});
+
+describe('a step of the prebuilt DEV app build', () => {
+ test('a step that prints nothing for the whole window is stopped with every process in its group, and the error names it and keeps its output', async () => {
+ vi.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout'] });
+ const dir = scratch();
+ const lock = join(dir, 'holder.sqlite');
+ const run = runBuildStep({
+ label: 'the stalled build step',
+ command: process.execPath,
+ args: [fixture(dir, 'stalled.mjs', STALLED_BUILD), fixture(dir, 'holder.mjs', HOLDER), lock],
+ env: {},
+ });
+ handled(run);
+ await until(() => lockHeld(lock), 'the child of the stalled build taking its lock');
+
+ vi.advanceTimersByTime(BUILD_STEP_SILENCE_MS);
+
+ await expect(run).rejects.toThrow(
+ `the stalled build step printed nothing for ${BUILD_STEP_SILENCE_MS / 1000} s, so this run stopped it and every process in its process group; its last output:\ntransforming...\n`,
+ );
+ vi.useRealTimers();
+ await until(() => !lockHeld(lock), 'the child of the stalled build exiting');
+ });
+
+ test('a step that keeps printing is not stopped while each silence stays shorter than the window', async () => {
+ vi.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout'] });
+ const dir = scratch();
+ const run = runBuildStep({
+ label: 'the progressing build step',
+ command: process.execPath,
+ args: [fixture(dir, 'progressing.mjs', PROGRESSING_BUILD), dir],
+ env: {},
+ });
+ handled(run);
+ await until(() => existsSync(join(dir, 'first-printed')), 'the first output');
+
+ vi.advanceTimersByTime(BUILD_STEP_SILENCE_MS - 1);
+ writeFileSync(join(dir, 'print-again'), '');
+ await until(() => existsSync(join(dir, 'second-printed')), 'the second output');
+ vi.advanceTimersByTime(BUILD_STEP_SILENCE_MS - 1);
+ writeFileSync(join(dir, 'finish'), '');
+
+ await expect(run).resolves.toBeUndefined();
+ });
+
+ test('a step run through pnpm exec that hangs is stopped, and pnpm ends the command it started', async () => {
+ vi.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout'] });
+ const dir = scratch();
+ const lock = join(dir, 'holder.sqlite');
+ const run = runBuildStep({
+ label: 'the stalled pnpm step',
+ command: 'pnpm',
+ args: ['exec', 'node', fixture(dir, 'holder.mjs', HOLDER), lock],
+ env: {},
+ });
+ handled(run);
+ await until(() => lockHeld(lock), 'the command pnpm started taking its lock');
+
+ vi.advanceTimersByTime(BUILD_STEP_SILENCE_MS);
+
+ await expect(run).rejects.toThrow(
+ `the stalled pnpm step printed nothing for ${BUILD_STEP_SILENCE_MS / 1000} s, so this run stopped it`,
+ );
+ vi.useRealTimers();
+ await until(() => !lockHeld(lock), 'the command pnpm started exiting');
+ });
+
+ test('a step that fails reports its exit code with the end of its output', async () => {
+ const dir = scratch();
+ const failing = fixture(
+ dir,
+ 'failing.mjs',
+ `process.stderr.write('vite build failed: missing entry\\n'); process.exitCode = 3;`,
+ );
+
+ await expect(
+ runBuildStep({
+ label: 'the failing build step',
+ command: process.execPath,
+ args: [failing],
+ env: {},
+ }),
+ ).rejects.toThrow('the failing build step exited with 3\nvite build failed: missing entry\n');
+ });
+});
diff --git a/packages/app/tests/foundation/prebuilt-build-use.node.test.ts b/packages/app/tests/foundation/prebuilt-build-use.node.test.ts
new file mode 100644
index 000000000..54ad1bf5f
--- /dev/null
+++ b/packages/app/tests/foundation/prebuilt-build-use.node.test.ts
@@ -0,0 +1,141 @@
+import { type ChildProcess, spawn } from 'node:child_process';
+import { once } from 'node:events';
+import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { dirname, join } from 'node:path';
+import { fileURLToPath } from 'node:url';
+import { afterEach, describe, expect, test } from 'vitest';
+import { resolvePrebuiltDevApp } from '../stress/_helpers/prebuilt/prebuilt-dev-app';
+import { PREBUILT_BUILD_OPTIONS } from '../stress/_helpers/prebuilt/prebuilt-serving';
+
+const OK_TREE: Readonly<Record<string, string>> = {
+ 'package.json': '{"name":"open-knowledge"}\n',
+ 'pnpm-lock.yaml': "lockfileVersion: '9.0'\n",
+ 'packages/app/package.json': '{"name":"@inkeep/open-knowledge-app"}\n',
+ 'packages/app/index.html': '<!doctype html><div id="root"></div>\n',
+ 'packages/app/src/App.tsx': 'export const App = () => 1;\n',
+ 'packages/core/src/index.ts': 'export const core = 1;\n',
+ 'packages/server/src/index.ts': 'export const server = 1;\n',
+};
+
+const BUILD_USE_MODULE = fileURLToPath(
+ new URL('../stress/_helpers/prebuilt/prebuilt-build-use.ts', import.meta.url),
+);
+
+const HOLDER = `
+import { holdPrebuiltBuildUse } from ${JSON.stringify(BUILD_USE_MODULE)};
+holdPrebuiltBuildUse(process.argv[1], process.argv[2]);
+process.stdout.write('held\\n');
+process.stdin.on('data', () => {});
+process.stdin.on('end', () => process.exit(0));
+`;
+
+const roots: string[] = [];
+const holders: ChildProcess[] = [];
+
+function scratch(prefix: string): string {
+ const root = mkdtempSync(join(tmpdir(), prefix));
+ roots.push(root);
+ return root;
+}
+
+function write(root: string, path: string, content: string): void {
+ mkdirSync(dirname(join(root, path)), { recursive: true });
+ writeFileSync(join(root, path), content);
+}
+
+function okTree(): string {
+ const root = scratch('ok-prebuilt-use-tree-');
+ for (const [path, content] of Object.entries(OK_TREE)) write(root, path, content);
+ return root;
+}
+
+function setApp(okRoot: string, value: number): void {
+ write(okRoot, 'packages/app/src/App.tsx', `export const App = () => ${value};\n`);
+}
+
+async function build(okRoot: string, distDir: string): Promise<void> {
+ mkdirSync(distDir, { recursive: true });
+ const app = readFileSync(join(okRoot, 'packages/app/src/App.tsx'), 'utf8');
+ writeFileSync(join(distDir, 'index.html'), `<!doctype html><script>${app}</script>\n`);
+}
+
+function resolveApp(okRoot: string, outRoot: string) {
+ return resolvePrebuiltDevApp({
+ okRoot,
+ outRoot,
+ options: PREBUILT_BUILD_OPTIONS,
+ build: (distDir) => build(okRoot, distDir),
+ });
+}
+
+async function holdInAnotherProcess(outRoot: string, digest: string): Promise<ChildProcess> {
+ const holder = spawn(process.execPath, ['--input-type=module', '-e', HOLDER, outRoot, digest], {
+ stdio: ['pipe', 'pipe', 'inherit'],
+ });
+ holders.push(holder);
+ const [chunk] = await once(holder.stdout, 'data');
+ expect(String(chunk)).toBe('held\n');
+ return holder;
+}
+
+async function endWithoutRelease(holder: ChildProcess): Promise<number | null> {
+ const exited = once(holder, 'exit');
+ holder.stdin?.end();
+ const [code] = await exited;
+ return code;
+}
+
+afterEach(async () => {
+ for (const holder of holders.splice(0)) {
+ if (holder.exitCode === null && holder.signalCode === null) await endWithoutRelease(holder);
+ }
+ for (const root of roots.splice(0)) rmSync(root, { recursive: true, force: true });
+});
+
+describe('a prebuilt DEV build in use', () => {
+ test('a build a run serves after a newer build replaced it survives another run building after it, and goes once that run releases it', async () => {
+ const okRoot = okTree();
+ const outRoot = scratch('ok-prebuilt-use-out-');
+ (await resolveApp(okRoot, outRoot)).use.release();
+ setApp(okRoot, 2);
+ (await resolveApp(okRoot, outRoot)).use.release();
+
+ setApp(okRoot, 1);
+ const served = await resolveApp(okRoot, outRoot);
+ setApp(okRoot, 3);
+ const next = await resolveApp(okRoot, outRoot);
+ next.use.release();
+
+ expect(served.built).toBe(false);
+ expect(next.built).toBe(true);
+ expect(existsSync(join(served.distDir, 'index.html'))).toBe(true);
+ expect(readFileSync(join(served.distDir, 'index.html'), 'utf8')).toContain('App = () => 1');
+
+ served.use.release();
+ setApp(okRoot, 4);
+ (await resolveApp(okRoot, outRoot)).use.release();
+
+ expect(existsSync(served.outputDir)).toBe(false);
+ });
+
+ test('a build another process holds stays while that process runs, and goes at the next build after it exits without releasing', async () => {
+ const okRoot = okTree();
+ const outRoot = scratch('ok-prebuilt-use-out-');
+ const held = await resolveApp(okRoot, outRoot);
+ held.use.release();
+ const holder = await holdInAnotherProcess(outRoot, held.manifest.digest);
+
+ for (const value of [2, 3]) {
+ setApp(okRoot, value);
+ (await resolveApp(okRoot, outRoot)).use.release();
+ }
+ expect(existsSync(join(held.distDir, 'index.html'))).toBe(true);
+
+ expect(await endWithoutRelease(holder)).toBe(0);
+ setApp(okRoot, 4);
+ (await resolveApp(okRoot, outRoot)).use.release();
+
+ expect(existsSync(held.outputDir)).toBe(false);
+ });
+});
diff --git a/packages/app/tests/foundation/prebuilt-concurrent-builders.node.test.ts b/packages/app/tests/foundation/prebuilt-concurrent-builders.node.test.ts
new file mode 100644
index 000000000..7da5a9fec
--- /dev/null
+++ b/packages/app/tests/foundation/prebuilt-concurrent-builders.node.test.ts
@@ -0,0 +1,190 @@
+import { spawnSync } from 'node:child_process';
+import {
+ existsSync,
+ mkdirSync,
+ mkdtempSync,
+ readdirSync,
+ readFileSync,
+ renameSync,
+ rmSync,
+ writeFileSync,
+} from 'node:fs';
+import { tmpdir } from 'node:os';
+import { dirname, join } from 'node:path';
+import { fileURLToPath } from 'node:url';
+import { afterEach, describe, expect, test, vi } from 'vitest';
+import { resolvePrebuiltDevApp } from '../stress/_helpers/prebuilt/prebuilt-dev-app';
+import {
+ PREBUILT_BUILD_OPTIONS,
+ PREBUILT_MANIFEST_FILE,
+} from '../stress/_helpers/prebuilt/prebuilt-serving';
+import { prebuiltManifest } from './test-server-inputs';
+
+vi.mock('node:fs', async (importOriginal) => {
+ const fs = await importOriginal<typeof import('node:fs')>();
+ return { ...fs, renameSync: vi.fn(fs.renameSync) };
+});
+
+const actualFs = await vi.importActual<typeof import('node:fs')>('node:fs');
+
+const OK_TREE: Readonly<Record<string, string>> = {
+ 'package.json': '{"name":"open-knowledge"}\n',
+ 'pnpm-lock.yaml': "lockfileVersion: '9.0'\n",
+ 'packages/app/package.json': '{"name":"@inkeep/open-knowledge-app"}\n',
+ 'packages/app/index.html': '<!doctype html><div id="root"></div>\n',
+ 'packages/app/src/App.tsx': 'export const App = () => 1;\n',
+ 'packages/core/src/index.ts': 'export const core = 1;\n',
+ 'packages/server/src/index.ts': 'export const server = 1;\n',
+};
+
+const APP_MODULE = fileURLToPath(
+ new URL('../stress/_helpers/prebuilt/prebuilt-dev-app.ts', import.meta.url),
+);
+const SERVING_MODULE = fileURLToPath(
+ new URL('../stress/_helpers/prebuilt/prebuilt-serving.ts', import.meta.url),
+);
+
+const OTHER_BUILDER = `
+import { mkdirSync, writeFileSync } from 'node:fs';
+import { join } from 'node:path';
+import { resolvePrebuiltDevApp } from ${JSON.stringify(APP_MODULE)};
+import { PREBUILT_BUILD_OPTIONS } from ${JSON.stringify(SERVING_MODULE)};
+const [okRoot, outRoot] = process.argv.slice(1);
+const app = await resolvePrebuiltDevApp({
+ okRoot,
+ outRoot,
+ options: PREBUILT_BUILD_OPTIONS,
+ build: async (distDir) => {
+ mkdirSync(distDir, { recursive: true });
+ writeFileSync(join(distDir, 'index.html'), '<!doctype html><p>landed first</p>\\n');
+ },
+});
+app.use.release();
+process.stdout.write(JSON.stringify({ built: app.built, distDir: app.distDir }));
+`;
+
+const OTHER_BUILDER_DEADLINE_MS = 20_000;
+
+type OtherBuild = { readonly built: boolean; readonly distDir: string };
+
+const roots: string[] = [];
+
+function scratch(prefix: string): string {
+ const root = mkdtempSync(join(tmpdir(), prefix));
+ roots.push(root);
+ return root;
+}
+
+function write(root: string, path: string, content: string): void {
+ mkdirSync(dirname(join(root, path)), { recursive: true });
+ writeFileSync(join(root, path), content);
+}
+
+function okTree(): string {
+ const root = scratch('ok-prebuilt-race-tree-');
+ for (const [path, content] of Object.entries(OK_TREE)) write(root, path, content);
+ return root;
+}
+
+function resolveSecond(okRoot: string, outRoot: string, beforeLanding: () => void = () => {}) {
+ return resolvePrebuiltDevApp({
+ okRoot,
+ outRoot,
+ options: PREBUILT_BUILD_OPTIONS,
+ build: async (distDir) => {
+ mkdirSync(distDir, { recursive: true });
+ writeFileSync(join(distDir, 'index.html'), '<!doctype html><p>built second</p>\n');
+ beforeLanding();
+ },
+ });
+}
+
+function beforeTheRename(land: (outputDir: string) => void): void {
+ vi.mocked(renameSync).mockImplementationOnce((from, to) => {
+ land(String(to));
+ actualFs.renameSync(from, to);
+ });
+}
+
+function otherBuilderLandsFirst(okRoot: string, outRoot: string) {
+ let other: OtherBuild | undefined;
+ const land = () => {
+ const run = spawnSync(
+ process.execPath,
+ ['--input-type=module', '-e', OTHER_BUILDER, okRoot, outRoot],
+ { encoding: 'utf8', timeout: OTHER_BUILDER_DEADLINE_MS },
+ );
+ process.stdout.write(
+ `other builder terminal: ${JSON.stringify({ pid: run.pid, code: run.status, signal: run.signal })}\n`,
+ );
+ if (run.status !== 0) {
+ throw new Error(`the other builder exited with ${run.status ?? run.signal}: ${run.stderr}`);
+ }
+ other = JSON.parse(run.stdout) as OtherBuild;
+ };
+ return Object.assign(() => other, { land });
+}
+
+function stagingLeft(outRoot: string): string[] {
+ return readdirSync(outRoot).filter((name) => name.startsWith('.staging-'));
+}
+
+afterEach(() => {
+ vi.mocked(renameSync).mockReset();
+ for (const root of roots.splice(0)) {
+ rmSync(root, { recursive: true, force: true });
+ if (existsSync(root)) throw new Error(`the owned fixture root remains: ${root}`);
+ process.stdout.write(`concurrent-builder fixture removed: ${root}\n`);
+ }
+});
+
+describe('two builders of the same inputs at once', () => {
+ test('a builder joins another complete same-input build that lands during its build', async () => {
+ const okRoot = okTree();
+ const outRoot = scratch('ok-prebuilt-race-out-');
+ const other = otherBuilderLandsFirst(okRoot, outRoot);
+
+ const second = await resolveSecond(okRoot, outRoot, other.land);
+ second.use.release();
+
+ expect(other()).toEqual({ built: true, distDir: second.distDir });
+ expect(second.built).toBe(false);
+ expect(readFileSync(join(second.distDir, 'index.html'), 'utf8')).toContain('landed first');
+ expect(stagingLeft(outRoot)).toEqual([]);
+ });
+
+ test('a builder whose rename finds a directory without a build manifest there still fails', async () => {
+ const okRoot = okTree();
+ const outRoot = scratch('ok-prebuilt-race-out-');
+ beforeTheRename((outputDir) => {
+ write(outputDir, 'dist/index.html', '<!doctype html><p>no manifest</p>\n');
+ });
+
+ await expect(resolveSecond(okRoot, outRoot)).rejects.toMatchObject({
+ code: expect.stringMatching(/^(ENOTEMPTY|EEXIST)$/),
+ });
+ expect(stagingLeft(outRoot)).toEqual([]);
+ });
+
+ test('a builder whose rename finds a build of other inputs there still fails', async () => {
+ const okRoot = okTree();
+ const outRoot = scratch('ok-prebuilt-race-out-');
+ const manifest = prebuiltManifest(okRoot, PREBUILT_BUILD_OPTIONS);
+ beforeTheRename((outputDir) => {
+ write(outputDir, 'dist/index.html', '<!doctype html><p>other inputs</p>\n');
+ write(
+ outputDir,
+ PREBUILT_MANIFEST_FILE,
+ JSON.stringify({
+ ...manifest,
+ inputs: { ...manifest.inputs, 'packages/app/src/App.tsx': 'other' },
+ }),
+ );
+ });
+
+ await expect(resolveSecond(okRoot, outRoot)).rejects.toMatchObject({
+ code: expect.stringMatching(/^(ENOTEMPTY|EEXIST)$/),
+ });
+ expect(stagingLeft(outRoot)).toEqual([]);
+ });
+});
diff --git a/packages/app/tests/foundation/prebuilt-concurrent-prune.node.test.ts b/packages/app/tests/foundation/prebuilt-concurrent-prune.node.test.ts
new file mode 100644
index 000000000..91a694272
--- /dev/null
+++ b/packages/app/tests/foundation/prebuilt-concurrent-prune.node.test.ts
@@ -0,0 +1,98 @@
+import { mkdirSync, mkdtempSync, readdirSync, readFileSync, writeFileSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { dirname, join } from 'node:path';
+import { afterEach, describe, expect, test, vi } from 'vitest';
+import { resolvePrebuiltDevApp } from '../stress/_helpers/prebuilt/prebuilt-dev-app';
+import {
+ PREBUILT_BUILD_OPTIONS,
+ readPrebuiltManifest,
+} from '../stress/_helpers/prebuilt/prebuilt-serving';
+
+vi.mock('node:fs', async (importOriginal) => {
+ const fs = await importOriginal<typeof import('node:fs')>();
+ return { ...fs, readdirSync: vi.fn(fs.readdirSync) };
+});
+
+const actualFs = await vi.importActual<typeof import('node:fs')>('node:fs');
+
+const OK_TREE: Readonly<Record<string, string>> = {
+ 'package.json': '{"name":"open-knowledge"}\n',
+ 'pnpm-lock.yaml': "lockfileVersion: '9.0'\n",
+ 'packages/app/package.json': '{"name":"@inkeep/open-knowledge-app"}\n',
+ 'packages/app/index.html': '<!doctype html><div id="root"></div>\n',
+ 'packages/app/src/App.tsx': 'export const App = () => 1;\n',
+ 'packages/core/src/index.ts': 'export const core = 1;\n',
+ 'packages/server/src/index.ts': 'export const server = 1;\n',
+};
+
+const roots: string[] = [];
+
+function scratch(prefix: string): string {
+ const root = mkdtempSync(join(tmpdir(), prefix));
+ roots.push(root);
+ return root;
+}
+
+function write(root: string, path: string, content: string): void {
+ mkdirSync(dirname(join(root, path)), { recursive: true });
+ writeFileSync(join(root, path), content);
+}
+
+function okTree(): string {
+ const root = scratch('ok-prebuilt-concurrent-prune-tree-');
+ for (const [path, content] of Object.entries(OK_TREE)) write(root, path, content);
+ return root;
+}
+
+function resolveWith(okRoot: string, outRoot: string) {
+ return resolvePrebuiltDevApp({
+ okRoot,
+ outRoot,
+ options: PREBUILT_BUILD_OPTIONS,
+ build: async (distDir: string) => {
+ const app = readFileSync(join(okRoot, 'packages/app/src/App.tsx'), 'utf8');
+ write(distDir, 'index.html', `<!doctype html>${app}`);
+ },
+ });
+}
+
+function anotherBuilderRetiresAfterTheListing(outRoot: string, digest: string): void {
+ vi.mocked(readdirSync).mockImplementation(((path: string, options?: unknown) => {
+ const listing = actualFs.readdirSync(path, options as never);
+ if (path === outRoot && (options as { withFileTypes?: boolean })?.withFileTypes === true) {
+ vi.mocked(readdirSync).mockImplementation(actualFs.readdirSync);
+ actualFs.rmSync(join(outRoot, digest), { recursive: true, force: true });
+ }
+ return listing;
+ }) as typeof readdirSync);
+}
+
+afterEach(() => {
+ vi.mocked(readdirSync).mockImplementation(actualFs.readdirSync);
+ for (const root of roots.splice(0)) actualFs.rmSync(root, { recursive: true, force: true });
+});
+
+describe('builders that prune the same output root together', () => {
+ test('a stale build another builder retires after this prune listed it is skipped, and the build still records itself as latest', async () => {
+ const okRoot = okTree();
+ const outRoot = scratch('ok-prebuilt-concurrent-prune-out-');
+ const stale = await resolveWith(okRoot, outRoot);
+ stale.use.release();
+ write(okRoot, 'packages/app/src/App.tsx', 'export const App = () => 2;\n');
+ const kept = await resolveWith(okRoot, outRoot);
+ kept.use.release();
+
+ write(okRoot, 'packages/app/src/App.tsx', 'export const App = () => 3;\n');
+ anotherBuilderRetiresAfterTheListing(outRoot, stale.manifest.digest);
+ const current = await resolveWith(okRoot, outRoot);
+ current.use.release();
+
+ expect(current.built).toBe(true);
+ expect(readPrebuiltManifest(join(outRoot, 'latest.json'))?.digest).toBe(
+ current.manifest.digest,
+ );
+ expect(actualFs.readdirSync(outRoot).sort()).toEqual(
+ ['.uses', 'latest.json', kept.manifest.digest, current.manifest.digest].sort(),
+ );
+ });
+});
diff --git a/packages/app/tests/foundation/prebuilt-dev-app.node.test.ts b/packages/app/tests/foundation/prebuilt-dev-app.node.test.ts
new file mode 100644
index 000000000..ae0c83fe5
--- /dev/null
+++ b/packages/app/tests/foundation/prebuilt-dev-app.node.test.ts
@@ -0,0 +1,298 @@
+import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { dirname, join } from 'node:path';
+import { afterEach, describe, expect, test } from 'vitest';
+import { resolvePrebuiltDevApp } from '../stress/_helpers/prebuilt/prebuilt-dev-app';
+import {
+ E2E_SERVING_ENV,
+ e2eServingMode,
+ PREBUILT_APP_ENV,
+ PREBUILT_BUILD_OPTIONS,
+ type PrebuiltServing,
+ prebuiltServerCommand,
+ workerServingFromEnv,
+} from '../stress/_helpers/prebuilt/prebuilt-serving';
+
+const OK_TREE: Readonly<Record<string, string>> = {
+ 'package.json': '{"name":"open-knowledge"}\n',
+ 'pnpm-lock.yaml': "lockfileVersion: '9.0'\n",
+ 'packages/app/package.json': '{"name":"@inkeep/open-knowledge-app"}\n',
+ 'packages/app/index.html': '<!doctype html><div id="root"></div>\n',
+ 'packages/app/src/App.tsx': 'export const App = () => null;\n',
+ 'packages/core/src/index.ts': 'export const core = 1;\n',
+ 'packages/server/src/index.ts': 'export const server = 1;\n',
+};
+
+const OPTIONS = PREBUILT_BUILD_OPTIONS;
+
+function prebuiltServingFrom(env: Readonly<Record<string, string>>): PrebuiltServing {
+ const serving = workerServingFromEnv(env);
+ if (serving.kind === 'prebuilt') return serving;
+ throw new Error(`expected prebuilt serving, got ${serving.kind}`);
+}
+
+const roots: string[] = [];
+
+function scratch(prefix: string): string {
+ const root = mkdtempSync(join(tmpdir(), prefix));
+ roots.push(root);
+ return root;
+}
+
+function okTree(): string {
+ const root = scratch('ok-prebuilt-tree-');
+ for (const [path, content] of Object.entries(OK_TREE)) write(root, path, content);
+ return root;
+}
+
+function write(root: string, path: string, content: string): void {
+ mkdirSync(dirname(join(root, path)), { recursive: true });
+ writeFileSync(join(root, path), content);
+}
+
+function bundler(okRoot: string) {
+ const builds: string[] = [];
+ return {
+ builds,
+ build: async (distDir: string) => {
+ builds.push(distDir);
+ mkdirSync(distDir, { recursive: true });
+ const app = readFileSync(join(okRoot, 'packages/app/src/App.tsx'), 'utf8');
+ writeFileSync(join(distDir, 'index.html'), `<!doctype html><script>${app}</script>\n`);
+ },
+ };
+}
+
+afterEach(() => {
+ for (const root of roots.splice(0)) rmSync(root, { recursive: true, force: true });
+});
+
+describe('the prebuilt DEV app output', () => {
+ test('a first run builds it once and a later run with the same inputs serves that build', async () => {
+ const okRoot = okTree();
+ const outRoot = scratch('ok-prebuilt-out-');
+ const { builds, build } = bundler(okRoot);
+
+ const first = await resolvePrebuiltDevApp({ okRoot, outRoot, options: OPTIONS, build });
+ const second = await resolvePrebuiltDevApp({ okRoot, outRoot, options: OPTIONS, build });
+
+ expect(first.built).toBe(true);
+ expect(first.decision).toEqual({ serve: false, reason: 'missing' });
+ expect(second.built).toBe(false);
+ expect(second.decision).toEqual({ serve: true });
+ expect(second.distDir).toBe(first.distDir);
+ expect(builds).toHaveLength(1);
+ expect(readFileSync(join(second.distDir, 'index.html'), 'utf8')).toContain('App = () => null');
+ expect(existsSync(builds[0] ?? '')).toBe(false);
+ });
+
+ test('a server edit refuses the earlier build, names the file and serves a new build', async () => {
+ const okRoot = okTree();
+ const outRoot = scratch('ok-prebuilt-out-');
+ const { builds, build } = bundler(okRoot);
+ const before = await resolvePrebuiltDevApp({ okRoot, outRoot, options: OPTIONS, build });
+
+ write(okRoot, 'packages/server/src/index.ts', 'export const server = 2;\n');
+ const after = await resolvePrebuiltDevApp({ okRoot, outRoot, options: OPTIONS, build });
+
+ expect(after.decision).toEqual({
+ serve: false,
+ reason: 'stale',
+ changedInputs: ['packages/server/src/index.ts'],
+ changedOptions: [],
+ });
+ expect(after.built).toBe(true);
+ expect(after.distDir).not.toBe(before.distDir);
+ expect(builds).toHaveLength(2);
+ });
+
+ test('an app edit is served from a build that contains it', async () => {
+ const okRoot = okTree();
+ const outRoot = scratch('ok-prebuilt-out-');
+ const { build } = bundler(okRoot);
+ await resolvePrebuiltDevApp({ okRoot, outRoot, options: OPTIONS, build });
+
+ write(okRoot, 'packages/app/src/App.tsx', 'export const App = () => "edited";\n');
+ const after = await resolvePrebuiltDevApp({ okRoot, outRoot, options: OPTIONS, build });
+
+ expect(readFileSync(join(after.distDir, 'index.html'), 'utf8')).toContain('"edited"');
+ });
+
+ test('a changed build option refuses the earlier build and names the option', async () => {
+ const okRoot = okTree();
+ const outRoot = scratch('ok-prebuilt-out-');
+ const { build } = bundler(okRoot);
+ await resolvePrebuiltDevApp({ okRoot, outRoot, options: OPTIONS, build });
+
+ const after = await resolvePrebuiltDevApp({
+ okRoot,
+ outRoot,
+ options: { ...OPTIONS, mode: 'production' },
+ build,
+ });
+
+ expect(after.decision).toEqual({
+ serve: false,
+ reason: 'stale',
+ changedInputs: [],
+ changedOptions: ['mode'],
+ });
+ expect(after.built).toBe(true);
+ });
+
+ test('a failed build records nothing, so the next run builds again', async () => {
+ const okRoot = okTree();
+ const outRoot = scratch('ok-prebuilt-out-');
+ const { builds, build } = bundler(okRoot);
+ const failing = async (distDir: string) => {
+ await build(distDir);
+ throw new Error('vite build exited 1');
+ };
+
+ await expect(
+ resolvePrebuiltDevApp({ okRoot, outRoot, options: OPTIONS, build: failing }),
+ ).rejects.toThrow('vite build exited 1');
+ const retried = await resolvePrebuiltDevApp({ okRoot, outRoot, options: OPTIONS, build });
+
+ expect(retried.built).toBe(true);
+ expect(retried.decision).toEqual({ serve: false, reason: 'missing' });
+ expect(builds).toHaveLength(2);
+ });
+
+ test('a build that writes no index.html is refused and not served later', async () => {
+ const okRoot = okTree();
+ const outRoot = scratch('ok-prebuilt-out-');
+ const { builds, build } = bundler(okRoot);
+ const empty = async (distDir: string) => {
+ mkdirSync(distDir, { recursive: true });
+ };
+
+ await expect(
+ resolvePrebuiltDevApp({ okRoot, outRoot, options: OPTIONS, build: empty }),
+ ).rejects.toThrow('wrote no index.html');
+ const retried = await resolvePrebuiltDevApp({ okRoot, outRoot, options: OPTIONS, build });
+
+ expect(retried.built).toBe(true);
+ expect(builds).toHaveLength(1);
+ });
+
+ test('an unreadable build manifest is not served and the app is rebuilt in its place', async () => {
+ const okRoot = okTree();
+ const outRoot = scratch('ok-prebuilt-out-');
+ const { builds, build } = bundler(okRoot);
+ const first = await resolvePrebuiltDevApp({ okRoot, outRoot, options: OPTIONS, build });
+
+ writeFileSync(join(first.outputDir, 'prebuilt-manifest.json'), '{"digest":');
+ const second = await resolvePrebuiltDevApp({ okRoot, outRoot, options: OPTIONS, build });
+
+ expect(second.built).toBe(true);
+ expect(second.outputDir).toBe(first.outputDir);
+ expect(builds).toHaveLength(2);
+ });
+
+ test('two runs building the same inputs at once both serve the build that landed first', async () => {
+ const okRoot = okTree();
+ const outRoot = scratch('ok-prebuilt-out-');
+ const { builds, build } = bundler(okRoot);
+ let overtaken: Promise<unknown> | undefined;
+ const slow = async (distDir: string) => {
+ overtaken = resolvePrebuiltDevApp({ okRoot, outRoot, options: OPTIONS, build });
+ await overtaken;
+ await build(distDir);
+ };
+
+ const late = await resolvePrebuiltDevApp({ okRoot, outRoot, options: OPTIONS, build: slow });
+ const early = await overtaken;
+
+ expect(late.built).toBe(false);
+ expect(early).toMatchObject({ built: true, distDir: late.distDir });
+ expect(builds).toHaveLength(2);
+ expect(readFileSync(join(late.distDir, 'index.html'), 'utf8')).toContain('App = () => null');
+ });
+
+ test('a new build keeps the one it replaces and discards older ones', async () => {
+ const okRoot = okTree();
+ const outRoot = scratch('ok-prebuilt-out-');
+ const { build } = bundler(okRoot);
+ const oldest = await resolvePrebuiltDevApp({ okRoot, outRoot, options: OPTIONS, build });
+ oldest.use.release();
+ write(okRoot, 'packages/app/src/App.tsx', 'export const App = () => 2;\n');
+ const previous = await resolvePrebuiltDevApp({ okRoot, outRoot, options: OPTIONS, build });
+ write(okRoot, 'packages/app/src/App.tsx', 'export const App = () => 3;\n');
+ const current = await resolvePrebuiltDevApp({ okRoot, outRoot, options: OPTIONS, build });
+
+ expect(existsSync(current.distDir)).toBe(true);
+ expect(existsSync(previous.distDir)).toBe(true);
+ expect(existsSync(oldest.outputDir)).toBe(false);
+ });
+});
+
+describe('the serving mode a worker starts', () => {
+ test('without the flag a worker runs the dev server', () => {
+ expect(workerServingFromEnv({})).toEqual({ kind: 'dev-server' });
+ });
+
+ test('a fresh prebuilt app is served by vite preview from its own directory', async () => {
+ const okRoot = okTree();
+ const app = await resolvePrebuiltDevApp({
+ okRoot,
+ outRoot: scratch('ok-prebuilt-out-'),
+ options: OPTIONS,
+ build: bundler(okRoot).build,
+ });
+ const serving = prebuiltServingFrom({
+ [E2E_SERVING_ENV]: 'prebuilt',
+ [PREBUILT_APP_ENV]: app.outputDir,
+ });
+
+ expect(prebuiltServerCommand('::1', serving, okRoot)).toEqual({
+ args: ['exec', 'vite', 'preview', '--config', 'vite.prebuilt-dev.ts', '--host', '::1'],
+ env: { OK_TEST_PREBUILT_OUT_DIR: app.distDir },
+ });
+ });
+
+ test('a core edit after the build makes a worker refuse the prebuilt app and name the file', async () => {
+ const okRoot = okTree();
+ const app = await resolvePrebuiltDevApp({
+ okRoot,
+ outRoot: scratch('ok-prebuilt-out-'),
+ options: OPTIONS,
+ build: bundler(okRoot).build,
+ });
+ const serving = prebuiltServingFrom({
+ [E2E_SERVING_ENV]: 'prebuilt',
+ [PREBUILT_APP_ENV]: app.outputDir,
+ });
+
+ write(okRoot, 'packages/core/src/index.ts', 'export const core = 2;\n');
+
+ expect(() => prebuiltServerCommand('127.0.0.1', serving, okRoot)).toThrow(
+ `the prebuilt DEV app at ${app.outputDir} is stale, because packages/core/src/index.ts changed since it was built`,
+ );
+ });
+
+ test('a prebuilt directory without a build manifest is refused', () => {
+ const outputDir = scratch('ok-prebuilt-unbuilt-');
+ const serving = prebuiltServingFrom({
+ [E2E_SERVING_ENV]: 'prebuilt',
+ [PREBUILT_APP_ENV]: outputDir,
+ });
+
+ expect(() => prebuiltServerCommand('127.0.0.1', serving, okTree())).toThrow(
+ `the prebuilt DEV app at ${outputDir} has no build manifest`,
+ );
+ });
+
+ test('the prebuilt flag without a prepared app refuses rather than falling back', () => {
+ expect(() => workerServingFromEnv({ [E2E_SERVING_ENV]: 'prebuilt' })).toThrow(
+ `${E2E_SERVING_ENV}=prebuilt serves the prebuilt DEV app its globalSetup records in ${PREBUILT_APP_ENV}`,
+ );
+ });
+
+ test('an unknown serving flag is refused', () => {
+ expect(() => e2eServingMode({ [E2E_SERVING_ENV]: 'prebuild' })).toThrow(
+ `${E2E_SERVING_ENV}=prebuild names no serving mode`,
+ );
+ expect(e2eServingMode({ [E2E_SERVING_ENV]: '' })).toBe('dev-server');
+ });
+});
diff --git a/packages/app/tests/foundation/prebuilt-dev-output.node.test.ts b/packages/app/tests/foundation/prebuilt-dev-output.node.test.ts
new file mode 100644
index 000000000..a5ad6915e
--- /dev/null
+++ b/packages/app/tests/foundation/prebuilt-dev-output.node.test.ts
@@ -0,0 +1,77 @@
+import { mkdirSync, mkdtempSync, realpathSync, rmSync, writeFileSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { dirname, join } from 'node:path';
+import { pathToFileURL } from 'node:url';
+import { build, type Rolldown } from 'vite';
+import { afterAll, describe, expect, test } from 'vitest';
+import { PREBUILT_OUTPUT_OPTIONS } from '../../src/build/prebuilt-dev-contract';
+
+const APP: Readonly<Record<string, string>> = {
+ 'src/selection.ts': "export class NodeSelection {\n readonly from = 'selection';\n}\n",
+ 'src/other-selection.ts':
+ "export class NodeSelection {\n readonly from = 'other-selection';\n}\n",
+ 'src/page-modules.ts':
+ "import { NodeSelection as OtherSelection } from './other-selection.ts';\nimport { NodeSelection } from './selection.ts';\n\nexport const selections = [new NodeSelection(), new OtherSelection()];\n",
+};
+
+const roots: string[] = [];
+
+function scratch(prefix: string): string {
+ const root = realpathSync(mkdtempSync(join(tmpdir(), prefix)));
+ roots.push(root);
+ return root;
+}
+
+function app(): string {
+ const root = scratch('ok-prebuilt-output-app-');
+ for (const [path, content] of Object.entries(APP)) {
+ mkdirSync(dirname(join(root, path)), { recursive: true });
+ writeFileSync(join(root, path), content);
+ }
+ return root;
+}
+
+function entryChunk(output: Awaited<ReturnType<typeof build>>): Rolldown.OutputChunk {
+ const outputs = Array.isArray(output) ? output : [output];
+ const chunk = outputs
+ .flatMap((result) => ('output' in result ? result.output : []))
+ .find((item) => item.type === 'chunk' && item.isEntry && item.name === 'page-modules');
+ if (chunk?.type !== 'chunk') throw new Error('the build wrote no page-modules entry chunk');
+ return chunk;
+}
+
+afterAll(() => {
+ for (const root of roots.splice(0)) rmSync(root, { recursive: true, force: true });
+});
+
+describe('the prebuilt DEV build output', () => {
+ test('classes keep their source names, as the dev server serves them, where bundling and minifying would rename them', async () => {
+ const appRoot = app();
+ const outDir = join(scratch('ok-prebuilt-output-dist-'), 'dist');
+
+ const chunk = entryChunk(
+ await build({
+ configFile: false,
+ root: appRoot,
+ logLevel: 'silent',
+ build: {
+ outDir,
+ emptyOutDir: true,
+ rolldownOptions: {
+ input: { 'page-modules': join(appRoot, 'src/page-modules.ts') },
+ preserveEntrySignatures: 'exports-only',
+ output: PREBUILT_OUTPUT_OPTIONS,
+ },
+ },
+ }),
+ );
+ const { selections } = (await import(pathToFileURL(join(outDir, chunk.fileName)).href)) as {
+ selections: readonly object[];
+ };
+
+ expect(selections.map((selection) => selection.constructor.name)).toEqual([
+ 'NodeSelection',
+ 'NodeSelection',
+ ]);
+ });
+});
diff --git a/packages/app/tests/foundation/prebuilt-interrupted-prune.node.test.ts b/packages/app/tests/foundation/prebuilt-interrupted-prune.node.test.ts
new file mode 100644
index 000000000..bc6086096
--- /dev/null
+++ b/packages/app/tests/foundation/prebuilt-interrupted-prune.node.test.ts
@@ -0,0 +1,180 @@
+import {
+ existsSync,
+ mkdirSync,
+ mkdtempSync,
+ readdirSync,
+ readFileSync,
+ rmSync,
+ statSync,
+ writeFileSync,
+} from 'node:fs';
+import { tmpdir } from 'node:os';
+import { dirname, join } from 'node:path';
+import { afterEach, describe, expect, test, vi } from 'vitest';
+import { resolvePrebuiltDevApp } from '../stress/_helpers/prebuilt/prebuilt-dev-app';
+import {
+ E2E_SERVING_ENV,
+ PREBUILT_APP_ENV,
+ PREBUILT_BUILD_OPTIONS,
+ PREBUILT_MANIFEST_FILE,
+ type PrebuiltServing,
+ prebuiltServerCommand,
+ readPrebuiltManifest,
+ workerServingFromEnv,
+} from '../stress/_helpers/prebuilt/prebuilt-serving';
+
+vi.mock('node:fs', async (importOriginal) => {
+ const fs = await importOriginal<typeof import('node:fs')>();
+ return { ...fs, rmSync: vi.fn(fs.rmSync) };
+});
+
+const actualFs = await vi.importActual<typeof import('node:fs')>('node:fs');
+
+const OK_TREE: Readonly<Record<string, string>> = {
+ 'package.json': '{"name":"open-knowledge"}\n',
+ 'pnpm-lock.yaml': "lockfileVersion: '9.0'\n",
+ 'packages/app/package.json': '{"name":"@inkeep/open-knowledge-app"}\n',
+ 'packages/app/index.html': '<!doctype html><div id="root"></div>\n',
+ 'packages/app/src/App.tsx': 'export const App = () => 1;\n',
+ 'packages/core/src/index.ts': 'export const core = 1;\n',
+ 'packages/server/src/index.ts': 'export const server = 1;\n',
+};
+
+const roots: string[] = [];
+
+function scratch(prefix: string): string {
+ const root = mkdtempSync(join(tmpdir(), prefix));
+ roots.push(root);
+ return root;
+}
+
+function write(root: string, path: string, content: string): void {
+ mkdirSync(dirname(join(root, path)), { recursive: true });
+ writeFileSync(join(root, path), content);
+}
+
+function okTree(): string {
+ const root = scratch('ok-prebuilt-prune-tree-');
+ for (const [path, content] of Object.entries(OK_TREE)) write(root, path, content);
+ return root;
+}
+
+function bundler(okRoot: string) {
+ return async (distDir: string) => {
+ const app = readFileSync(join(okRoot, 'packages/app/src/App.tsx'), 'utf8');
+ write(distDir, 'index.html', `<!doctype html><script src="/assets/app.js"></script>${app}`);
+ write(distDir, 'assets/app.js', app);
+ write(distDir, 'assets/vendor.js', 'export {};\n');
+ };
+}
+
+function resolveWith(okRoot: string, outRoot: string) {
+ return resolvePrebuiltDevApp({
+ okRoot,
+ outRoot,
+ options: PREBUILT_BUILD_OPTIONS,
+ build: bundler(okRoot),
+ });
+}
+
+function buildDirHolding(path: string, digest: string): string | undefined {
+ const candidates = [path];
+ if (statSync(path, { throwIfNoEntry: false })?.isDirectory() === true) {
+ candidates.push(...readdirSync(path).map((name) => join(path, name)));
+ }
+ return candidates.find(
+ (dir) => readPrebuiltManifest(join(dir, PREBUILT_MANIFEST_FILE))?.digest === digest,
+ );
+}
+
+class Interrupted extends Error {}
+
+function interruptTheDeletionOf(digest: string, removedBeforeTheStop: readonly string[]): void {
+ vi.mocked(rmSync).mockImplementation((path, options) => {
+ const buildDir = buildDirHolding(String(path), digest);
+ if (buildDir === undefined) {
+ actualFs.rmSync(path, options);
+ return;
+ }
+ vi.mocked(rmSync).mockImplementation(actualFs.rmSync);
+ for (const entry of removedBeforeTheStop) {
+ actualFs.rmSync(join(buildDir, entry), { recursive: true, force: true });
+ }
+ throw new Interrupted(`the deletion of ${buildDir} stopped part-way`);
+ });
+}
+
+function workerServing(outputDir: string): PrebuiltServing {
+ const serving = workerServingFromEnv({
+ [E2E_SERVING_ENV]: 'prebuilt',
+ [PREBUILT_APP_ENV]: outputDir,
+ });
+ if (serving.kind === 'prebuilt') return serving;
+ throw new Error(`expected prebuilt serving, got ${serving.kind}`);
+}
+
+afterEach(() => {
+ vi.mocked(rmSync).mockImplementation(actualFs.rmSync);
+ for (const root of roots.splice(0)) actualFs.rmSync(root, { recursive: true, force: true });
+});
+
+async function interruptedPruneThenRevisit(removedBeforeTheStop: readonly string[]) {
+ const okRoot = okTree();
+ const outRoot = scratch('ok-prebuilt-prune-out-');
+ const first = await resolveWith(okRoot, outRoot);
+ first.use.release();
+ write(okRoot, 'packages/app/src/App.tsx', 'export const App = () => 2;\n');
+ (await resolveWith(okRoot, outRoot)).use.release();
+
+ write(okRoot, 'packages/app/src/App.tsx', 'export const App = () => 3;\n');
+ interruptTheDeletionOf(first.manifest.digest, removedBeforeTheStop);
+ await expect(resolveWith(okRoot, outRoot)).rejects.toBeInstanceOf(Interrupted);
+
+ write(okRoot, 'packages/app/src/App.tsx', 'export const App = () => 1;\n');
+ const again = await resolveWith(okRoot, outRoot);
+ again.use.release();
+
+ expect(again.manifest.digest).toBe(first.manifest.digest);
+ expect(again.built).toBe(true);
+ expect(readFileSync(join(again.distDir, 'index.html'), 'utf8')).toContain('App = () => 1');
+ expect(readFileSync(join(again.distDir, 'assets', 'app.js'), 'utf8')).toContain('App = () => 1');
+ expect(prebuiltServerCommand('127.0.0.1', workerServing(again.outputDir), okRoot).env).toEqual({
+ OK_TEST_PREBUILT_OUT_DIR: again.distDir,
+ });
+}
+
+describe('a prune that stops part-way through deleting a build', () => {
+ test('when it leaves the build manifest over a dist without its index.html, the next run with those inputs builds again and serves a whole build', async () => {
+ await interruptedPruneThenRevisit(['dist/index.html']);
+ });
+
+ test('when it leaves the build manifest over a dist missing a chunk, the next run with those inputs builds again and serves a whole build', async () => {
+ await interruptedPruneThenRevisit(['dist/assets/app.js']);
+ });
+
+ test('when it leaves the build manifest and no dist, the next run with those inputs builds again and serves a whole build', async () => {
+ await interruptedPruneThenRevisit(['dist']);
+ });
+
+ test('when it leaves a dist without its build manifest, the next run with those inputs builds again and serves a whole build', async () => {
+ await interruptedPruneThenRevisit([PREBUILT_MANIFEST_FILE]);
+ });
+
+ test('a prune that runs to the end leaves nothing of the discarded build in the output root', async () => {
+ const okRoot = okTree();
+ const outRoot = scratch('ok-prebuilt-prune-out-');
+ const first = await resolveWith(okRoot, outRoot);
+ first.use.release();
+ write(okRoot, 'packages/app/src/App.tsx', 'export const App = () => 2;\n');
+ const second = await resolveWith(okRoot, outRoot);
+ second.use.release();
+ write(okRoot, 'packages/app/src/App.tsx', 'export const App = () => 3;\n');
+ const third = await resolveWith(okRoot, outRoot);
+ third.use.release();
+
+ expect(existsSync(first.outputDir)).toBe(false);
+ expect(readdirSync(outRoot).sort()).toEqual(
+ ['.uses', 'latest.json', second.manifest.digest, third.manifest.digest].sort(),
+ );
+ });
+});
diff --git a/packages/app/tests/foundation/served-modules.node.test.ts b/packages/app/tests/foundation/served-modules.node.test.ts
new file mode 100644
index 000000000..a5ba34c76
--- /dev/null
+++ b/packages/app/tests/foundation/served-modules.node.test.ts
@@ -0,0 +1,227 @@
+import { mkdirSync, mkdtempSync, realpathSync, rmSync, writeFileSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { dirname, join } from 'node:path';
+import { build } from 'vite';
+import { afterAll, beforeAll, describe, expect, test } from 'vitest';
+import { PREBUILT_MODULE_MANIFEST } from '../../src/build/prebuilt-dev-contract';
+import { prebuiltModuleChunksPlugin } from '../../src/build/prebuilt-module-chunks-plugin';
+import {
+ parseModuleChunks,
+ parseModuleManifest,
+ resolveServedModules,
+} from '../stress/_helpers/prebuilt/served-modules';
+
+const APP: Readonly<Record<string, string>> = {
+ 'index.html':
+ '<!doctype html><script>document.documentElement.lang = "en";</script><script type="module" src="/src/main.ts"></script>\n',
+ 'src/main.ts':
+ "import { shared } from './shared.ts';\ndocument.title = shared;\nvoid import('./lazy.ts');\nvoid import('./locales/es/messages.json');\nvoid import('./locales/fr/messages.json');\n",
+ 'src/lazy.ts': "import { child } from './lazy-child.ts';\nexport const lazy = 'lazy ' + child;\n",
+ 'src/lazy-child.ts': "export const child = 'child';\n",
+ 'src/shared.ts': "export const shared = 'shared';\n",
+ 'src/unused.ts': "export const unused = 'unused';\n",
+ 'src/page-modules.ts': "export { shared } from './shared.ts';\n",
+ 'src/locales/es/messages.json': '{"messages":{"hello":"hola"}}\n',
+ 'src/locales/fr/messages.json': '{"messages":{"hello":"bonjour"}}\n',
+};
+
+const roots: string[] = [];
+
+function scratch(prefix: string): string {
+ const root = realpathSync(mkdtempSync(join(tmpdir(), prefix)));
+ roots.push(root);
+ return root;
+}
+
+function app(files: Readonly<Record<string, string>> = APP): string {
+ const root = scratch('ok-served-modules-app-');
+ for (const [path, content] of Object.entries(files)) {
+ mkdirSync(dirname(join(root, path)), { recursive: true });
+ writeFileSync(join(root, path), content);
+ }
+ return root;
+}
+
+let appRoot: string;
+let outputDir: string;
+
+beforeAll(async () => {
+ appRoot = app();
+ outputDir = scratch('ok-served-modules-output-');
+ await build({
+ configFile: false,
+ root: appRoot,
+ logLevel: 'silent',
+ plugins: [prebuiltModuleChunksPlugin(appRoot)],
+ build: {
+ outDir: join(outputDir, 'dist'),
+ emptyOutDir: true,
+ manifest: PREBUILT_MODULE_MANIFEST,
+ rolldownOptions: {
+ input: {
+ index: join(appRoot, 'index.html'),
+ 'page-modules': join(appRoot, 'src/page-modules.ts'),
+ },
+ preserveEntrySignatures: 'exports-only',
+ },
+ },
+ });
+});
+
+afterAll(() => {
+ for (const root of roots.splice(0)) rmSync(root, { recursive: true, force: true });
+});
+
+const at = (path: string) => new URL(path, 'http://127.0.0.1:51234');
+
+describe('modules served by the dev server', () => {
+ test('a script module is requested at its source path', () => {
+ const modules = resolveServedModules({ kind: 'dev-server' }, appRoot);
+ expect(modules.moduleUrl('src/page-modules.ts')).toBe('/src/page-modules.ts');
+ });
+
+ test('a JSON module is requested as an import, the way the app’s own import requests it', () => {
+ const modules = resolveServedModules({ kind: 'dev-server' }, appRoot);
+ expect(modules.moduleUrl('src/locales/es/messages.json')).toBe(
+ '/src/locales/es/messages.json?import',
+ );
+ });
+
+ test('a source the app does not have is refused rather than given a URL nothing serves', () => {
+ const modules = resolveServedModules({ kind: 'dev-server' }, appRoot);
+ expect(() => modules.moduleUrl('src/locales/de/messages.json')).toThrow(
+ 'src/locales/de/messages.json is not a source file of the app, so the dev server serves no module for it',
+ );
+ });
+
+ test('the entry script is the shell’s one module script', () => {
+ const modules = resolveServedModules({ kind: 'dev-server' }, appRoot);
+ expect(modules.entryScriptUrl()).toBe('/src/main.ts');
+ });
+
+ test('a shell with two module scripts names no single entry script', () => {
+ const twoEntries = app({
+ 'index.html':
+ '<!doctype html><script type="module" src="/src/main.ts"></script><script src="/classic.js"></script><script type="module" src="/src/second.ts"></script>\n',
+ });
+ const modules = resolveServedModules({ kind: 'dev-server' }, twoEntries);
+ expect(() => modules.entryScriptUrl()).toThrow(
+ 'index.html has 2 module scripts, so it names no single entry script',
+ );
+ });
+
+ test('a module route matches the request with any query and nothing else', () => {
+ const route = resolveServedModules({ kind: 'dev-server' }, appRoot).moduleRoute(
+ 'src/locales/es/messages.json',
+ );
+ expect(route(at('/src/locales/es/messages.json?import'))).toBe(true);
+ expect(route(at('/src/locales/es/messages.json?import&t=1791000000000'))).toBe(true);
+ expect(route(at('/src/locales/fr/messages.json?import'))).toBe(false);
+ expect(route(at('/locales/es/messages.json'))).toBe(false);
+ });
+
+ test('a module another module imports is delayed at its own request', () => {
+ const route = resolveServedModules({ kind: 'dev-server' }, appRoot).moduleRoute(
+ 'src/lazy-child.ts',
+ );
+ expect(route(at('/src/lazy-child.ts'))).toBe(true);
+ expect(route(at('/src/lazy.ts'))).toBe(false);
+ });
+});
+
+describe('modules served by the prebuilt DEV app', () => {
+ test('a lazily loaded module is requested at its own chunk', () => {
+ const modules = resolveServedModules({ kind: 'prebuilt', outputDir }, appRoot);
+ const route = modules.moduleRoute('src/locales/es/messages.json');
+ const url = modules.moduleUrl('src/locales/es/messages.json');
+ expect(url).toMatch(/^\/assets\/messages-[\w-]+\.js$/);
+ expect(route(at(url))).toBe(true);
+ expect(route(at(modules.moduleUrl('src/locales/fr/messages.json')))).toBe(false);
+ });
+
+ test('a separate entry keeps the exports a page imports from it', async () => {
+ const modules = resolveServedModules({ kind: 'prebuilt', outputDir }, appRoot);
+ const chunk: { shared?: unknown } = await import(
+ join(outputDir, 'dist', modules.moduleUrl('src/page-modules.ts'))
+ );
+ expect(chunk.shared).toBe('shared');
+ });
+
+ test('the entry script is the chunk the shell starts', () => {
+ const modules = resolveServedModules({ kind: 'prebuilt', outputDir }, appRoot);
+ const entry = modules.entryScriptUrl();
+ expect(entry).toMatch(/^\/assets\/index-[\w-]+\.js$/);
+ expect(modules.entryScriptRoute()(at(entry))).toBe(true);
+ expect(modules.entryScriptRoute()(at(modules.moduleUrl('src/lazy.ts')))).toBe(false);
+ });
+
+ test('a module bundled into another chunk is refused, since no request serves it alone', () => {
+ const modules = resolveServedModules({ kind: 'prebuilt', outputDir }, appRoot);
+ expect(() => modules.moduleUrl('src/main.ts')).toThrow(
+ /^src\/main\.ts names no chunk of the prebuilt DEV app \(.+\), so no request serves it alone$/,
+ );
+ });
+
+ test('a module bundled into its importer’s chunk is delayed through that chunk when its importer is too', () => {
+ const modules = resolveServedModules({ kind: 'prebuilt', outputDir }, appRoot);
+ const lazyChunk = modules.moduleUrl('src/lazy.ts');
+ const route = modules.moduleRoute('src/lazy.ts', 'src/lazy-child.ts');
+ expect(route(at(lazyChunk))).toBe(true);
+ expect(route(at(modules.moduleUrl('src/locales/es/messages.json')))).toBe(false);
+ });
+
+ test('a module bundled into a chunk the route does not otherwise delay is refused', () => {
+ const modules = resolveServedModules({ kind: 'prebuilt', outputDir }, appRoot);
+ expect(() => modules.moduleRoute('src/lazy-child.ts')).toThrow(
+ /^src\/lazy-child\.ts is bundled into assets\/lazy-[\w-]+\.js, the chunk of src\/lazy\.ts, which this route does not delay, so delaying it would delay more than src\/lazy-child\.ts$/,
+ );
+ });
+
+ test('a module bundled into a chunk two entries share is refused, since no module loads that chunk on its own', () => {
+ const modules = resolveServedModules({ kind: 'prebuilt', outputDir }, appRoot);
+ expect(() => modules.moduleRoute('src/shared.ts')).toThrow(
+ /^src\/shared\.ts is bundled into assets\/[\w-]+\.js, which no single module loads on its own, so delaying it would delay more than src\/shared\.ts$/,
+ );
+ });
+
+ test('a module bundled into the entry chunk is refused, since delaying it would hold the whole app', () => {
+ const modules = resolveServedModules({ kind: 'prebuilt', outputDir }, appRoot);
+ expect(() => modules.moduleRoute('src/main.ts')).toThrow(
+ /^src\/main\.ts is bundled into assets\/index-[\w-]+\.js, the chunk of index\.html, which this route does not delay/,
+ );
+ });
+
+ test('a source the build never bundled is refused rather than given a route that matches nothing', () => {
+ const modules = resolveServedModules({ kind: 'prebuilt', outputDir }, appRoot);
+ expect(() => modules.moduleRoute('src/unused.ts')).toThrow(
+ /^src\/unused\.ts is in no chunk of the prebuilt DEV app \(.+\)$/,
+ );
+ });
+
+ test('a route with no source to match is refused', () => {
+ const modules = resolveServedModules({ kind: 'prebuilt', outputDir }, appRoot);
+ expect(() => modules.moduleRoute()).toThrow(
+ 'a module route needs at least one source to match',
+ );
+ });
+});
+
+describe('the prebuilt module manifest', () => {
+ test('an entry naming no file is refused', () => {
+ expect(() => parseModuleManifest({ 'src/lazy.ts': { isDynamicEntry: true } })).toThrow(
+ 'the prebuilt module manifest entry for src/lazy.ts names no file',
+ );
+ });
+
+ test('a chunk map entry naming no chunk is refused', () => {
+ expect(() => parseModuleChunks({ 'src/lazy.ts': '' })).toThrow(
+ 'the prebuilt module chunk map names no chunk for src/lazy.ts',
+ );
+ });
+
+ test('a manifest that is not an object is refused', () => {
+ expect(() => parseModuleManifest(['assets/index.js'])).toThrow(
+ 'the prebuilt module manifest is not a JSON object',
+ );
+ });
+});
diff --git a/packages/app/tests/foundation/server-lease-acquire-failure.node.test.ts b/packages/app/tests/foundation/server-lease-acquire-failure.node.test.ts
new file mode 100644
index 000000000..b6ad03ebe
--- /dev/null
+++ b/packages/app/tests/foundation/server-lease-acquire-failure.node.test.ts
@@ -0,0 +1,197 @@
+import { mkdtempSync, realpathSync, rmSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import type { Browser, WorkerInfo } from '@playwright/test';
+import { afterEach, describe, expect, test, vi } from 'vitest';
+import { WORKER_SERVER_SETUP_STARVATION_LINE_MS } from '../stress/_helpers/fixtures.ts';
+import { E2E_SERVING_ENV } from '../stress/_helpers/prebuilt/prebuilt-serving.ts';
+import { leaseWorkerServer, type ServerLease } from '../stress/_helpers/reuse/lease-fixture.ts';
+import { REUSE_RUN_ENV, REUSE_SOCKET_ENV } from '../stress/_helpers/reuse/lease-protocol.ts';
+import {
+ type LaunchOwnedServer,
+ WorkerServerSupervisor,
+} from '../stress/_helpers/reuse/server-supervisor.ts';
+import {
+ connectLeaseClient,
+ type SupervisorEndpoint,
+ serveSupervisor,
+} from '../stress/_helpers/reuse/supervisor-socket.ts';
+import { APP_PACKAGE_ROOT } from '../stress/_helpers/seed-key.ts';
+import { contractServerLauncher } from './contract-dev-server.test-helper.ts';
+
+const WORKTREE = realpathSync(join(APP_PACKAGE_ROOT, '..', '..'));
+const CALL_MS = 30_000;
+const DISCARD_MS = 30_000;
+const LEASE_ENV_NAMES = [REUSE_SOCKET_ENV, REUSE_RUN_ENV, E2E_SERVING_ENV] as const;
+
+const WORKER_INFO = {
+ parallelIndex: 0,
+ workerIndex: 0,
+ config: { configFile: '/worktree/packages/app/playwright.config.ts' },
+ project: { name: 'chromium' },
+} as WorkerInfo;
+
+const BROWSER_THE_FAILED_LEASE_NEVER_OPENS = {} as Browser;
+
+const savedEnv = new Map(LEASE_ENV_NAMES.map((name) => [name, process.env[name]]));
+const openEndpoints: SupervisorEndpoint[] = [];
+const openSupervisors: WorkerServerSupervisor[] = [];
+const scratchDirs: string[] = [];
+const openGates: Array<(launch: boolean) => void> = [];
+
+afterEach(async () => {
+ vi.useRealTimers();
+ for (const [name, value] of savedEnv) {
+ if (value === undefined) delete process.env[name];
+ else process.env[name] = value;
+ }
+ for (const openGate of openGates.splice(0)) openGate(false);
+ for (const endpoint of openEndpoints.splice(0)) await endpoint.close();
+ for (const supervisor of openSupervisors.splice(0)) await supervisor.shutdown();
+ for (const dir of scratchDirs.splice(0)) rmSync(dir, { recursive: true, force: true });
+});
+
+interface GatedSupervisor {
+ readonly supervisor: WorkerServerSupervisor;
+ readonly endpoint: SupervisorEndpoint;
+ readonly launcher: ReturnType<typeof contractServerLauncher>;
+ readonly launchStarted: Promise<void>;
+ openGate(launch: boolean): void;
+}
+
+async function supervisorWithGatedLaunch(): Promise<GatedSupervisor> {
+ const scratch = realpathSync(mkdtempSync(join(tmpdir(), 'ok-lease-acquire-failure-')));
+ scratchDirs.push(scratch);
+ const launcher = contractServerLauncher();
+ let markStarted: () => void = () => {};
+ const launchStarted = new Promise<void>((resolve) => {
+ markStarted = resolve;
+ });
+ let openGate: (launch: boolean) => void = () => {};
+ const gate = new Promise<boolean>((resolve) => {
+ openGate = resolve;
+ });
+ openGates.push(openGate);
+ const launch: LaunchOwnedServer = async (slot, env) => {
+ markStarted();
+ if (!(await gate)) throw new Error('the test closed the launch gate');
+ return launcher.launch(slot, env);
+ };
+ const supervisor = new WorkerServerSupervisor({
+ okRoot: WORKTREE,
+ launch,
+ identityTimeoutMs: 2_000,
+ });
+ openSupervisors.push(supervisor);
+ const endpoint = await serveSupervisor(supervisor, join(scratch, 'supervisor.sock'));
+ openEndpoints.push(endpoint);
+ return { supervisor, endpoint, launcher, launchStarted, openGate };
+}
+
+function leaseThroughWorkerFixture(
+ socketPath: string,
+ whileInUse: (lease: ServerLease | undefined) => Promise<void>,
+): Promise<void> {
+ process.env[REUSE_SOCKET_ENV] = socketPath;
+ process.env[REUSE_RUN_ENV] = 'run-b';
+ delete process.env[E2E_SERVING_ENV];
+ return leaseWorkerServer(
+ { workerServerEnv: {}, browser: BROWSER_THE_FAILED_LEASE_NEVER_OPENS },
+ whileInUse,
+ WORKER_INFO,
+ );
+}
+
+function failureOf(work: Promise<void>): Promise<unknown> {
+ return work.then(
+ () => undefined,
+ (error: unknown) => error,
+ );
+}
+
+async function serving(baseURL: string): Promise<boolean> {
+ try {
+ await fetch(`${baseURL}/__ok-test-server-identity`, { signal: AbortSignal.timeout(2_000) });
+ return true;
+ } catch {
+ return false;
+ }
+}
+
+describe('a worker whose lease request fails', () => {
+ test('a supervisor that shuts down while the slot is starting leaves the worker on its own server, with a note naming the closed connection', async () => {
+ const rig = await supervisorWithGatedLaunch();
+ const offered: Array<ServerLease | undefined> = [];
+
+ const fixtureFailure = failureOf(
+ leaseThroughWorkerFixture(rig.endpoint.socketPath, async (lease) => {
+ offered.push(lease);
+ }),
+ );
+ await rig.launchStarted;
+ const operator = await connectLeaseClient(rig.endpoint.socketPath, 5_000);
+ try {
+ await operator.shutdown(CALL_MS);
+ } finally {
+ operator.close();
+ }
+
+ expect(await fixtureFailure).toBeUndefined();
+ expect(offered).toEqual([
+ {
+ kind: 'fresh',
+ note: `slot 0: the lease request failed (the supervisor at ${rig.endpoint.socketPath} closed the lease connection), so this worker starts its own server`,
+ },
+ ]);
+ rig.openGate(false);
+ expect(await rig.endpoint.closed).toEqual({ released: true });
+ await expect.poll(() => rig.supervisor.status()[0]?.leasedBy).toBeNull();
+ expect(rig.launcher.launched).toEqual([]);
+ });
+
+ test('a supervisor that does not answer before the setup deadline leaves the worker on its own server, and the server it launches afterwards is discarded rather than held for that worker', async () => {
+ vi.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout', 'Date'] });
+ const rig = await supervisorWithGatedLaunch();
+ let finishUse: () => void = () => {};
+ const useMayFinish = new Promise<void>((resolve) => {
+ finishUse = resolve;
+ });
+ let markOffered: (lease: ServerLease | undefined) => void = () => {};
+ const offered = new Promise<ServerLease | undefined>((resolve) => {
+ markOffered = resolve;
+ });
+
+ const fixtureFailure = failureOf(
+ leaseThroughWorkerFixture(rig.endpoint.socketPath, async (lease) => {
+ markOffered(lease);
+ await useMayFinish;
+ }),
+ );
+ await rig.launchStarted;
+ await vi.advanceTimersByTimeAsync(WORKER_SERVER_SETUP_STARVATION_LINE_MS);
+ vi.useRealTimers();
+ const first = await Promise.race([
+ offered.then((lease) => ({ offered: lease })),
+ fixtureFailure.then((failure) => ({ fixtureEndedWith: failure })),
+ ]);
+
+ expect(first).toEqual({
+ offered: {
+ kind: 'fresh',
+ note: `slot 0: the lease request failed (the supervisor did not answer acquire within ${WORKER_SERVER_SETUP_STARVATION_LINE_MS}ms), so this worker starts its own server`,
+ },
+ });
+ expect(rig.supervisor.status()[0]?.leasedBy).toBe('run-b');
+ rig.openGate(true);
+ await expect
+ .poll(() => rig.supervisor.status()[0]?.leasedBy, { timeout: DISCARD_MS })
+ .toBeNull();
+ const [lateServer] = rig.launcher.launched;
+ if (lateServer === undefined) throw new Error('the supervisor launched no server for the slot');
+ await expect.poll(() => serving(lateServer.baseURL), { timeout: DISCARD_MS }).toBe(false);
+ expect(rig.supervisor.status()[0]?.baseURL).toBeNull();
+
+ finishUse();
+ expect(await fixtureFailure).toBeUndefined();
+ }, 60_000);
+});
diff --git a/packages/app/tests/foundation/server-lease-safety.node.test.ts b/packages/app/tests/foundation/server-lease-safety.node.test.ts
new file mode 100644
index 000000000..8148d3ade
--- /dev/null
+++ b/packages/app/tests/foundation/server-lease-safety.node.test.ts
@@ -0,0 +1,323 @@
+import { mkdtempSync, realpathSync, rmSync } from 'node:fs';
+import { connect, createServer, type Server } from 'node:net';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { createInterface } from 'node:readline';
+import type { Browser, WorkerInfo } from '@playwright/test';
+import { afterEach, describe, expect, test } from 'vitest';
+import { seedRequiredFixtureFiles } from '../stress/_helpers/fixtures.ts';
+import { E2E_SERVING_ENV } from '../stress/_helpers/prebuilt/prebuilt-serving.ts';
+import { leaseWorkerServer, type ServerLease } from '../stress/_helpers/reuse/lease-fixture.ts';
+import {
+ type LeaseOutcome,
+ type LeaseRequest,
+ REUSE_RUN_ENV,
+ REUSE_SOCKET_ENV,
+} from '../stress/_helpers/reuse/lease-protocol.ts';
+import { WorkerServerSupervisor } from '../stress/_helpers/reuse/server-supervisor.ts';
+import {
+ connectLeaseClient,
+ type SupervisorEndpoint,
+ serveSupervisor,
+} from '../stress/_helpers/reuse/supervisor-socket.ts';
+import { APP_PACKAGE_ROOT } from '../stress/_helpers/seed-key.ts';
+import {
+ contractServerLauncher,
+ failContractServerReadiness,
+} from './contract-dev-server.test-helper.ts';
+
+const CONFIG_FILE = '/worktree/packages/app/playwright.config.ts';
+const PROJECT = 'chromium';
+const WORKTREE = realpathSync(join(APP_PACKAGE_ROOT, '..', '..'));
+const CALL_MS = 30_000;
+const LEASE_ENV_NAMES = [REUSE_SOCKET_ENV, REUSE_RUN_ENV, E2E_SERVING_ENV] as const;
+
+const WORKER_INFO = {
+ parallelIndex: 0,
+ workerIndex: 0,
+ config: { configFile: CONFIG_FILE },
+ project: { name: PROJECT },
+} as WorkerInfo;
+
+const BROWSER_THE_REUSED_PATH_NEVER_OPENS = {} as Browser;
+
+interface Rig {
+ readonly releaseFails: boolean;
+ readonly supervisor: WorkerServerSupervisor;
+ readonly launcher: ReturnType<typeof contractServerLauncher>;
+ readonly scratch: string;
+ request(overrides: Partial<LeaseRequest>): LeaseRequest;
+}
+
+const openRigs: Rig[] = [];
+const openEndpoints: SupervisorEndpoint[] = [];
+const openProxies: Server[] = [];
+const savedEnv = new Map(LEASE_ENV_NAMES.map((name) => [name, process.env[name]]));
+
+const RELEASE_FAILURE = 'the server stopped, but its release reported a failure';
+
+function makeRig({ releaseFails = false }: { releaseFails?: boolean } = {}): Rig {
+ const scratch = realpathSync(mkdtempSync(join(tmpdir(), 'ok-lease-safety-scratch-')));
+ const launcher = contractServerLauncher();
+ const supervisor = new WorkerServerSupervisor({
+ okRoot: WORKTREE,
+ launch: releaseFails
+ ? async (slot, env) => {
+ const owned = await launcher.launch(slot, env);
+ return {
+ ...owned,
+ release: async () => {
+ await owned.release();
+ throw new Error(RELEASE_FAILURE);
+ },
+ };
+ }
+ : launcher.launch,
+ identityTimeoutMs: 2_000,
+ });
+ const rig: Rig = {
+ releaseFails,
+ supervisor,
+ launcher,
+ scratch,
+ request: (overrides) => ({
+ worktree: WORKTREE,
+ config: `${CONFIG_FILE}#${PROJECT}`,
+ run: 'run-a',
+ slot: 0,
+ workerServerEnv: {},
+ serverMode: 'dev',
+ ...overrides,
+ }),
+ };
+ openRigs.push(rig);
+ return rig;
+}
+
+afterEach(async () => {
+ for (const [name, value] of savedEnv) {
+ if (value === undefined) delete process.env[name];
+ else process.env[name] = value;
+ }
+ for (const proxy of openProxies.splice(0)) {
+ await new Promise<void>((resolve) => proxy.close(() => resolve()));
+ }
+ for (const endpoint of openEndpoints.splice(0)) await endpoint.close();
+ for (const rig of openRigs.splice(0)) {
+ await rig.supervisor.shutdown().catch((error: unknown) => {
+ if (!rig.releaseFails || !String(error).includes(RELEASE_FAILURE)) throw error;
+ });
+ rmSync(rig.scratch, { recursive: true, force: true });
+ }
+});
+
+function granted(outcome: LeaseOutcome): Extract<LeaseOutcome, { kind: 'granted' }> {
+ if (outcome.kind !== 'granted') {
+ throw new Error(`expected a granted lease, got ${outcome.reason}: ${outcome.detail}`);
+ }
+ return outcome;
+}
+
+function refusal(outcome: LeaseOutcome): Extract<LeaseOutcome, { kind: 'refused' }> {
+ if (outcome.kind !== 'refused') {
+ throw new Error(`expected a refusal, got a lease on ${outcome.baseURL}`);
+ }
+ return outcome;
+}
+
+async function serving(baseURL: string): Promise<boolean> {
+ try {
+ await fetch(`${baseURL}/__ok-test-server-identity`, { signal: AbortSignal.timeout(2_000) });
+ return true;
+ } catch {
+ return false;
+ }
+}
+
+async function identityNonce(baseURL: string): Promise<unknown> {
+ const response = await fetch(`${baseURL}/__ok-test-server-identity`);
+ return ((await response.json()) as { nonce?: unknown }).nonce;
+}
+
+async function idleSeededSlot(options: { releaseFails?: boolean } = {}): Promise<{
+ rig: Rig;
+ endpoint: SupervisorEndpoint;
+ slot: Extract<LeaseOutcome, { kind: 'granted' }>;
+}> {
+ const rig = makeRig(options);
+ const endpoint = await serveSupervisor(rig.supervisor, join(rig.scratch, 'supervisor.sock'));
+ openEndpoints.push(endpoint);
+ const client = await connectLeaseClient(endpoint.socketPath, 5_000);
+ try {
+ const slot = granted(await client.acquire(rig.request({ run: 'run-a' }), CALL_MS));
+ seedRequiredFixtureFiles(slot.contentDir);
+ await client.release(slot.token, false, CALL_MS);
+ return { rig, endpoint, slot };
+ } finally {
+ client.close();
+ }
+}
+
+async function connectionDroppedAtRelease(
+ endpoint: SupervisorEndpoint,
+ scratch: string,
+): Promise<string> {
+ const proxyPath = join(scratch, 'drop.sock');
+ const proxy = createServer((worker) => {
+ const supervisor = connect(endpoint.socketPath);
+ supervisor.on('error', () => worker.destroy());
+ worker.on('error', () => supervisor.destroy());
+ worker.on('end', () => supervisor.end());
+ supervisor.pipe(worker);
+ const lines = createInterface({ input: worker, crlfDelay: Number.POSITIVE_INFINITY });
+ lines.on('line', (line) => {
+ if ((JSON.parse(line) as { op?: unknown }).op === 'release') {
+ supervisor.destroy();
+ worker.destroy();
+ return;
+ }
+ supervisor.write(`${line}\n`);
+ });
+ });
+ openProxies.push(proxy);
+ await new Promise<void>((resolve, reject) => {
+ proxy.once('error', reject);
+ proxy.listen({ path: proxyPath }, () => resolve());
+ });
+ return proxyPath;
+}
+
+async function leaseThroughWorkerFixture(
+ socketPath: string,
+ run: string,
+): Promise<{ lease: ServerLease | undefined; heldDuringUse: boolean | undefined }> {
+ process.env[REUSE_SOCKET_ENV] = socketPath;
+ process.env[REUSE_RUN_ENV] = run;
+ delete process.env[E2E_SERVING_ENV];
+ let lease: ServerLease | undefined;
+ let heldDuringUse: boolean | undefined;
+ await leaseWorkerServer(
+ { workerServerEnv: {}, browser: BROWSER_THE_REUSED_PATH_NEVER_OPENS },
+ async (offered) => {
+ lease = offered;
+ if (offered?.kind === 'leased') heldDuringUse = (await offered.verify()).held;
+ },
+ WORKER_INFO,
+ );
+ return { lease, heldDuringUse };
+}
+
+describe('a reused slot that fails readiness after its reset', () => {
+ test('a slot whose reset and readiness checks pass is reused by the next run', async () => {
+ const { endpoint, slot } = await idleSeededSlot();
+
+ const { lease, heldDuringUse } = await leaseThroughWorkerFixture(endpoint.socketPath, 'run-b');
+
+ expect(lease?.kind).toBe('leased');
+ expect(lease?.note).toBe(`slot 0: reused the owned server at ${slot.baseURL}`);
+ expect(heldDuringUse).toBe(true);
+ expect(await serving(slot.baseURL)).toBe(true);
+ });
+
+ test('a slot whose API config stops answering after its reset is discarded and the worker starts its own server', async () => {
+ const { rig, endpoint, slot } = await idleSeededSlot();
+ const [server] = rig.launcher.launched;
+ if (server === undefined) throw new Error('no server was launched');
+ await failContractServerReadiness(server, 'api-config');
+
+ const { lease, heldDuringUse } = await leaseThroughWorkerFixture(endpoint.socketPath, 'run-b');
+
+ expect(lease).toEqual({
+ kind: 'fresh',
+ note: 'slot 0: the leased server could not be reset (/api/config returned status 503, expected 200), so it was discarded and this worker starts its own server',
+ });
+ expect(heldDuringUse).toBeUndefined();
+ expect(await serving(slot.baseURL)).toBe(false);
+ });
+
+ test('a slot whose collaboration endpoint stops syncing after its reset is discarded and the worker starts its own server', async () => {
+ const { rig, endpoint, slot } = await idleSeededSlot();
+ const [server] = rig.launcher.launched;
+ if (server === undefined) throw new Error('no server was launched');
+ await failContractServerReadiness(server, 'collab');
+
+ const { lease, heldDuringUse } = await leaseThroughWorkerFixture(endpoint.socketPath, 'run-b');
+
+ expect(lease).toEqual({
+ kind: 'fresh',
+ note: 'slot 0: the leased server could not be reset (/collab sync round-trip did not complete within 10000ms), so it was discarded and this worker starts its own server',
+ });
+ expect(heldDuringUse).toBeUndefined();
+ expect(await serving(slot.baseURL)).toBe(false);
+ });
+});
+
+describe('a discarded slot whose release fails', () => {
+ test('the reset failure stays the reason, the release failure is reported beside it, and the worker still starts its own server', async () => {
+ const { rig, endpoint, slot } = await idleSeededSlot({ releaseFails: true });
+ const [server] = rig.launcher.launched;
+ if (server === undefined) throw new Error('no server was launched');
+ await failContractServerReadiness(server, 'api-config');
+
+ const { lease, heldDuringUse } = await leaseThroughWorkerFixture(endpoint.socketPath, 'run-b');
+
+ expect(lease).toEqual({
+ kind: 'fresh',
+ note: `slot 0: the leased server could not be reset (/api/config returned status 503, expected 200), so this worker starts its own server; the supervisor did not confirm discarding the leased server (the supervisor refused the request: ${RELEASE_FAILURE})`,
+ });
+ expect(heldDuringUse).toBeUndefined();
+ expect(await serving(slot.baseURL)).toBe(false);
+ });
+});
+
+describe('a discarded slot whose lease connection fails as the worker releases it', () => {
+ test('the note claims no discard, and the supervisor discards the server rather than returning it to the slot', async () => {
+ const { rig, endpoint, slot } = await idleSeededSlot();
+ const [server] = rig.launcher.launched;
+ if (server === undefined) throw new Error('no server was launched');
+ const servedNonce = await identityNonce(slot.baseURL);
+ await failContractServerReadiness(server, 'api-config');
+ const dropping = await connectionDroppedAtRelease(endpoint, rig.scratch);
+
+ const { lease, heldDuringUse } = await leaseThroughWorkerFixture(dropping, 'run-b');
+
+ expect(lease).toEqual({
+ kind: 'fresh',
+ note: `slot 0: the leased server could not be reset (/api/config returned status 503, expected 200), so this worker starts its own server; the supervisor did not confirm discarding the leased server (the supervisor at ${dropping} closed the lease connection)`,
+ });
+ expect(heldDuringUse).toBeUndefined();
+ await expect.poll(() => rig.supervisor.status()[0]?.leasedBy).toBeNull();
+ const next = granted(await rig.supervisor.acquire(rig.request({ run: 'run-c' })));
+ expect(next.launched).toBe(true);
+ expect(next.relaunchedBecause).toBe('its holder disconnected without releasing it');
+ expect(await identityNonce(next.baseURL)).not.toBe(servedNonce);
+ }, 60_000);
+});
+
+describe('a worker environment change on a held slot', () => {
+ test('a run with a different worker environment is refused as busy while the slot is held, and restarts it once the slot is idle', async () => {
+ const rig = makeRig();
+ const env = { OK_TEST_FEATURE: 'on' };
+ const changed = { OK_TEST_FEATURE: 'off' };
+ const held = granted(await rig.supervisor.acquire(rig.request({ workerServerEnv: env })));
+ const nonce = await identityNonce(held.baseURL);
+
+ const whileHeld = refusal(
+ await rig.supervisor.acquire(rig.request({ run: 'run-b', workerServerEnv: changed })),
+ );
+ expect(whileHeld.reason).toBe('busy');
+ expect(whileHeld.detail).toBe('slot 0 is leased to run run-a');
+ expect(await rig.supervisor.verify(held.token)).toEqual({ held: true });
+ expect(await identityNonce(held.baseURL)).toBe(nonce);
+ expect(rig.launcher.environments).toEqual([env]);
+
+ await rig.supervisor.release(held.token, false);
+ const onceIdle = granted(
+ await rig.supervisor.acquire(rig.request({ run: 'run-b', workerServerEnv: changed })),
+ );
+ expect(onceIdle.launched).toBe(true);
+ expect(onceIdle.relaunchedBecause).toBe('its workerServerEnv differs in OK_TEST_FEATURE');
+ expect(await serving(held.baseURL)).toBe(false);
+ expect(rig.launcher.environments).toEqual([env, changed]);
+ }, 60_000);
+});
diff --git a/packages/app/tests/foundation/server-lease.node.test.ts b/packages/app/tests/foundation/server-lease.node.test.ts
new file mode 100644
index 000000000..e009b06ca
--- /dev/null
+++ b/packages/app/tests/foundation/server-lease.node.test.ts
@@ -0,0 +1,726 @@
+import { type ChildProcess, execFileSync } from 'node:child_process';
+import {
+ existsSync,
+ lstatSync,
+ mkdirSync,
+ mkdtempSync,
+ readFileSync,
+ readlinkSync,
+ realpathSync,
+ rmSync,
+ symlinkSync,
+ writeFileSync,
+} from 'node:fs';
+import { createServer, type IncomingMessage, type Server } from 'node:http';
+import { type AddressInfo, connect } from 'node:net';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { afterEach, describe, expect, onTestFinished, test, vi } from 'vitest';
+import type {
+ LeaseOutcome,
+ LeaseRequest,
+ WorkerServerEnv,
+} from '../stress/_helpers/reuse/lease-protocol.ts';
+import {
+ type LaunchOwnedServer,
+ WorkerServerSupervisor,
+} from '../stress/_helpers/reuse/server-supervisor.ts';
+import {
+ connectLeaseClient,
+ type LeaseClient,
+ type SupervisorEndpoint,
+ serveSupervisor,
+} from '../stress/_helpers/reuse/supervisor-socket.ts';
+import {
+ contractServerLauncher,
+ exitContractServer,
+ forgeContractServerIdentity,
+} from './contract-dev-server.test-helper.ts';
+
+const STRESS_CONFIG = '/worktree/packages/app/playwright.config.ts#chromium';
+
+interface Rig {
+ readonly supervisor: WorkerServerSupervisor;
+ readonly launcher: ReturnType<typeof contractServerLauncher>;
+ readonly okRoot: string;
+ readonly releaseFailure: string | undefined;
+ request(overrides: Partial<LeaseRequest>): LeaseRequest;
+}
+
+interface RigOptions {
+ readonly launch?: (launch: LaunchOwnedServer) => LaunchOwnedServer;
+ readonly releaseFailure?: string;
+}
+
+const openRigs: Rig[] = [];
+const openListeners: Server[] = [];
+const openEndpoints: SupervisorEndpoint[] = [];
+const openClients: LeaseClient[] = [];
+const openScratch: string[] = [];
+
+function makeRig({ launch, releaseFailure }: RigOptions = {}): Rig {
+ const okRoot = realpathSync(mkdtempSync(join(tmpdir(), 'ok-lease-root-')));
+ mkdirSync(join(okRoot, 'packages', 'server', 'src'), { recursive: true });
+ mkdirSync(join(okRoot, 'packages', 'app', 'src'), { recursive: true });
+ writeFileSync(join(okRoot, 'packages', 'server', 'src', 'boot.ts'), 'export const boot = 1;\n');
+ writeFileSync(join(okRoot, 'packages', 'app', 'src', 'App.tsx'), 'export const app = 1;\n');
+ const launcher = contractServerLauncher();
+ const supervisor = new WorkerServerSupervisor({
+ okRoot,
+ launch: launch === undefined ? launcher.launch : launch(launcher.launch),
+ identityTimeoutMs: 2_000,
+ });
+ const rig: Rig = {
+ supervisor,
+ launcher,
+ okRoot,
+ releaseFailure,
+ request: (overrides) => ({
+ worktree: okRoot,
+ config: STRESS_CONFIG,
+ run: 'run-a',
+ slot: 0,
+ workerServerEnv: {},
+ serverMode: 'dev',
+ ...overrides,
+ }),
+ };
+ openRigs.push(rig);
+ return rig;
+}
+
+afterEach(async () => {
+ for (const client of openClients.splice(0)) client.close();
+ for (const endpoint of openEndpoints.splice(0)) await endpoint.close();
+ for (const listener of openListeners.splice(0)) {
+ listener.closeAllConnections();
+ await new Promise<void>((resolve) => listener.close(() => resolve()));
+ }
+ for (const rig of openRigs.splice(0)) {
+ await rig.supervisor.shutdown().catch((error: unknown) => {
+ if (rig.releaseFailure === undefined || !String(error).includes(rig.releaseFailure)) {
+ throw error;
+ }
+ });
+ rmSync(rig.okRoot, { recursive: true, force: true });
+ }
+ for (const dir of openScratch.splice(0)) rmSync(dir, { recursive: true, force: true });
+});
+
+function granted(outcome: LeaseOutcome): Extract<LeaseOutcome, { kind: 'granted' }> {
+ if (outcome.kind !== 'granted') {
+ throw new Error(`expected a granted lease, got ${outcome.reason}: ${outcome.detail}`);
+ }
+ return outcome;
+}
+
+function refusal(outcome: LeaseOutcome): Extract<LeaseOutcome, { kind: 'refused' }> {
+ if (outcome.kind !== 'refused')
+ throw new Error(`expected a refusal, got a lease on ${outcome.baseURL}`);
+ return outcome;
+}
+
+async function serving(baseURL: string): Promise<boolean> {
+ try {
+ await fetch(`${baseURL}/__ok-test-server-identity`, { signal: AbortSignal.timeout(2_000) });
+ return true;
+ } catch {
+ return false;
+ }
+}
+
+function processExit(proc: ChildProcess): Promise<void> {
+ if (proc.exitCode !== null || proc.signalCode !== null) return Promise.resolve();
+ return new Promise((resolve) => proc.once('exit', () => resolve()));
+}
+
+async function occupyWithForgedIdentity(
+ port: number,
+ identity: unknown,
+): Promise<{ requests: string[] }> {
+ const requests: string[] = [];
+ const listener = createServer((request: IncomingMessage, response) => {
+ requests.push(request.url ?? '');
+ response.setHeader('Content-Type', 'application/json');
+ response.end(JSON.stringify(identity));
+ });
+ openListeners.push(listener);
+ await new Promise<void>((resolve, reject) => {
+ listener.once('error', reject);
+ listener.listen(port, '127.0.0.1', () => resolve());
+ });
+ expect((listener.address() as AddressInfo).port).toBe(port);
+ return { requests };
+}
+
+async function identityNonce(baseURL: string): Promise<unknown> {
+ const response = await fetch(`${baseURL}/__ok-test-server-identity`);
+ return ((await response.json()) as { nonce?: unknown }).nonce;
+}
+
+describe('worker server leases', () => {
+ test('a slot launched for one run is leased to the next run without a second launch', async () => {
+ const rig = makeRig();
+ const first = granted(await rig.supervisor.acquire(rig.request({ run: 'run-a' })));
+ expect(first.launched).toBe(true);
+ const firstNonce = await identityNonce(first.baseURL);
+ await rig.supervisor.release(first.token, false);
+
+ const second = granted(await rig.supervisor.acquire(rig.request({ run: 'run-b' })));
+ expect(second.launched).toBe(false);
+ expect(second.relaunchedBecause).toBeNull();
+ expect(second.baseURL).toBe(first.baseURL);
+ expect(second.contentDir).toBe(first.contentDir);
+ expect(await identityNonce(second.baseURL)).toBe(firstNonce);
+ expect(rig.launcher.launched).toHaveLength(1);
+ });
+
+ test('an overlapping run asking for a held slot is refused as busy and the holder keeps its server', async () => {
+ const rig = makeRig();
+ const held = granted(await rig.supervisor.acquire(rig.request({ run: 'run-a' })));
+ const nonce = await identityNonce(held.baseURL);
+
+ const overlapping = refusal(await rig.supervisor.acquire(rig.request({ run: 'run-b' })));
+ expect(overlapping.reason).toBe('busy');
+ expect(overlapping.detail).toContain('run-a');
+ expect(await rig.supervisor.verify(held.token)).toEqual({ held: true });
+ expect(await identityNonce(held.baseURL)).toBe(nonce);
+ expect(rig.launcher.launched).toHaveLength(1);
+ });
+
+ test('overlapping workers of one run lease distinct slots with their own servers and content roots', async () => {
+ const rig = makeRig();
+ const [zero, one] = await Promise.all([
+ rig.supervisor.acquire(rig.request({ slot: 0 })),
+ rig.supervisor.acquire(rig.request({ slot: 1 })),
+ ]);
+ const first = granted(zero);
+ const second = granted(one);
+ expect(second.baseURL).not.toBe(first.baseURL);
+ expect(second.contentDir).not.toBe(first.contentDir);
+ expect(await identityNonce(second.baseURL)).not.toBe(await identityNonce(first.baseURL));
+ expect(rig.launcher.launched).toHaveLength(2);
+ });
+
+ test('a run whose worker environment differs restarts the idle slot with that environment', async () => {
+ const rig = makeRig();
+ const env: WorkerServerEnv = { OK_TEST_FEATURE: 'on' };
+ const first = granted(await rig.supervisor.acquire(rig.request({ workerServerEnv: env })));
+ await rig.supervisor.release(first.token, false);
+
+ const changed: WorkerServerEnv = { OK_TEST_FEATURE: 'off' };
+ const differing = granted(
+ await rig.supervisor.acquire(rig.request({ run: 'run-b', workerServerEnv: changed })),
+ );
+ expect(differing.launched).toBe(true);
+ expect(differing.relaunchedBecause).toBe('its workerServerEnv differs in OK_TEST_FEATURE');
+ expect(differing.baseURL).not.toBe(first.baseURL);
+ expect(await serving(first.baseURL)).toBe(false);
+ await rig.supervisor.release(differing.token, false);
+
+ const same = granted(
+ await rig.supervisor.acquire(rig.request({ run: 'run-c', workerServerEnv: changed })),
+ );
+ expect(same.launched).toBe(false);
+ expect(same.baseURL).toBe(differing.baseURL);
+ expect(rig.launcher.environments).toEqual([env, changed]);
+ });
+
+ test('a run from another invocation config is refused as incompatible', async () => {
+ const rig = makeRig();
+ const first = granted(await rig.supervisor.acquire(rig.request({})));
+ await rig.supervisor.release(first.token, false);
+
+ const other = refusal(
+ await rig.supervisor.acquire(
+ rig.request({
+ run: 'run-b',
+ config: '/worktree/packages/app/playwright.a11y.config.ts#a11y',
+ }),
+ ),
+ );
+ expect(other.reason).toBe('incompatible');
+ expect(other.detail).toContain('invocation config');
+ expect(rig.launcher.launched).toHaveLength(1);
+ });
+
+ test('an admission whose discard of the previous server fails is refused with that failure and leaves the slot unleased', async () => {
+ const rig = makeRig({ launch: failingRelease, releaseFailure: RELEASE_FAILURE });
+ const first = granted(
+ await rig.supervisor.acquire(rig.request({ workerServerEnv: { OK_TEST_FEATURE: 'on' } })),
+ );
+ await rig.supervisor.release(first.token, false);
+
+ const failed = refusal(
+ await rig.supervisor.acquire(
+ rig.request({ run: 'run-b', workerServerEnv: { OK_TEST_FEATURE: 'off' } }),
+ ),
+ );
+ expect(failed.reason).toBe('launch-failed');
+ expect(failed.detail).toContain(RELEASE_FAILURE);
+ expect(slotStatus(rig, 0)?.leasedBy).toBeNull();
+
+ const next = granted(
+ await rig.supervisor.acquire(
+ rig.request({ run: 'run-c', workerServerEnv: { OK_TEST_FEATURE: 'off' } }),
+ ),
+ );
+ expect(next.launched).toBe(true);
+ });
+
+ test('an admission whose reuse check throws is refused with that error and leaves the slot unleased', async () => {
+ const rig = makeRig();
+ const first = granted(await rig.supervisor.acquire(rig.request({})));
+ await rig.supervisor.release(first.token, false);
+ const dangling = join(rig.okRoot, 'packages', 'server', 'src', 'dangling.ts');
+ symlinkSync(join(rig.okRoot, 'missing.ts'), dangling);
+
+ const failed = refusal(await rig.supervisor.acquire(rig.request({ run: 'run-b' })));
+ expect(failed.reason).toBe('launch-failed');
+ expect(failed.detail).toContain('ENOENT');
+ expect(slotStatus(rig, 0)?.leasedBy).toBeNull();
+
+ rmSync(dangling);
+ const next = granted(await rig.supervisor.acquire(rig.request({ run: 'run-c' })));
+ expect(next.launched).toBe(false);
+ expect(next.baseURL).toBe(first.baseURL);
+ });
+
+ test('a run from another worktree is refused before an empty slot launches a server from this one', async () => {
+ const rig = makeRig();
+ const elsewhere = realpathSync(mkdtempSync(join(tmpdir(), 'ok-lease-other-worktree-')));
+ openScratch.push(elsewhere);
+
+ const other = refusal(await rig.supervisor.acquire(rig.request({ worktree: elsewhere })));
+ expect(other.reason).toBe('incompatible');
+ expect(other.detail).toBe(
+ `worktree ${elsewhere} is not this supervisor's worktree ${rig.okRoot}; it would test that checkout's code`,
+ );
+ expect(rig.launcher.launched).toHaveLength(0);
+
+ const own = granted(await rig.supervisor.acquire(rig.request({})));
+ expect(own.launched).toBe(true);
+ });
+
+ test('a server that exits after its receipt during a lease loses the lease and aborts that run', async () => {
+ const rig = makeRig();
+ const held = granted(await rig.supervisor.acquire(rig.request({ run: 'run-a' })));
+ const [server] = rig.launcher.launched;
+ if (server === undefined) throw new Error('no server was launched');
+ await exitContractServer(server);
+
+ const verdict = await rig.supervisor.verify(held.token);
+ expect(verdict.held).toBe(false);
+ expect(verdict).toMatchObject({
+ reason: expect.stringContaining('exited with code 0 during the lease'),
+ });
+ const again = refusal(await rig.supervisor.acquire(rig.request({ run: 'run-a', slot: 1 })));
+ expect(again.reason).toBe('run-aborted');
+
+ await rig.supervisor.release(held.token, false);
+ const next = granted(await rig.supervisor.acquire(rig.request({ run: 'run-b' })));
+ expect(next.launched).toBe(true);
+ expect(next.baseURL).not.toBe(held.baseURL);
+ expect(rig.launcher.launched).toHaveLength(2);
+ });
+
+ test('a server whose lifetime owner is lost during a lease loses the lease and aborts that run', async () => {
+ const rig = makeRig();
+ const held = granted(await rig.supervisor.acquire(rig.request({ run: 'run-a' })));
+ const [server] = rig.launcher.launched;
+ if (server === undefined) throw new Error('no server was launched');
+ const ownerLost = new Error(
+ 'worker server lifetime closed without completed release (code=1, signal=null)',
+ );
+
+ expect(() => server.proc.emit('error', ownerLost)).not.toThrow();
+
+ expect(await rig.supervisor.verify(held.token)).toEqual({
+ held: false,
+ reason: `its server lifetime failed during the lease: ${ownerLost.message}`,
+ });
+ await processExit(server.proc);
+ expect(
+ refusal(await rig.supervisor.acquire(rig.request({ run: 'run-a', slot: 1 }))).reason,
+ ).toBe('run-aborted');
+ });
+
+ test('an idle slot whose lifetime owner is lost is discarded and relaunched for the next run', async () => {
+ const rig = makeRig();
+ const first = granted(await rig.supervisor.acquire(rig.request({ run: 'run-a' })));
+ await rig.supervisor.release(first.token, false);
+ const [server] = rig.launcher.launched;
+ if (server === undefined) throw new Error('no server was launched');
+ const ownerLost = new Error('write EPIPE');
+
+ expect(() => server.proc.emit('error', ownerLost)).not.toThrow();
+
+ const next = granted(await rig.supervisor.acquire(rig.request({ run: 'run-b' })));
+ expect(next.launched).toBe(true);
+ expect(next.relaunchedBecause).toBe(`its server lifetime failed: ${ownerLost.message}`);
+ await processExit(server.proc);
+ });
+
+ test("a foreign listener answering an exited slot server's old identity on its port is never attached", async () => {
+ const rig = makeRig();
+ const first = granted(await rig.supervisor.acquire(rig.request({})));
+ await rig.supervisor.release(first.token, false);
+ const [server] = rig.launcher.launched;
+ if (server === undefined) throw new Error('no server was launched');
+ await exitContractServer(server);
+ const foreign = await occupyWithForgedIdentity(first.port, server.identity);
+
+ const next = granted(await rig.supervisor.acquire(rig.request({ run: 'run-b' })));
+ expect(next.launched).toBe(true);
+ expect(next.relaunchedBecause).toContain('exited with code 0');
+ expect(next.baseURL).not.toBe(first.baseURL);
+ expect(foreign.requests).toEqual([]);
+ });
+
+ test('a server whose identity changes during a lease loses the lease, aborts the run and is released', async () => {
+ const rig = makeRig();
+ const held = granted(await rig.supervisor.acquire(rig.request({ run: 'run-a' })));
+ const [server] = rig.launcher.launched;
+ if (server === undefined) throw new Error('no server was launched');
+ await forgeContractServerIdentity(server);
+
+ const verdict = await rig.supervisor.verify(held.token);
+ expect(verdict).toEqual({
+ held: false,
+ reason:
+ 'its server identity changed during the lease: its identity endpoint answers for a different server',
+ });
+ expect(server.proc.exitCode !== null || server.proc.signalCode !== null).toBe(true);
+ expect(await serving(held.baseURL)).toBe(false);
+ expect(existsSync(held.contentDir)).toBe(false);
+ expect(refusal(await rig.supervisor.acquire(rig.request({ run: 'run-a' }))).reason).toBe(
+ 'run-aborted',
+ );
+ });
+
+ test('a server or core edit relaunches the slot while an app-only edit keeps it', async () => {
+ const rig = makeRig();
+ const first = granted(await rig.supervisor.acquire(rig.request({})));
+ await rig.supervisor.release(first.token, false);
+ writeFileSync(join(rig.okRoot, 'packages', 'app', 'src', 'App.tsx'), 'export const app = 2;\n');
+ const appEdit = granted(await rig.supervisor.acquire(rig.request({ run: 'run-b' })));
+ expect(appEdit.launched).toBe(false);
+ await rig.supervisor.release(appEdit.token, false);
+
+ writeFileSync(
+ join(rig.okRoot, 'packages', 'server', 'src', 'boot.ts'),
+ 'export const boot = 2;\n',
+ );
+ const serverEdit = granted(await rig.supervisor.acquire(rig.request({ run: 'run-c' })));
+ expect(serverEdit.launched).toBe(true);
+ expect(serverEdit.relaunchedBecause).toBe(
+ 'its consumed inputs changed: packages/server/src/boot.ts',
+ );
+ expect(await serving(first.baseURL)).toBe(false);
+ });
+
+ test('a slot whose content root disappeared is relaunched with a new one', async () => {
+ const rig = makeRig();
+ const first = granted(await rig.supervisor.acquire(rig.request({})));
+ await rig.supervisor.release(first.token, false);
+ rmSync(first.contentDir, { recursive: true, force: true });
+
+ const next = granted(await rig.supervisor.acquire(rig.request({ run: 'run-b' })));
+ expect(next.launched).toBe(true);
+ expect(next.relaunchedBecause).toBe(`its content root ${first.contentDir} is gone`);
+ expect(existsSync(next.contentDir)).toBe(true);
+ });
+
+ test('shutdown aborts the holding run, releases every owned server and refuses new leases', async () => {
+ const rig = makeRig();
+ const held = granted(await rig.supervisor.acquire(rig.request({ slot: 0 })));
+ const idle = granted(await rig.supervisor.acquire(rig.request({ slot: 1 })));
+ await rig.supervisor.release(idle.token, false);
+
+ await rig.supervisor.shutdown();
+ expect(await serving(held.baseURL)).toBe(false);
+ expect(await serving(idle.baseURL)).toBe(false);
+ expect(existsSync(held.contentDir)).toBe(false);
+ expect(existsSync(idle.contentDir)).toBe(false);
+ expect(await rig.supervisor.verify(held.token)).toEqual({
+ held: false,
+ reason: 'the supervisor shut down during the lease',
+ });
+ expect(refusal(await rig.supervisor.acquire(rig.request({ run: 'run-b' }))).reason).toBe(
+ 'supervisor-closed',
+ );
+ });
+});
+
+const CALL_MS = 30_000;
+
+async function servedRig(
+ options: RigOptions = {},
+): Promise<{ rig: Rig; endpoint: SupervisorEndpoint }> {
+ const rig = makeRig(options);
+ const endpoint = await serveSupervisor(rig.supervisor, join(rig.okRoot, 'supervisor.sock'));
+ openEndpoints.push(endpoint);
+ return { rig, endpoint };
+}
+
+async function client(endpoint: SupervisorEndpoint): Promise<LeaseClient> {
+ const connected = await connectLeaseClient(endpoint.socketPath, 5_000);
+ openClients.push(connected);
+ return connected;
+}
+
+const RELEASE_FAILURE = 'the server stopped, but its release reported a failure';
+const SETTLE_MS = 10_000;
+
+function failingRelease(launch: LaunchOwnedServer): LaunchOwnedServer {
+ return async (slot, workerServerEnv) => {
+ const owned = await launch(slot, workerServerEnv);
+ return {
+ ...owned,
+ release: async () => {
+ await owned.release();
+ throw new Error(RELEASE_FAILURE);
+ },
+ };
+ };
+}
+
+interface HeldLaunch {
+ readonly started: Promise<void>;
+ proceed(): void;
+ wrap(launch: LaunchOwnedServer): LaunchOwnedServer;
+}
+
+function holdLaunch(heldSlot: number): HeldLaunch {
+ let markStarted: () => void = () => {};
+ const started = new Promise<void>((resolve) => {
+ markStarted = resolve;
+ });
+ let proceed: () => void = () => {};
+ const gate = new Promise<void>((resolve) => {
+ proceed = resolve;
+ });
+ return {
+ started,
+ proceed: () => proceed(),
+ wrap: (launch) => async (slot, workerServerEnv) => {
+ if (slot === heldSlot) {
+ markStarted();
+ await gate;
+ }
+ return launch(slot, workerServerEnv);
+ },
+ };
+}
+
+function slotStatus(rig: Rig, slot: number) {
+ return rig.supervisor.status().find((entry) => entry.slot === slot);
+}
+
+function discardReports(): string[] {
+ const reports: string[] = [];
+ const passThrough = console.warn.bind(console);
+ const warn = vi.spyOn(console, 'warn').mockImplementation((...parts: unknown[]) => {
+ const line = parts.map(String).join(' ');
+ if (line.includes('discarding the server')) reports.push(line);
+ else passThrough(...parts);
+ });
+ onTestFinished(() => warn.mockRestore());
+ return reports;
+}
+
+async function closeWhileSlotZeroLaunches(
+ rig: Rig,
+ endpoint: SupervisorEndpoint,
+ held: HeldLaunch,
+): Promise<string> {
+ const holder = await client(endpoint);
+ granted(await holder.acquire(rig.request({ slot: 1 }), CALL_MS));
+ const late = holder.acquire(rig.request({ slot: 0 }), CALL_MS).then(
+ () => 'answered',
+ (error: unknown) => String(error),
+ );
+ await held.started;
+ holder.close();
+ await expect.poll(() => slotStatus(rig, 1)?.leasedBy, { timeout: SETTLE_MS }).toBeNull();
+ held.proceed();
+ return late;
+}
+
+function rawExchange(socketPath: string, line: string): Promise<string> {
+ return new Promise((resolve, reject) => {
+ const socket = connect(socketPath);
+ let received = '';
+ socket.once('error', reject);
+ socket.on('data', (chunk: Buffer) => {
+ received += chunk.toString('utf8');
+ if (received.includes('\n')) {
+ socket.destroy();
+ resolve(received.trim());
+ }
+ });
+ socket.once('connect', () => socket.write(`${line}\n`));
+ });
+}
+
+describe('worker server leases over the supervisor socket', () => {
+ test('a lease whose holding connection closes without releasing it is freed and its server discarded, so the next run starts a new one', async () => {
+ const { rig, endpoint } = await servedRig();
+ const first = await client(endpoint);
+ const held = granted(await first.acquire(rig.request({ run: 'run-a' }), CALL_MS));
+ const heldNonce = await identityNonce(held.baseURL);
+ const second = await client(endpoint);
+ expect(refusal(await second.acquire(rig.request({ run: 'run-b' }), CALL_MS)).reason).toBe(
+ 'busy',
+ );
+
+ first.close();
+ await expect.poll(() => rig.supervisor.status()[0]?.leasedBy).toBeNull();
+ const relaunched = granted(await second.acquire(rig.request({ run: 'run-b' }), CALL_MS));
+ expect(relaunched.launched).toBe(true);
+ expect(relaunched.relaunchedBecause).toBe('its holder disconnected without releasing it');
+ expect(rig.launcher.launched).toHaveLength(2);
+ expect(await identityNonce(relaunched.baseURL)).not.toBe(heldNonce);
+ });
+
+ test('an acquire answered after its connection closed leaves the slot unleased and discards the server it launched', async () => {
+ const held = holdLaunch(0);
+ const { rig, endpoint } = await servedRig({ launch: held.wrap });
+
+ expect(await closeWhileSlotZeroLaunches(rig, endpoint, held)).toContain(
+ 'closed the lease connection',
+ );
+ await expect.poll(() => slotStatus(rig, 0)?.leasedBy, { timeout: SETTLE_MS }).toBeNull();
+ expect(rig.supervisor.status()).toEqual([
+ { slot: 0, baseURL: null, config: null, leasedBy: null },
+ { slot: 1, baseURL: null, config: null, leasedBy: null },
+ ]);
+ const [, lateServer] = rig.launcher.launched;
+ if (lateServer === undefined) throw new Error('the acquire for slot 0 launched no server');
+ await expect.poll(() => serving(lateServer.baseURL), { timeout: SETTLE_MS }).toBe(false);
+
+ const next = await client(endpoint);
+ const relaunched = granted(await next.acquire(rig.request({ run: 'run-b' }), CALL_MS));
+ expect(relaunched.launched).toBe(true);
+ expect(relaunched.relaunchedBecause).toBe('its holder disconnected without releasing it');
+ });
+
+ test('a failed discard after a lease connection closes is reported with its slot and run, whether the lease was held or granted after the close', async () => {
+ const reports = discardReports();
+ const held = holdLaunch(0);
+ const { rig, endpoint } = await servedRig({
+ launch: (launch) => held.wrap(failingRelease(launch)),
+ releaseFailure: RELEASE_FAILURE,
+ });
+
+ await closeWhileSlotZeroLaunches(rig, endpoint, held);
+ await expect.poll(() => reports.length, { timeout: SETTLE_MS }).toBe(2);
+ expect([...reports].sort()).toEqual([
+ `[server reuse] slot 0: discarding the server of run run-a, whose lease connection closed without a release, failed: ${RELEASE_FAILURE}`,
+ `[server reuse] slot 1: discarding the server of run run-a, whose lease connection closed without a release, failed: ${RELEASE_FAILURE}`,
+ ]);
+ });
+
+ test('only the connection holding a lease can verify or release it', async () => {
+ const { rig, endpoint } = await servedRig();
+ const holder = await client(endpoint);
+ const held = granted(await holder.acquire(rig.request({}), CALL_MS));
+ const stranger = await client(endpoint);
+
+ expect(await stranger.verify(held.token, CALL_MS)).toEqual({
+ held: false,
+ reason: 'the lease is not held by this connection',
+ });
+ await stranger.release(held.token, true, CALL_MS);
+ expect(await holder.verify(held.token, CALL_MS)).toEqual({ held: true });
+ expect(rig.supervisor.status()[0]?.leasedBy).toBe('run-a');
+ });
+
+ test('a malformed request is answered with an error and grants no lease', async () => {
+ const { rig, endpoint } = await servedRig();
+ const answer = JSON.parse(
+ await rawExchange(
+ endpoint.socketPath,
+ JSON.stringify({ id: 4, op: 'acquire', request: { ...rig.request({}), slot: -1 } }),
+ ),
+ ) as { ok: boolean; error: string };
+ expect(answer).toEqual({
+ id: 0,
+ ok: false,
+ error: 'malformed request: lease request.slot is not a non-negative integer',
+ });
+ expect(rig.supervisor.status()).toEqual([]);
+ expect(rig.launcher.launched).toHaveLength(0);
+ });
+
+ test('a socket already served by a live supervisor is refused and a stale socket file is reclaimed', async () => {
+ const { rig, endpoint } = await servedRig();
+ const rival = new WorkerServerSupervisor({
+ okRoot: rig.okRoot,
+ launch: rig.launcher.launch,
+ identityTimeoutMs: 2_000,
+ });
+ await expect(serveSupervisor(rival, endpoint.socketPath)).rejects.toThrow(
+ `another supervisor already serves ${endpoint.socketPath}`,
+ );
+ expect(await (await client(endpoint)).status(CALL_MS)).toEqual([]);
+
+ const stale = join(rig.okRoot, 'stale.sock');
+ execFileSync(process.execPath, [
+ '-e',
+ `require('node:net').createServer().listen(${JSON.stringify(stale)}, () => process.exit(0))`,
+ ]);
+ expect(existsSync(stale)).toBe(true);
+ const reclaimed = await serveSupervisor(rival, stale);
+ openEndpoints.push(reclaimed);
+ expect(await (await client(reclaimed)).status(CALL_MS)).toEqual([]);
+ });
+
+ test('an entry at the socket path that is not a socket is refused by its kind and left in place', async () => {
+ const rig = makeRig();
+ const rival = new WorkerServerSupervisor({
+ okRoot: rig.okRoot,
+ launch: rig.launcher.launch,
+ identityTimeoutMs: 2_000,
+ });
+ const regular = join(rig.okRoot, 'regular.sock');
+ writeFileSync(regular, 'a regular file\n');
+ const directory = join(rig.okRoot, 'directory.sock');
+ mkdirSync(directory);
+ const target = join(rig.okRoot, 'link-target.txt');
+ writeFileSync(target, 'the link target\n');
+ const link = join(rig.okRoot, 'link.sock');
+ symlinkSync(target, link);
+ const inodes = [regular, directory, link].map((path) => lstatSync(path).ino);
+
+ await expect(serveSupervisor(rival, regular)).rejects.toThrow(
+ `cannot claim ${regular}: it is a regular file, not a socket`,
+ );
+ await expect(serveSupervisor(rival, directory)).rejects.toThrow(
+ `cannot claim ${directory}: it is a directory, not a socket`,
+ );
+ await expect(serveSupervisor(rival, link)).rejects.toThrow(
+ `cannot claim ${link}: it is a symbolic link, not a socket`,
+ );
+
+ expect(readFileSync(regular, 'utf8')).toBe('a regular file\n');
+ expect(lstatSync(directory).isDirectory()).toBe(true);
+ expect(readlinkSync(link)).toBe(target);
+ expect(readFileSync(target, 'utf8')).toBe('the link target\n');
+ expect([regular, directory, link].map((path) => lstatSync(path).ino)).toEqual(inodes);
+ });
+
+ test('a shutdown request answers, closes the endpoint and releases every owned server', async () => {
+ const { rig, endpoint } = await servedRig();
+ const holder = await client(endpoint);
+ const held = granted(await holder.acquire(rig.request({}), CALL_MS));
+
+ await holder.shutdown(CALL_MS);
+ await endpoint.closed;
+ expect(await serving(held.baseURL)).toBe(false);
+ expect(existsSync(held.contentDir)).toBe(false);
+ await expect(connectLeaseClient(endpoint.socketPath, 2_000)).rejects.toThrow(
+ 'is not reachable',
+ );
+ });
+});
diff --git a/packages/app/tests/foundation/server-supervisor-shutdown.node.test.ts b/packages/app/tests/foundation/server-supervisor-shutdown.node.test.ts
new file mode 100644
index 000000000..dae218c93
--- /dev/null
+++ b/packages/app/tests/foundation/server-supervisor-shutdown.node.test.ts
@@ -0,0 +1,301 @@
+import { mkdirSync, mkdtempSync, realpathSync, rmSync, writeFileSync } from 'node:fs';
+import { createServer, type Server } from 'node:http';
+import type { AddressInfo } from 'node:net';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { setImmediate as nextTurn } from 'node:timers/promises';
+import { afterEach, describe, expect, onTestFinished, test, vi } from 'vitest';
+import type { LeaseOutcome, LeaseRequest } from '../stress/_helpers/reuse/lease-protocol.ts';
+import {
+ type LaunchOwnedServer,
+ type OwnedServer,
+ WorkerServerSupervisor,
+} from '../stress/_helpers/reuse/server-supervisor.ts';
+import {
+ connectLeaseClient,
+ type SupervisorEndpoint,
+ serveSupervisor,
+} from '../stress/_helpers/reuse/supervisor-socket.ts';
+import { contractServerLauncher } from './contract-dev-server.test-helper.ts';
+
+const STRESS_CONFIG = '/worktree/packages/app/playwright.config.ts#chromium';
+const CALL_MS = 30_000;
+const RELEASE_FAILURE = 'the server stopped, but its release reported a failure';
+
+interface Rig {
+ readonly supervisor: WorkerServerSupervisor;
+ readonly launcher: ReturnType<typeof contractServerLauncher>;
+ readonly okRoot: string;
+ request(overrides: Partial<LeaseRequest>): LeaseRequest;
+}
+
+const openRigs: Rig[] = [];
+const openEndpoints: SupervisorEndpoint[] = [];
+const openListeners: Server[] = [];
+const openGates: Array<() => void> = [];
+
+afterEach(async () => {
+ for (const open of openGates.splice(0)) open();
+ for (const listener of openListeners.splice(0)) {
+ listener.closeAllConnections();
+ await new Promise<void>((resolve) => listener.close(() => resolve()));
+ }
+ for (const endpoint of openEndpoints.splice(0)) await endpoint.close();
+ for (const rig of openRigs.splice(0)) {
+ await rig.supervisor.shutdown().catch((error: unknown) => {
+ if (!String(error).includes(RELEASE_FAILURE)) throw error;
+ });
+ rmSync(rig.okRoot, { recursive: true, force: true });
+ }
+});
+
+function makeRig(wrap: (launch: LaunchOwnedServer) => LaunchOwnedServer = (launch) => launch): Rig {
+ const okRoot = realpathSync(mkdtempSync(join(tmpdir(), 'ok-lease-shutdown-root-')));
+ mkdirSync(join(okRoot, 'packages', 'server', 'src'), { recursive: true });
+ mkdirSync(join(okRoot, 'packages', 'app', 'src'), { recursive: true });
+ writeFileSync(join(okRoot, 'packages', 'server', 'src', 'boot.ts'), 'export const boot = 1;\n');
+ writeFileSync(join(okRoot, 'packages', 'app', 'src', 'App.tsx'), 'export const app = 1;\n');
+ const launcher = contractServerLauncher();
+ const supervisor = new WorkerServerSupervisor({
+ okRoot,
+ launch: wrap(launcher.launch),
+ identityTimeoutMs: 5_000,
+ });
+ const rig: Rig = {
+ supervisor,
+ launcher,
+ okRoot,
+ request: (overrides) => ({
+ worktree: okRoot,
+ config: STRESS_CONFIG,
+ run: 'run-a',
+ slot: 0,
+ workerServerEnv: {},
+ serverMode: 'dev',
+ ...overrides,
+ }),
+ };
+ openRigs.push(rig);
+ return rig;
+}
+
+interface Gate {
+ readonly reached: Promise<void>;
+ readonly opened: Promise<void>;
+ open(): void;
+ markReached(): void;
+}
+
+function gate(): Gate {
+ let markReached: () => void = () => {};
+ const reached = new Promise<void>((resolve) => {
+ markReached = resolve;
+ });
+ let open: () => void = () => {};
+ const opened = new Promise<void>((resolve) => {
+ open = resolve;
+ });
+ openGates.push(open);
+ return { reached, opened, open, markReached };
+}
+
+function granted(outcome: LeaseOutcome): Extract<LeaseOutcome, { kind: 'granted' }> {
+ if (outcome.kind !== 'granted') {
+ throw new Error(`expected a granted lease, got ${outcome.reason}: ${outcome.detail}`);
+ }
+ return outcome;
+}
+
+async function serving(baseURL: string): Promise<boolean> {
+ try {
+ await fetch(`${baseURL}/__ok-test-server-identity`, { signal: AbortSignal.timeout(2_000) });
+ return true;
+ } catch {
+ return false;
+ }
+}
+
+function settledFlag(work: Promise<unknown>): () => boolean {
+ let settled = false;
+ work.then(
+ () => {
+ settled = true;
+ },
+ () => {
+ settled = true;
+ },
+ );
+ return () => settled;
+}
+
+async function afterPendingTurns(): Promise<void> {
+ for (let turn = 0; turn < 5; turn += 1) await nextTurn();
+}
+
+function warnings(): string[] {
+ const lines: string[] = [];
+ const passThrough = console.warn.bind(console);
+ const warn = vi.spyOn(console, 'warn').mockImplementation((...parts: unknown[]) => {
+ const line = parts.map(String).join(' ');
+ if (line.startsWith('[server reuse]')) lines.push(line);
+ else passThrough(...parts);
+ });
+ onTestFinished(() => warn.mockRestore());
+ return lines;
+}
+
+function releaseFailsAfterReleasing(launch: LaunchOwnedServer, held?: Gate): LaunchOwnedServer {
+ return async (slot, workerServerEnv) => {
+ const owned = await launch(slot, workerServerEnv);
+ return {
+ ...owned,
+ release: async () => {
+ await owned.release();
+ if (held !== undefined && slot === 1) {
+ held.markReached();
+ await held.opened;
+ }
+ throw new Error(`${RELEASE_FAILURE} for slot ${slot}`);
+ },
+ };
+ };
+}
+
+async function idleSlots(rig: Rig, slots: readonly number[]): Promise<OwnedServerURL[]> {
+ const urls: OwnedServerURL[] = [];
+ for (const slot of slots) {
+ const lease = granted(await rig.supervisor.acquire(rig.request({ slot })));
+ await rig.supervisor.release(lease.token, false);
+ urls.push({ slot, baseURL: lease.baseURL });
+ }
+ return urls;
+}
+
+interface OwnedServerURL {
+ readonly slot: number;
+ readonly baseURL: string;
+}
+
+async function identityHeldBehind(held: Gate, server: OwnedServer): Promise<string> {
+ const listener = createServer((request, response) => {
+ if (request.url !== '/__ok-test-server-identity') {
+ response.statusCode = 404;
+ response.end();
+ return;
+ }
+ held.markReached();
+ void held.opened.then(() => {
+ response.setHeader('Content-Type', 'application/json');
+ response.end(JSON.stringify(server.identity));
+ });
+ });
+ openListeners.push(listener);
+ await new Promise<void>((resolve, reject) => {
+ listener.once('error', reject);
+ listener.listen(0, '127.0.0.1', () => resolve());
+ });
+ return `http://127.0.0.1:${(listener.address() as AddressInfo).port}`;
+}
+
+describe('a supervisor that shuts down', () => {
+ test('an endpoint whose owned servers all release closes reporting a clean release', async () => {
+ const rig = makeRig();
+ const [idle] = await idleSlots(rig, [0]);
+ const endpoint = await serveSupervisor(rig.supervisor, join(rig.okRoot, 'supervisor.sock'));
+ openEndpoints.push(endpoint);
+
+ await endpoint.close();
+
+ expect(await endpoint.closed).toEqual({ released: true });
+ expect(await serving(idle?.baseURL ?? '')).toBe(false);
+ });
+
+ test('an endpoint whose owned server fails to release closes reporting that failure, with a warning naming the slot', async () => {
+ const reports = warnings();
+ const rig = makeRig((launch) => releaseFailsAfterReleasing(launch));
+ await idleSlots(rig, [0]);
+ const endpoint = await serveSupervisor(rig.supervisor, join(rig.okRoot, 'supervisor.sock'));
+ openEndpoints.push(endpoint);
+ const operator = await connectLeaseClient(endpoint.socketPath, 5_000);
+
+ await operator.shutdown(CALL_MS);
+ operator.close();
+
+ const failure = `releasing the owned servers failed: slot 0: ${RELEASE_FAILURE} for slot 0`;
+ expect(await endpoint.closed).toEqual({ released: false, failure });
+ expect(reports).toEqual([`[server reuse] ${failure}`]);
+ });
+
+ test('a shutdown whose two releases fail settles only after both have run, and reports each by slot', async () => {
+ const slowRelease = gate();
+ const rig = makeRig((launch) => releaseFailsAfterReleasing(launch, slowRelease));
+ const idle = await idleSlots(rig, [0, 1]);
+
+ const shutdown = rig.supervisor.shutdown();
+ const shutdownSettled = settledFlag(shutdown);
+ await slowRelease.reached;
+ await afterPendingTurns();
+
+ expect(shutdownSettled()).toBe(false);
+ slowRelease.open();
+ await expect(shutdown).rejects.toThrow(
+ `releasing the owned servers failed: slot 0: ${RELEASE_FAILURE} for slot 0; slot 1: ${RELEASE_FAILURE} for slot 1`,
+ );
+ for (const { baseURL } of idle) expect(await serving(baseURL)).toBe(false);
+ });
+
+ test('an admission checking an idle slot when shutdown begins is refused and launches no new server', async () => {
+ const heldIdentity = gate();
+ let launches = 0;
+ const rig = makeRig((launch) => async (slot, workerServerEnv) => {
+ launches += 1;
+ const owned = await launch(slot, workerServerEnv);
+ return { ...owned, baseURL: await identityHeldBehind(heldIdentity, owned) };
+ });
+ const first = granted(await rig.supervisor.acquire(rig.request({ run: 'run-a' })));
+ await rig.supervisor.release(first.token, false);
+ const [firstServer] = rig.launcher.launched;
+ if (firstServer === undefined) throw new Error('the first acquire launched no server');
+
+ const admission = rig.supervisor.acquire(rig.request({ run: 'run-b' }));
+ await heldIdentity.reached;
+ const shutdown = rig.supervisor.shutdown();
+ heldIdentity.open();
+
+ expect(await admission).toEqual({
+ kind: 'refused',
+ reason: 'supervisor-closed',
+ detail: 'the supervisor shut down while admission checked the slot',
+ });
+ await shutdown;
+ expect(launches).toBe(1);
+ expect(await serving(firstServer.baseURL)).toBe(false);
+ });
+
+ test('a shutdown that begins while a slot is starting its server settles only after that server is released', async () => {
+ const heldLaunch = gate();
+ const rig = makeRig((launch) => async (slot, workerServerEnv) => {
+ heldLaunch.markReached();
+ await heldLaunch.opened;
+ return launch(slot, workerServerEnv);
+ });
+
+ const admission = rig.supervisor.acquire(rig.request({}));
+ await heldLaunch.reached;
+ const shutdown = rig.supervisor.shutdown();
+ const shutdownSettled = settledFlag(shutdown);
+ await afterPendingTurns();
+
+ expect(shutdownSettled()).toBe(false);
+ heldLaunch.open();
+ await shutdown;
+ const [lateServer] = rig.launcher.launched;
+ if (lateServer === undefined) throw new Error('the held launch started no server');
+ expect(await serving(lateServer.baseURL)).toBe(false);
+ expect(await admission).toEqual({
+ kind: 'refused',
+ reason: 'supervisor-closed',
+ detail: 'the supervisor shut down while the slot was starting',
+ });
+ });
+});
diff --git a/packages/app/tests/foundation/startup-restart.node.test.ts b/packages/app/tests/foundation/startup-restart.node.test.ts
new file mode 100644
index 000000000..278de43de
--- /dev/null
+++ b/packages/app/tests/foundation/startup-restart.node.test.ts
@@ -0,0 +1,214 @@
+import { mkdirSync, mkdtempSync, realpathSync, rmSync, writeFileSync } from 'node:fs';
+import { type AddressInfo, createServer } from 'node:net';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { afterEach, expect, test } from 'vitest';
+import type {
+ LeaseOutcome,
+ LeaseRequest,
+ WorkerServerEnv,
+} from '../stress/_helpers/reuse/lease-protocol.ts';
+import { WorkerServerSupervisor } from '../stress/_helpers/reuse/server-supervisor.ts';
+import {
+ CONTRACT_SERVER_CONFIG,
+ contractServerGeneration,
+ contractServerLauncher,
+ restartContractServer,
+} from './contract-dev-server.test-helper.ts';
+import { CONTRACT_SERVER_EXCLUSIVE_RESOURCE_ENV } from './fixtures/contract-dev-server-routes.ts';
+
+const RESTART_SETTLE_MS = 10_000;
+
+interface Rig {
+ readonly supervisor: WorkerServerSupervisor;
+ readonly launcher: ReturnType<typeof contractServerLauncher>;
+ readonly okRoot: string;
+ request(run: string): LeaseRequest;
+}
+
+interface ImportingConfig {
+ readonly configFile: string;
+ readonly importedModule: string;
+ edit(value: string): void;
+}
+
+const openRigs: Rig[] = [];
+const openDirs: string[] = [];
+
+function importingConfig(): ImportingConfig {
+ const dir = realpathSync(mkdtempSync(join(tmpdir(), 'ok-restart-config-')));
+ openDirs.push(dir);
+ const importedModule = join(dir, 'imported-input.ts');
+ const configFile = join(dir, 'vite.config.mts');
+ const edit = (value: string) => {
+ writeFileSync(importedModule, `export const importedInput = ${JSON.stringify(value)};\n`);
+ };
+ edit('v1');
+ writeFileSync(
+ configFile,
+ [
+ `import contract from ${JSON.stringify(CONTRACT_SERVER_CONFIG)};`,
+ "import { importedInput } from './imported-input.ts';",
+ 'export default { ...contract, define: { __IMPORTED_INPUT__: JSON.stringify(importedInput) } };',
+ '',
+ ].join('\n'),
+ );
+ return { configFile, importedModule, edit };
+}
+
+function makeRig({
+ configFile,
+ workerServerEnv = {},
+}: {
+ readonly configFile?: string;
+ readonly workerServerEnv?: WorkerServerEnv;
+} = {}): Rig {
+ const okRoot = realpathSync(mkdtempSync(join(tmpdir(), 'ok-restart-root-')));
+ mkdirSync(join(okRoot, 'packages', 'app', 'src'), { recursive: true });
+ writeFileSync(join(okRoot, 'packages', 'app', 'src', 'App.tsx'), 'export const app = 1;\n');
+ const launcher = contractServerLauncher({ configFile });
+ const rig: Rig = {
+ supervisor: new WorkerServerSupervisor({
+ okRoot,
+ launch: launcher.launch,
+ identityTimeoutMs: 2_000,
+ }),
+ launcher,
+ okRoot,
+ request: (run) => ({
+ worktree: okRoot,
+ config: '/worktree/packages/app/playwright.config.ts#chromium',
+ run,
+ slot: 0,
+ workerServerEnv,
+ serverMode: 'dev',
+ }),
+ };
+ openRigs.push(rig);
+ return rig;
+}
+
+afterEach(async () => {
+ for (const rig of openRigs.splice(0)) {
+ await rig.supervisor.shutdown();
+ rmSync(rig.okRoot, { recursive: true, force: true });
+ }
+ for (const dir of openDirs.splice(0)) rmSync(dir, { recursive: true, force: true });
+});
+
+function granted(outcome: LeaseOutcome): Extract<LeaseOutcome, { kind: 'granted' }> {
+ if (outcome.kind !== 'granted') {
+ throw new Error(`expected a granted lease, got ${outcome.reason}: ${outcome.detail}`);
+ }
+ return outcome;
+}
+
+async function freePort(): Promise<number> {
+ const probe = createServer();
+ await new Promise<void>((resolve) => probe.listen(0, '127.0.0.1', resolve));
+ const { port } = probe.address() as AddressInfo;
+ await new Promise<void>((resolve) => probe.close(() => resolve()));
+ return port;
+}
+
+test('a dev server that restarts in-process on its own port keeps its startup receipt, and the next run reuses it', async () => {
+ const rig = makeRig();
+ const first = granted(await rig.supervisor.acquire(rig.request('run-a')));
+ await rig.supervisor.release(first.token, false);
+ const [server] = rig.launcher.launched;
+ if (server === undefined) throw new Error('the supervisor launched no server');
+
+ await restartContractServer(server);
+ await expect
+ .poll(() => contractServerGeneration(server).catch(() => 0), { timeout: RESTART_SETTLE_MS })
+ .toBe(2);
+
+ const second = granted(await rig.supervisor.acquire(rig.request('run-b')));
+ expect(second).toMatchObject({ launched: false, baseURL: first.baseURL, port: first.port });
+ expect(rig.launcher.launched).toHaveLength(1);
+ expect(rig.launcher.stderrOf(server)).not.toContain('automatic Vite startup');
+});
+
+test('a dev server that restarts in-process onto another port exits naming the endpoint change, and the next run starts a new one', async () => {
+ const rig = makeRig();
+ const first = granted(await rig.supervisor.acquire(rig.request('run-a')));
+ await rig.supervisor.release(first.token, false);
+ const [server] = rig.launcher.launched;
+ if (server === undefined) throw new Error('the supervisor launched no server');
+ const exited = new Promise<number | null>((resolve) => {
+ server.proc.once('exit', (code) => resolve(code));
+ });
+
+ await restartContractServer(server, await freePort());
+
+ expect(await exited).toBe(1);
+ expect(rig.launcher.stderrOf(server)).toContain(
+ `automatic Vite startup restarted in-process and bound {"nonce":"${server.identity.nonce}"`,
+ );
+ const second = granted(await rig.supervisor.acquire(rig.request('run-b')));
+ expect(second).toMatchObject({
+ launched: true,
+ relaunchedBecause: 'its server process exited with code 1',
+ });
+});
+
+test('a slot whose in-process restart failed after an edit to a module its configuration imports is relaunched, naming that module', async () => {
+ const config = importingConfig();
+ const rig = makeRig({
+ configFile: config.configFile,
+ workerServerEnv: { [CONTRACT_SERVER_EXCLUSIVE_RESOURCE_ENV]: '1' },
+ });
+ const first = granted(await rig.supervisor.acquire(rig.request('run-a')));
+ await rig.supervisor.release(first.token, false);
+ const [server] = rig.launcher.launched;
+ if (server === undefined) throw new Error('the supervisor launched no server');
+
+ config.edit('v2');
+ await restartContractServer(server);
+ await expect
+ .poll(() => rig.launcher.stderrOf(server), { timeout: RESTART_SETTLE_MS })
+ .toContain('server restart failed');
+ expect(await contractServerGeneration(server)).toBe(1);
+
+ const second = granted(await rig.supervisor.acquire(rig.request('run-b')));
+ expect(second).toMatchObject({
+ launched: true,
+ relaunchedBecause: `its loaded configuration inputs changed: ${config.importedModule}`,
+ });
+});
+
+test('a slot whose env file changed after its server loaded its configuration is relaunched, naming that file', async () => {
+ const rig = makeRig();
+ const first = granted(await rig.supervisor.acquire(rig.request('run-a')));
+ await rig.supervisor.release(first.token, false);
+ const [server] = rig.launcher.launched;
+ if (server === undefined) throw new Error('the supervisor launched no server');
+
+ const envFile = join(server.contentDir, '.env.local');
+ writeFileSync(envFile, 'VITE_CONTRACT_INPUT=edited\n');
+
+ const second = granted(await rig.supervisor.acquire(rig.request('run-b')));
+ expect(second).toMatchObject({
+ launched: true,
+ relaunchedBecause: `its loaded configuration inputs changed: ${envFile}`,
+ });
+});
+
+test('a slot that restarted in-process after an edit to a module its configuration imports serves what it reloaded, and the next run reuses it', async () => {
+ const config = importingConfig();
+ const rig = makeRig({ configFile: config.configFile });
+ const first = granted(await rig.supervisor.acquire(rig.request('run-a')));
+ await rig.supervisor.release(first.token, false);
+ const [server] = rig.launcher.launched;
+ if (server === undefined) throw new Error('the supervisor launched no server');
+
+ config.edit('v2');
+ await restartContractServer(server);
+ await expect
+ .poll(() => contractServerGeneration(server).catch(() => 0), { timeout: RESTART_SETTLE_MS })
+ .toBe(2);
+
+ const second = granted(await rig.supervisor.acquire(rig.request('run-b')));
+ expect(second).toMatchObject({ launched: false, baseURL: first.baseURL, port: first.port });
+ expect(rig.launcher.launched).toHaveLength(1);
+});
diff --git a/packages/app/tests/foundation/test-server-inputs.node.test.ts b/packages/app/tests/foundation/test-server-inputs.node.test.ts
index db13f7fde..ee39ad8d3 100644
--- a/packages/app/tests/foundation/test-server-inputs.node.test.ts
+++ b/packages/app/tests/foundation/test-server-inputs.node.test.ts
@@ -179,6 +179,21 @@ describe('prebuilt DEV output freshness', () => {
);
});
+ test('an edited env file Vite reads at build time refuses the recorded output and names the file', () => {
+ const root = okTree();
+ write(root, 'packages/app/.env.local', 'VITE_OTEL_ENABLED=false\n');
+ const recorded = prebuiltManifest(root, BUILD_OPTIONS);
+
+ write(root, 'packages/app/.env.local', 'VITE_OTEL_ENABLED=true\n');
+
+ expect(prebuiltServingDecision(recorded, prebuiltManifest(root, BUILD_OPTIONS))).toEqual({
+ serve: false,
+ reason: 'stale',
+ changedInputs: ['packages/app/.env.local'],
+ changedOptions: [],
+ });
+ });
+
test('a changed build option refuses the recorded output and names the option', () => {
const root = okTree();
const recorded = prebuiltManifest(root, BUILD_OPTIONS);
diff --git a/packages/app/tests/foundation/test-server-inputs.ts b/packages/app/tests/foundation/test-server-inputs.ts
index 243226415..5641d5570 100644
--- a/packages/app/tests/foundation/test-server-inputs.ts
+++ b/packages/app/tests/foundation/test-server-inputs.ts
@@ -27,6 +27,7 @@ export const SERVER_RESTART_INPUTS = [
export const PREBUILT_BUILD_INPUTS = [
...SERVER_RESTART_INPUTS,
'packages/app/*.html',
+ 'packages/app/.env*',
'packages/app/public/**',
'packages/app/src/**',
] as const;
diff --git a/packages/app/tests/foundation/worker-lifetime/failing-ipc-send.cjs b/packages/app/tests/foundation/worker-lifetime/failing-ipc-send.cjs
new file mode 100644
index 000000000..77b8e9ca5
--- /dev/null
+++ b/packages/app/tests/foundation/worker-lifetime/failing-ipc-send.cjs
@@ -0,0 +1,12 @@
+process.send = function send(message, ...rest) {
+ process.stderr.write(`failing ipc send: ${message?.type}\n`);
+ const error = Object.assign(new Error('write EPIPE'), {
+ code: 'EPIPE',
+ errno: -32,
+ syscall: 'write',
+ });
+ const callback = rest.find((part) => typeof part === 'function');
+ if (callback === undefined) process.nextTick(() => process.emit('error', error));
+ else process.nextTick(callback, error);
+ return false;
+};
diff --git a/packages/app/tests/foundation/worker-lifetime/observe-standalone-prebuilt-lifetime.test-helper.ts b/packages/app/tests/foundation/worker-lifetime/observe-standalone-prebuilt-lifetime.test-helper.ts
new file mode 100644
index 000000000..058777bd9
--- /dev/null
+++ b/packages/app/tests/foundation/worker-lifetime/observe-standalone-prebuilt-lifetime.test-helper.ts
@@ -0,0 +1,372 @@
+import { createHash } from 'node:crypto';
+import { existsSync, mkdirSync, readFileSync, symlinkSync, writeFileSync } from 'node:fs';
+import { createRequire } from 'node:module';
+import { dirname, join, resolve } from 'node:path';
+import { setTimeout as pause } from 'node:timers/promises';
+import { isDeepStrictEqual } from 'node:util';
+import { z } from 'zod';
+import { WORKER_SERVER_BUDGET_TOTAL_MS } from '../../stress/_helpers/fixtures.ts';
+import { BUILD_STEP_SILENCE_MS } from '../../stress/_helpers/prebuilt/prebuilt-dev-build.ts';
+import {
+ APP_DIR,
+ type CleanupReceipt,
+ identities,
+ identitySchema,
+ isSameLiveIdentity,
+ makeRunDir,
+ type ProcessIdentity,
+ startResourceOwner,
+} from './owner-launch.test-helper.ts';
+
+const INNER_RUN_LIMIT_MS = 2 * BUILD_STEP_SILENCE_MS + 2 * WORKER_SERVER_BUDGET_TOTAL_MS + 120_000;
+export const OBSERVATION_LIMIT_MS = INNER_RUN_LIMIT_MS + 60_000 + 20_000;
+export const INNER_EXPECTED = [
+ {
+ file: 'peer.e2e.ts',
+ name: 'lifetime subject: peer uses its separate real server',
+ outcome: 'passed',
+ },
+ {
+ file: 'exit.e2e.ts',
+ name: 'lifetime subject: worker exits after both real servers are ready',
+ outcome: 'failed',
+ },
+] as const;
+
+const readySchema = z.object({
+ pid: z.number().int().positive(),
+ at: z.number(),
+ apiStatus: z.literal(200),
+ identity: z.record(z.string(), z.unknown()),
+ server: z.object({
+ baseURL: z.url(),
+ port: z.number().int().positive(),
+ contentDir: z.string(),
+ }),
+});
+type ReadyServer = z.infer<typeof readySchema>;
+
+const eventSchema = z.object({
+ event: z.string(),
+ runId: z.string(),
+ identity: identitySchema.optional(),
+ environment: z.record(z.string(), z.string()).optional(),
+});
+
+type NativeRow = {
+ file: string;
+ name: string;
+ project: string;
+ status: string;
+ workerIndex: number;
+ errors: string[];
+};
+const nativeSpecSchema = z.object({
+ file: z.string(),
+ title: z.string(),
+ tests: z.array(
+ z.object({
+ projectName: z.string(),
+ results: z.array(
+ z.object({
+ status: z.string(),
+ workerIndex: z.number(),
+ errors: z.array(z.object({ message: z.string().optional() })).default([]),
+ }),
+ ),
+ }),
+ ),
+});
+
+function nativeRows(report: unknown): NativeRow[] {
+ const root = z
+ .object({ suites: z.array(z.unknown()), errors: z.array(z.unknown()) })
+ .parse(report);
+ if (root.errors.length !== 0) throw new Error('inner runner has setup or collection errors');
+ const rows: NativeRow[] = [];
+ function walk(raw: unknown): void {
+ const suite = z
+ .object({
+ specs: z.array(nativeSpecSchema).default([]),
+ suites: z.array(z.unknown()).default([]),
+ })
+ .parse(raw);
+ for (const spec of suite.specs) {
+ for (const test of spec.tests) {
+ for (const result of test.results) {
+ rows.push({
+ file: spec.file,
+ name: spec.title,
+ project: test.projectName,
+ status: result.status,
+ workerIndex: result.workerIndex,
+ errors: result.errors.map((error) => error.message ?? ''),
+ });
+ }
+ }
+ }
+ for (const child of suite.suites) walk(child);
+ }
+ for (const suite of root.suites) walk(suite);
+ return rows;
+}
+
+type ServerSample = {
+ kind: 'exit' | 'peer';
+ livePids: number[];
+ processes: { birth: ProcessIdentity; current: ProcessIdentity }[];
+ apiStatus: number | null;
+ identityStatus: number | null;
+ identityBody: unknown;
+ ownedListener: boolean;
+};
+export type LifetimeObservation = {
+ dir: string;
+ runId: string;
+ exit: { code: number | null; signal: string | null; at: number };
+ native: NativeRow[];
+ ready: { exit: ReadyServer; peer: ReadyServer };
+ serverBirths: { exit: ProcessIdentity[]; peer: ProcessIdentity[] };
+ samples: { at: number; observedAt: number; elapsedMs: number; servers: ServerSample[] }[];
+ cleanup: CleanupReceipt;
+};
+
+export async function observeStandalonePrebuiltWorkerLifetime(): Promise<LifetimeObservation> {
+ const mode = 'prebuilt';
+ if (process.version !== 'v24.21.0')
+ throw new Error(`worker lifetime substrate requires Node24.21.0, got ${process.version}`);
+ const require = createRequire(import.meta.url);
+ const browser: { chromium: { executablePath(): string } } = require('@playwright/test');
+ if (!existsSync(browser.chromium.executablePath()))
+ throw new Error('worker lifetime substrate has no pinned Chromium executable');
+ const run = makeRunDir('standalone-prebuilt');
+ const casesDir = join(run.dir, 'cases');
+ mkdirSync(casesDir);
+ writeFileSync(join(casesDir, 'package.json'), `${JSON.stringify({ type: 'module' })}\n`);
+ symlinkSync(join(APP_DIR, 'node_modules'), join(casesDir, 'node_modules'), 'dir');
+ const fixturePath = join(APP_DIR, 'tests', 'stress', 'prebuilt-dev-serving.e2e.ts');
+ const fixtureSource = readFileSync(fixturePath, 'utf8');
+ const resolvedFixture = fixtureSource.replace(
+ /\bfrom\s+(['"])(\.[^'"]+)\1/g,
+ (_match, _quote, path: string) => `from ${JSON.stringify(resolve(dirname(fixturePath), path))}`,
+ );
+ const casePath = join(
+ APP_DIR,
+ 'tests',
+ 'meta',
+ '_helpers',
+ 'worker-lifetime',
+ 'standalone-case.test-helper.ts',
+ );
+ const caseSource = readFileSync(casePath);
+ const caseInputs = INNER_EXPECTED.map((required) => {
+ const kind = required.file === 'exit.e2e.ts' ? 'exit' : 'peer';
+ const appended = `
+import { runStandaloneLifetimeCase } from ${JSON.stringify(casePath)};
+test(${JSON.stringify(required.name)}, async ({ prebuilt }) => {
+ await runStandaloneLifetimeCase(prebuilt.server, ${JSON.stringify(kind)});
+});
+`;
+ const executableSource = resolvedFixture + appended;
+ const executablePath = join(casesDir, required.file);
+ writeFileSync(executablePath, executableSource);
+ return {
+ sourcePath: fixturePath,
+ sourceSha256: createHash('sha256').update(fixtureSource).digest('hex'),
+ casePath,
+ caseSha256: createHash('sha256').update(caseSource).digest('hex'),
+ executablePath,
+ executableSha256: createHash('sha256').update(executableSource).digest('hex'),
+ retainedFixtureSource: fixtureSource,
+ originalAssertionsRetained: true,
+ alterations:
+ 'Only static relative imports resolve to current source, and the standalone lifetime case is appended.',
+ };
+ });
+ writeFileSync(join(run.dir, 'case-inputs.json'), `${JSON.stringify(caseInputs, null, 2)}\n`);
+ const args = [
+ require.resolve('@playwright/test/cli'),
+ 'test',
+ '--config',
+ join(APP_DIR, 'playwright.worker-lifetime.config.ts'),
+ 'peer.e2e.ts',
+ 'exit.e2e.ts',
+ '--grep',
+ 'lifetime subject:',
+ '--workers=2',
+ '--retries=0',
+ '--reporter=list,json',
+ `--global-timeout=${INNER_RUN_LIMIT_MS}`,
+ `--output=${join(run.dir, 'test-results')}`,
+ ];
+ writeFileSync(
+ join(run.dir, 'inner-intent.json'),
+ `${JSON.stringify({ at: Date.now(), cwd: APP_DIR, args, expected: INNER_EXPECTED, mode, outerLifetimeMs: OBSERVATION_LIMIT_MS, deadlines: [0, 10, 60] }, null, 2)}\n`,
+ );
+ const owner = startResourceOwner({
+ ...run,
+ cwd: APP_DIR,
+ args,
+ environment: {
+ OK_TEST_LIFETIME_DIR: run.dir,
+ OK_TEST_E2E_SERVING: mode,
+ OK_TEST_E2E_REUSE_SOCKET: '',
+ OK_TEST_E2E_REUSE_RUN: '',
+ OK_TEST_PREBUILT_APP: '',
+ PLAYWRIGHT_JSON_OUTPUT_FILE: join(run.dir, 'native.json'),
+ PLAYWRIGHT_JSON_OUTPUT_NAME: '',
+ PLAYWRIGHT_JSON_OUTPUT_DIR: '',
+ },
+ deadlineMs: OBSERVATION_LIMIT_MS,
+ });
+ let observation: Omit<LifetimeObservation, 'cleanup'> | undefined;
+ let observationError: unknown;
+ try {
+ const exit = await owner.subjectExit;
+ const native = nativeRows(JSON.parse(readFileSync(join(run.dir, 'native.json'), 'utf8')));
+ for (const required of INNER_EXPECTED) {
+ const matches = native.filter(
+ (row) => row.file.endsWith(required.file) && row.name === required.name,
+ );
+ if (matches.length !== 1 || matches[0]?.status !== required.outcome)
+ throw new Error(
+ `inner membership or execution mismatch: ${JSON.stringify({ required, matches })}`,
+ );
+ if (
+ required.outcome === 'failed' &&
+ !matches[0].errors.some((error) => error.includes('code=83, signal=null'))
+ )
+ throw new Error(
+ `inner exit is not the intended code83 trigger: ${JSON.stringify(matches)}`,
+ );
+ }
+ const ready = {
+ exit: readySchema.parse(JSON.parse(readFileSync(join(run.dir, 'exit-ready.json'), 'utf8'))),
+ peer: readySchema.parse(JSON.parse(readFileSync(join(run.dir, 'peer-ready.json'), 'utf8'))),
+ };
+ if (
+ ready.exit.server.contentDir === ready.peer.server.contentDir ||
+ ready.exit.server.port === ready.peer.server.port
+ )
+ throw new Error('inner ready servers are not isolated');
+ const events = readFileSync(join(run.dir, 'events.jsonl'), 'utf8')
+ .split('\n')
+ .filter(Boolean)
+ .map((line) => eventSchema.parse(JSON.parse(line)));
+ if (events.some((event) => event.runId !== run.runId))
+ throw new Error('foreign native startup record');
+ const serverBirths = { exit: [] as ProcessIdentity[], peer: [] as ProcessIdentity[] };
+ for (const kind of ['exit', 'peer'] as const) {
+ const byIdentity = new Map<string, ProcessIdentity>();
+ for (const event of events) {
+ if (event.event !== 'spawn' && event.event !== 'runtime') continue;
+ if (
+ event.environment?.OK_TEST_CONTENT_DIR !== ready[kind].server.contentDir ||
+ !event.identity?.found
+ )
+ continue;
+ byIdentity.set(`${event.identity.pid}:${event.identity.stamp}`, event.identity);
+ }
+ serverBirths[kind] = [...byIdentity.values()];
+ if (serverBirths[kind].length < 2)
+ throw new Error(`no real leader/server startup identities for ${kind}`);
+ }
+ const prebuiltDirectories = [
+ ...new Set(
+ events
+ .map((event) => event.environment?.OK_TEST_PREBUILT_APP)
+ .filter((path): path is string => path !== undefined && path !== ''),
+ ),
+ ];
+ writeFileSync(
+ join(run.dir, 'served-manifests.json'),
+ `${JSON.stringify(
+ prebuiltDirectories.map((path) => {
+ const manifest = join(path, 'prebuilt-manifest.json');
+ const bytes = readFileSync(manifest);
+ return {
+ path: manifest,
+ sha256: createHash('sha256').update(bytes).digest('hex'),
+ value: JSON.parse(bytes.toString('utf8')),
+ };
+ }),
+ null,
+ 2,
+ )}\n`,
+ );
+ const samples: LifetimeObservation['samples'] = [];
+ for (const at of [0, 10, 60]) {
+ const remaining = exit.at + at * 1_000 - Date.now();
+ if (remaining > 0) await pause(remaining);
+ const observedAt = Date.now();
+ const servers: ServerSample[] = [];
+ for (const kind of ['exit', 'peer'] as const) {
+ const births = serverBirths[kind];
+ const current = identities(
+ births.map((birth) => birth.pid),
+ run.dir,
+ run.runId,
+ );
+ const processes = births.map((birth, index) => ({
+ birth,
+ current: identitySchema.parse(current[index]),
+ }));
+ const response = await fetch(`${ready[kind].server.baseURL}/api/config`, {
+ signal: AbortSignal.timeout(1_000),
+ }).catch(() => null);
+ const identity = await fetch(`${ready[kind].server.baseURL}/__ok-test-server-identity`, {
+ signal: AbortSignal.timeout(1_000),
+ }).catch(() => null);
+ const identityBody: unknown = identity === null ? null : await identity.json();
+ servers.push({
+ kind,
+ processes,
+ livePids: processes
+ .filter((row) => isSameLiveIdentity(row.birth, row.current))
+ .map((row) => row.birth.pid),
+ apiStatus: response?.status ?? null,
+ identityStatus: identity?.status ?? null,
+ identityBody,
+ ownedListener:
+ identity?.status === 200 && isDeepStrictEqual(identityBody, ready[kind].identity),
+ });
+ }
+ samples.push({ at, observedAt, elapsedMs: observedAt - exit.at, servers });
+ writeFileSync(
+ join(run.dir, 'samples.json'),
+ `${JSON.stringify({ runId: run.runId, exit, ready, serverBirths, samples }, null, 2)}\n`,
+ );
+ }
+ observation = { dir: run.dir, runId: run.runId, exit, native, ready, serverBirths, samples };
+ } catch (error) {
+ observationError = error;
+ writeFileSync(
+ join(run.dir, 'observation-error.json'),
+ `${JSON.stringify({ message: String(error), stack: error instanceof Error ? error.stack : undefined }, null, 2)}\n`,
+ );
+ }
+ let cleanup: CleanupReceipt;
+ try {
+ cleanup = await owner.release();
+ } catch (error) {
+ if (observationError !== undefined)
+ throw new AggregateError(
+ [observationError, error],
+ `observation and cleanup failed; see ${run.dir}`,
+ );
+ throw error;
+ }
+ if (cleanup.liveSurvivors.length !== 0 || !cleanup.workDirAbsent) {
+ throw new AggregateError(
+ [
+ ...(observationError === undefined ? [] : [observationError]),
+ new Error('worker lifetime observer cleanup incomplete'),
+ ],
+ `see ${run.dir}`,
+ );
+ }
+ if (observationError !== undefined) throw observationError;
+ if (observation === undefined)
+ throw new Error(`worker lifetime observation not established; see ${run.dir}`);
+ return { ...observation, cleanup };
+}
diff --git a/packages/app/tests/foundation/worker-lifetime/observe-worker-lifetime.test-helper.ts b/packages/app/tests/foundation/worker-lifetime/observe-worker-lifetime.test-helper.ts
new file mode 100644
index 000000000..d7ae58dae
--- /dev/null
+++ b/packages/app/tests/foundation/worker-lifetime/observe-worker-lifetime.test-helper.ts
@@ -0,0 +1,357 @@
+import { createHash } from 'node:crypto';
+import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs';
+import { createRequire } from 'node:module';
+import { join } from 'node:path';
+import { setTimeout as pause } from 'node:timers/promises';
+import { isDeepStrictEqual } from 'node:util';
+import { z } from 'zod';
+import { WORKER_SERVER_BUDGET_TOTAL_MS } from '../../stress/_helpers/fixtures.ts';
+import { BUILD_STEP_SILENCE_MS } from '../../stress/_helpers/prebuilt/prebuilt-dev-build.ts';
+import {
+ APP_DIR,
+ type CleanupReceipt,
+ identities,
+ identitySchema,
+ isSameLiveIdentity,
+ makeRunDir,
+ type ProcessIdentity,
+ startResourceOwner,
+} from './owner-launch.test-helper.ts';
+
+const INNER_RUN_LIMIT_MS = 2 * BUILD_STEP_SILENCE_MS + 2 * WORKER_SERVER_BUDGET_TOTAL_MS + 120_000;
+export const OBSERVATION_LIMIT_MS = INNER_RUN_LIMIT_MS + 60_000 + 20_000;
+export const INNER_EXPECTED = [
+ {
+ file: 'peer.e2e.ts',
+ name: 'lifetime subject: peer uses its separate real server',
+ outcome: 'passed',
+ },
+ {
+ file: 'exit.e2e.ts',
+ name: 'lifetime subject: worker exits after both real servers are ready',
+ outcome: 'failed',
+ },
+] as const;
+
+const readySchema = z.object({
+ pid: z.number().int().positive(),
+ at: z.number(),
+ apiStatus: z.literal(200),
+ identity: z.record(z.string(), z.unknown()),
+ server: z.object({
+ baseURL: z.url(),
+ port: z.number().int().positive(),
+ contentDir: z.string(),
+ }),
+});
+type ReadyServer = z.infer<typeof readySchema>;
+
+const eventSchema = z.object({
+ event: z.string(),
+ runId: z.string(),
+ identity: identitySchema.optional(),
+ environment: z.record(z.string(), z.string()).optional(),
+});
+
+type NativeRow = {
+ file: string;
+ name: string;
+ project: string;
+ status: string;
+ workerIndex: number;
+ errors: string[];
+};
+const nativeSpecSchema = z.object({
+ file: z.string(),
+ title: z.string(),
+ tests: z.array(
+ z.object({
+ projectName: z.string(),
+ results: z.array(
+ z.object({
+ status: z.string(),
+ workerIndex: z.number(),
+ errors: z.array(z.object({ message: z.string().optional() })).default([]),
+ }),
+ ),
+ }),
+ ),
+});
+
+function nativeRows(report: unknown): NativeRow[] {
+ const root = z
+ .object({ suites: z.array(z.unknown()), errors: z.array(z.unknown()) })
+ .parse(report);
+ if (root.errors.length !== 0) throw new Error('inner runner has setup or collection errors');
+ const rows: NativeRow[] = [];
+ function walk(raw: unknown): void {
+ const suite = z
+ .object({
+ specs: z.array(nativeSpecSchema).default([]),
+ suites: z.array(z.unknown()).default([]),
+ })
+ .parse(raw);
+ for (const spec of suite.specs) {
+ for (const test of spec.tests) {
+ for (const result of test.results) {
+ rows.push({
+ file: spec.file,
+ name: spec.title,
+ project: test.projectName,
+ status: result.status,
+ workerIndex: result.workerIndex,
+ errors: result.errors.map((error) => error.message ?? ''),
+ });
+ }
+ }
+ }
+ for (const child of suite.suites) walk(child);
+ }
+ for (const suite of root.suites) walk(suite);
+ return rows;
+}
+
+type ServerSample = {
+ kind: 'exit' | 'peer';
+ livePids: number[];
+ processes: { birth: ProcessIdentity; current: ProcessIdentity }[];
+ apiStatus: number | null;
+ identityStatus: number | null;
+ identityBody: unknown;
+ ownedListener: boolean;
+};
+export type LifetimeObservation = {
+ dir: string;
+ runId: string;
+ exit: { code: number | null; signal: string | null; at: number };
+ native: NativeRow[];
+ ready: { exit: ReadyServer; peer: ReadyServer };
+ serverBirths: { exit: ProcessIdentity[]; peer: ProcessIdentity[] };
+ samples: { at: number; observedAt: number; elapsedMs: number; servers: ServerSample[] }[];
+ cleanup: CleanupReceipt;
+};
+
+export async function observeWorkerLifetime(
+ mode: 'dev-server' | 'prebuilt',
+): Promise<LifetimeObservation> {
+ if (process.version !== 'v24.21.0')
+ throw new Error(`worker lifetime substrate requires Node24.21.0, got ${process.version}`);
+ const require = createRequire(import.meta.url);
+ const browser: { chromium: { executablePath(): string } } = require('@playwright/test');
+ if (!existsSync(browser.chromium.executablePath()))
+ throw new Error('worker lifetime substrate has no pinned Chromium executable');
+ const run = makeRunDir(mode);
+ const casesDir = join(run.dir, 'cases');
+ mkdirSync(casesDir);
+ writeFileSync(join(casesDir, 'package.json'), `${JSON.stringify({ type: 'module' })}\n`);
+ const caseInputs = ['peer', 'exit'].map((name) => {
+ const sourcePath = join(
+ APP_DIR,
+ 'tests',
+ 'meta',
+ '_helpers',
+ 'worker-lifetime',
+ `${name}-case.test-helper.ts`,
+ );
+ const source = readFileSync(sourcePath, 'utf8');
+ const fixtureImport = "'../../../stress/_helpers/fixtures.ts'";
+ if (source.split(fixtureImport).length !== 2)
+ throw new Error('lifetime case has no unique current fixture import');
+ const executableSource = source.replace(
+ fixtureImport,
+ JSON.stringify(join(APP_DIR, 'tests', 'stress', '_helpers', 'fixtures.ts')),
+ );
+ const executablePath = join(casesDir, `${name}.e2e.ts`);
+ writeFileSync(executablePath, executableSource);
+ return {
+ sourcePath,
+ sourceSha256: createHash('sha256').update(source).digest('hex'),
+ executablePath,
+ executableSha256: createHash('sha256').update(executableSource).digest('hex'),
+ };
+ });
+ writeFileSync(join(run.dir, 'case-inputs.json'), `${JSON.stringify(caseInputs, null, 2)}\n`);
+ const args = [
+ require.resolve('@playwright/test/cli'),
+ 'test',
+ '--config',
+ join(APP_DIR, 'playwright.worker-lifetime.config.ts'),
+ 'peer.e2e.ts',
+ 'exit.e2e.ts',
+ '--workers=2',
+ '--retries=0',
+ '--reporter=list,json',
+ `--global-timeout=${INNER_RUN_LIMIT_MS}`,
+ `--output=${join(run.dir, 'test-results')}`,
+ ];
+ writeFileSync(
+ join(run.dir, 'inner-intent.json'),
+ `${JSON.stringify({ at: Date.now(), cwd: APP_DIR, args, expected: INNER_EXPECTED, mode, outerLifetimeMs: OBSERVATION_LIMIT_MS, deadlines: [0, 10, 60] }, null, 2)}\n`,
+ );
+ const owner = startResourceOwner({
+ ...run,
+ cwd: APP_DIR,
+ args,
+ environment: {
+ OK_TEST_LIFETIME_DIR: run.dir,
+ OK_TEST_E2E_SERVING: mode,
+ OK_TEST_E2E_REUSE_SOCKET: '',
+ OK_TEST_E2E_REUSE_RUN: '',
+ OK_TEST_PREBUILT_APP: '',
+ PLAYWRIGHT_JSON_OUTPUT_FILE: join(run.dir, 'native.json'),
+ PLAYWRIGHT_JSON_OUTPUT_NAME: '',
+ PLAYWRIGHT_JSON_OUTPUT_DIR: '',
+ },
+ deadlineMs: OBSERVATION_LIMIT_MS,
+ });
+ let observation: Omit<LifetimeObservation, 'cleanup'> | undefined;
+ let observationError: unknown;
+ try {
+ const exit = await owner.subjectExit;
+ const native = nativeRows(JSON.parse(readFileSync(join(run.dir, 'native.json'), 'utf8')));
+ for (const required of INNER_EXPECTED) {
+ const matches = native.filter(
+ (row) => row.file.endsWith(required.file) && row.name === required.name,
+ );
+ if (matches.length !== 1 || matches[0]?.status !== required.outcome)
+ throw new Error(
+ `inner membership or execution mismatch: ${JSON.stringify({ required, matches })}`,
+ );
+ if (
+ required.outcome === 'failed' &&
+ !matches[0].errors.some((error) => error.includes('code=83, signal=null'))
+ )
+ throw new Error(
+ `inner exit is not the intended code83 trigger: ${JSON.stringify(matches)}`,
+ );
+ }
+ const ready = {
+ exit: readySchema.parse(JSON.parse(readFileSync(join(run.dir, 'exit-ready.json'), 'utf8'))),
+ peer: readySchema.parse(JSON.parse(readFileSync(join(run.dir, 'peer-ready.json'), 'utf8'))),
+ };
+ if (
+ ready.exit.server.contentDir === ready.peer.server.contentDir ||
+ ready.exit.server.port === ready.peer.server.port
+ )
+ throw new Error('inner ready servers are not isolated');
+ const events = readFileSync(join(run.dir, 'events.jsonl'), 'utf8')
+ .split('\n')
+ .filter(Boolean)
+ .map((line) => eventSchema.parse(JSON.parse(line)));
+ if (events.some((event) => event.runId !== run.runId))
+ throw new Error('foreign native startup record');
+ const serverBirths = { exit: [] as ProcessIdentity[], peer: [] as ProcessIdentity[] };
+ for (const kind of ['exit', 'peer'] as const) {
+ const byIdentity = new Map<string, ProcessIdentity>();
+ for (const event of events) {
+ if (event.event !== 'spawn' && event.event !== 'runtime') continue;
+ if (
+ event.environment?.OK_TEST_CONTENT_DIR !== ready[kind].server.contentDir ||
+ !event.identity?.found
+ )
+ continue;
+ byIdentity.set(`${event.identity.pid}:${event.identity.stamp}`, event.identity);
+ }
+ serverBirths[kind] = [...byIdentity.values()];
+ if (serverBirths[kind].length < 2)
+ throw new Error(`no real leader/server startup identities for ${kind}`);
+ }
+ const prebuiltDirectories = [
+ ...new Set(
+ events
+ .map((event) => event.environment?.OK_TEST_PREBUILT_APP)
+ .filter((path): path is string => path !== undefined && path !== ''),
+ ),
+ ];
+ writeFileSync(
+ join(run.dir, 'served-manifests.json'),
+ `${JSON.stringify(
+ prebuiltDirectories.map((path) => {
+ const manifest = join(path, 'prebuilt-manifest.json');
+ const bytes = readFileSync(manifest);
+ return {
+ path: manifest,
+ sha256: createHash('sha256').update(bytes).digest('hex'),
+ value: JSON.parse(bytes.toString('utf8')),
+ };
+ }),
+ null,
+ 2,
+ )}\n`,
+ );
+ const samples: LifetimeObservation['samples'] = [];
+ for (const at of [0, 10, 60]) {
+ const remaining = exit.at + at * 1_000 - Date.now();
+ if (remaining > 0) await pause(remaining);
+ const observedAt = Date.now();
+ const servers: ServerSample[] = [];
+ for (const kind of ['exit', 'peer'] as const) {
+ const births = serverBirths[kind];
+ const current = identities(
+ births.map((birth) => birth.pid),
+ run.dir,
+ run.runId,
+ );
+ const processes = births.map((birth, index) => ({
+ birth,
+ current: identitySchema.parse(current[index]),
+ }));
+ const response = await fetch(`${ready[kind].server.baseURL}/api/config`, {
+ signal: AbortSignal.timeout(1_000),
+ }).catch(() => null);
+ const identity = await fetch(`${ready[kind].server.baseURL}/__ok-test-server-identity`, {
+ signal: AbortSignal.timeout(1_000),
+ }).catch(() => null);
+ const identityBody: unknown = identity === null ? null : await identity.json();
+ servers.push({
+ kind,
+ processes,
+ livePids: processes
+ .filter((row) => isSameLiveIdentity(row.birth, row.current))
+ .map((row) => row.birth.pid),
+ apiStatus: response?.status ?? null,
+ identityStatus: identity?.status ?? null,
+ identityBody,
+ ownedListener:
+ identity?.status === 200 && isDeepStrictEqual(identityBody, ready[kind].identity),
+ });
+ }
+ samples.push({ at, observedAt, elapsedMs: observedAt - exit.at, servers });
+ writeFileSync(
+ join(run.dir, 'samples.json'),
+ `${JSON.stringify({ runId: run.runId, exit, ready, serverBirths, samples }, null, 2)}\n`,
+ );
+ }
+ observation = { dir: run.dir, runId: run.runId, exit, native, ready, serverBirths, samples };
+ } catch (error) {
+ observationError = error;
+ writeFileSync(
+ join(run.dir, 'observation-error.json'),
+ `${JSON.stringify({ message: String(error), stack: error instanceof Error ? error.stack : undefined }, null, 2)}\n`,
+ );
+ }
+ let cleanup: CleanupReceipt;
+ try {
+ cleanup = await owner.release();
+ } catch (error) {
+ if (observationError !== undefined)
+ throw new AggregateError(
+ [observationError, error],
+ `observation and cleanup failed; see ${run.dir}`,
+ );
+ throw error;
+ }
+ if (cleanup.liveSurvivors.length !== 0 || !cleanup.workDirAbsent) {
+ throw new AggregateError(
+ [
+ ...(observationError === undefined ? [] : [observationError]),
+ new Error('worker lifetime observer cleanup incomplete'),
+ ],
+ `see ${run.dir}`,
+ );
+ }
+ if (observationError !== undefined) throw observationError;
+ if (observation === undefined)
+ throw new Error(`worker lifetime observation not established; see ${run.dir}`);
+ return { ...observation, cleanup };
+}
diff --git a/packages/app/tests/foundation/worker-lifetime/observe-worker-spawn-error.test-helper.ts b/packages/app/tests/foundation/worker-lifetime/observe-worker-spawn-error.test-helper.ts
new file mode 100644
index 000000000..196152bfa
--- /dev/null
+++ b/packages/app/tests/foundation/worker-lifetime/observe-worker-spawn-error.test-helper.ts
@@ -0,0 +1,206 @@
+import { type ChildProcess, spawn } from 'node:child_process';
+import { mkdirSync, mkdtempSync, readFileSync, writeFileSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { fileURLToPath } from 'node:url';
+import { z } from 'zod';
+import {
+ APP_DIR,
+ makeRunDir,
+ type OwnerConfig,
+ startResourceOwner,
+} from './owner-launch.test-helper.ts';
+
+export const FAILED_SPAWN_COMMANDS = {
+ 'dev-server': ['run', 'dev', '--host', '127.0.0.1'],
+ prebuilt: ['exec', 'vite', 'preview', '--config', 'vite.prebuilt-dev.ts', '--host', '127.0.0.1'],
+} as const;
+
+export type FailedSpawnMode = keyof typeof FAILED_SPAWN_COMMANDS;
+export type FailedSpawnBackend = 'exported' | 'native';
+
+const terminalSchema = z.object({
+ code: z.number().nullable(),
+ signal: z.string().nullable(),
+});
+const errorSchema = z.object({
+ name: z.string(),
+ message: z.string(),
+ errno: z.number().optional(),
+ code: z.string().optional(),
+ syscall: z.string().optional(),
+ path: z.string().optional(),
+ spawnargs: z.array(z.string()).optional(),
+});
+const afterErrorSchema = z.object({
+ exitCode: z.number().nullable(),
+ signalCode: z.string().nullable(),
+});
+const contractSchema = z.object({
+ error: errorSchema,
+ afterError: afterErrorSchema,
+ exits: z.array(terminalSchema),
+ close: terminalSchema,
+});
+const resourcesSchema = z.object({
+ contentDir: z.string(),
+ viteCacheDir: z.string(),
+ receiptDir: z.string(),
+ logPath: z.string(),
+ emptyPath: z.string(),
+});
+export const failedSpawnObservationSchema = z.object({
+ mode: z.enum(['dev-server', 'prebuilt']),
+ backend: z.enum(['exported', 'native']),
+ node: z.string(),
+ resources: resourcesSchema,
+ contract: contractSchema,
+ at: z.number(),
+});
+
+export function failedSpawnOwnerConfig(
+ mode: FailedSpawnMode,
+ backend: FailedSpawnBackend,
+ ending: 'complete' | 'hold' = 'complete',
+): OwnerConfig {
+ const run = makeRunDir(`spawn-error-${mode}-${backend}-${ending}`);
+ return {
+ ...run,
+ cwd: APP_DIR,
+ args: [fileURLToPath(import.meta.url), 'actor', mode, backend, ending, run.dir],
+ environment: {},
+ deadlineMs: 30_000,
+ };
+}
+
+export function readFailedSpawnObservation(dir: string) {
+ return failedSpawnObservationSchema.parse(
+ JSON.parse(readFileSync(join(dir, 'spawn-error-result.json'), 'utf8')),
+ );
+}
+
+export async function observeWorkerSpawnError(mode: FailedSpawnMode, backend: FailedSpawnBackend) {
+ const config = failedSpawnOwnerConfig(mode, backend);
+ const owner = startResourceOwner(config);
+ try {
+ const exit = await owner.subjectExit;
+ if (exit.code !== 0 || exit.signal !== null) {
+ throw new Error(
+ `failed-spawn observation ended before its result: ${JSON.stringify(exit)}; ${config.dir}`,
+ );
+ }
+ const observation = readFailedSpawnObservation(config.dir);
+ const cleanup = await owner.release();
+ return { observation, cleanup, exit, dir: config.dir };
+ } finally {
+ await owner.release();
+ }
+}
+
+async function runFailedSpawnActor(
+ mode: FailedSpawnMode,
+ backend: FailedSpawnBackend,
+ ending: 'complete' | 'hold',
+ dir: string,
+): Promise<void> {
+ const fixture = await import('../../stress/_helpers/fixtures.ts');
+ const {
+ closeServerLog,
+ createViteStartupRequest,
+ killGracefully: killNative,
+ openServerLog,
+ prepareViteCacheDir,
+ } = await import('../../stress/_helpers/server-process.ts');
+ const { completeWorkerServerRelease, killGracefully } = await import(
+ '../../stress/_helpers/worker-server-lifetime.ts'
+ );
+ const { removeAllDuringTeardown, runTeardownPhases } = await import(
+ '../../stress/_helpers/teardown-fs.ts'
+ );
+ const request = createViteStartupRequest('127.0.0.1');
+ const contentDir = mkdtempSync(join(tmpdir(), 'ok-w412-spawn-error-'));
+ const viteCacheDir = prepareViteCacheDir('w412');
+ const serverLog = openServerLog('w412-spawn-error');
+ const emptyPath = join(contentDir, 'empty-bin');
+ mkdirSync(emptyPath);
+ const outputDir = join(contentDir, 'prebuilt');
+ if (mode === 'prebuilt') {
+ const { prebuiltManifest } = await import('../test-server-inputs.ts');
+ const { PREBUILT_BUILD_OPTIONS, PREBUILT_MANIFEST_FILE } = await import(
+ '../../stress/_helpers/prebuilt/prebuilt-serving.ts'
+ );
+ mkdirSync(outputDir);
+ writeFileSync(
+ join(outputDir, PREBUILT_MANIFEST_FILE),
+ JSON.stringify(prebuiltManifest(join(APP_DIR, '..', '..'), PREBUILT_BUILD_OPTIONS)),
+ );
+ }
+ const proc: ChildProcess =
+ backend === 'native'
+ ? spawn('pnpm', [...FAILED_SPAWN_COMMANDS[mode]], {
+ cwd: APP_DIR,
+ env: { ...process.env, PATH: emptyPath },
+ stdio: ['ignore', serverLog.fd, 'inherit'],
+ })
+ : fixture.spawnWorkerDevServer(
+ request,
+ { PATH: emptyPath },
+ contentDir,
+ viteCacheDir,
+ serverLog,
+ mode === 'prebuilt' ? { kind: 'prebuilt', outputDir } : { kind: 'dev-server' },
+ );
+ const exits: Array<z.infer<typeof terminalSchema>> = [];
+ proc.on('exit', (code, signal) => exits.push({ code, signal }));
+ const closed = new Promise<z.infer<typeof terminalSchema>>((resolve) =>
+ proc.once('close', (code, signal) => resolve({ code, signal })),
+ );
+ const resources = {
+ contentDir,
+ viteCacheDir,
+ receiptDir: request.receiptDir,
+ logPath: serverLog.path,
+ emptyPath,
+ };
+ let error: z.infer<typeof errorSchema>;
+ let afterError: z.infer<typeof afterErrorSchema>;
+ try {
+ const nativeError = await new Promise<Error>((resolve) => proc.once('error', resolve));
+ error = errorSchema.parse(nativeError);
+ afterError = { exitCode: proc.exitCode, signalCode: proc.signalCode };
+ writeFileSync(
+ join(dir, 'spawn-error-ready.json'),
+ JSON.stringify({ mode, backend, error, afterError, resources, at: Date.now() }),
+ );
+ if (ending === 'hold') {
+ await new Promise<never>(() => setInterval(() => undefined, 1_000));
+ }
+ } finally {
+ await runTeardownPhases(
+ () => (backend === 'native' ? killNative(proc, 5_000) : killGracefully(proc, 5_000)),
+ () => {
+ closeServerLog(serverLog);
+ removeAllDuringTeardown(contentDir, viteCacheDir, request.receiptDir, serverLog.path);
+ },
+ () => (backend === 'exported' ? completeWorkerServerRelease(proc) : undefined),
+ );
+ }
+ writeFileSync(
+ join(dir, 'spawn-error-result.json'),
+ `${JSON.stringify({ mode, backend, node: process.version, resources, contract: { error, afterError, exits, close: await closed }, at: Date.now() }, null, 2)}\n`,
+ );
+}
+
+if (process.argv[1] === fileURLToPath(import.meta.url)) {
+ const [, , role, mode, backend, ending, dir] = process.argv;
+ if (
+ role !== 'actor' ||
+ (mode !== 'dev-server' && mode !== 'prebuilt') ||
+ (backend !== 'exported' && backend !== 'native') ||
+ (ending !== 'complete' && ending !== 'hold') ||
+ dir === undefined
+ ) {
+ throw new Error('failed-spawn actor requires its recorded mode, backend, ending and directory');
+ }
+ await runFailedSpawnActor(mode, backend, ending, dir);
+}
diff --git a/packages/app/tests/foundation/worker-lifetime/owner-launch.test-helper.ts b/packages/app/tests/foundation/worker-lifetime/owner-launch.test-helper.ts
new file mode 100644
index 000000000..ef550bb60
--- /dev/null
+++ b/packages/app/tests/foundation/worker-lifetime/owner-launch.test-helper.ts
@@ -0,0 +1,183 @@
+import { type ChildProcess, fork } from 'node:child_process';
+import { randomUUID } from 'node:crypto';
+import { existsSync, mkdirSync, mkdtempSync, readFileSync, writeFileSync } from 'node:fs';
+import { createRequire } from 'node:module';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { setTimeout as pause } from 'node:timers/promises';
+import { z } from 'zod';
+
+export const HELPERS_DIR = import.meta.dirname;
+export const APP_DIR = join(HELPERS_DIR, '..', '..', '..');
+export const identitySchema = z.object({
+ pid: z.number().int().positive(),
+ found: z.boolean(),
+ uid: z.number().int().optional(),
+ stamp: z.string().optional(),
+ state: z.string().optional(),
+ ppid: z.number().int().optional(),
+ executable: z.string().optional(),
+});
+export type ProcessIdentity = z.infer<typeof identitySchema>;
+
+const nativeIdentity: {
+ identifyAll(pids: number[], eventsPath?: string, runId?: string): unknown;
+} = createRequire(import.meta.url)('./process-identity.cjs');
+
+export function identities(pids: number[], dir: string, runId: string): ProcessIdentity[] {
+ return z
+ .array(identitySchema)
+ .parse(nativeIdentity.identifyAll(pids, join(dir, 'events.jsonl'), runId));
+}
+
+export function isSameLiveIdentity(birth: ProcessIdentity, current: ProcessIdentity): boolean {
+ return (
+ birth.found &&
+ current.found &&
+ birth.pid === current.pid &&
+ birth.uid === current.uid &&
+ birth.stamp === current.stamp &&
+ current.state !== 'Z'
+ );
+}
+
+export const cleanupSchema = z.object({
+ runId: z.string(),
+ reason: z.string(),
+ at: z.number(),
+ before: z.array(z.object({ birth: identitySchema, current: identitySchema })),
+ after: z.array(z.object({ birth: identitySchema, current: identitySchema })),
+ actions: z.array(
+ z.object({
+ recorded: identitySchema,
+ verified: identitySchema,
+ signal: z.string(),
+ at: z.number(),
+ error: z.string().optional(),
+ }),
+ ),
+ liveSurvivors: z.array(z.object({ birth: identitySchema, current: identitySchema })),
+ workDirAbsent: z.boolean(),
+});
+export type CleanupReceipt = z.infer<typeof cleanupSchema>;
+
+export function readCleanup(dir: string): CleanupReceipt {
+ return cleanupSchema.parse(JSON.parse(readFileSync(join(dir, 'cleanup.json'), 'utf8')));
+}
+
+export async function waitUntil(condition: () => boolean, timeoutMs: number): Promise<void> {
+ const deadline = Date.now() + timeoutMs;
+ while (!condition()) {
+ if (Date.now() >= deadline) throw new Error(`condition did not settle in ${timeoutMs}ms`);
+ await pause(25);
+ }
+}
+
+export function makeRunDir(label: string): { dir: string; workDir: string; runId: string } {
+ const root = process.env.OK_WORKER_LIFETIME_ARTIFACT_ROOT ?? tmpdir();
+ mkdirSync(root, { recursive: true });
+ const dir = mkdtempSync(join(root, `ok-worker-lifetime-${label}-`));
+ const workDir = join(dir, 'scratch');
+ mkdirSync(workDir);
+ const runId = randomUUID();
+ writeFileSync(
+ join(dir, 'intent.json'),
+ `${JSON.stringify({ label, runId, at: Date.now(), outerPid: process.pid, node: process.version }, null, 2)}\n`,
+ );
+ return { dir, workDir, runId };
+}
+
+export type OwnerConfig = {
+ dir: string;
+ workDir: string;
+ runId: string;
+ cwd: string;
+ args: string[];
+ environment: Record<string, string>;
+ deadlineMs: number;
+};
+
+export function writeOwnerConfig(config: OwnerConfig): string {
+ const path = join(config.dir, 'owner-config.json');
+ writeFileSync(path, `${JSON.stringify(config, null, 2)}\n`);
+ return path;
+}
+
+export function launchRecorded(
+ script: string,
+ args: string[],
+ dir: string,
+ runId: string,
+): {
+ child: ChildProcess;
+ exited: Promise<{ code: number | null; signal: NodeJS.Signals | null }>;
+} {
+ identities([process.pid], dir, runId);
+ const child = fork(script, args, { execArgv: [], stdio: ['ignore', 'ignore', 'ignore', 'ipc'] });
+ if (child.pid === undefined) throw new Error(`process did not launch: ${script}`);
+ const birth = identities([child.pid], dir, runId)[0];
+ writeFileSync(
+ join(dir, `outer-launch-${child.pid}.json`),
+ `${JSON.stringify({ script, args, pid: child.pid, birth, at: Date.now(), runId }, null, 2)}\n`,
+ );
+ const exited = new Promise<{ code: number | null; signal: NodeJS.Signals | null }>(
+ (resolve, reject) => {
+ child.once('error', reject);
+ child.once('exit', (code, signal) => resolve({ code, signal }));
+ },
+ );
+ return { child, exited };
+}
+
+const ownerMessageSchema = z.discriminatedUnion('type', [
+ z.object({
+ runId: z.string(),
+ type: z.literal('started'),
+ pid: z.number(),
+ childPid: z.number(),
+ birth: identitySchema,
+ }),
+ z.object({
+ runId: z.string(),
+ type: z.literal('subject-exit'),
+ exit: z.object({ code: z.number().nullable(), signal: z.string().nullable(), at: z.number() }),
+ }),
+ z.object({ runId: z.string(), type: z.literal('cleaned'), receipt: cleanupSchema }),
+]);
+
+export function startResourceOwner(config: OwnerConfig): {
+ child: ChildProcess;
+ subjectExit: Promise<{ code: number | null; signal: string | null; at: number }>;
+ release: () => Promise<CleanupReceipt>;
+} {
+ const path = writeOwnerConfig(config);
+ const { child, exited } = launchRecorded(
+ join(HELPERS_DIR, 'resource-owner.cjs'),
+ [path],
+ config.dir,
+ config.runId,
+ );
+ const {
+ promise: subjectExit,
+ resolve: resolveSubject,
+ reject: rejectSubject,
+ } = Promise.withResolvers<{ code: number | null; signal: string | null; at: number }>();
+ child.on('message', (message: unknown) => {
+ const parsed = ownerMessageSchema.parse(message);
+ if (parsed.runId !== config.runId) throw new Error('foreign resource owner message');
+ if (parsed.type === 'subject-exit') resolveSubject(parsed.exit);
+ });
+ void exited.then(() => {
+ if (!existsSync(join(config.dir, 'exit.json')))
+ rejectSubject(new Error(`resource owner exited before its subject; see ${config.dir}`));
+ }, rejectSubject);
+ return {
+ child,
+ subjectExit,
+ release: async () => {
+ if (child.connected) child.send({ type: 'release', runId: config.runId }, () => {});
+ await exited;
+ return readCleanup(config.dir);
+ },
+ };
+}
diff --git a/packages/app/tests/foundation/worker-lifetime/process-identity.cjs b/packages/app/tests/foundation/worker-lifetime/process-identity.cjs
new file mode 100644
index 000000000..2dc1fa90e
--- /dev/null
+++ b/packages/app/tests/foundation/worker-lifetime/process-identity.cjs
@@ -0,0 +1,60 @@
+const { spawnSync } = require('node:child_process');
+const { readFileSync } = require('node:fs');
+const { join } = require('node:path');
+
+function identifyAll(pids, eventsPath, runId) {
+ if (pids.some((pid) => !Number.isInteger(pid) || pid <= 1)) {
+ throw new Error('identity requires positive process IDs');
+ }
+ if (pids.length === 0) return [];
+ if (process.platform === 'darwin') {
+ const result = spawnSync(
+ '/usr/bin/python3',
+ [join(__dirname, 'process-identity.py'), ...pids.map(String)],
+ {
+ encoding: 'utf8',
+ env: {
+ ...process.env,
+ ...(eventsPath === undefined
+ ? {}
+ : { OK_LIFETIME_IDENTITY_EVENTS: eventsPath, OK_LIFETIME_IDENTITY_RUN: runId }),
+ },
+ },
+ );
+ if (result.status !== 0) throw new Error(`identity sampler failed: ${result.stderr}`);
+ return JSON.parse(result.stdout);
+ }
+ if (process.platform === 'linux') {
+ return pids.map((pid) => {
+ try {
+ const stat = readFileSync(`/proc/${pid}/stat`, 'utf8');
+ const fields = stat.slice(stat.lastIndexOf(')') + 2).split(' ');
+ const status = readFileSync(`/proc/${pid}/status`, 'utf8');
+ return {
+ pid,
+ ppid: Number(fields[1]),
+ uid: Number(status.match(/^Uid:\s+(\d+)/m)?.[1]),
+ stamp: fields[19],
+ state: fields[0],
+ found: true,
+ };
+ } catch (error) {
+ if (error.code !== 'ENOENT' && error.code !== 'ESRCH') throw error;
+ return { pid, found: false };
+ }
+ });
+ }
+ throw new Error(`no native process identity substrate for ${process.platform}`);
+}
+
+function sameIdentity(recorded, current) {
+ return (
+ recorded.found === true &&
+ current.found === true &&
+ recorded.pid === current.pid &&
+ recorded.uid === current.uid &&
+ recorded.stamp === current.stamp
+ );
+}
+
+module.exports = { identifyAll, sameIdentity };
diff --git a/packages/app/tests/foundation/worker-lifetime/process-identity.py b/packages/app/tests/foundation/worker-lifetime/process-identity.py
new file mode 100644
index 000000000..fd345bf35
--- /dev/null
+++ b/packages/app/tests/foundation/worker-lifetime/process-identity.py
@@ -0,0 +1,60 @@
+import ctypes
+import json
+import os
+import sys
+
+
+class Info(ctypes.Structure):
+ _fields_ = [
+ (name, ctypes.c_uint32)
+ for name in [
+ "flags", "status", "xstatus", "pid", "ppid", "uid", "gid", "ruid",
+ "rgid", "svuid", "svgid", "reserved",
+ ]
+ ] + [("comm", ctypes.c_char * 16), ("name", ctypes.c_char * 32)] + [
+ (name, ctypes.c_uint32)
+ for name in ["nfiles", "pgid", "pjobc", "tdev", "tpgid"]
+ ] + [
+ ("nice", ctypes.c_int32), ("startSec", ctypes.c_uint64),
+ ("startUsec", ctypes.c_uint64),
+ ]
+
+
+lib = ctypes.CDLL("/usr/lib/libproc.dylib", use_errno=True)
+lib.proc_pidinfo.argtypes = [
+ ctypes.c_int, ctypes.c_int, ctypes.c_uint64, ctypes.c_void_p, ctypes.c_int,
+]
+lib.proc_pidpath.argtypes = [ctypes.c_int, ctypes.c_void_p, ctypes.c_uint32]
+
+
+def identify(pid):
+ info = Info()
+ count = lib.proc_pidinfo(pid, 3, 0, ctypes.byref(info), ctypes.sizeof(info))
+ if count != ctypes.sizeof(info):
+ return {"pid": pid, "found": False, "errno": ctypes.get_errno()}
+ executable = ctypes.create_string_buffer(4096)
+ lib.proc_pidpath(pid, executable, 4096)
+ return {
+ "pid": info.pid, "ppid": info.ppid, "uid": info.uid,
+ "stamp": f"{info.startSec}:{info.startUsec}",
+ "state": "Z" if info.status == 5 else str(info.status),
+ "executable": os.fsdecode(executable.value), "found": True,
+ }
+
+
+destination = os.environ.get("OK_LIFETIME_IDENTITY_EVENTS")
+if destination:
+ with open(destination, "a") as stream:
+ stream.write(json.dumps({
+ "event": "sampler-start", "identity": identify(os.getpid()),
+ "runId": os.environ["OK_LIFETIME_IDENTITY_RUN"],
+ }) + "\n")
+
+print(json.dumps([identify(int(value)) for value in sys.argv[1:]]))
+
+if destination:
+ with open(destination, "a") as stream:
+ stream.write(json.dumps({
+ "event": "sampler-settled", "pid": os.getpid(), "code": 0,
+ "runId": os.environ["OK_LIFETIME_IDENTITY_RUN"],
+ }) + "\n")
diff --git a/packages/app/tests/foundation/worker-lifetime/resource-owner.cjs b/packages/app/tests/foundation/worker-lifetime/resource-owner.cjs
new file mode 100644
index 000000000..1ba686787
--- /dev/null
+++ b/packages/app/tests/foundation/worker-lifetime/resource-owner.cjs
@@ -0,0 +1,199 @@
+const { spawn } = require('node:child_process');
+const {
+ appendFileSync,
+ closeSync,
+ copyFileSync,
+ existsSync,
+ lstatSync,
+ openSync,
+ readFileSync,
+ renameSync,
+ rmSync,
+ writeFileSync,
+} = require('node:fs');
+const { join, relative, isAbsolute } = require('node:path');
+const { setTimeout: pause } = require('node:timers/promises');
+const { identifyAll, sameIdentity } = require('./process-identity.cjs');
+
+const config = JSON.parse(readFileSync(process.argv[2], 'utf8'));
+const { dir, workDir, runId } = config;
+const eventsPath = join(dir, 'events.jsonl');
+const write = (name, value) => {
+ const path = join(dir, name);
+ writeFileSync(`${path}.pending`, `${JSON.stringify(value, null, 2)}\n`);
+ renameSync(`${path}.pending`, path);
+};
+const record = (value) =>
+ appendFileSync(
+ eventsPath,
+ `${JSON.stringify({ at: Date.now(), pid: process.pid, runId, ...value })}\n`,
+ );
+const identify = (pids) => identifyAll(pids, eventsPath, runId);
+const send = (value) => {
+ if (process.connected) process.send({ runId, ...value }, () => {});
+};
+const events = () =>
+ readFileSync(eventsPath, 'utf8')
+ .split('\n')
+ .filter(Boolean)
+ .map((line) => JSON.parse(line));
+const owned = () => {
+ const ids = new Map();
+ for (const event of events()) {
+ if (event.runId !== runId) throw new Error('foreign startup receipt');
+ if (event.event !== 'spawn' && event.event !== 'runtime') continue;
+ const id = event.identity;
+ if (id?.found && id.pid !== process.pid) ids.set(`${id.pid}:${id.stamp}`, id);
+ }
+ return [...ids.values()];
+};
+const inventory = () => {
+ const births = owned();
+ const current = identify(births.map((birth) => birth.pid));
+ return births.map((birth, index) => ({ birth, current: current[index] }));
+};
+const live = (rows) =>
+ rows.filter((row) => sameIdentity(row.birth, row.current) && row.current.state !== 'Z');
+
+record({ event: 'guardian-start', identity: identify([process.pid])[0] });
+const bootstrap = join(dir, 'preload.cjs');
+writeFileSync(
+ bootstrap,
+ `require(${JSON.stringify(join(__dirname, 'startup-observer.cjs'))}).install(${JSON.stringify({ eventsPath, runId })});\n`,
+);
+const fd = openSync(join(dir, 'subject.log'), 'a');
+const command = {
+ exe: process.execPath,
+ args: config.args,
+ cwd: config.cwd,
+ environment: config.environment,
+};
+const child = spawn(command.exe, command.args, {
+ cwd: command.cwd,
+ env: {
+ ...process.env,
+ ...command.environment,
+ TMPDIR: workDir,
+ TMP: workDir,
+ TEMP: workDir,
+ NODE_OPTIONS: `${process.env.NODE_OPTIONS ?? ''} --require=${bootstrap}`.trim(),
+ },
+ stdio: ['ignore', fd, fd],
+});
+const birth = identify([child.pid])[0];
+record({
+ event: 'spawn',
+ identity: birth,
+ command: command.exe,
+ args: command.args,
+ cwd: command.cwd,
+ environment: command.environment,
+});
+write('command.json', { ...command, pid: child.pid, birth, runId, at: Date.now() });
+child.once('exit', (code, signal) => {
+ const exit = { code, signal, at: Date.now() };
+ write('exit.json', exit);
+ send({ type: 'subject-exit', exit });
+});
+child.once('error', (error) => {
+ write('subject-error.json', { message: String(error) });
+ void cleanup('subject spawn error');
+});
+
+let cleanupPromise;
+let deadline;
+async function performCleanup(reason) {
+ clearTimeout(deadline);
+ const before = inventory();
+ const actions = [];
+ for (const signal of ['SIGTERM', 'SIGKILL']) {
+ for (const row of live(inventory()).reverse()) {
+ const verified = identify([row.birth.pid])[0];
+ if (!sameIdentity(row.birth, verified) || verified.state === 'Z') continue;
+ try {
+ process.kill(row.birth.pid, signal);
+ actions.push({ recorded: row.birth, verified, signal, at: Date.now() });
+ } catch (error) {
+ actions.push({
+ recorded: row.birth,
+ verified,
+ signal,
+ at: Date.now(),
+ error: String(error),
+ });
+ if (error.code !== 'ESRCH') throw error;
+ }
+ }
+ const until = Date.now() + (signal === 'SIGTERM' ? 3_000 : 2_000);
+ while (live(inventory()).length !== 0 && Date.now() < until) await pause(50);
+ if (live(inventory()).length === 0) break;
+ }
+ const after = inventory();
+ const remaining = live(after);
+ const resources = events().filter((event) => event.event === 'resource-created');
+ const removals = [];
+ if (remaining.length === 0) {
+ for (const resource of resources) {
+ if (!existsSync(resource.path)) continue;
+ const within = relative(workDir, resource.path);
+ const inWork = !isAbsolute(within) && within !== '..' && !within.startsWith('../');
+ const workerCache =
+ /^\.vite-(?:w\d+|prebuilt-serving)-/.test(resource.path.split('/').at(-1)) &&
+ resource.path.includes('/packages/app/node_modules/');
+ if ((!inWork && !workerCache) || lstatSync(resource.path).isSymbolicLink()) continue;
+ if (resource.kind === 'file') {
+ copyFileSync(
+ resource.path,
+ join(dir, `retained-server-${resource.pid}-${resource.fd}.log`),
+ );
+ }
+ rmSync(resource.path, { recursive: resource.kind === 'directory', force: true });
+ removals.push({ path: resource.path, absent: !existsSync(resource.path) });
+ }
+ rmSync(workDir, { recursive: true, force: true });
+ }
+ closeSync(fd);
+ const receipt = {
+ runId,
+ reason,
+ at: Date.now(),
+ before,
+ actions,
+ after,
+ liveSurvivors: remaining,
+ resources,
+ removals,
+ workDirAbsent: !existsSync(workDir),
+ };
+ write('cleanup.json', receipt);
+ send({ type: 'cleaned', receipt });
+ if (process.connected) process.disconnect();
+ process.exitCode = remaining.length === 0 ? 0 : 1;
+ return receipt;
+}
+
+function cleanup(reason) {
+ cleanupPromise ??= performCleanup(reason).catch((error) => {
+ write('cleanup-error.json', { reason, message: String(error), stack: error.stack });
+ if (process.connected) process.disconnect();
+ process.exitCode = 1;
+ throw error;
+ });
+ return cleanupPromise;
+}
+
+process.once('disconnect', () => void cleanup('outer owner disconnected'));
+process.on('message', (message) => {
+ if (message?.runId !== runId || message.type !== 'release') {
+ throw new Error('invalid resource owner message');
+ }
+ void cleanup('normal release after observation');
+});
+for (const signal of ['SIGINT', 'SIGTERM']) {
+ process.once(signal, () => void cleanup(`guardian received ${signal}`));
+}
+deadline = setTimeout(
+ () => void cleanup('independent maximum lifetime reached'),
+ config.deadlineMs,
+);
+send({ type: 'started', pid: process.pid, childPid: child.pid, birth });
diff --git a/packages/app/tests/foundation/worker-lifetime/safety-outer-owner.cjs b/packages/app/tests/foundation/worker-lifetime/safety-outer-owner.cjs
new file mode 100644
index 000000000..699726232
--- /dev/null
+++ b/packages/app/tests/foundation/worker-lifetime/safety-outer-owner.cjs
@@ -0,0 +1,22 @@
+const { fork } = require('node:child_process');
+const { readFileSync } = require('node:fs');
+const { join } = require('node:path');
+
+const [configPath, guardianPath = join(__dirname, 'resource-owner.cjs')] = process.argv.slice(2);
+const { runId } = JSON.parse(readFileSync(configPath, 'utf8'));
+const guardian = fork(guardianPath, [configPath], {
+ execArgv: [],
+ stdio: ['ignore', 'ignore', 'ignore', 'ipc'],
+});
+guardian.on('message', (message) => {
+ if (process.connected) process.send(message);
+});
+guardian.once('exit', (code) => {
+ if (process.connected) process.disconnect();
+ process.exitCode = code;
+});
+process.on('message', (message) => {
+ if (message?.runId !== runId || message.type !== 'release')
+ throw new Error('invalid safety owner message');
+ guardian.send(message);
+});
diff --git a/packages/app/tests/foundation/worker-lifetime/safety-subject.cjs b/packages/app/tests/foundation/worker-lifetime/safety-subject.cjs
new file mode 100644
index 000000000..41b0fd72c
--- /dev/null
+++ b/packages/app/tests/foundation/worker-lifetime/safety-subject.cjs
@@ -0,0 +1,18 @@
+const { spawn } = require('node:child_process');
+const { writeFileSync } = require('node:fs');
+const { createServer } = require('node:http');
+const { join } = require('node:path');
+
+const dir = process.env.OK_LIFETIME_SAFETY_DIR;
+const role = process.argv[2] ?? 'parent';
+const server = createServer((_request, response) => response.end(role));
+server.listen(0, '127.0.0.1', () => {
+ writeFileSync(
+ join(dir, `safety-${role}.json`),
+ JSON.stringify({ pid: process.pid, port: server.address().port }),
+ );
+ if (role === 'parent') {
+ spawn(process.execPath, [__filename, 'child'], { stdio: 'ignore', env: process.env });
+ }
+});
+process.on('SIGTERM', () => {});
diff --git a/packages/app/tests/foundation/worker-lifetime/startup-observer.cjs b/packages/app/tests/foundation/worker-lifetime/startup-observer.cjs
new file mode 100644
index 000000000..92afdf82e
--- /dev/null
+++ b/packages/app/tests/foundation/worker-lifetime/startup-observer.cjs
@@ -0,0 +1,76 @@
+const cp = require('node:child_process');
+const fs = require('node:fs');
+const { syncBuiltinESMExports } = require('node:module');
+const { identifyAll } = require('./process-identity.cjs');
+
+function install({ eventsPath, runId }) {
+ const append = fs.appendFileSync;
+ const record = (value) =>
+ append(
+ eventsPath,
+ `${JSON.stringify({ at: Date.now(), pid: process.pid, runId, ...value })}\n`,
+ );
+ const identity = (pid) => identifyAll([pid], eventsPath, runId)[0];
+ const selectedEnvironment = (pairs) =>
+ Object.fromEntries(
+ pairs
+ .filter((pair) => pair.startsWith('OK_TEST_') || pair.startsWith('VITE_PORT='))
+ .map((pair) => {
+ const separator = pair.indexOf('=');
+ return [pair.slice(0, separator), pair.slice(separator + 1)];
+ }),
+ );
+ record({
+ event: 'runtime',
+ identity: identity(process.pid),
+ ppid: process.ppid,
+ argv: process.argv,
+ cwd: process.cwd(),
+ node: process.version,
+ environment: selectedEnvironment(
+ Object.entries(process.env).map(([key, value]) => `${key}=${value}`),
+ ),
+ });
+ const spawn = cp.ChildProcess.prototype.spawn;
+ cp.ChildProcess.prototype.spawn = function (options) {
+ const result = spawn.call(this, options);
+ if (Number.isInteger(this.pid) && this.pid > 1) {
+ record({
+ event: 'spawn',
+ identity: identity(this.pid),
+ command: options.file,
+ args: options.args,
+ cwd: options.cwd,
+ detached: options.detached,
+ environment: selectedEnvironment(options.envPairs ?? []),
+ });
+ this.once('exit', (code, signal) =>
+ record({ event: 'child-exit', child: this.pid, code, signal }),
+ );
+ }
+ return result;
+ };
+ const kill = process.kill;
+ process.kill = (pid, signal) => {
+ record({ event: 'subject-signal', target: pid, signal });
+ return kill.call(process, pid, signal);
+ };
+ const mkdtemp = fs.mkdtempSync;
+ fs.mkdtempSync = (prefix, options) => {
+ const path = mkdtemp.call(fs, prefix, options);
+ record({ event: 'resource-created', kind: 'directory', path: String(path) });
+ return path;
+ };
+ const open = fs.openSync;
+ fs.openSync = (path, flags, mode) => {
+ const fd = open.call(fs, path, flags, mode);
+ if (String(path).includes('/ok-e2e-') && String(flags).includes('w')) {
+ record({ event: 'resource-created', kind: 'file', path: String(path), fd });
+ }
+ return fd;
+ };
+ process.on('exit', (code) => record({ event: 'runtime-exit', code }));
+ syncBuiltinESMExports();
+}
+
+module.exports = { install };
diff --git a/packages/app/tests/foundation/worker-server-lifetime-ipc.node.test.ts b/packages/app/tests/foundation/worker-server-lifetime-ipc.node.test.ts
new file mode 100644
index 000000000..b70d94e4b
--- /dev/null
+++ b/packages/app/tests/foundation/worker-server-lifetime-ipc.node.test.ts
@@ -0,0 +1,68 @@
+import { spawn } from 'node:child_process';
+import { existsSync, mkdirSync, mkdtempSync, realpathSync, rmSync, writeFileSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { afterEach, describe, expect, test } from 'vitest';
+import { APP_PACKAGE_ROOT } from '../stress/_helpers/seed-key.ts';
+
+const OWNER = join(APP_PACKAGE_ROOT, 'tests', 'stress', '_helpers', 'worker-server-lifetime.cjs');
+const FAILING_SEND = join(import.meta.dirname, 'worker-lifetime', 'failing-ipc-send.cjs');
+
+const scratchDirs: string[] = [];
+
+afterEach(() => {
+ for (const dir of scratchDirs.splice(0)) rmSync(dir, { recursive: true, force: true });
+});
+
+describe('a worker server lifetime owner whose caller is gone', () => {
+ test('a failed IPC send does not crash the owner, and the disconnect still runs its teardown', async () => {
+ const scratch = realpathSync(mkdtempSync(join(tmpdir(), 'ok-lifetime-ipc-')));
+ scratchDirs.push(scratch);
+ const contentDir = join(scratch, 'content');
+ const viteCacheDir = join(scratch, 'vite-cache');
+ const receiptDir = join(scratch, 'receipt');
+ const serverLogPath = join(scratch, 'server.log');
+ for (const dir of [contentDir, viteCacheDir, receiptDir]) mkdirSync(dir);
+ writeFileSync(serverLogPath, '');
+ const launch = {
+ request: {
+ host: '127.0.0.1',
+ nonce: 'lifetime-ipc',
+ receiptDir,
+ receiptPath: join(receiptDir, 'receipt.json'),
+ environment: {},
+ },
+ workerServerEnv: {},
+ contentDir,
+ viteCacheDir,
+ serverLog: { path: serverLogPath, fd: 1 },
+ serving: { kind: 'prebuilt' },
+ command: null,
+ reapMs: 1_000,
+ };
+
+ const owner = spawn(
+ process.execPath,
+ ['--require', FAILING_SEND, OWNER, JSON.stringify(launch)],
+ { cwd: APP_PACKAGE_ROOT, stdio: ['ignore', 'ignore', 'pipe', 'ipc'] },
+ );
+ let stderr = '';
+ const sendFailed = new Promise<void>((resolve) => {
+ owner.stderr?.on('data', (chunk: Buffer) => {
+ stderr += chunk.toString('utf8');
+ if (stderr.includes('failing ipc send: launch-error')) resolve();
+ });
+ });
+ const exited = new Promise<{ code: number | null; signal: NodeJS.Signals | null }>((resolve) =>
+ owner.once('exit', (code, signal) => resolve({ code, signal })),
+ );
+
+ await sendFailed;
+ if (owner.connected) owner.disconnect();
+
+ expect(await exited, stderr).toEqual({ code: 0, signal: null });
+ expect(
+ [contentDir, viteCacheDir, receiptDir, serverLogPath].filter((path) => existsSync(path)),
+ ).toEqual([]);
+ });
+});
diff --git a/packages/app/tests/integration/e2e-stop-rules.test.ts b/packages/app/tests/integration/e2e-stop-rules.test.ts
index c031f8274..307c5e1d4 100644
--- a/packages/app/tests/integration/e2e-stop-rules.test.ts
+++ b/packages/app/tests/integration/e2e-stop-rules.test.ts
@@ -105,16 +105,21 @@ const CHILD_PROCESS_CALL_PATTERN =
/(?<![\w$])(?:spawnSync|spawn|execFileSync|execFile|execSync|fork)\s*\(|(?<![.\w$])exec\s*\(/;
const MIRROR_EXCLUDED_INFIX = '.private.';
const DEV_SERVER_SPAWN_SITES = [
- 'packages/app/tests/stress/_helpers/fixtures.ts',
+ 'packages/app/tests/stress/_helpers/worker-server-command.ts',
'packages/app/tests/stress/_helpers/global-warm-cache.ts',
] as const;
const NON_DEV_SERVER_SPAWN_SITES = [
+ 'packages/app/tests/stress/_helpers/worker-server-lifetime.ts',
'packages/app/tests/stress/_helpers/i18n-catalog-freshness.ts',
'packages/app/tests/stress/_helpers/port-ownership/cleanup-order-control.test-helper.ts',
'packages/app/tests/stress/_helpers/port-ownership/lifetime-global-setup.ts',
+ 'packages/app/tests/stress/_helpers/port-ownership/nested-playwright-run.test-helper.ts',
'packages/app/tests/stress/_helpers/port-ownership/owner-loss.test-helper.ts',
'packages/app/tests/stress/_helpers/port-ownership/run-case.test-helper.ts',
+ 'packages/app/tests/stress/_helpers/prebuilt/prebuilt-dev-build.ts',
+ 'packages/app/tests/stress/_helpers/reuse/reuse-run.test-helper.ts',
'packages/app/tests/stress/okf-generated-index-settings.e2e.ts',
+ 'packages/app/tests/stress/prebuilt-dev-serving.e2e.ts',
] as const;
const PINNED_SPAWN_SITES = [...DEV_SERVER_SPAWN_SITES, ...NON_DEV_SERVER_SPAWN_SITES].sort();
diff --git a/packages/app/tests/integration/loopback-bind-discipline.test.ts b/packages/app/tests/integration/loopback-bind-discipline.test.ts
index 6f83dfc81..ec014649e 100644
--- a/packages/app/tests/integration/loopback-bind-discipline.test.ts
+++ b/packages/app/tests/integration/loopback-bind-discipline.test.ts
@@ -44,6 +44,7 @@ function isCommentOnlyLine(line: string): boolean {
}
const LOOPBACK_HOST_LITERAL = /['"](?:127\.0\.0\.1|::1)['"]/;
+const UNIX_SOCKET_LISTEN = /^\.listen\(\s*\{(?=[^}]*\bpath\s*[:,}])(?![^}]*\bport\b)/;
export function findNonLoopbackListenCalls(lines: string[]): Array<{ line: number; text: string }> {
const violations: Array<{ line: number; text: string }> = [];
@@ -58,7 +59,7 @@ export function findNonLoopbackListenCalls(lines: string[]): Array<{ line: numbe
}
const paren = window.indexOf(')');
if (paren !== -1) window = window.slice(0, paren + 1);
- if (!LOOPBACK_HOST_LITERAL.test(window)) {
+ if (!LOOPBACK_HOST_LITERAL.test(window) && !UNIX_SOCKET_LISTEN.test(window)) {
violations.push({ line: i + 1, text: line.trim() });
}
}
@@ -302,4 +303,22 @@ describe('loopback bind discipline (app test sources)', () => {
expect(findAmbiguousLocalhostDials([" host: 'localhost:5173',"]).length).toBe(0);
});
+
+ test('a listen on a Unix-socket path is not a host bind, while the object form with a port still needs a loopback host', () => {
+ const planted = [
+ ' s.listen(0);',
+ ' server.listen(port, () => resolve());',
+ ' server.listen({ port }, cb);',
+ " server.listen({ port: 0, host: '0.0.0.0' }, cb);",
+ ' server.listen({ path: socketPath, port: 0 }, cb);',
+ ' server.listen({ path: socketPath }, () => {',
+ ' server.listen({ path }, cb);',
+ ' server.listen(',
+ ' { path: join(dir, "supervisor.sock") },',
+ ' () => resolve(),',
+ ' );',
+ " server.listen({ port: 0, host: '127.0.0.1' }, cb);",
+ ];
+ expect(findNonLoopbackListenCalls(planted).map((v) => v.line)).toEqual([1, 2, 3, 4, 5]);
+ });
});
diff --git a/packages/app/tests/integration/test-reset-server-scope.test.ts b/packages/app/tests/integration/test-reset-server-scope.test.ts
new file mode 100644
index 000000000..5ae3dc9c1
--- /dev/null
+++ b/packages/app/tests/integration/test-reset-server-scope.test.ts
@@ -0,0 +1,422 @@
+import {
+ existsSync,
+ mkdirSync,
+ mkdtempSync,
+ readdirSync,
+ readFileSync,
+ realpathSync,
+ writeFileSync,
+} from 'node:fs';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { HocuspocusProvider, HocuspocusProviderWebsocket } from '@hocuspocus/provider';
+import {
+ bindConfigDoc,
+ CONFIG_DOC_NAME_PROJECT,
+ CONFIG_DOC_NAME_PROJECT_LOCAL,
+ CONFIG_DOC_NAME_USER,
+ type ConfigBinding,
+ ConfigSchema,
+} from '@inkeep/open-knowledge-core';
+import { afterEach, describe, expect, test } from 'vitest';
+import * as Y from 'yjs';
+import { removeAllDuringTeardown } from '../stress/_helpers/teardown-fs.ts';
+import { createTestClient, createTestServer, pollUntil, type TestServer } from './test-harness';
+
+interface BoundConfig {
+ readonly binding: ConfigBinding;
+ close(): void;
+}
+
+interface IsolatedServer extends TestServer {
+ readonly home: string;
+}
+
+interface ServerResetBody {
+ readonly scope: 'server';
+ readonly residue: {
+ readonly loadedDocuments: readonly string[];
+ readonly clientConnections: number;
+ readonly agentSessions: number;
+ readonly agentPresence: readonly string[];
+ readonly staleExternalWrites: readonly string[];
+ readonly removedDocuments: readonly string[];
+ };
+}
+
+const NO_RESIDUE: ServerResetBody['residue'] = {
+ loadedDocuments: [],
+ clientConnections: 0,
+ agentSessions: 0,
+ agentPresence: [],
+ staleExternalWrites: [],
+ removedDocuments: [],
+};
+
+const opened: Array<() => Promise<void> | void> = [];
+
+afterEach(async () => {
+ for (const close of opened.splice(0).reverse()) await close();
+});
+
+async function isolatedServer(): Promise<IsolatedServer> {
+ const home = realpathSync(mkdtempSync(join(tmpdir(), 'ok-reset-home-')));
+ opened.push(() => removeAllDuringTeardown(home));
+ const server = await createTestServer({
+ configHomedirOverride: home,
+ seedProjectConfigYml: '{}\n',
+ });
+ opened.push(() => server.cleanup());
+ return { ...server, home };
+}
+
+async function bindConfig(
+ server: TestServer,
+ docName: string,
+ scope: Parameters<typeof bindConfigDoc>[1],
+): Promise<BoundConfig> {
+ const document = new Y.Doc();
+ const provider = new HocuspocusProvider({
+ url: `${server.wsUrl}/collab`,
+ name: docName,
+ document,
+ });
+ const binding = bindConfigDoc(provider, scope);
+ const close = () => {
+ binding.dispose();
+ provider.destroy();
+ document.destroy();
+ };
+ opened.push(close);
+ await pollUntil(() => binding.hasSynced(), 10_000, 100, `${docName} to sync`);
+ return { binding, close };
+}
+
+async function openClientThatStaysAway(
+ server: TestServer,
+ docName: string,
+): Promise<{ readonly provider: HocuspocusProvider; readonly document: Y.Doc }> {
+ const websocket = new HocuspocusProviderWebsocket({
+ url: `${server.wsUrl}/collab`,
+ delay: 600_000,
+ minDelay: 600_000,
+ });
+ const document = new Y.Doc();
+ const provider = new HocuspocusProvider({
+ websocketProvider: websocket,
+ name: docName,
+ document,
+ });
+ provider.attach();
+ opened.push(() => {
+ provider.destroy();
+ websocket.destroy();
+ document.destroy();
+ });
+ await pollUntil(() => provider.isSynced, 10_000, 100, `${docName} to sync`);
+ return { provider, document };
+}
+
+async function semanticSearchEnabled(server: TestServer): Promise<boolean> {
+ const response = await fetch(`${server.baseUrl}/api/semantic-status`);
+ const body = (await response.json()) as { enabled: boolean };
+ return body.enabled;
+}
+
+async function pageNames(server: TestServer): Promise<string[]> {
+ const response = await fetch(`${server.baseUrl}/api/pages`);
+ const body = (await response.json()) as { pages: Array<{ docName: string }> };
+ return body.pages.map((page) => page.docName);
+}
+
+async function resetServer(server: TestServer): Promise<Response> {
+ return fetch(`${server.baseUrl}/api/test-reset?scope=server`, { method: 'POST' });
+}
+
+async function agentWrite(server: TestServer, docName: string, markdown: string): Promise<void> {
+ const response = await fetch(`${server.baseUrl}/api/agent-write-md`, {
+ method: 'POST',
+ headers: { 'Content-Type': 'application/json' },
+ body: JSON.stringify({
+ docName,
+ markdown,
+ position: 'replace',
+ agentId: 'reset-probe',
+ agentName: 'Reset probe',
+ clientName: 'reset-probe',
+ }),
+ });
+ if (!response.ok) throw new Error(`agent-write-md returned ${response.status}`);
+}
+
+async function saveEverySettingScope(server: IsolatedServer): Promise<void> {
+ const project = await bindConfig(server, CONFIG_DOC_NAME_PROJECT, 'project');
+ const local = await bindConfig(server, CONFIG_DOC_NAME_PROJECT_LOCAL, 'project-local');
+ const user = await bindConfig(server, CONFIG_DOC_NAME_USER, 'user');
+ expect(project.binding.patch({ content: { attachmentFolderPath: 'assets' } }).ok).toBe(true);
+ expect(local.binding.patch({ search: { semantic: { enabled: true } } }).ok).toBe(true);
+ expect(user.binding.patch({ appearance: { theme: 'dark' } }).ok).toBe(true);
+ await pollUntil(
+ () =>
+ existsSync(projectConfigPath(server)) &&
+ readFileSync(projectConfigPath(server), 'utf-8').includes('assets') &&
+ existsSync(localConfigPath(server)) &&
+ existsSync(userConfigPath(server)),
+ 15_000,
+ 100,
+ 'all three setting scopes to persist',
+ );
+ await pollUntil(
+ () => !server.instance.contentFilter.isExcluded('assets/new.png'),
+ 15_000,
+ 100,
+ 'the attachment folder to apply',
+ );
+ await pollUntil(() => semanticSearchEnabled(server), 15_000, 100, 'semantic search to enable');
+ project.close();
+ local.close();
+ user.close();
+}
+
+function projectConfigPath(server: TestServer): string {
+ return join(server.contentDir, '.ok', 'config.yml');
+}
+
+function localConfigPath(server: TestServer): string {
+ return join(server.contentDir, '.ok', 'local', 'config.yml');
+}
+
+function userConfigPath(server: IsolatedServer): string {
+ return join(server.home, '.ok', 'global.yml');
+}
+
+describe('test-reset with scope=server returns an isolated server to its booted state', () => {
+ test('a local setting saved by one run is gone from disk and from its consumers after the reset', async () => {
+ const server = await isolatedServer();
+ const local = await bindConfig(server, CONFIG_DOC_NAME_PROJECT_LOCAL, 'project-local');
+ expect(local.binding.patch({ search: { semantic: { enabled: true } } }).ok).toBe(true);
+ await pollUntil(() => existsSync(localConfigPath(server)), 15_000, 100, 'the local config');
+ await pollUntil(() => semanticSearchEnabled(server), 15_000, 100, 'semantic search to enable');
+ local.close();
+
+ const response = await resetServer(server);
+
+ expect(response.status).toBe(200);
+ expect(existsSync(localConfigPath(server))).toBe(false);
+ expect(await semanticSearchEnabled(server)).toBe(false);
+ }, 60_000);
+
+ test('project, local and user settings saved by one run read as defaults to the next run', async () => {
+ const server = await isolatedServer();
+ await saveEverySettingScope(server);
+
+ const response = await resetServer(server);
+
+ expect(response.status).toBe(200);
+ expect(existsSync(projectConfigPath(server))).toBe(false);
+ expect(existsSync(localConfigPath(server))).toBe(false);
+ expect(existsSync(userConfigPath(server))).toBe(false);
+ expect(server.instance.contentFilter.isExcluded('assets/new.png')).toBe(true);
+ expect(await semanticSearchEnabled(server)).toBe(false);
+ const defaults = ConfigSchema.parse({});
+ for (const [docName, scope] of [
+ [CONFIG_DOC_NAME_PROJECT, 'project'],
+ [CONFIG_DOC_NAME_PROJECT_LOCAL, 'project-local'],
+ [CONFIG_DOC_NAME_USER, 'user'],
+ ] as const) {
+ const next = await bindConfig(server, docName, scope);
+ expect(next.binding.current(), docName).toEqual(defaults);
+ }
+ }, 90_000);
+
+ test('a connected client, an agent session, a removal record and the documents they loaded are gone after the reset', async () => {
+ const server = await isolatedServer();
+ writeFileSync(join(server.contentDir, 'scratch.md'), 'kept on disk\n', 'utf-8');
+ await openClientThatStaysAway(server, 'scratch');
+ await openClientThatStaysAway(server, CONFIG_DOC_NAME_PROJECT);
+ await agentWrite(server, 'agent-notes', 'written by an agent\n');
+ await agentWrite(server, 'gone', 'deleted before the reset\n');
+ const deleted = await fetch(`${server.baseUrl}/api/delete-path`, {
+ method: 'POST',
+ headers: { 'Content-Type': 'application/json' },
+ body: JSON.stringify({ kind: 'file', path: 'gone' }),
+ });
+ expect(deleted.status).toBe(200);
+ expect(server.instance.hocuspocus.documents.has('scratch')).toBe(true);
+ expect(server.instance.sessionManager.liveSessionCount).toBeGreaterThan(0);
+
+ const response = await resetServer(server);
+
+ expect(response.status).toBe(200);
+ const body = (await response.json()) as ServerResetBody;
+ expect(body).toEqual({ scope: 'server', residue: NO_RESIDUE });
+ expect(readFileSync(join(server.contentDir, 'scratch.md'), 'utf-8')).toBe('kept on disk\n');
+ }, 60_000);
+
+ test('a page deleted just before the reset leaves no removal record once the server has seen the deletion on disk', async () => {
+ const contentDir = realpathSync(mkdtempSync(join(tmpdir(), 'ok-reset-removal-')));
+ opened.push(() => removeAllDuringTeardown(contentDir));
+ const home = realpathSync(mkdtempSync(join(tmpdir(), 'ok-reset-home-')));
+ opened.push(() => removeAllDuringTeardown(home));
+ writeFileSync(join(contentDir, 'test-doc.md'), '', 'utf-8');
+ mkdirSync(join(contentDir, '.ok'));
+ writeFileSync(join(contentDir, '.ok', 'config.yml'), '{}\n', 'utf-8');
+ mkdirSync(join(contentDir, '.claude', 'skills'), { recursive: true });
+ mkdirSync(join(contentDir, 'sidebar-folder'));
+ writeFileSync(join(contentDir, 'sidebar-folder', 'nested-doc.md'), '', 'utf-8');
+ const server = await createTestServer({
+ contentDir,
+ keepContentDir: true,
+ configHomedirOverride: home,
+ });
+ let running = true;
+ opened.push(async () => {
+ if (running) await server.cleanup();
+ });
+ writeFileSync(join(contentDir, 'sidebar-folder', 'left-behind.md'), '# earlier run\n', 'utf-8');
+ await pollUntil(
+ async () => (await pageNames(server)).includes('sidebar-folder/left-behind'),
+ 30_000,
+ 100,
+ 'the server to see a page an earlier run left on disk',
+ );
+
+ const deleted = await fetch(`${server.baseUrl}/api/delete-path`, {
+ method: 'POST',
+ headers: { 'Content-Type': 'application/json' },
+ body: JSON.stringify({ kind: 'file', path: 'sidebar-folder/left-behind' }),
+ });
+ expect(deleted.status).toBe(200);
+ const response = await resetServer(server);
+
+ expect(response.status).toBe(200);
+ expect(await response.json()).toEqual({ scope: 'server', residue: NO_RESIDUE });
+ expect(
+ readdirSync(join(contentDir, '.ok')).filter((name) =>
+ name.startsWith('watcher-settle-fence-'),
+ ),
+ ).toEqual([]);
+ writeFileSync(join(contentDir, 'sidebar-folder', 'after-reset.md'), '', 'utf-8');
+ await pollUntil(
+ async () => (await pageNames(server)).includes('sidebar-folder/after-reset'),
+ 30_000,
+ 100,
+ 'the server to see a page written after the reset',
+ );
+ running = false;
+ await server.cleanup();
+ const journal = JSON.parse(
+ readFileSync(join(contentDir, '.ok', 'local', 'removed-docs.json'), 'utf-8'),
+ ) as { entries: Array<{ docName: string }> };
+ expect(journal.entries.map((entry) => entry.docName)).toEqual([]);
+ }, 120_000);
+
+ test('a server-scope reset that asks to keep .okignore is refused and changes nothing, and a plain one clears .okignore', async () => {
+ const server = await isolatedServer();
+ const local = await bindConfig(server, CONFIG_DOC_NAME_PROJECT_LOCAL, 'project-local');
+ expect(local.binding.patch({ search: { semantic: { enabled: true } } }).ok).toBe(true);
+ await pollUntil(() => existsSync(localConfigPath(server)), 15_000, 100, 'the local config');
+ local.close();
+ const okignorePath = join(server.contentDir, '.okignore');
+ writeFileSync(okignorePath, 'kept-by-request/\n', 'utf-8');
+ await pollUntil(
+ () => server.instance.contentFilter.isExcluded('kept-by-request/note.md'),
+ 15_000,
+ 100,
+ 'the .okignore pattern to apply',
+ );
+
+ const refused = await fetch(
+ `${server.baseUrl}/api/test-reset?scope=server&reset-okignore=false`,
+ { method: 'POST' },
+ );
+
+ expect(refused.status).toBe(400);
+ expect(await refused.json()).toMatchObject({
+ type: 'urn:ok:error:invalid-request',
+ title:
+ 'reset-okignore cannot be combined with scope=server: a server-scope reset always restores .okignore.',
+ });
+ expect(readFileSync(okignorePath, 'utf-8')).toBe('kept-by-request/\n');
+ expect(server.instance.contentFilter.isExcluded('kept-by-request/note.md')).toBe(true);
+ expect(existsSync(localConfigPath(server))).toBe(true);
+ expect(await semanticSearchEnabled(server)).toBe(true);
+
+ expect((await resetServer(server)).status).toBe(200);
+
+ expect(existsSync(okignorePath)).toBe(false);
+ expect(server.instance.contentFilter.isExcluded('kept-by-request/note.md')).toBe(false);
+ expect(existsSync(localConfigPath(server))).toBe(false);
+ }, 60_000);
+
+ test('a project config the next run writes again with the same bytes reaches its consumers after the reset', async () => {
+ const server = await isolatedServer();
+ const seeded = 'content:\n attachmentFolderPath: assets\n';
+ writeFileSync(projectConfigPath(server), seeded, 'utf-8');
+ await pollUntil(
+ () => !server.instance.contentFilter.isExcluded('assets/new.png'),
+ 15_000,
+ 100,
+ 'the seeded attachment folder to apply',
+ );
+
+ expect((await resetServer(server)).status).toBe(200);
+ expect(server.instance.contentFilter.isExcluded('assets/new.png')).toBe(true);
+ writeFileSync(projectConfigPath(server), seeded, 'utf-8');
+
+ await pollUntil(
+ () => !server.instance.contentFilter.isExcluded('assets/new.png'),
+ 15_000,
+ 100,
+ 'the same attachment folder, written again after the reset, to apply',
+ );
+ const next = await bindConfig(server, CONFIG_DOC_NAME_PROJECT, 'project');
+ expect(next.binding.current().content.attachmentFolderPath).toBe('assets');
+ }, 60_000);
+
+ test('an .okignore the next run writes again with the same bytes applies after the reset', async () => {
+ const server = await isolatedServer();
+ const okignorePath = join(server.contentDir, '.okignore');
+ writeFileSync(okignorePath, 'seeded-again/\n', 'utf-8');
+ await pollUntil(
+ () => server.instance.contentFilter.isExcluded('seeded-again/note.md'),
+ 15_000,
+ 100,
+ 'the seeded .okignore pattern to apply',
+ );
+
+ expect((await resetServer(server)).status).toBe(200);
+ expect(server.instance.contentFilter.isExcluded('seeded-again/note.md')).toBe(false);
+ writeFileSync(okignorePath, 'seeded-again/\n', 'utf-8');
+
+ await pollUntil(
+ () => server.instance.contentFilter.isExcluded('seeded-again/note.md'),
+ 15_000,
+ 100,
+ 'the same .okignore pattern, written again after the reset, to apply',
+ );
+ }, 60_000);
+
+ test('resetting one server leaves another server’s settings, documents and content alone', async () => {
+ const reset = await isolatedServer();
+ const other = await isolatedServer();
+ await saveEverySettingScope(reset);
+ await saveEverySettingScope(other);
+ const client = await createTestClient(other.port, 'test-doc', { skipInvariantWatcher: true });
+ opened.push(() => client.cleanup());
+ client.ytext.insert(0, 'live in the other server\n');
+ await agentWrite(other, 'agent-notes', 'other server agent\n');
+ const otherSettings = [projectConfigPath(other), localConfigPath(other), userConfigPath(other)];
+ const settingsBefore = otherSettings.map((path) => readFileSync(path, 'utf-8'));
+
+ expect((await resetServer(reset)).status).toBe(200);
+
+ expect(otherSettings.map((path) => readFileSync(path, 'utf-8'))).toEqual(settingsBefore);
+ expect(other.instance.contentFilter.isExcluded('assets/new.png')).toBe(false);
+ expect(await semanticSearchEnabled(other)).toBe(true);
+ expect(other.instance.hocuspocus.documents.has('test-doc')).toBe(true);
+ expect(other.instance.sessionManager.liveSessionCount).toBeGreaterThan(0);
+ expect(client.provider.isSynced).toBe(true);
+ expect(client.ytext.toString()).toContain('live in the other server');
+ }, 120_000);
+});
diff --git a/packages/app/tests/integration/vite-worker-cachedir-isolation.test.ts b/packages/app/tests/integration/vite-worker-cachedir-isolation.test.ts
index 808c407d1..8df4273f7 100644
--- a/packages/app/tests/integration/vite-worker-cachedir-isolation.test.ts
+++ b/packages/app/tests/integration/vite-worker-cachedir-isolation.test.ts
@@ -10,150 +10,615 @@ const WORKER_FIXTURES = resolve(APP_PACKAGE_ROOT, 'tests/stress/_helpers/fixture
const CACHE_DIR_ENV_VAR = 'OK_TEST_VITE_CACHE_DIR';
-function inspectWorkerCache(source: string) {
- const project = new Project({ useInMemoryFileSystem: true, compilerOptions: { noLib: true } });
- const syntax = project.createSourceFile('/input.ts', source);
- const file = project.createSourceFile(
- '/fixtures.ts',
+function readWorkerCacheGraph(entry: string): Record<string, string> {
+ const modules: Record<string, string> = {};
+ const visit = (path: string) => {
+ if (path in modules) return;
+ const source = readFileSync(path, 'utf8');
+ modules[path] = source;
+ const project = new Project({
+ useInMemoryFileSystem: true,
+ compilerOptions: { allowJs: true },
+ });
+ const file = project.createSourceFile(path, source);
+ for (const declaration of file.getImportDeclarations()) {
+ const module = declaration.getModuleSpecifierValue();
+ if (module.startsWith('.')) visit(resolve(file.getDirectoryPath(), module));
+ }
+ for (const call of file.getDescendantsOfKind(SyntaxKind.CallExpression)) {
+ const first = call.getArguments()[0];
+ if (
+ call.getExpression().getText() === 'require' &&
+ Node.isStringLiteral(first) &&
+ first.getLiteralValue().startsWith('.')
+ )
+ visit(resolve(file.getDirectoryPath(), first.getLiteralValue()));
+ if (call.getExpression().getText() !== 'join' || first?.getText() !== 'import.meta.dirname')
+ continue;
+ const parts = call.getArguments().slice(1);
+ if (
+ parts.length === 1 &&
+ Node.isStringLiteral(parts[0]) &&
+ parts[0].getLiteralValue().endsWith('.cjs')
+ )
+ visit(resolve(file.getDirectoryPath(), parts[0].getLiteralValue()));
+ }
+ };
+ visit(entry);
+ return modules;
+}
+
+function inspectWorkerCache(
+ source: string,
+ modules: Readonly<Record<string, string>> = {},
+ entry = '/fixtures.ts',
+) {
+ type Path = 'failure' | 'normal' | 'acquisition';
+ type FunctionNode =
+ | import('ts-morph').FunctionDeclaration
+ | import('ts-morph').ArrowFunction
+ | import('ts-morph').FunctionExpression;
+ type Frame = {
+ scope: Node;
+ parent?: Frame;
+ key: string;
+ path: Path;
+ bindings: Map<Node, Value>;
+ assignments: Map<import('ts-morph').Symbol, Value>;
+ };
+ type Value = { node: Node; frame: Frame };
+ const project = new Project({
+ useInMemoryFileSystem: true,
+ compilerOptions: {
+ noLib: true,
+ allowJs: true,
+ target: ts.ScriptTarget.ESNext,
+ module: ts.ModuleKind.ESNext,
+ },
+ });
+ for (const [path, text] of Object.entries(modules)) {
+ if (path !== entry) project.createSourceFile(path, text);
+ }
+ const syntax = project.createSourceFile(`${entry}.syntax.ts`, source);
+ const input = project.createSourceFile(
+ entry,
ts.transpile(source, {
target: ts.ScriptTarget.ESNext,
module: ts.ModuleKind.ESNext,
verbatimModuleSyntax: true,
}),
);
- const imported = (module: string, name: string) => {
- const declaration =
- file.getImportDeclaration(module) ?? file.getImportDeclaration(module.replace('node:', ''));
- const entry = declaration?.getNamedImports().find((item) => item.getName() === name);
- return (entry?.getAliasNode() ?? entry?.getNameNode())?.getSymbol();
+ const exported = input.getExportedDeclarations().get('test')?.filter(Node.isVariableDeclaration);
+ const file = exported?.length === 1 ? exported[0].getSourceFile() : input;
+ const functions = (node: Node): node is FunctionNode =>
+ Node.isFunctionDeclaration(node) ||
+ Node.isArrowFunction(node) ||
+ Node.isFunctionExpression(node);
+ const nodeKey = (node: Node) =>
+ `${node.getSourceFile().getFilePath()}:${node.getStart()}:${node.getKind()}`;
+ const key = (value: Value) => `${nodeKey(value.node)}@${value.frame.key}`;
+ const ownFunction = (node: Node) =>
+ node.getFirstAncestor((ancestor) => ts.isFunctionLike(ancestor.compilerNode));
+ const frames = new Map<import('ts-morph').SourceFile, Frame>();
+ const frame = (scope: Node, parent: Frame | undefined, identity: string, path: Path): Frame => {
+ const result: Frame = {
+ scope,
+ parent,
+ key: identity,
+ path,
+ bindings: new Map(),
+ assignments: new Map(),
+ };
+ for (const assignment of scope.getDescendantsOfKind(SyntaxKind.BinaryExpression)) {
+ if (
+ ownFunction(assignment) !== (functions(scope) ? scope : undefined) ||
+ assignment.getOperatorToken().getKind() !== SyntaxKind.EqualsToken
+ )
+ continue;
+ const symbol = assignment.getLeft().getSymbol();
+ if (symbol) result.assignments.set(symbol, { node: assignment.getRight(), frame: result });
+ }
+ return result;
};
- const returned = new Map<import('ts-morph').Symbol, Node>();
- const value = (node: Node | undefined, seen = new Set<Node>()): Node | undefined => {
- if (!node || seen.has(node)) return undefined;
- const next = new Set(seen).add(node);
- if (Node.isParenthesizedExpression(node) || Node.isAwaitExpression(node))
- return value(node.getExpression(), next);
- if (Node.isVariableDeclaration(node))
- return value(node.getInitializer() ?? returned.get(node.getSymbolOrThrow()), next);
- if (Node.isPropertyAssignment(node)) return value(node.getInitializer(), next);
- if (Node.isShorthandPropertyAssignment(node))
- return value(node.getValueSymbol()?.getDeclarations()[0], next);
+ const moduleFrame = (module: import('ts-morph').SourceFile): Frame => {
+ let result = frames.get(module);
+ if (!result) {
+ result = frame(module, undefined, module.getFilePath(), 'acquisition');
+ frames.set(module, result);
+ }
+ return result;
+ };
+ const lexical = (node: Node, context: Frame): Frame => {
+ const owner = ownFunction(node);
+ for (let current: Frame | undefined = context; current; current = current.parent) {
+ if (
+ owner
+ ? current.scope === owner
+ : Node.isSourceFile(current.scope) && current.scope === node.getSourceFile()
+ )
+ return current;
+ }
+ return moduleFrame(node.getSourceFile());
+ };
+ const imported = (node: Node): { module: string; name: string } | undefined => {
+ const declaration = node.getSymbol()?.getDeclarations()[0];
+ if (Node.isImportSpecifier(declaration))
+ return {
+ module: declaration.getImportDeclaration().getModuleSpecifierValue(),
+ name: declaration.getName(),
+ };
+ if (Node.isBindingElement(declaration)) {
+ const owner = declaration.getParent().getParent();
+ const call = Node.isVariableDeclaration(owner)
+ ? owner.getInitializerIfKind(SyntaxKind.CallExpression)
+ : undefined;
+ const path = call?.getArguments()[0];
+ if (call?.getExpression().getText() === 'require' && Node.isStringLiteral(path))
+ return {
+ module: path.getLiteralValue(),
+ name: declaration.getPropertyNameNode()?.getText() ?? declaration.getName(),
+ };
+ }
+ return undefined;
+ };
+ const localModule = (from: Node, path: string) =>
+ project.getSourceFile(resolve(from.getSourceFile().getDirectoryPath(), path));
+ const failures = new Set<string>();
+ const unwrap = (value: Value): Value => {
+ let node = value.node;
+ while (
+ Node.isParenthesizedExpression(node) ||
+ Node.isAwaitExpression(node) ||
+ Node.isAsExpression(node) ||
+ Node.isSatisfiesExpression(node) ||
+ Node.isNonNullExpression(node)
+ )
+ node = node.getExpression();
+ return { ...value, node };
+ };
+ const fn = (value: Value): { node: FunctionNode; closure: Frame } | undefined => {
+ const current = unwrap(value);
+ if (functions(current.node)) return { node: current.node, closure: current.frame };
+ const binding = imported(current.node);
+ if (binding?.module.startsWith('.')) {
+ const target = localModule(current.node, binding.module);
+ const declaration =
+ target?.getExportedDeclarations().get(binding.name)?.find(functions) ??
+ target?.getFunction(binding.name);
+ if (declaration && functions(declaration))
+ return { node: declaration, closure: moduleFrame(declaration.getSourceFile()) };
+ }
+ const declaration = current.node.getSymbol()?.getDeclarations()[0];
+ if (functions(declaration))
+ return { node: declaration, closure: lexical(declaration, current.frame) };
+ return undefined;
+ };
+ const boundFunction = (
+ call: import('ts-morph').CallExpression,
+ context: Frame,
+ target: { node: FunctionNode; closure: Frame },
+ ): Frame => {
+ const result = frame(
+ target.node,
+ target.closure,
+ `${context.key}/${nodeKey(call)}`,
+ context.path,
+ );
+ for (const [index, parameter] of target.node.getParameters().entries()) {
+ const argument = call.getArguments()[index];
+ if (argument) result.bindings.set(parameter, { node: argument, frame: context });
+ else {
+ const initializer = parameter.getInitializer();
+ if (initializer) result.bindings.set(parameter, { node: initializer, frame: result });
+ }
+ }
+ return result;
+ };
+ const value = (input: Value | undefined, seen = new Set<string>()): Value | undefined => {
+ if (!input) return undefined;
+ const current = unwrap(input);
+ const identity = key(current);
+ if (seen.has(identity)) return undefined;
+ const next = new Set(seen).add(identity);
+ const node = current.node;
+ if (Node.isVariableDeclaration(node) || Node.isParameterDeclaration(node)) {
+ const context = lexical(node, current.frame);
+ const bound = context.bindings.get(node);
+ if (bound) return value(bound, next);
+ const initializer = node.getInitializer();
+ const symbol = node.getSymbol();
+ const assigned = symbol && context.assignments.get(symbol);
+ return initializer
+ ? value({ node: initializer, frame: context }, next)
+ : assigned
+ ? value(assigned, next)
+ : { node, frame: context };
+ }
if (Node.isIdentifier(node)) {
const declaration = node.getSymbol()?.getDeclarations()[0];
return declaration &&
- (Node.isVariableDeclaration(declaration) || Node.isBindingElement(declaration))
- ? value(declaration, next)
- : node;
+ (Node.isVariableDeclaration(declaration) ||
+ Node.isParameterDeclaration(declaration) ||
+ Node.isBindingElement(declaration))
+ ? value({ node: declaration, frame: current.frame }, next)
+ : current;
+ }
+ if (Node.isPropertyAssignment(node))
+ return value({ node: node.getInitializerOrThrow(), frame: current.frame }, next);
+ if (Node.isShorthandPropertyAssignment(node)) {
+ const declaration = node.getValueSymbol()?.getDeclarations()[0];
+ return declaration ? value({ node: declaration, frame: current.frame }, next) : undefined;
}
if (Node.isBindingElement(node)) {
const owner = node.getParent().getParent();
- const object = Node.isVariableDeclaration(owner)
- ? value(owner.getInitializer(), next)
- : undefined;
- const name = node.getPropertyNameNode();
- const key = name && Node.isIdentifier(name) ? ts.idText(name.compilerNode) : node.getName();
- const field = object?.getType().getProperty(key)?.getDeclarations()[0];
- return field ? value(field, next) : node;
+ const object =
+ Node.isVariableDeclaration(owner) || Node.isParameterDeclaration(owner)
+ ? value({ node: owner, frame: current.frame }, next)
+ : undefined;
+ return (
+ property(object, node.getPropertyNameNode()?.getText() ?? node.getName(), next) ?? current
+ );
}
- return node;
+ if (Node.isPropertyAccessExpression(node))
+ return (
+ property(
+ value({ node: node.getExpression(), frame: current.frame }, next),
+ node.getName(),
+ next,
+ ) ?? current
+ );
+ if (Node.isCallExpression(node)) {
+ const target = fn(
+ value({ node: node.getExpression(), frame: current.frame }, next) ?? {
+ node: node.getExpression(),
+ frame: current.frame,
+ },
+ );
+ if (target) {
+ const called = boundFunction(node, current.frame, target);
+ const body = target.node.getBody();
+ const returns = body
+ ?.getDescendantsOfKind(SyntaxKind.ReturnStatement)
+ .filter((statement) => ownFunction(statement) === target.node);
+ const returned = Node.isBlock(body)
+ ? returns?.length === 1
+ ? returns[0].getExpression()
+ : undefined
+ : body;
+ if (returned) return value({ node: returned, frame: called }, next);
+ }
+ }
+ return current;
};
- const property = (node: Node | undefined, key: string) =>
- value(
- value(node)
- ?.asKind(SyntaxKind.ObjectLiteralExpression)
- ?.getProperties()
- .find((field) => field.getSymbol()?.getName() === key),
- );
- const calls = file.getDescendantsOfKind(SyntaxKind.CallExpression);
- const phase = file.getFunction('spendOnBudgetPhase')?.getSymbol();
- for (const call of calls) {
- if (!phase || value(call.getExpression())?.getSymbol() !== phase) continue;
- const callback = call.getArguments()[1];
- const result = callback
- ?.getDescendantsOfKind(SyntaxKind.ReturnStatement)
+ const property = (
+ input: Value | undefined,
+ name: string,
+ seen = new Set<string>(),
+ ): Value | undefined => {
+ const object = input && value(input, seen);
+ if (!object || !Node.isObjectLiteralExpression(object.node)) return undefined;
+ const field = object.node
+ .getProperties()
.find(
- (statement) =>
- statement.getFirstAncestor((node) => ts.isFunctionLike(node.compilerNode)) === callback,
+ (field) =>
+ (Node.isPropertyAssignment(field) || Node.isShorthandPropertyAssignment(field)) &&
+ field.getName() === name,
+ );
+ return field ? value({ node: field, frame: object.frame }, seen) : undefined;
+ };
+ const primitive = (input: Value | undefined): string | number | boolean | undefined => {
+ const current = input && value(input);
+ if (current && Node.isStringLiteral(current.node)) return current.node.getLiteralValue();
+ if (current && Node.isNumericLiteral(current.node)) return current.node.getLiteralValue();
+ if (current?.node.getKind() === SyntaxKind.TrueKeyword) return true;
+ if (current?.node.getKind() === SyntaxKind.FalseKeyword) return false;
+ return undefined;
+ };
+ const condition = (node: Node, context: Frame): boolean | undefined => {
+ if (
+ Node.isBinaryExpression(node) &&
+ [SyntaxKind.EqualsEqualsEqualsToken, SyntaxKind.ExclamationEqualsEqualsToken].includes(
+ node.getOperatorToken().getKind(),
)
- ?.getExpression();
- const assignment = call.getFirstAncestorByKind(SyntaxKind.BinaryExpression);
- const binding = assignment?.getLeft().getSymbol();
- if (result && binding && assignment?.getOperatorToken().getKind() === SyntaxKind.EqualsToken)
- returned.set(binding, result);
- }
+ ) {
+ const left = primitive({ node: node.getLeft(), frame: context });
+ const right = primitive({ node: node.getRight(), frame: context });
+ if (left !== undefined && right !== undefined)
+ return node.getOperatorToken().getKind() === SyntaxKind.EqualsEqualsEqualsToken
+ ? left === right
+ : left !== right;
+ }
+ const result = primitive({ node, frame: context });
+ return typeof result === 'boolean' ? result : undefined;
+ };
+ const callsOf = (scope: Node, context: Frame) =>
+ scope.getDescendantsOfKind(SyntaxKind.CallExpression).filter((call) => {
+ if (ownFunction(call) !== (functions(scope) ? scope : undefined)) return false;
+ for (const ancestor of call.getAncestors()) {
+ if (ancestor === scope) break;
+ if (!Node.isIfStatement(ancestor)) continue;
+ const choice = condition(ancestor.getExpression(), context);
+ if (choice === undefined) continue;
+ const selected = choice ? ancestor.getThenStatement() : ancestor.getElseStatement();
+ if (!selected || call.getStart() < selected.getStart() || call.getEnd() > selected.getEnd())
+ return false;
+ }
+ return true;
+ });
const declaration = file.getVariableDeclaration('test');
const extend = declaration?.getInitializerIfKind(SyntaxKind.CallExpression);
const callee = extend?.getExpression().asKind(SyntaxKind.PropertyAccessExpression);
- const fixture = property(extend?.getArguments()[0], 'workerServer')
- ?.asKind(SyntaxKind.ArrayLiteralExpression)
+ const registrationFrame = moduleFrame(file);
+ const workerProperty = property(
+ extend?.getArguments()[0]
+ ? { node: extend.getArguments()[0], frame: registrationFrame }
+ : undefined,
+ 'workerServer',
+ );
+ const fixture = workerProperty?.node
+ .asKind(SyntaxKind.ArrayLiteralExpression)
?.getElements()[0]
?.asKind(SyntaxKind.ArrowFunction);
- const base = imported('@playwright/test', 'test');
- const worker = fixture?.getParameters()[2]?.getSymbol();
- const spawn = imported('node:child_process', 'spawn');
- const fresh = imported('node:fs', 'mkdtempSync');
- const removals = [
- imported('node:fs', 'rmSync'),
- imported('./teardown-fs.ts', 'removeAllDuringTeardown'),
- ].filter((symbol) => symbol !== undefined);
- const spawns = (fixture?.getDescendantsOfKind(SyntaxKind.CallExpression) ?? []).filter(
- (call) => spawn && value(call.getExpression())?.getSymbol() === spawn,
- );
+ const worker = fixture?.getParameters()[2];
+ const workerFrame = fixture ? frame(fixture, registrationFrame, 'worker', 'normal') : undefined;
+ const spawns: Array<{ call: import('ts-morph').CallExpression; frame: Frame }> = [];
+ const reclaims: Array<{ path: Path; resources: Value[] }> = [];
+ const arrays = new Map<string, Value[]>();
+ const owners = new Map<string, { frame: Frame; handler: Value }>();
+ const reached = new Set<string>();
+ const active = new Set<Node>();
+ const callbacks = (argument: Node, context: Frame): Value[] => {
+ if (Node.isSpreadElement(argument)) return callbacks(argument.getExpression(), context);
+ const current = value({ node: argument, frame: context });
+ if (!current) return [];
+ if (Node.isCallExpression(current.node)) {
+ const expression = current.node.getExpression();
+ if (Node.isPropertyAccessExpression(expression) && expression.getName() === 'reverse')
+ return callbacks(expression.getExpression(), current.frame).reverse();
+ }
+ if (Node.isArrayLiteralExpression(current.node)) {
+ const registered = arrays.get(key(current));
+ return registered
+ ? registered
+ : current.node.getElements().flatMap((item) => callbacks(item, current.frame));
+ }
+ return [current];
+ };
+ const nativeName = (node: Node, module: string, name: string) => {
+ const binding = imported(node);
+ return (
+ binding?.name === name &&
+ (binding.module === module || binding.module === module.replace('node:', ''))
+ );
+ };
+ const entryFor = (call: import('ts-morph').CallExpression): string | undefined => {
+ const args = call.getArguments()[1]?.asKind(SyntaxKind.ArrayLiteralExpression)?.getElements();
+ const script = args?.[0]?.asKind(SyntaxKind.CallExpression);
+ if (!script || !nativeName(script.getExpression(), 'node:path', 'join')) return undefined;
+ const parts = script.getArguments();
+ if (
+ parts[0]?.getText() !== 'import.meta.dirname' ||
+ !parts.slice(1).every(Node.isStringLiteral)
+ )
+ return undefined;
+ return resolve(
+ call.getSourceFile().getDirectoryPath(),
+ ...parts.slice(1).map((part) => (part as import('ts-morph').StringLiteral).getLiteralValue()),
+ );
+ };
+ const invoke = (
+ target: { node: FunctionNode; closure: Frame },
+ call: import('ts-morph').CallExpression,
+ context: Frame,
+ ) => {
+ if (active.has(target.node)) return;
+ active.add(target.node);
+ walk(target.node, boundFunction(call, context, target));
+ active.delete(target.node);
+ };
+ const walk = (scope: Node, context: Frame): void => {
+ for (const call of callsOf(scope, context)) {
+ const expression = call.getExpression();
+ const callContext = {
+ ...context,
+ path: call.getFirstAncestorByKind(SyntaxKind.CatchClause)
+ ? ('failure' as const)
+ : context.path,
+ };
+ const current = value({ node: expression, frame: context }) ?? {
+ node: expression,
+ frame: context,
+ };
+ const binding = imported(current.node);
+ if (nativeName(current.node, 'node:child_process', 'spawn')) {
+ if (call.getArguments()[0]?.getText() !== 'process.execPath') {
+ spawns.push({ call, frame: context });
+ continue;
+ }
+ const entryPath = entryFor(call);
+ const owner = entryPath && project.getSourceFile(entryPath);
+ const args = call
+ .getArguments()[1]
+ ?.asKind(SyntaxKind.ArrayLiteralExpression)
+ ?.getElements();
+ const encoded = args?.[1]?.asKind(SyntaxKind.CallExpression);
+ const payload =
+ encoded?.getExpression().getText() === 'JSON.stringify' && encoded.getArguments()[0]
+ ? value({ node: encoded.getArguments()[0], frame: context })
+ : undefined;
+ const decoders = owner
+ ?.getVariableDeclarations()
+ .filter(
+ (variable) => variable.getInitializer()?.getText() === 'JSON.parse(process.argv[2])',
+ );
+ const handlers = owner
+ ?.getDescendantsOfKind(SyntaxKind.CallExpression)
+ .filter(
+ (candidate) =>
+ candidate.getExpression().getText() === 'process.on' &&
+ candidate.getArguments()[0]?.getText() === "'message'",
+ );
+ if (
+ !owner ||
+ !payload ||
+ !Node.isObjectLiteralExpression(payload.node) ||
+ decoders?.length !== 1 ||
+ handlers?.length !== 1 ||
+ !Node.isArrowFunction(handlers[0].getArguments()[1])
+ ) {
+ failures.add(`process-entry:${entryPath ?? nodeKey(call)}`);
+ continue;
+ }
+ const owned = frame(owner, context, `${context.key}/${nodeKey(call)}/entry`, 'acquisition');
+ owned.bindings.set(decoders[0], payload);
+ owners.set(key({ node: call, frame: context }), {
+ frame: owned,
+ handler: { node: handlers[0].getArguments()[1], frame: owned },
+ });
+ walk(owner, owned);
+ continue;
+ }
+ const removal =
+ nativeName(current.node, 'node:fs', 'rmSync') ||
+ (binding?.name === 'removeAllDuringTeardown' && binding.module.endsWith('/teardown-fs.ts'));
+ if (removal) {
+ reclaims.push({
+ path: callContext.path,
+ resources: call
+ .getArguments()
+ .map((arg) => value({ node: arg, frame: context }))
+ .filter((item): item is Value => item !== undefined),
+ });
+ continue;
+ }
+ if (binding?.name === 'runTeardownPhases' && binding.module.endsWith('/teardown-fs.ts')) {
+ for (const argument of call.getArguments())
+ for (const callback of callbacks(argument, context)) {
+ const target = fn(callback);
+ if (target) invoke(target, call, callContext);
+ else failures.add(`release-binding:${nodeKey(call)}`);
+ }
+ continue;
+ }
+ if (Node.isPropertyAccessExpression(expression) && expression.getName() === 'push') {
+ const receiver = value({ node: expression.getExpression(), frame: context });
+ if (receiver && Node.isArrayLiteralExpression(receiver.node)) {
+ const additions = call
+ .getArguments()
+ .map((node) => ({ node, frame: context }))
+ .filter((item) => fn(item));
+ arrays.set(key(receiver), [...(arrays.get(key(receiver)) ?? []), ...additions]);
+ }
+ }
+ if (Node.isPropertyAccessExpression(expression) && expression.getName() === 'send') {
+ const receiver = value({ node: expression.getExpression(), frame: context });
+ const owned = receiver && owners.get(key(receiver));
+ if (owned) {
+ const target = fn(owned.handler);
+ if (target) invoke(target, call, callContext);
+ else failures.add(`control-binding:${nodeKey(call)}`);
+ }
+ }
+ const target = fn(current);
+ if (target) invoke(target, call, callContext);
+ }
+ };
+ if (fixture && workerFrame) walk(fixture, workerFrame);
+ const baseBinding = callee && imported(callee.getExpression());
const census = Object.entries({
- syntax: project.getProgram().getSyntacticDiagnostics(syntax).length === 0,
+ syntax: project
+ .getSourceFiles()
+ .every((sourceFile) => project.getProgram().getSyntacticDiagnostics(sourceFile).length === 0),
test: !!declaration?.getVariableStatement()?.isExported(),
- extend: callee?.getName() === 'extend' && !!base && callee.getExpression().getSymbol() === base,
+ extend:
+ callee?.getName() === 'extend' &&
+ baseBinding?.module === '@playwright/test' &&
+ baseBinding.name === 'test',
workerServer: !!fixture,
workerInfo: !!worker,
spawn: spawns.length > 0,
}).flatMap(([name, holds]) => (holds ? [] : [name]));
- const unique = (input: Node | undefined): boolean => {
- const node = value(input);
- return (
- !!node &&
- [node, ...node.getDescendants()].some(
- (part) =>
- (Node.isCallExpression(part) &&
- !!fresh &&
- value(part.getExpression())?.getSymbol() === fresh) ||
- (Node.isPropertyAccessExpression(part) &&
- part.getName() === 'workerIndex' &&
- value(part.getExpression())?.getSymbol() === worker),
- )
- );
+ census.push(...failures);
+ const unique = (input: Value | undefined): boolean => {
+ const current = input && value(input);
+ if (!current) return false;
+ if (
+ Node.isCallExpression(current.node) &&
+ nativeName(current.node.getExpression(), 'node:fs', 'mkdtempSync')
+ )
+ return true;
+ for (const part of [current.node, ...current.node.getDescendants()]) {
+ if (Node.isPropertyAccessExpression(part) && part.getName() === 'workerIndex') {
+ const object = value({ node: part.getExpression(), frame: current.frame });
+ if (object?.node === worker) return true;
+ }
+ }
+ return false;
};
- const cacheProperties = file
- .getDescendants()
- .filter(
- (node) =>
- (Node.isPropertyAssignment(node) || Node.isShorthandPropertyAssignment(node)) &&
- node.getSymbol()?.getName() === CACHE_DIR_ENV_VAR,
- );
- const nonUnique = cacheProperties.filter((node) => !unique(node)).length;
let missingEnv = 0,
+ nonUnique = 0,
missingCleanup = 0,
teardowns = 0;
- const reclaims = calls.filter((call) =>
- removals.some((symbol) => value(call.getExpression())?.getSymbol() === symbol),
- );
- const blocksOf = (resource: Node | undefined) =>
- resource
- ? reclaims
- .filter((call) => call.getArguments().some((arg) => value(arg) === resource))
- .map((call) => call.getFirstAncestorByKind(SyntaxKind.Block))
- : [];
- for (const call of spawns) {
- const env = property(call.getArguments()[2], 'env');
+ for (const { call, frame: context } of spawns) {
+ const options = call.getArguments()[2];
+ const env = property(options ? { node: options, frame: context } : undefined, 'env');
const cache = property(env, CACHE_DIR_ENV_VAR);
+ const object = env && value(env);
+ const field =
+ object && Node.isObjectLiteralExpression(object.node)
+ ? object.node
+ .getProperties()
+ .find(
+ (field) =>
+ (Node.isPropertyAssignment(field) || Node.isShorthandPropertyAssignment(field)) &&
+ field.getName() === CACHE_DIR_ENV_VAR,
+ )
+ : undefined;
+ if (field) reached.add(nodeKey(field));
if (!cache) {
missingEnv++;
continue;
}
- const contents = blocksOf(property(env, 'OK_TEST_CONTENT_DIR'));
- const caches = new Set(blocksOf(cache));
- teardowns += contents.length;
- missingCleanup +=
- Number(contents.length < 2) + contents.filter((block) => !block || !caches.has(block)).length;
+ if (!unique(cache)) nonUnique++;
+ const content = property(env, 'OK_TEST_CONTENT_DIR');
+ for (const path of ['failure', 'normal'] as const) {
+ const removals = reclaims
+ .filter((reclaim) => reclaim.path === path)
+ .flatMap((reclaim) => reclaim.resources.map(key));
+ const hasContent = !!content && removals.includes(key(content));
+ const hasCache = removals.includes(key(cache));
+ teardowns += Number(hasContent);
+ missingCleanup += Number(!hasContent || !hasCache);
+ }
+ }
+ for (const subject of project.getSourceFiles().filter((subject) => subject !== syntax)) {
+ for (const field of subject
+ .getDescendants()
+ .filter(
+ (node) =>
+ (Node.isPropertyAssignment(node) || Node.isShorthandPropertyAssignment(node)) &&
+ node.getName() === CACHE_DIR_ENV_VAR,
+ )) {
+ if (reached.has(nodeKey(field))) continue;
+ if (
+ !unique({
+ node: field,
+ frame: subject === file ? (workerFrame ?? registrationFrame) : moduleFrame(subject),
+ })
+ )
+ nonUnique++;
+ }
+ if (spawns.length > 0) {
+ const accounted = new Set(spawns.map((spawn) => nodeKey(spawn.call)));
+ for (const owner of owners.keys()) accounted.add(owner.split('@')[0]);
+ for (const call of subject.getDescendantsOfKind(SyntaxKind.CallExpression)) {
+ if (
+ nativeName(call.getExpression(), 'node:child_process', 'spawn') &&
+ !accounted.has(nodeKey(call))
+ )
+ census.push(`unreached-spawn:${nodeKey(call)}`);
+ }
+ }
}
return { census, missingEnv, nonUnique, missingCleanup, spawns: spawns.length, teardowns };
}
@@ -213,7 +678,10 @@ describe('per-worker Vite cacheDir isolation — vite.config.ts side', () => {
});
describe('per-worker Vite cacheDir isolation — workerServer fixture side', () => {
- const inspect = () => inspectWorkerCache(readFileSync(WORKER_FIXTURES, 'utf8'));
+ const inspect = () => {
+ const graph = readWorkerCacheGraph(WORKER_FIXTURES);
+ return inspectWorkerCache(graph[WORKER_FIXTURES], graph, WORKER_FIXTURES);
+ };
test(`B1: workerServer spawn() env declares ${CACHE_DIR_ENV_VAR}`, () => {
const result = inspect();
expect(result.census).toEqual([]);
@@ -228,7 +696,7 @@ describe('per-worker Vite cacheDir isolation — workerServer fixture side', ()
expect(result.missingEnv).toBe(0);
expect(
result.nonUnique,
- `${CACHE_DIR_ENV_VAR} must be, or be bound through variables to, an expression that itself calls the imported mkdtempSync or reads the fixture's workerInfo.workerIndex. A static string does not isolate workers, and an index first read into another variable is not followed.`,
+ `${CACHE_DIR_ENV_VAR} must be, or be bound through variables to, an expression that itself calls the imported mkdtempSync or reads the fixture's workerInfo.workerIndex. A static string does not isolate workers.`,
).toBe(0);
});
test(`B3: workerServer teardown reclaims the per-worker ${CACHE_DIR_ENV_VAR} at both teardown sites`, () => {
@@ -237,7 +705,7 @@ describe('per-worker Vite cacheDir isolation — workerServer fixture side', ()
expect(result.missingEnv).toBe(0);
expect(
result.missingCleanup,
- `Bind ${CACHE_DIR_ENV_VAR} to a local variable and reclaim the same resource alongside contentDir at BOTH failure and happy teardown sites. Recognised primitives are node:fs rmSync and teardown-fs removeAllDuringTeardown; admit a new removal primitive by adding its import to the removals list in inspectWorkerCache. Missing cleanup leaves orphan Vite cache directories under packages/app/node_modules.`,
+ `Bind ${CACHE_DIR_ENV_VAR} to a local variable and reclaim the same resource alongside contentDir on BOTH failure and normal release paths. Recognised primitives are node:fs rmSync and teardown-fs removeAllDuringTeardown. Missing cleanup leaves orphan Vite cache directories under packages/app/node_modules.`,
).toBe(0);
expect(result.teardowns).toBeGreaterThanOrEqual(2);
});
@@ -273,7 +741,173 @@ const phasedFixture = `import { test as base } from '@playwright/test';
rmSync(contentDir); rmSync(viteCacheDir);
}, { scope: 'worker' }] });`;
+const helperFixture = `import { test as base } from '@playwright/test';
+ import { spawn } from 'node:child_process';
+ import { mkdtempSync, rmSync } from 'node:fs';
+ function startServer(contentDir, viteCacheDir) {
+ return spawn('pnpm', [], { env: { OK_TEST_CONTENT_DIR: contentDir, OK_TEST_VITE_CACHE_DIR: viteCacheDir } });
+ }
+ export const test = base.extend({ workerServer: [async ({}, use, workerInfo) => {
+ const contentDir = mkdtempSync('content');
+ const viteCacheDir = mkdtempSync('cache');
+ startServer(contentDir, viteCacheDir);
+ try { await use(); } catch { rmSync(contentDir); rmSync(viteCacheDir); }
+ rmSync(contentDir); rmSync(viteCacheDir);
+ }, { scope: 'worker' }] });`;
+
describe('worker cache rule self-test', () => {
+ test('follows a reached exported spawn across modules and refuses a missing or disconnected implementation', () => {
+ const start = helperFixture.indexOf(' function startServer');
+ const end = helperFixture.indexOf(' export const test');
+ const implementation = `import { spawn } from 'node:child_process';\nexport ${helperFixture.slice(start, end).trim()}`;
+ const caller =
+ helperFixture.slice(0, start) +
+ "import { startServer } from './moved.ts';\n" +
+ helperFixture.slice(end);
+ const expected = {
+ census: [],
+ missingEnv: 0,
+ nonUnique: 0,
+ missingCleanup: 0,
+ spawns: 1,
+ teardowns: 2,
+ };
+ expect(inspectWorkerCache(caller, { '/moved.ts': implementation })).toEqual(expected);
+ expect(
+ inspectWorkerCache("export { test } from './registered.ts';", {
+ '/registered.ts': caller,
+ '/moved.ts': implementation,
+ }),
+ ).toEqual(expected);
+ expect(inspectWorkerCache(caller).census).toContain('spawn');
+ expect(
+ inspectWorkerCache(caller.replace('startServer(contentDir, viteCacheDir);', ''), {
+ '/moved.ts': implementation,
+ }).census,
+ ).toContain('spawn');
+ });
+
+ test('accounts for failure and normal reclamation through one shared cleanup helper', () => {
+ const helper =
+ 'function reclaim(contentDir, viteCacheDir) { rmSync(contentDir); rmSync(viteCacheDir); }';
+ const paired = 'rmSync(contentDir); rmSync(viteCacheDir);';
+ const factored = workerFixture
+ .replaceAll(paired, 'reclaim(contentDir, viteCacheDir);')
+ .replace(' export const test', `${helper}\n export const test`);
+ expect(inspectWorkerCache(factored)).toEqual({
+ census: [],
+ missingEnv: 0,
+ nonUnique: 0,
+ missingCleanup: 0,
+ spawns: 1,
+ teardowns: 2,
+ });
+ expect(
+ inspectWorkerCache(
+ factored.replace('catch { reclaim(contentDir, viteCacheDir); }', 'catch {}'),
+ ).missingCleanup,
+ ).toBe(1);
+ expect(
+ inspectWorkerCache(factored.replace('reclaim(contentDir, viteCacheDir);\n', '\n'))
+ .missingCleanup,
+ ).toBe(1);
+ expect(inspectWorkerCache(factored.replace('rmSync(viteCacheDir);', '')).missingCleanup).toBe(
+ 2,
+ );
+ expect(
+ inspectWorkerCache(factored.replace('rmSync(viteCacheDir);', "rmSync('other');"))
+ .missingCleanup,
+ ).toBe(2);
+ });
+ test('examines an additional bad spawn and helper call outside the runtime sample', () => {
+ const spawn =
+ "spawn('pnpm', [], { env: { OK_TEST_CONTENT_DIR: contentDir, OK_TEST_VITE_CACHE_DIR: viteCacheDir } });";
+ const added = workerFixture.replace(
+ spawn,
+ `if (workerInfo.workerIndex === 999) { ${spawn.replace('OK_TEST_VITE_CACHE_DIR: viteCacheDir', "OK_TEST_VITE_CACHE_DIR: 'shared'")} } ${spawn}`,
+ );
+ expect(inspectWorkerCache(added)).toMatchObject({
+ census: [],
+ spawns: 2,
+ nonUnique: 1,
+ missingCleanup: 2,
+ });
+ const helper = helperFixture.replace(
+ 'startServer(contentDir, viteCacheDir);',
+ "if (workerInfo.workerIndex === 999) startServer(contentDir, 'shared'); startServer(contentDir, viteCacheDir);",
+ );
+ expect(inspectWorkerCache(helper)).toMatchObject({ census: [], spawns: 2, nonUnique: 1 });
+ expect(inspectWorkerCache(workerFixture)).toMatchObject({
+ census: [],
+ spawns: 1,
+ nonUnique: 0,
+ missingCleanup: 0,
+ });
+ });
+ test('binds the cache handoff to the actual owner entry and service call', () => {
+ const graph = readWorkerCacheGraph(WORKER_FIXTURES);
+ const source = graph[WORKER_FIXTURES];
+ expect(inspectWorkerCache(source, graph, WORKER_FIXTURES)).toEqual({
+ census: [],
+ missingEnv: 0,
+ nonUnique: 0,
+ missingCleanup: 0,
+ spawns: 2,
+ teardowns: 4,
+ });
+ const command = resolve(APP_PACKAGE_ROOT, 'tests/stress/_helpers/worker-server-command.ts');
+ const adapter = resolve(APP_PACKAGE_ROOT, 'tests/stress/_helpers/worker-server-lifetime.ts');
+ const owner = resolve(APP_PACKAGE_ROOT, 'tests/stress/_helpers/worker-server-lifetime.cjs');
+ const absentKey = graph[command].replaceAll('OK_TEST_VITE_CACHE_DIR:', 'OTHER_CACHE:');
+ expect(absentKey).not.toBe(graph[command]);
+ expect(
+ inspectWorkerCache(source, { ...graph, [command]: absentKey }, WORKER_FIXTURES).missingEnv,
+ ).toBe(2);
+ const noOwnerCache = graph[owner].replace('launch.viteCacheDir,', '');
+ const noFailureCache = source.replace(
+ 'releaseSetupResources.push(() => removeAllDuringTeardown(viteCacheDir));',
+ '',
+ );
+ expect(noOwnerCache).not.toBe(graph[owner]);
+ expect(noFailureCache).not.toBe(source);
+ expect(
+ inspectWorkerCache(noFailureCache, { ...graph, [owner]: noOwnerCache }, WORKER_FIXTURES)
+ .missingCleanup,
+ ).toBe(2);
+ const missing = { ...graph };
+ delete missing[owner];
+ expect(inspectWorkerCache(source, missing, WORKER_FIXTURES).census).not.toEqual([]);
+ expect(
+ inspectWorkerCache(
+ source,
+ { ...graph, [owner]: graph[owner].replace('process.argv[2]', 'process.argv[3]') },
+ WORKER_FIXTURES,
+ ).census,
+ ).not.toEqual([]);
+ expect(
+ inspectWorkerCache(
+ source,
+ {
+ ...graph,
+ [command]: graph[command].replaceAll(
+ 'OK_TEST_VITE_CACHE_DIR: viteCacheDir',
+ "OK_TEST_VITE_CACHE_DIR: 'shared'",
+ ),
+ },
+ WORKER_FIXTURES,
+ ).nonUnique,
+ ).toBe(2);
+ expect(
+ inspectWorkerCache(
+ source,
+ {
+ ...graph,
+ [adapter]: graph[adapter].replace(' viteCacheDir,', " viteCacheDir: 'shared',"),
+ },
+ WORKER_FIXTURES,
+ ).nonUnique,
+ ).toBe(2);
+ });
test('accepts a unique reclaimed cache and rejects the adjacent shared cache', () => {
expect(inspectWorkerCache(workerFixture)).toEqual({
census: [],
@@ -324,6 +958,44 @@ describe('worker cache rule self-test', () => {
teardowns: 2,
});
});
+ test('follows the spawn into a same-module helper the fixture calls and reads the call arguments', () => {
+ expect(inspectWorkerCache(helperFixture)).toEqual({
+ census: [],
+ missingEnv: 0,
+ nonUnique: 0,
+ missingCleanup: 0,
+ spawns: 1,
+ teardowns: 2,
+ });
+ expect(
+ inspectWorkerCache(
+ helperFixture.replace(
+ 'startServer(contentDir, viteCacheDir);',
+ "startServer(contentDir, 'shared');",
+ ),
+ ).nonUnique,
+ ).toBe(1);
+ expect(
+ inspectWorkerCache(
+ helperFixture.replace(
+ 'OK_TEST_VITE_CACHE_DIR: viteCacheDir',
+ "OK_TEST_VITE_CACHE_DIR: 'shared'",
+ ),
+ ).nonUnique,
+ ).toBe(1);
+ expect(
+ inspectWorkerCache(
+ helperFixture.replace(
+ 'startServer(contentDir, viteCacheDir);',
+ "startServer(contentDir, 'shared'); startServer(contentDir, viteCacheDir);",
+ ),
+ ).nonUnique,
+ ).toBe(1);
+ expect(
+ inspectWorkerCache(helperFixture.replace('startServer(contentDir, viteCacheDir);', ''))
+ .census,
+ ).toEqual(['spawn']);
+ });
test('ignores comments and refuses a forwarding or unparseable fixture', () => {
expect(
inspectWorkerCache(`${workerFixture} /* OK_TEST_VITE_CACHE_DIR: 'shared' */`).nonUnique,
diff --git a/packages/app/tests/meta/_helpers/worker-lifetime/exit-case.test-helper.ts b/packages/app/tests/meta/_helpers/worker-lifetime/exit-case.test-helper.ts
new file mode 100644
index 000000000..129c67d07
--- /dev/null
+++ b/packages/app/tests/meta/_helpers/worker-lifetime/exit-case.test-helper.ts
@@ -0,0 +1,27 @@
+import { existsSync, writeFileSync } from 'node:fs';
+import { join } from 'node:path';
+import { expect, test } from '../../../stress/_helpers/fixtures.ts';
+
+const directory = process.env.OK_TEST_LIFETIME_DIR;
+if (directory === undefined) throw new Error('OK_TEST_LIFETIME_DIR is required');
+
+test('lifetime subject: worker exits after both real servers are ready', async ({
+ workerServer,
+}) => {
+ const response = await fetch(`${workerServer.baseURL}/api/config`);
+ expect(response.status).toBe(200);
+ const identity = await fetch(`${workerServer.baseURL}/__ok-test-server-identity`);
+ expect(identity.status).toBe(200);
+ writeFileSync(
+ join(directory, 'exit-ready.json'),
+ JSON.stringify({
+ pid: process.pid,
+ server: workerServer,
+ at: Date.now(),
+ identity: await identity.json(),
+ apiStatus: response.status,
+ }),
+ );
+ await expect.poll(() => existsSync(join(directory, 'peer-ready.json'))).toBe(true);
+ process.exit(83);
+});
diff --git a/packages/app/tests/meta/_helpers/worker-lifetime/peer-case.test-helper.ts b/packages/app/tests/meta/_helpers/worker-lifetime/peer-case.test-helper.ts
new file mode 100644
index 000000000..daeee2f94
--- /dev/null
+++ b/packages/app/tests/meta/_helpers/worker-lifetime/peer-case.test-helper.ts
@@ -0,0 +1,24 @@
+import { existsSync, writeFileSync } from 'node:fs';
+import { join } from 'node:path';
+import { expect, test } from '../../../stress/_helpers/fixtures.ts';
+
+const directory = process.env.OK_TEST_LIFETIME_DIR;
+if (directory === undefined) throw new Error('OK_TEST_LIFETIME_DIR is required');
+
+test('lifetime subject: peer uses its separate real server', async ({ workerServer }) => {
+ const response = await fetch(`${workerServer.baseURL}/api/config`);
+ expect(response.status).toBe(200);
+ const identity = await fetch(`${workerServer.baseURL}/__ok-test-server-identity`);
+ expect(identity.status).toBe(200);
+ writeFileSync(
+ join(directory, 'peer-ready.json'),
+ JSON.stringify({
+ pid: process.pid,
+ server: workerServer,
+ at: Date.now(),
+ identity: await identity.json(),
+ apiStatus: response.status,
+ }),
+ );
+ await expect.poll(() => existsSync(join(directory, 'exit-ready.json'))).toBe(true);
+});
diff --git a/packages/app/tests/meta/_helpers/worker-lifetime/standalone-case.test-helper.ts b/packages/app/tests/meta/_helpers/worker-lifetime/standalone-case.test-helper.ts
new file mode 100644
index 000000000..15c831fb2
--- /dev/null
+++ b/packages/app/tests/meta/_helpers/worker-lifetime/standalone-case.test-helper.ts
@@ -0,0 +1,33 @@
+import { existsSync, writeFileSync } from 'node:fs';
+import { join } from 'node:path';
+import { expect } from '@playwright/test';
+import type { OwnedServer } from '../../../stress/_helpers/reuse/server-supervisor.ts';
+
+export async function runStandaloneLifetimeCase(
+ server: OwnedServer,
+ kind: 'exit' | 'peer',
+): Promise<void> {
+ const directory = process.env.OK_TEST_LIFETIME_DIR;
+ if (directory === undefined) throw new Error('OK_TEST_LIFETIME_DIR is required');
+ const response = await fetch(`${server.baseURL}/api/config`);
+ expect(response.status).toBe(200);
+ const identity = await fetch(`${server.baseURL}/__ok-test-server-identity`);
+ expect(identity.status).toBe(200);
+ writeFileSync(
+ join(directory, `${kind}-ready.json`),
+ JSON.stringify({
+ pid: process.pid,
+ server: {
+ baseURL: server.baseURL,
+ port: server.identity.port,
+ contentDir: server.contentDir,
+ },
+ at: Date.now(),
+ identity: await identity.json(),
+ apiStatus: response.status,
+ }),
+ );
+ const other = kind === 'exit' ? 'peer' : 'exit';
+ await expect.poll(() => existsSync(join(directory, `${other}-ready.json`))).toBe(true);
+ if (kind === 'exit') process.exit(83);
+}
diff --git a/packages/app/tests/meta/nested-playwright-env.test.ts b/packages/app/tests/meta/nested-playwright-env.test.ts
new file mode 100644
index 000000000..c2b4b0b18
--- /dev/null
+++ b/packages/app/tests/meta/nested-playwright-env.test.ts
@@ -0,0 +1,52 @@
+import { afterEach, describe, expect, test, vi } from 'vitest';
+import { createNestedPlaywrightEnv } from '../stress/_helpers/port-ownership/nested-playwright-env.test-helper.ts';
+
+const OUTER_RUN_SERVING = {
+ OK_TEST_E2E_REUSE_SOCKET: '/tmp/outer-supervisor.sock',
+ OK_TEST_E2E_REUSE_RUN: 'outer-run',
+ OK_TEST_E2E_SERVING: 'prebuilt',
+ OK_TEST_PREBUILT_APP: '/tmp/outer-prebuilt-app',
+};
+
+afterEach(() => {
+ vi.unstubAllEnvs();
+});
+
+function stubOuterRun(): void {
+ for (const [name, value] of Object.entries(OUTER_RUN_SERVING)) vi.stubEnv(name, value);
+ vi.stubEnv('OK_NESTED_ENV_UNRELATED', 'kept');
+ vi.stubEnv('PLAYWRIGHT_JSON_OUTPUT_FILE', '/tmp/outer-results.json');
+}
+
+describe('a nested Playwright run starts from the outer environment without its server reuse or serving mode', () => {
+ test('the outer run’s reuse socket, run id, serving mode and prebuilt app are not inherited', () => {
+ stubOuterRun();
+ expect(process.env.OK_TEST_E2E_REUSE_SOCKET).toBe(OUTER_RUN_SERVING.OK_TEST_E2E_REUSE_SOCKET);
+
+ const env = createNestedPlaywrightEnv({ OK_NESTED_ENV_OVERRIDE: 'set' });
+
+ expect(Object.keys(env).filter((name) => name in OUTER_RUN_SERVING)).toEqual([]);
+ });
+
+ test('everything else the outer run had is kept, the overrides apply and the JSON report path is dropped', () => {
+ stubOuterRun();
+
+ const env = createNestedPlaywrightEnv({ OK_NESTED_ENV_OVERRIDE: 'set' });
+
+ expect(env.OK_NESTED_ENV_UNRELATED).toBe('kept');
+ expect(env.OK_NESTED_ENV_OVERRIDE).toBe('set');
+ expect(env.PATH).toBe(process.env.PATH);
+ expect(env.PLAYWRIGHT_JSON_OUTPUT_FILE).toBeUndefined();
+ });
+
+ test('a nested run that names its own reuse socket keeps that socket', () => {
+ stubOuterRun();
+
+ const env = createNestedPlaywrightEnv({
+ OK_TEST_E2E_REUSE_SOCKET: '/tmp/nested-supervisor.sock',
+ });
+
+ expect(env.OK_TEST_E2E_REUSE_SOCKET).toBe('/tmp/nested-supervisor.sock');
+ expect(env.OK_TEST_E2E_REUSE_RUN).toBeUndefined();
+ });
+});
diff --git a/packages/app/tests/meta/playwright-assertion-timeout.test.ts b/packages/app/tests/meta/playwright-assertion-timeout.test.ts
new file mode 100644
index 000000000..e62927493
--- /dev/null
+++ b/packages/app/tests/meta/playwright-assertion-timeout.test.ts
@@ -0,0 +1,49 @@
+import { describe, expect, test, vi } from 'vitest';
+
+const ASSERTION_TIMEOUT_MS = 15_000;
+
+const FIXTURE_SHARING_CONFIGS = {
+ e2e: () => import('../../playwright.config'),
+ a11y: () => import('../../playwright.a11y.config'),
+ visual: () => import('../../playwright.visual.config'),
+} as const;
+
+type ConfigName = keyof typeof FIXTURE_SHARING_CONFIGS;
+
+async function loadConfig(which: ConfigName, ci: string | undefined) {
+ vi.stubEnv('CI', ci);
+ vi.resetModules();
+ try {
+ return (await FIXTURE_SHARING_CONFIGS[which]()).default;
+ } finally {
+ vi.unstubAllEnvs();
+ vi.resetModules();
+ }
+}
+
+const MODES = [
+ { mode: 'locally', ci: undefined },
+ { mode: 'under CI', ci: '1' },
+] as const;
+
+const CASES = (Object.keys(FIXTURE_SHARING_CONFIGS) as ConfigName[]).flatMap((which) =>
+ MODES.map(({ mode, ci }) => ({ which, mode, ci })),
+);
+
+describe('playwright assertion timeout', () => {
+ test.each(CASES)(
+ '$mode the $which config gives web-first assertions a 15 s default',
+ async ({ which, ci }) => {
+ const config = await loadConfig(which, ci);
+ expect(config.expect?.timeout).toBe(ASSERTION_TIMEOUT_MS);
+ },
+ );
+
+ test.each(CASES)(
+ '$mode the $which config keeps its per-test budget above the assertion default',
+ async ({ which, ci }) => {
+ const config = await loadConfig(which, ci);
+ expect(config.timeout).toBeGreaterThan(ASSERTION_TIMEOUT_MS);
+ },
+ );
+});
diff --git a/packages/app/tests/meta/select-all-wait-band.test.ts b/packages/app/tests/meta/select-all-wait-band.test.ts
index 346edd8b3..c3e529c24 100644
--- a/packages/app/tests/meta/select-all-wait-band.test.ts
+++ b/packages/app/tests/meta/select-all-wait-band.test.ts
@@ -26,10 +26,4 @@ describe('select-all barrier wait band', () => {
const { configured, helper } = await loadBand(undefined);
expect(helper).toBe(configured);
});
-
- test('the CI band and the local band stay distinct, so neither regime can go vacuous', async () => {
- const ci = await loadBand('1');
- const local = await loadBand(undefined);
- expect(ci.helper).toBeGreaterThan(local.helper);
- });
});
diff --git a/packages/app/tests/meta/standalone-prebuilt-worker-lifetime.test.ts b/packages/app/tests/meta/standalone-prebuilt-worker-lifetime.test.ts
new file mode 100644
index 000000000..98633b438
--- /dev/null
+++ b/packages/app/tests/meta/standalone-prebuilt-worker-lifetime.test.ts
@@ -0,0 +1,63 @@
+import { beforeAll, describe, expect, it } from 'vitest';
+import {
+ type LifetimeObservation,
+ OBSERVATION_LIMIT_MS,
+ observeStandalonePrebuiltWorkerLifetime,
+} from '../foundation/worker-lifetime/observe-standalone-prebuilt-lifetime.test-helper.ts';
+
+describe('standalone prebuilt worker server lifetime:', () => {
+ let observation: LifetimeObservation;
+ beforeAll(async () => {
+ observation = await observeStandalonePrebuiltWorkerLifetime();
+ }, OBSERVATION_LIMIT_MS + 10_000);
+
+ it('an abrupt worker exit leaves no ready server at 10s and 60s', () => {
+ expect(observation.ready.exit.apiStatus).toBe(200);
+ expect(observation.serverBirths.exit.length).toBeGreaterThanOrEqual(2);
+ expect(
+ observation.native.find(
+ (row) => row.name === 'lifetime subject: worker exits after both real servers are ready',
+ )?.errors,
+ ).toContain('Error: worker process exited unexpectedly (code=83, signal=null)');
+ expect(
+ observation.samples
+ .filter((sample) => sample.at === 10 || sample.at === 60)
+ .map((sample) => {
+ const server = sample.servers.find((entry) => entry.kind === 'exit');
+ return {
+ at: sample.at,
+ livePids: server?.livePids,
+ ownedListener: server?.ownedListener,
+ };
+ }),
+ ).toEqual([
+ { at: 10, livePids: [], ownedListener: false },
+ { at: 60, livePids: [], ownedListener: false },
+ ]);
+ });
+
+ it('a normal worker releases its ready server at 10s and 60s', () => {
+ expect(observation.ready.peer.apiStatus).toBe(200);
+ expect(observation.serverBirths.peer.length).toBeGreaterThanOrEqual(2);
+ expect(
+ observation.native.find(
+ (row) => row.name === 'lifetime subject: peer uses its separate real server',
+ )?.status,
+ ).toBe('passed');
+ expect(
+ observation.samples
+ .filter((sample) => sample.at === 10 || sample.at === 60)
+ .map((sample) => {
+ const server = sample.servers.find((entry) => entry.kind === 'peer');
+ return {
+ at: sample.at,
+ livePids: server?.livePids,
+ ownedListener: server?.ownedListener,
+ };
+ }),
+ ).toEqual([
+ { at: 10, livePids: [], ownedListener: false },
+ { at: 60, livePids: [], ownedListener: false },
+ ]);
+ });
+});
diff --git a/packages/app/tests/meta/worker-lifetime-owner-safety.test.ts b/packages/app/tests/meta/worker-lifetime-owner-safety.test.ts
new file mode 100644
index 000000000..55bc97293
--- /dev/null
+++ b/packages/app/tests/meta/worker-lifetime-owner-safety.test.ts
@@ -0,0 +1,92 @@
+import { existsSync, readFileSync, writeFileSync } from 'node:fs';
+import { join } from 'node:path';
+import { describe, expect, it } from 'vitest';
+import { z } from 'zod';
+import {
+ HELPERS_DIR,
+ launchRecorded,
+ makeRunDir,
+ readCleanup,
+ waitUntil,
+ writeOwnerConfig,
+} from '../foundation/worker-lifetime/owner-launch.test-helper.ts';
+
+const readySchema = z.object({
+ pid: z.number().int().positive(),
+ port: z.number().int().positive(),
+});
+
+describe('worker lifetime observation owner safety', () => {
+ for (const ending of ['normal release', 'outer owner killed'] as const) {
+ it(`${ending}: both owned listeners and descendants are reclaimed`, async () => {
+ const run = makeRunDir(ending.replaceAll(' ', '-'));
+ const path = writeOwnerConfig({
+ ...run,
+ cwd: HELPERS_DIR,
+ args: [join(HELPERS_DIR, 'safety-subject.cjs')],
+ environment: { OK_LIFETIME_SAFETY_DIR: run.dir },
+ deadlineMs: 30_000,
+ });
+ const guardian =
+ process.env.OK_LIFETIME_SAFETY_GUARDIAN ?? join(HELPERS_DIR, 'resource-owner.cjs');
+ const { child, exited } = launchRecorded(
+ join(HELPERS_DIR, 'safety-outer-owner.cjs'),
+ [path, guardian],
+ run.dir,
+ run.runId,
+ );
+ try {
+ await waitUntil(
+ () =>
+ ['parent', 'child'].every((role) => existsSync(join(run.dir, `safety-${role}.json`))),
+ 10_000,
+ );
+ const ready = ['parent', 'child'].map((role) =>
+ readySchema.parse(JSON.parse(readFileSync(join(run.dir, `safety-${role}.json`), 'utf8'))),
+ );
+ const before = await Promise.all(
+ ready.map(
+ async ({ port }) =>
+ (await fetch(`http://127.0.0.1:${port}`, { signal: AbortSignal.timeout(1_000) }))
+ .status,
+ ),
+ );
+ expect(before).toEqual([200, 200]);
+ const endedAt = Date.now();
+ if (ending === 'normal release') child.send({ type: 'release', runId: run.runId });
+ else child.kill('SIGKILL');
+ await expect
+ .poll(() => existsSync(join(run.dir, 'cleanup.json')), { timeout: 8_000 })
+ .toBe(true);
+ const cleanup = readCleanup(run.dir);
+ expect(cleanup.reason).toBe(
+ ending === 'normal release'
+ ? 'normal release after observation'
+ : 'outer owner disconnected',
+ );
+ expect(cleanup.before.filter((row) => row.birth.found).map((row) => row.birth.pid)).toEqual(
+ expect.arrayContaining(ready.map((item) => item.pid)),
+ );
+ expect(cleanup.liveSurvivors).toEqual([]);
+ expect(cleanup.workDirAbsent).toBe(true);
+ const after = await Promise.all(
+ ready.map(async ({ port }) =>
+ fetch(`http://127.0.0.1:${port}`, { signal: AbortSignal.timeout(1_000) }).then(
+ (response) => response.status,
+ () => null,
+ ),
+ ),
+ );
+ expect(after).toEqual([null, null]);
+ writeFileSync(
+ join(run.dir, 'safety-verdict.json'),
+ `${JSON.stringify({ ending, ready, before, after, cleanupElapsedMs: cleanup.at - endedAt, cleanupReason: cleanup.reason }, null, 2)}\n`,
+ );
+ } finally {
+ if (child.connected) child.send({ type: 'release', runId: run.runId }, () => {});
+ await exited;
+ await waitUntil(() => existsSync(join(run.dir, 'cleanup.json')), 35_000);
+ }
+ }, 50_000);
+ }
+});
diff --git a/packages/app/tests/meta/worker-server-lifetime.test.ts b/packages/app/tests/meta/worker-server-lifetime.test.ts
new file mode 100644
index 000000000..47c333c57
--- /dev/null
+++ b/packages/app/tests/meta/worker-server-lifetime.test.ts
@@ -0,0 +1,68 @@
+import { beforeAll, describe, expect, it } from 'vitest';
+import {
+ type LifetimeObservation,
+ OBSERVATION_LIMIT_MS,
+ observeWorkerLifetime,
+} from '../foundation/worker-lifetime/observe-worker-lifetime.test-helper.ts';
+
+describe('ordinary worker server lifetime', () => {
+ for (const mode of ['dev-server', 'prebuilt'] as const) {
+ describe(`${mode}:`, () => {
+ let observation: LifetimeObservation;
+ beforeAll(async () => {
+ observation = await observeWorkerLifetime(mode);
+ }, OBSERVATION_LIMIT_MS + 10_000);
+
+ it('an abrupt worker exit leaves no ready server at 10s and 60s', () => {
+ expect(observation.ready.exit.apiStatus).toBe(200);
+ expect(observation.serverBirths.exit.length).toBeGreaterThanOrEqual(2);
+ expect(
+ observation.native.find(
+ (row) =>
+ row.name === 'lifetime subject: worker exits after both real servers are ready',
+ )?.errors,
+ ).toContain('Error: worker process exited unexpectedly (code=83, signal=null)');
+ expect(
+ observation.samples
+ .filter((sample) => sample.at === 10 || sample.at === 60)
+ .map((sample) => {
+ const server = sample.servers.find((entry) => entry.kind === 'exit');
+ return {
+ at: sample.at,
+ livePids: server?.livePids,
+ ownedListener: server?.ownedListener,
+ };
+ }),
+ ).toEqual([
+ { at: 10, livePids: [], ownedListener: false },
+ { at: 60, livePids: [], ownedListener: false },
+ ]);
+ });
+
+ it('the normal peer executes and releases its ready server', () => {
+ expect(observation.ready.peer.apiStatus).toBe(200);
+ expect(observation.serverBirths.peer.length).toBeGreaterThanOrEqual(2);
+ expect(
+ observation.native.find(
+ (row) => row.name === 'lifetime subject: peer uses its separate real server',
+ )?.status,
+ ).toBe('passed');
+ expect(
+ observation.samples
+ .filter((sample) => sample.at === 10 || sample.at === 60)
+ .map((sample) => {
+ const server = sample.servers.find((entry) => entry.kind === 'peer');
+ return {
+ at: sample.at,
+ livePids: server?.livePids,
+ ownedListener: server?.ownedListener,
+ };
+ }),
+ ).toEqual([
+ { at: 10, livePids: [], ownedListener: false },
+ { at: 60, livePids: [], ownedListener: false },
+ ]);
+ });
+ });
+ }
+});
diff --git a/packages/app/tests/meta/worker-server-spawn-error.test.ts b/packages/app/tests/meta/worker-server-spawn-error.test.ts
new file mode 100644
index 000000000..a0859cfd4
--- /dev/null
+++ b/packages/app/tests/meta/worker-server-spawn-error.test.ts
@@ -0,0 +1,51 @@
+import { existsSync } from 'node:fs';
+import { describe, expect, it } from 'vitest';
+import {
+ FAILED_SPAWN_COMMANDS,
+ type FailedSpawnMode,
+ observeWorkerSpawnError,
+} from '../foundation/worker-lifetime/observe-worker-spawn-error.test-helper.ts';
+
+async function nativeFailure(mode: FailedSpawnMode) {
+ const result = await observeWorkerSpawnError(mode, 'native');
+ const { error, afterError, exits, close } = result.observation.contract;
+ expect(error).toEqual({
+ name: 'Error',
+ message: 'spawn pnpm ENOENT',
+ errno: expect.any(Number),
+ code: 'ENOENT',
+ syscall: 'spawn pnpm',
+ path: 'pnpm',
+ spawnargs: [...FAILED_SPAWN_COMMANDS[mode]],
+ });
+ expect(error.errno).toBeLessThan(0);
+ expect(afterError).toEqual({ exitCode: error.errno, signalCode: null });
+ expect(exits).toEqual([]);
+ expect(close).toEqual({ code: error.errno, signal: null });
+ expect(result.cleanup.before.some((row) => row.birth.found)).toBe(true);
+ expect(result.cleanup.liveSurvivors).toEqual([]);
+ expect(result.cleanup.workDirAbsent).toBe(true);
+ expect(Object.values(result.observation.resources).every((path) => !existsSync(path))).toBe(true);
+ return result.observation.contract;
+}
+
+describe('worker server failed native spawn', () => {
+ for (const mode of ['dev-server', 'prebuilt'] as const) {
+ it(`${mode}: the exported caller preserves native error fields, failure status and terminal events`, async () => {
+ const expected = await nativeFailure(mode);
+ const actual = await observeWorkerSpawnError(mode, 'exported');
+ expect(actual.observation.contract.error.code).toBe('ENOENT');
+ expect(actual.cleanup.before.some((row) => row.birth.found)).toBe(true);
+ expect(actual.cleanup.liveSurvivors).toEqual([]);
+ expect(actual.cleanup.workDirAbsent).toBe(true);
+ expect(Object.values(actual.observation.resources).every((path) => !existsSync(path))).toBe(
+ true,
+ );
+ expect(actual.observation.contract).toEqual(expected);
+ }, 50_000);
+
+ it(`${mode}: the native reference reports a failed spawn without an exit event`, async () => {
+ await nativeFailure(mode);
+ }, 50_000);
+ }
+});
diff --git a/packages/app/tests/stress/_helpers/assertion-timeout/assertion-timeout-run.config.ts b/packages/app/tests/stress/_helpers/assertion-timeout/assertion-timeout-run.config.ts
new file mode 100644
index 000000000..a4960d7a3
--- /dev/null
+++ b/packages/app/tests/stress/_helpers/assertion-timeout/assertion-timeout-run.config.ts
@@ -0,0 +1,20 @@
+import { join } from 'node:path';
+import { fileURLToPath } from 'node:url';
+import { defineConfig } from '@playwright/test';
+import stock from '../../../../playwright.config.ts';
+
+const runDir = process.env.OK_ASSERTION_TIMEOUT_RUN_DIR;
+if (!runDir) throw new Error('OK_ASSERTION_TIMEOUT_RUN_DIR is required');
+
+export default defineConfig({
+ ...stock,
+ testDir: fileURLToPath(new URL('.', import.meta.url)),
+ testMatch: /.*\.assertion-timeout-case\.ts$/,
+ globalSetup: [],
+ globalTimeout: 60_000,
+ outputDir: join(runDir, 'test-results'),
+ reporter: [['json', { outputFile: join(runDir, 'results.json') }]],
+ retries: 0,
+ workers: 1,
+ use: { ...stock.use, headless: true, video: 'off', trace: 'off', screenshot: 'off' },
+});
diff --git a/packages/app/tests/stress/_helpers/assertion-timeout/cases.ts b/packages/app/tests/stress/_helpers/assertion-timeout/cases.ts
new file mode 100644
index 000000000..146471094
--- /dev/null
+++ b/packages/app/tests/stress/_helpers/assertion-timeout/cases.ts
@@ -0,0 +1,3 @@
+export const MET_CASE = 'an assertion that already holds passes at once';
+export const UNMET_CASE = 'an assertion that never holds waits out the default timeout';
+export const ASSERTION_ELAPSED_ANNOTATION = 'ok:assertion-elapsed-ms';
diff --git a/packages/app/tests/stress/_helpers/assertion-timeout/run-assertion-timeout.test-helper.ts b/packages/app/tests/stress/_helpers/assertion-timeout/run-assertion-timeout.test-helper.ts
new file mode 100644
index 000000000..0dcf5db7e
--- /dev/null
+++ b/packages/app/tests/stress/_helpers/assertion-timeout/run-assertion-timeout.test-helper.ts
@@ -0,0 +1,68 @@
+import { join } from 'node:path';
+import { stripVTControlCharacters } from 'node:util';
+import type { JSONReport, JSONReportTestResult } from '@playwright/test/reporter';
+import { createNestedPlaywrightEnv } from '../port-ownership/nested-playwright-env.test-helper.ts';
+import {
+ runNestedPlaywright,
+ specsOf,
+} from '../port-ownership/nested-playwright-run.test-helper.ts';
+import { APP_PACKAGE_ROOT } from '../server-process.ts';
+
+const RUN_CONFIG = join(
+ APP_PACKAGE_ROOT,
+ 'tests/stress/_helpers/assertion-timeout/assertion-timeout-run.config.ts',
+);
+
+export type ConfigurationMode = 'local' | 'ci';
+
+export interface CaseResult {
+ status: JSONReportTestResult['status'];
+ message: string;
+ annotations: { type: string; description?: string }[];
+}
+
+export interface AssertionTimeoutRun {
+ exitCode: number | null;
+ transcript: string;
+ forbidOnly: boolean | undefined;
+ cases: Map<string, CaseResult[]>;
+}
+
+function casesOf(report: JSONReport | undefined): Map<string, CaseResult[]> {
+ const cases = new Map<string, CaseResult[]>();
+ for (const spec of specsOf(report?.suites ?? [])) {
+ const results = spec.tests.flatMap((t) =>
+ t.results.map((result) => ({
+ status: result.status,
+ message: stripVTControlCharacters(result.error?.message ?? ''),
+ annotations: result.annotations,
+ })),
+ );
+ cases.set(spec.title, [...(cases.get(spec.title) ?? []), ...results]);
+ }
+ return cases;
+}
+
+export async function runAssertionTimeoutCases(
+ mode: ConfigurationMode,
+): Promise<AssertionTimeoutRun> {
+ const { exitCode, transcript, report } = await runNestedPlaywright({
+ args: ['--config', RUN_CONFIG],
+ runDirPrefix: 'ok-assertion-timeout-',
+ env: (runDir) => {
+ const env = createNestedPlaywrightEnv({
+ OK_ASSERTION_TIMEOUT_RUN_DIR: runDir,
+ PLAYWRIGHT_HTML_OPEN: 'never',
+ });
+ if (mode === 'ci') env.CI = '1';
+ else delete env.CI;
+ return env;
+ },
+ });
+ return {
+ exitCode,
+ transcript,
+ forbidOnly: report?.config.forbidOnly,
+ cases: casesOf(report),
+ };
+}
diff --git a/packages/app/tests/stress/_helpers/assertion-timeout/web-first.assertion-timeout-case.ts b/packages/app/tests/stress/_helpers/assertion-timeout/web-first.assertion-timeout-case.ts
new file mode 100644
index 000000000..fbcd4d652
--- /dev/null
+++ b/packages/app/tests/stress/_helpers/assertion-timeout/web-first.assertion-timeout-case.ts
@@ -0,0 +1,22 @@
+import { expect, test } from '@playwright/test';
+import { ASSERTION_ELAPSED_ANNOTATION, MET_CASE, UNMET_CASE } from './cases.ts';
+
+test.beforeEach(async ({ page }) => {
+ await page.setContent('<p>present</p>');
+});
+
+test(MET_CASE, async ({ page }) => {
+ await expect(page.getByText('present')).toBeVisible();
+});
+
+test(UNMET_CASE, async ({ page }) => {
+ const started = performance.now();
+ try {
+ await expect(page.getByText('absent')).toBeVisible();
+ } finally {
+ test.info().annotations.push({
+ type: ASSERTION_ELAPSED_ANNOTATION,
+ description: String(Math.round(performance.now() - started)),
+ });
+ }
+});
diff --git a/packages/app/tests/stress/_helpers/content-reset.ts b/packages/app/tests/stress/_helpers/content-reset.ts
index 29aeba3f2..49d793fcc 100644
--- a/packages/app/tests/stress/_helpers/content-reset.ts
+++ b/packages/app/tests/stress/_helpers/content-reset.ts
@@ -1,6 +1,6 @@
import { readdirSync, rmSync } from 'node:fs';
import { join } from 'node:path';
-import { REQUIRED_FIXTURE_ENTRY_NAMES } from './fixtures.ts';
+import { REQUIRED_FIXTURE_DOC_NAMES, REQUIRED_FIXTURE_ENTRY_NAMES } from './fixtures.ts';
async function deletePathIfExists(
baseURL: string,
@@ -16,23 +16,47 @@ async function deletePathIfExists(
throw new Error(`delete-path failed for ${kind}:${path}: ${res.status} ${await res.text()}`);
}
-export async function resetContentToFixtureBaseline(
+async function removeEntriesOutside(
baseURL: string,
contentDir: string,
+ folder: string,
+ preserved: ReadonlySet<string>,
): Promise<void> {
- const preserved = new Set<string>(REQUIRED_FIXTURE_ENTRY_NAMES);
- for (const entry of readdirSync(contentDir, { withFileTypes: true })) {
+ for (const entry of readdirSync(join(contentDir, folder), { withFileTypes: true })) {
if (entry.name.startsWith('.')) continue;
if (preserved.has(entry.name)) continue;
+ const path = folder === '' ? entry.name : `${folder}/${entry.name}`;
if (entry.isDirectory()) {
- await deletePathIfExists(baseURL, 'folder', entry.name);
+ await deletePathIfExists(baseURL, 'folder', path);
continue;
}
- const docName = entry.name.replace(/\.(md|mdx)$/i, '');
- if (docName !== entry.name) {
+ const docName = path.replace(/\.(md|mdx)$/i, '');
+ if (docName !== path) {
await deletePathIfExists(baseURL, 'file', docName);
continue;
}
- rmSync(join(contentDir, entry.name), { recursive: true, force: true });
+ rmSync(join(contentDir, path), { recursive: true, force: true });
+ }
+}
+
+export async function resetContentToFixtureBaseline(
+ baseURL: string,
+ contentDir: string,
+): Promise<void> {
+ await removeEntriesOutside(baseURL, contentDir, '', new Set(REQUIRED_FIXTURE_ENTRY_NAMES));
+}
+
+export async function resetFixtureFolders(baseURL: string, contentDir: string): Promise<void> {
+ const preservedByFolder = new Map<string, Set<string>>();
+ for (const docName of REQUIRED_FIXTURE_DOC_NAMES) {
+ const slash = docName.lastIndexOf('/');
+ if (slash === -1) continue;
+ const folder = docName.slice(0, slash);
+ const preserved = preservedByFolder.get(folder) ?? new Set<string>();
+ preserved.add(`${docName.slice(slash + 1)}.md`);
+ preservedByFolder.set(folder, preserved);
+ }
+ for (const [folder, preserved] of preservedByFolder) {
+ await removeEntriesOutside(baseURL, contentDir, folder, preserved);
}
}
diff --git a/packages/app/tests/stress/_helpers/editor-state.ts b/packages/app/tests/stress/_helpers/editor-state.ts
index d44e691e5..0b1a22be0 100644
--- a/packages/app/tests/stress/_helpers/editor-state.ts
+++ b/packages/app/tests/stress/_helpers/editor-state.ts
@@ -1,6 +1,6 @@
import { expect, type JSHandle, type Page } from '@playwright/test';
-export const SELECT_ALL_SETTLE_TIMEOUT_MS = process.env.CI ? 15_000 : 5_000;
+export const SELECT_ALL_SETTLE_TIMEOUT_MS = 15_000;
type ViewSelectionState = 'covers-document' | 'partial' | 'empty' | 'unreadable';
diff --git a/packages/app/tests/stress/_helpers/fixtures.ts b/packages/app/tests/stress/_helpers/fixtures.ts
index 540b880ef..d892fd804 100644
--- a/packages/app/tests/stress/_helpers/fixtures.ts
+++ b/packages/app/tests/stress/_helpers/fixtures.ts
@@ -1,4 +1,4 @@
-import { type ChildProcess, spawn } from 'node:child_process';
+import type { ChildProcess } from 'node:child_process';
import { mkdirSync, mkdtempSync, writeFileSync } from 'node:fs';
import { tmpdir } from 'node:os';
import { join } from 'node:path';
@@ -6,25 +6,31 @@ import type { ProblemType } from '@inkeep/open-knowledge-core';
import { test as base } from '@playwright/test';
import { resetContentToFixtureBaseline } from './content-reset.ts';
import { gotoWhileLoadProgresses, requireDeadlineAt } from './load-progress.ts';
+import { type WorkerServing, workerServingFromEnv } from './prebuilt/prebuilt-serving.ts';
+import { checkServerLease, leaseWorkerServer, type ServerLease } from './reuse/lease-fixture.ts';
import {
- APP_PACKAGE_ROOT,
type BoundViteEndpoint,
beginViteStartup,
checkCollabSync,
closeServerLog,
createViteStartupRequest,
- killGracefully,
openServerLog,
type PendingViteStartup,
prepareViteCacheDir,
requireBoundMs,
type ServerLog,
tailServerLog,
+ type ViteStartupRequest,
waitForBoundViteEndpoint,
waitForHttpReady,
} from './server-process.ts';
import { declareSetupNonResult } from './setup-non-result.ts';
import { removeAllDuringTeardown, runTeardownPhases } from './teardown-fs.ts';
+import {
+ acquireWorkerServer,
+ completeWorkerServerRelease,
+ killGracefully,
+} from './worker-server-lifetime.ts';
interface ProblemError extends Error {
status?: number;
@@ -78,10 +84,12 @@ export interface ApiHelpers {
type WorkerFixtures = {
workerServer: WorkerServer;
workerServerEnv: Record<string, string>;
+ serverLease: ServerLease | undefined;
};
type TestFixtures = {
api: ApiHelpers;
+ serverLeaseCheck: undefined;
};
export const WORKER_SERVER_BUDGET_TOTAL_MS = 240_000;
@@ -260,7 +268,7 @@ export async function checkApiConfig(baseURL: string, timeoutMs: number): Promis
}
}
-async function waitForServerReady(started: StartedWorkerServer): Promise<BoundViteEndpoint> {
+export async function waitForServerReady(started: StartedWorkerServer): Promise<BoundViteEndpoint> {
const { pending } = started;
const { baseURL, port } = await waitForBoundViteEndpoint(
pending,
@@ -370,7 +378,7 @@ export async function warmupAppFirstLoad(
export const REQUIRED_FIXTURE_ENTRY_NAMES = ['test-doc.md', 'sidebar-folder'] as const;
-const REQUIRED_FIXTURE_DOC_NAMES = ['test-doc', 'sidebar-folder/nested-doc'] as const;
+export const REQUIRED_FIXTURE_DOC_NAMES = ['test-doc', 'sidebar-folder/nested-doc'] as const;
async function waitForSeededPagesSettled(baseURL: string, seededNames: string[]): Promise<void> {
const missingSet = new Set(seededNames);
@@ -430,14 +438,33 @@ async function waitForSeededPagesSettled(baseURL: string, seededNames: string[])
);
}
-function seedRequiredFixtureFiles(contentDir: string): void {
+export function seedRequiredFixtureFiles(contentDir: string): void {
writeFileSync(join(contentDir, 'test-doc.md'), '', 'utf-8');
mkdirSync(join(contentDir, 'sidebar-folder'), { recursive: true });
writeFileSync(join(contentDir, 'sidebar-folder', 'nested-doc.md'), '', 'utf-8');
mkdirSync(join(contentDir, '.claude', 'skills'), { recursive: true });
}
-interface StartedWorkerServer {
+export function spawnWorkerDevServer(
+ request: ViteStartupRequest,
+ workerServerEnv: Readonly<Record<string, string>>,
+ contentDir: string,
+ viteCacheDir: string,
+ serverLog: ServerLog,
+ serving: WorkerServing = workerServingFromEnv(process.env),
+): ChildProcess {
+ return acquireWorkerServer(
+ request,
+ workerServerEnv,
+ contentDir,
+ viteCacheDir,
+ serverLog,
+ serving,
+ WORKER_SERVER_BUDGET_RESERVES.devServerReap,
+ );
+}
+
+export interface StartedWorkerServer {
pending: PendingViteStartup;
contentDir: string;
viteCacheDir: string;
@@ -451,8 +478,14 @@ function setupResidueOf(started: StartedWorkerServer): string {
export const test = base.extend<TestFixtures, WorkerFixtures>({
workerServerEnv: [{}, { scope: 'worker', option: true }],
+ serverLease: [leaseWorkerServer, { scope: 'worker', timeout: WORKER_SERVER_BUDGET_TOTAL_MS }],
+ serverLeaseCheck: [checkServerLease, { auto: true }],
workerServer: [
- async ({ workerServerEnv, browser }, use, workerInfo) => {
+ async ({ workerServerEnv, browser, serverLease }, use, workerInfo) => {
+ if (serverLease?.kind === 'leased') {
+ await use(serverLease.server);
+ return;
+ }
const fixtureStartedAt = Date.now();
const setupOverhead = openBudgetPhase(
SETUP_OVERHEAD_PHASE_NAME,
@@ -461,6 +494,7 @@ export const test = base.extend<TestFixtures, WorkerFixtures>({
const releaseSetupResources: Array<() => void | Promise<void>> = [];
let openedServerLog: ServerLog | undefined;
let started: StartedWorkerServer | undefined;
+ let ownedProc: ChildProcess | undefined;
let endpoint: BoundViteEndpoint;
try {
@@ -480,21 +514,14 @@ export const test = base.extend<TestFixtures, WorkerFixtures>({
openedServerLog = serverLog;
releaseSetupResources.push(() => closeServerLog(serverLog));
- const proc = spawn('pnpm', ['run', 'dev', '--host', request.host], {
- cwd: APP_PACKAGE_ROOT,
- detached: true,
- env: {
- ...process.env,
- ...workerServerEnv,
- ...request.environment,
- OK_TEST_CONTENT_DIR: contentDir,
- OK_TEST_VITE_CACHE_DIR: viteCacheDir,
- OK_TEST_SKIP_I18N_COMPILE: '1',
- OK_TEST_GIT_ENABLED: '1',
- NO_COLOR: process.env.NO_COLOR ?? '1',
- },
- stdio: ['ignore', serverLog.fd, 'inherit'],
- });
+ const proc = spawnWorkerDevServer(
+ request,
+ workerServerEnv,
+ contentDir,
+ viteCacheDir,
+ serverLog,
+ );
+ ownedProc = proc;
releaseSetupResources.push(() =>
killGracefully(proc, WORKER_SERVER_BUDGET_RESERVES.devServerReap),
);
@@ -540,6 +567,15 @@ export const test = base.extend<TestFixtures, WorkerFixtures>({
openedServerLog === undefined
? '--- no dev server log was open when this failure ended the setup ---'
: `--- dev server log tail (${openedServerLog.path}) ---\n${tailServerLog(openedServerLog)}`;
+ try {
+ if (ownedProc !== undefined) await completeWorkerServerRelease(ownedProc);
+ } catch (cleanupErr) {
+ const completionFailure = `--- cleanup after this failure did not complete: ${cleanupErr instanceof Error ? cleanupErr.message : String(cleanupErr)} ---`;
+ drainFailure =
+ drainFailure === undefined
+ ? completionFailure
+ : `${drainFailure}\n${completionFailure}`;
+ }
declareSetupNonResult(base.info(), reason);
throw new Error(
`${reason}${borrowed === undefined ? '' : `\n${borrowed}`}${drainFailure === undefined ? '' : `\n${drainFailure}`}\n${tail}`,
@@ -551,19 +587,21 @@ export const test = base.extend<TestFixtures, WorkerFixtures>({
await use({ ...endpoint, contentDir });
const teardown = openBudgetPhase(TEARDOWN_PHASE_NAME, WORKER_SERVER_BUDGET_RESERVES.teardown);
- try {
- await killGracefully(proc, WORKER_SERVER_BUDGET_RESERVES.devServerReap);
- } finally {
- await spendOnBudgetPhase(teardown, () => {
- closeServerLog(serverLog);
- removeAllDuringTeardown(
- serverLog.path,
- contentDir,
- viteCacheDir,
- pending.request.receiptDir,
- );
- });
- }
+ await completeWorkerServerRelease(proc, async () => {
+ try {
+ await killGracefully(proc, WORKER_SERVER_BUDGET_RESERVES.devServerReap);
+ } finally {
+ await spendOnBudgetPhase(teardown, () => {
+ closeServerLog(serverLog);
+ removeAllDuringTeardown(
+ serverLog.path,
+ contentDir,
+ viteCacheDir,
+ pending.request.receiptDir,
+ );
+ });
+ }
+ });
reportBudgetOverrun(
teardown,
workerInfo.workerIndex,
diff --git a/packages/app/tests/stress/_helpers/index.ts b/packages/app/tests/stress/_helpers/index.ts
index 1abc5ac25..e78a8cdd2 100644
--- a/packages/app/tests/stress/_helpers/index.ts
+++ b/packages/app/tests/stress/_helpers/index.ts
@@ -43,6 +43,12 @@ export {
toggleMode,
waitForLandingSettled,
} from './landing.ts';
+export {
+ appSourcesIn,
+ servedEntryScriptRoute,
+ servedModuleRoute,
+ servedModuleUrl,
+} from './prebuilt/served-modules.ts';
export {
installClockAfterSync,
type WaitForProviderOptions,
diff --git a/packages/app/tests/stress/_helpers/loop-markers.ts b/packages/app/tests/stress/_helpers/loop-markers.ts
new file mode 100644
index 000000000..2b840226d
--- /dev/null
+++ b/packages/app/tests/stress/_helpers/loop-markers.ts
@@ -0,0 +1,44 @@
+import { readFileSync } from 'node:fs';
+import { join } from 'node:path';
+import { APP_PACKAGE_ROOT } from './seed-key.ts';
+
+export const LOOP_MARKER_SOURCES = {
+ app: 'packages/app/src/test-utils/test-loop-marker.ts',
+ server: 'packages/server/src/test-loop-marker.ts',
+ core: 'packages/core/src/test-loop-marker.ts',
+} as const;
+
+export type LoopMarkerClass = keyof typeof LOOP_MARKER_SOURCES;
+
+export const EXPECTED_LOOP_MARKERS_ENV = 'OK_TEST_LOOP_EXPECTED_MARKERS';
+
+const OK_ROOT = join(APP_PACKAGE_ROOT, '..', '..');
+const MARKER_SOURCE = /^export const [A-Z_]+ = '([a-z0-9-]+)';\n$/;
+
+function markerOnDisk(markerClass: LoopMarkerClass): string {
+ const path = LOOP_MARKER_SOURCES[markerClass];
+ const match = MARKER_SOURCE.exec(readFileSync(join(OK_ROOT, path), 'utf8'));
+ if (match === null) throw new Error(`${path} is not a single exported marker literal`);
+ return match[1];
+}
+
+function instrumentExpectation(markerClass: LoopMarkerClass): string | undefined {
+ const raw = process.env[EXPECTED_LOOP_MARKERS_ENV];
+ if (raw === undefined || raw === '') return undefined;
+ const parsed: unknown = JSON.parse(raw);
+ const value =
+ typeof parsed === 'object' && parsed !== null && !Array.isArray(parsed)
+ ? (parsed as Record<string, unknown>)[markerClass]
+ : undefined;
+ if (typeof value !== 'string') {
+ throw new Error(`${EXPECTED_LOOP_MARKERS_ENV} names no ${markerClass} marker: ${raw}`);
+ }
+ return value;
+}
+
+export function expectedLoopMarker(markerClass: LoopMarkerClass): {
+ readonly onDisk: string;
+ readonly instrument: string | undefined;
+} {
+ return { onDisk: markerOnDisk(markerClass), instrument: instrumentExpectation(markerClass) };
+}
diff --git a/packages/app/tests/stress/_helpers/port-ownership/nested-playwright-env.test-helper.ts b/packages/app/tests/stress/_helpers/port-ownership/nested-playwright-env.test-helper.ts
index aabe83728..19b444161 100644
--- a/packages/app/tests/stress/_helpers/port-ownership/nested-playwright-env.test-helper.ts
+++ b/packages/app/tests/stress/_helpers/port-ownership/nested-playwright-env.test-helper.ts
@@ -1,5 +1,17 @@
+import { E2E_SERVING_ENV, PREBUILT_APP_ENV } from '../prebuilt/prebuilt-serving.ts';
+import { REUSE_RUN_ENV, REUSE_SOCKET_ENV } from '../reuse/lease-protocol.ts';
+
+const OUTER_RUN_SERVER_ENV = [
+ REUSE_SOCKET_ENV,
+ REUSE_RUN_ENV,
+ E2E_SERVING_ENV,
+ PREBUILT_APP_ENV,
+] as const;
+
export function createNestedPlaywrightEnv(overrides: NodeJS.ProcessEnv): NodeJS.ProcessEnv {
- const env = { ...process.env, ...overrides };
+ const inherited = { ...process.env };
+ for (const name of OUTER_RUN_SERVER_ENV) delete inherited[name];
+ const env = { ...inherited, ...overrides };
delete env.PLAYWRIGHT_JSON_OUTPUT_FILE;
delete env.PLAYWRIGHT_JSON_OUTPUT_NAME;
delete env.PLAYWRIGHT_JSON_OUTPUT_DIR;
diff --git a/packages/app/tests/stress/_helpers/port-ownership/nested-playwright-run.test-helper.ts b/packages/app/tests/stress/_helpers/port-ownership/nested-playwright-run.test-helper.ts
new file mode 100644
index 000000000..65da07059
--- /dev/null
+++ b/packages/app/tests/stress/_helpers/port-ownership/nested-playwright-run.test-helper.ts
@@ -0,0 +1,58 @@
+import { spawn } from 'node:child_process';
+import { existsSync, mkdtempSync, readFileSync, rmSync } from 'node:fs';
+import { createRequire } from 'node:module';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import type { JSONReport, JSONReportSpec, JSONReportSuite } from '@playwright/test/reporter';
+import { APP_PACKAGE_ROOT } from '../server-process.ts';
+
+const PLAYWRIGHT_CLI = createRequire(join(APP_PACKAGE_ROOT, 'package.json')).resolve(
+ '@playwright/test/cli',
+);
+
+export interface NestedPlaywrightRun {
+ readonly exitCode: number | null;
+ readonly transcript: string;
+ readonly report: JSONReport | undefined;
+}
+
+export function specsOf(suites: readonly JSONReportSuite[]): JSONReportSpec[] {
+ return suites.flatMap((suite) => [...suite.specs, ...specsOf(suite.suites ?? [])]);
+}
+
+export async function runNestedPlaywright({
+ args,
+ runDirPrefix,
+ env,
+}: {
+ readonly args: readonly string[];
+ readonly runDirPrefix: string;
+ readonly env: (runDir: string) => NodeJS.ProcessEnv;
+}): Promise<NestedPlaywrightRun> {
+ const runDir = mkdtempSync(join(tmpdir(), runDirPrefix));
+ try {
+ const child = spawn(process.execPath, [PLAYWRIGHT_CLI, 'test', ...args], {
+ cwd: APP_PACKAGE_ROOT,
+ env: env(runDir),
+ stdio: ['ignore', 'pipe', 'pipe'],
+ });
+ let transcript = '';
+ child.stdout.on('data', (chunk: Buffer) => {
+ transcript += chunk.toString('utf8');
+ });
+ child.stderr.on('data', (chunk: Buffer) => {
+ transcript += chunk.toString('utf8');
+ });
+ const exitCode = await new Promise<number | null>((resolve, reject) => {
+ child.once('error', reject);
+ child.once('close', (code) => resolve(code));
+ });
+ const reportPath = join(runDir, 'results.json');
+ const report = existsSync(reportPath)
+ ? (JSON.parse(readFileSync(reportPath, 'utf8')) as JSONReport)
+ : undefined;
+ return { exitCode, transcript, report };
+ } finally {
+ rmSync(runDir, { recursive: true, force: true });
+ }
+}
diff --git a/packages/app/tests/stress/_helpers/prebuilt/prebuilt-build-use.ts b/packages/app/tests/stress/_helpers/prebuilt/prebuilt-build-use.ts
new file mode 100644
index 000000000..d477bb4ff
--- /dev/null
+++ b/packages/app/tests/stress/_helpers/prebuilt/prebuilt-build-use.ts
@@ -0,0 +1,100 @@
+import { mkdirSync, readdirSync, rmSync } from 'node:fs';
+import { join } from 'node:path';
+import { DatabaseSync } from 'node:sqlite';
+
+export type PrebuiltBuildUse = { readonly release: () => void };
+
+const USE_LOCKS_DIR = '.uses';
+const ROOT_USE_LOCK = 'root';
+const DISCARD_WAIT_MS = 60_000;
+const SQLITE_BUSY = 5;
+
+function openUseLock(outRoot: string, digest: string, waitMs: number): DatabaseSync {
+ const dir = join(outRoot, USE_LOCKS_DIR);
+ mkdirSync(dir, { recursive: true });
+ return new DatabaseSync(join(dir, `${digest}.sqlite`), { timeout: waitMs });
+}
+
+function isBusy(error: unknown): boolean {
+ const code = (error as { errcode?: unknown }).errcode;
+ return typeof code === 'number' && (code & 0xff) === SQLITE_BUSY;
+}
+
+function holdReadUse(outRoot: string, name: string): DatabaseSync {
+ const lock = openUseLock(outRoot, name, DISCARD_WAIT_MS);
+ try {
+ lock.exec('BEGIN');
+ lock.prepare('SELECT count(*) FROM sqlite_schema').get();
+ return lock;
+ } catch (error) {
+ lock.close();
+ throw error;
+ }
+}
+
+export function holdPrebuiltBuildUse(outRoot: string, digest: string): PrebuiltBuildUse {
+ const root = holdReadUse(outRoot, ROOT_USE_LOCK);
+ try {
+ const lock = holdReadUse(outRoot, digest);
+ return {
+ release: () => {
+ try {
+ if (lock.isOpen) lock.close();
+ } finally {
+ if (root.isOpen) root.close();
+ }
+ },
+ };
+ } catch (error) {
+ root.close();
+ throw error;
+ }
+}
+
+export function withPrebuiltBuildMutation(
+ outRoot: string,
+ digest: string,
+ mutate: () => 'landed' | 'joined',
+): 'landed' | 'joined' {
+ const lock = openUseLock(outRoot, `${digest}.mutation`, DISCARD_WAIT_MS);
+ try {
+ lock.exec('BEGIN EXCLUSIVE');
+ return mutate();
+ } finally {
+ lock.close();
+ }
+}
+
+export function resetPrebuiltOutput(outRoot: string): boolean {
+ const lock = openUseLock(outRoot, ROOT_USE_LOCK, 0);
+ try {
+ try {
+ lock.exec('BEGIN EXCLUSIVE');
+ } catch (error) {
+ if (isBusy(error)) return false;
+ throw error;
+ }
+ for (const name of readdirSync(outRoot)) {
+ if (name !== USE_LOCKS_DIR) rmSync(join(outRoot, name), { recursive: true, force: true });
+ }
+ return readdirSync(outRoot).every((name) => name === USE_LOCKS_DIR);
+ } finally {
+ lock.close();
+ }
+}
+
+export function discardUnlessInUse(outRoot: string, digest: string, discard: () => void): boolean {
+ const lock = openUseLock(outRoot, digest, 0);
+ try {
+ try {
+ lock.exec('BEGIN EXCLUSIVE');
+ } catch (error) {
+ if (isBusy(error)) return false;
+ throw error;
+ }
+ discard();
+ return true;
+ } finally {
+ lock.close();
+ }
+}
diff --git a/packages/app/tests/stress/_helpers/prebuilt/prebuilt-dev-app.ts b/packages/app/tests/stress/_helpers/prebuilt/prebuilt-dev-app.ts
new file mode 100644
index 000000000..6bddea367
--- /dev/null
+++ b/packages/app/tests/stress/_helpers/prebuilt/prebuilt-dev-app.ts
@@ -0,0 +1,149 @@
+import { existsSync, mkdirSync, mkdtempSync, renameSync, statSync, writeFileSync } from 'node:fs';
+import { join } from 'node:path';
+import {
+ type BuildOptions,
+ type PrebuiltManifest,
+ type PrebuiltServingDecision,
+ prebuiltManifest,
+ prebuiltServingDecision,
+} from '../../../foundation/test-server-inputs.ts';
+import {
+ discardUnlessInUse,
+ holdPrebuiltBuildUse,
+ type PrebuiltBuildUse,
+ withPrebuiltBuildMutation,
+} from './prebuilt-build-use.ts';
+import { discardDirectory, outputDigests } from './prebuilt-output-fs.ts';
+import { PREBUILT_MANIFEST_FILE, readPrebuiltManifest } from './prebuilt-serving.ts';
+
+export type BuildPrebuiltDist = (distDir: string) => Promise<void>;
+
+export type PrebuiltDevApp = {
+ readonly outputDir: string;
+ readonly distDir: string;
+ readonly manifest: PrebuiltManifest;
+ readonly decision: PrebuiltServingDecision;
+ readonly built: boolean;
+ readonly use: PrebuiltBuildUse;
+};
+
+const LATEST_FILE = 'latest.json';
+const RETIRING_PREFIX = '.retiring-';
+
+function prebuiltDevApp(
+ outputDir: string,
+ manifest: PrebuiltManifest,
+ decision: PrebuiltServingDecision,
+ built: boolean,
+ use: PrebuiltBuildUse,
+): PrebuiltDevApp {
+ return { outputDir, distDir: join(outputDir, 'dist'), manifest, decision, built, use };
+}
+
+function servesManifest(outputDir: string, manifest: PrebuiltManifest): boolean {
+ return prebuiltServingDecision(
+ readPrebuiltManifest(join(outputDir, PREBUILT_MANIFEST_FILE)),
+ manifest,
+ ).serve;
+}
+
+function occupiedByAnotherDirectory(error: unknown): boolean {
+ const { code } = error as NodeJS.ErrnoException;
+ return code === 'ENOTEMPTY' || code === 'EEXIST';
+}
+
+function retireOutput(outRoot: string, digest: string): void {
+ const retiring = mkdtempSync(join(outRoot, `${RETIRING_PREFIX}${digest.slice(0, 12)}-`));
+ renameSync(join(outRoot, digest), join(retiring, digest));
+ discardDirectory(retiring);
+}
+
+function landOrJoin(
+ outRoot: string,
+ staging: string,
+ manifest: PrebuiltManifest,
+): 'landed' | 'joined' {
+ const outputDir = join(outRoot, manifest.digest);
+ if (existsSync(outputDir) && servesManifest(outputDir, manifest)) return 'joined';
+ return withPrebuiltBuildMutation(outRoot, manifest.digest, () => {
+ if (existsSync(outputDir)) {
+ if (servesManifest(outputDir, manifest)) return 'joined';
+ retireOutput(outRoot, manifest.digest);
+ }
+ try {
+ renameSync(staging, outputDir);
+ return 'landed';
+ } catch (error) {
+ if (occupiedByAnotherDirectory(error) && servesManifest(outputDir, manifest)) return 'joined';
+ throw error;
+ }
+ });
+}
+
+function recordLatest(outRoot: string, manifest: PrebuiltManifest): void {
+ const pending = join(outRoot, `.${LATEST_FILE}.${process.pid}`);
+ writeFileSync(pending, JSON.stringify(manifest));
+ renameSync(pending, join(outRoot, LATEST_FILE));
+}
+
+function discardUnusedOutputs(outRoot: string, keep: ReadonlySet<string>): void {
+ for (const digest of outputDigests(outRoot)) {
+ if (keep.has(digest)) continue;
+ discardUnlessInUse(outRoot, digest, () => {
+ if (existsSync(join(outRoot, digest))) retireOutput(outRoot, digest);
+ });
+ }
+}
+
+async function serveOrBuild(
+ outRoot: string,
+ manifest: PrebuiltManifest,
+ build: BuildPrebuiltDist,
+ use: PrebuiltBuildUse,
+): Promise<PrebuiltDevApp> {
+ const outputDir = join(outRoot, manifest.digest);
+ const recorded = readPrebuiltManifest(join(outputDir, PREBUILT_MANIFEST_FILE));
+ const latest = readPrebuiltManifest(join(outRoot, LATEST_FILE));
+ const previous = latest?.digest === manifest.digest ? undefined : latest;
+ const decision = prebuiltServingDecision(recorded ?? previous, manifest);
+ if (decision.serve) return prebuiltDevApp(outputDir, manifest, decision, false, use);
+
+ mkdirSync(outRoot, { recursive: true });
+ const staging = mkdtempSync(join(outRoot, `.staging-${manifest.digest.slice(0, 12)}-`));
+ try {
+ const distDir = join(staging, 'dist');
+ await build(distDir);
+ if (statSync(join(distDir, 'index.html'), { throwIfNoEntry: false })?.isFile() !== true) {
+ throw new Error(`the prebuilt DEV app build wrote no index.html under ${distDir}`);
+ }
+ writeFileSync(join(staging, PREBUILT_MANIFEST_FILE), JSON.stringify(manifest));
+ if (landOrJoin(outRoot, staging, manifest) === 'joined') {
+ return prebuiltDevApp(outputDir, manifest, decision, false, use);
+ }
+ } finally {
+ discardDirectory(staging);
+ }
+ discardUnusedOutputs(
+ outRoot,
+ new Set(latest === undefined ? [manifest.digest] : [manifest.digest, latest.digest]),
+ );
+ recordLatest(outRoot, manifest);
+ return prebuiltDevApp(outputDir, manifest, decision, true, use);
+}
+
+export async function resolvePrebuiltDevApp(request: {
+ readonly okRoot: string;
+ readonly outRoot: string;
+ readonly options: BuildOptions;
+ readonly build: BuildPrebuiltDist;
+}): Promise<PrebuiltDevApp> {
+ const { okRoot, outRoot, options, build } = request;
+ const manifest = prebuiltManifest(okRoot, options);
+ const use = holdPrebuiltBuildUse(outRoot, manifest.digest);
+ try {
+ return await serveOrBuild(outRoot, manifest, build, use);
+ } catch (error) {
+ use.release();
+ throw error;
+ }
+}
diff --git a/packages/app/tests/stress/_helpers/prebuilt/prebuilt-dev-build.ts b/packages/app/tests/stress/_helpers/prebuilt/prebuilt-dev-build.ts
new file mode 100644
index 000000000..8d114499b
--- /dev/null
+++ b/packages/app/tests/stress/_helpers/prebuilt/prebuilt-dev-build.ts
@@ -0,0 +1,127 @@
+import { spawn } from 'node:child_process';
+import { join } from 'node:path';
+import { watchForSilence } from '../../../../../../test-support/child-silence-bound.test-helper.ts';
+import { PREBUILT_OUT_DIR_ENV } from '../../../../src/build/prebuilt-dev-contract.ts';
+import { APP_PACKAGE_ROOT } from '../seed-key.ts';
+import { killGroup } from '../server-process.ts';
+import { type PrebuiltDevApp, resolvePrebuiltDevApp } from './prebuilt-dev-app.ts';
+import { PREBUILT_BUILD_OPTIONS } from './prebuilt-serving.ts';
+
+export const OK_ROOT = join(APP_PACKAGE_ROOT, '..', '..');
+
+export const PREBUILT_OUT_ROOT = join(
+ APP_PACKAGE_ROOT,
+ 'node_modules',
+ '.cache',
+ 'ok-prebuilt-dev',
+);
+
+const OUTPUT_TAIL_CHARS = 8_000;
+
+export const BUILD_STEP_SILENCE_MS = 300_000;
+
+export type BuildStep = {
+ readonly label: string;
+ readonly command: string;
+ readonly args: readonly string[];
+ readonly env: Readonly<Record<string, string>>;
+ readonly inherits?: (name: string) => boolean;
+};
+
+function inheritedEnv(inherits: (name: string) => boolean): NodeJS.ProcessEnv {
+ return Object.fromEntries(Object.entries(process.env).filter(([name]) => inherits(name)));
+}
+
+const notViteVariable = (name: string): boolean => !name.startsWith('VITE_');
+
+export function runBuildStep(step: BuildStep): Promise<void> {
+ const { label, command, args, env, inherits = () => true } = step;
+ return new Promise((resolve, reject) => {
+ const child = spawn(command, [...args], {
+ cwd: APP_PACKAGE_ROOT,
+ detached: true,
+ env: { ...inheritedEnv(inherits), ...env, NO_COLOR: '1' },
+ stdio: ['ignore', 'pipe', 'pipe'],
+ });
+ const endGroup = () => {
+ killGroup(child, 'SIGKILL');
+ };
+ let silent = false;
+ const releaseSilence = watchForSilence(
+ [child.stdout, child.stderr],
+ BUILD_STEP_SILENCE_MS,
+ () => {
+ silent = true;
+ endGroup();
+ },
+ );
+ process.once('exit', endGroup);
+ let output = '';
+ const keep = (chunk: Buffer) => {
+ output = (output + chunk.toString()).slice(-OUTPUT_TAIL_CHARS);
+ };
+ child.stdout.on('data', keep);
+ child.stderr.on('data', keep);
+ const settle = () => {
+ releaseSilence();
+ process.off('exit', endGroup);
+ };
+ child.once('error', (error) => {
+ settle();
+ reject(error);
+ });
+ child.once('close', (code, signal) => {
+ settle();
+ if (silent) {
+ reject(
+ new Error(
+ `${label} printed nothing for ${BUILD_STEP_SILENCE_MS / 1000} s, so this run stopped it and every process in its process group; its last output:\n${output}`,
+ ),
+ );
+ } else if (code === 0) resolve();
+ else reject(new Error(`${label} exited with ${code ?? signal}\n${output}`));
+ });
+ });
+}
+
+function runPnpm(
+ label: string,
+ args: readonly string[],
+ env: Readonly<Record<string, string>>,
+): Promise<void> {
+ return runBuildStep({ label, command: 'pnpm', args, env });
+}
+
+export function prebuiltBuildStep(distDir: string): BuildStep {
+ return {
+ label: 'the prebuilt DEV app build',
+ command: 'pnpm',
+ args: [
+ 'exec',
+ 'vite',
+ 'build',
+ '--config',
+ PREBUILT_BUILD_OPTIONS.config,
+ '--mode',
+ PREBUILT_BUILD_OPTIONS.mode,
+ ],
+ env: { NODE_ENV: PREBUILT_BUILD_OPTIONS.nodeEnv, [PREBUILT_OUT_DIR_ENV]: distDir },
+ inherits: notViteVariable,
+ };
+}
+
+export function buildPrebuiltDist(distDir: string): Promise<void> {
+ return runBuildStep(prebuiltBuildStep(distDir));
+}
+
+export async function preparePrebuiltDevApp(): Promise<PrebuiltDevApp> {
+ await runPnpm('predev for the prebuilt DEV app', ['run', 'predev'], {
+ OK_TEST_SKIP_I18N_COMPILE: '1',
+ });
+ return resolvePrebuiltDevApp({
+ okRoot: OK_ROOT,
+ outRoot: PREBUILT_OUT_ROOT,
+ options: PREBUILT_BUILD_OPTIONS,
+ build: buildPrebuiltDist,
+ });
+}
diff --git a/packages/app/tests/stress/_helpers/prebuilt/prebuilt-dev-setup.ts b/packages/app/tests/stress/_helpers/prebuilt/prebuilt-dev-setup.ts
new file mode 100644
index 000000000..9a698a3d5
--- /dev/null
+++ b/packages/app/tests/stress/_helpers/prebuilt/prebuilt-dev-setup.ts
@@ -0,0 +1,23 @@
+import { preparePrebuiltDevApp } from './prebuilt-dev-build.ts';
+import {
+ describeChanges,
+ E2E_SERVING_ENV,
+ e2eServingMode,
+ PREBUILT_APP_ENV,
+} from './prebuilt-serving.ts';
+
+export default async function preparePrebuiltServing(): Promise<(() => void) | undefined> {
+ if (e2eServingMode(process.env) !== 'prebuilt') return undefined;
+ const app = await preparePrebuiltDevApp();
+ const { decision } = app;
+ const how = decision.serve
+ ? 'serving the build already made for these inputs'
+ : !app.built
+ ? 'serving the build another run made for these inputs while this run was building'
+ : decision.reason === 'missing'
+ ? 'built it, since nothing was built for these inputs'
+ : `rebuilt it, because ${describeChanges(decision)} changed since the previous build`;
+ console.log(`[e2e ${E2E_SERVING_ENV}=prebuilt] ${app.outputDir}: ${how}`);
+ process.env[PREBUILT_APP_ENV] = app.outputDir;
+ return app.use.release;
+}
diff --git a/packages/app/tests/stress/_helpers/prebuilt/prebuilt-output-fs.ts b/packages/app/tests/stress/_helpers/prebuilt/prebuilt-output-fs.ts
new file mode 100644
index 000000000..7216e8a8d
--- /dev/null
+++ b/packages/app/tests/stress/_helpers/prebuilt/prebuilt-output-fs.ts
@@ -0,0 +1,13 @@
+import { readdirSync, rmSync } from 'node:fs';
+
+const OUTPUT_DIR_NAME = /^[0-9a-f]{64}$/;
+
+export function discardDirectory(path: string): void {
+ rmSync(path, { recursive: true, force: true });
+}
+
+export function outputDigests(outRoot: string): string[] {
+ return readdirSync(outRoot, { withFileTypes: true })
+ .filter((entry) => entry.isDirectory() && OUTPUT_DIR_NAME.test(entry.name))
+ .map((entry) => entry.name);
+}
diff --git a/packages/app/tests/stress/_helpers/prebuilt/prebuilt-serving.ts b/packages/app/tests/stress/_helpers/prebuilt/prebuilt-serving.ts
new file mode 100644
index 000000000..6526be647
--- /dev/null
+++ b/packages/app/tests/stress/_helpers/prebuilt/prebuilt-serving.ts
@@ -0,0 +1,133 @@
+import { readFileSync } from 'node:fs';
+import { join } from 'node:path';
+import { PREBUILT_OUT_DIR_ENV } from '../../../../src/build/prebuilt-dev-contract.ts';
+import {
+ type BuildOptions,
+ type PrebuiltManifest,
+ type PrebuiltServingDecision,
+ prebuiltManifest,
+ prebuiltServingDecision,
+} from '../../../foundation/test-server-inputs.ts';
+
+export const E2E_SERVING_ENV = 'OK_TEST_E2E_SERVING';
+export const PREBUILT_APP_ENV = 'OK_TEST_PREBUILT_APP';
+export const PREBUILT_MANIFEST_FILE = 'prebuilt-manifest.json';
+
+export const E2E_SERVING_MODES = ['dev-server', 'prebuilt'] as const;
+export type E2eServingMode = (typeof E2E_SERVING_MODES)[number];
+
+export const PREBUILT_BUILD_OPTIONS = {
+ config: 'vite.prebuilt-dev.ts',
+ mode: 'development',
+ nodeEnv: 'development',
+} as const satisfies BuildOptions;
+
+export type WorkerServing =
+ | { readonly kind: 'dev-server' }
+ | { readonly kind: 'prebuilt'; readonly outputDir: string };
+
+export type PrebuiltServing = Extract<WorkerServing, { readonly kind: 'prebuilt' }>;
+
+export type PrebuiltServerCommand = {
+ readonly args: readonly string[];
+ readonly env: Readonly<Record<string, string>>;
+};
+
+type Env = Readonly<Record<string, string | undefined>>;
+
+const LISTED_CHANGES = 5;
+
+export function e2eServingMode(env: Env): E2eServingMode {
+ const raw = env[E2E_SERVING_ENV];
+ if (raw === undefined || raw === '') return 'dev-server';
+ const mode = E2E_SERVING_MODES.find((candidate) => candidate === raw);
+ if (mode === undefined) {
+ throw new Error(
+ `${E2E_SERVING_ENV}=${raw} names no serving mode; use ${E2E_SERVING_MODES.join(' or ')}`,
+ );
+ }
+ return mode;
+}
+
+export function workerServingFromEnv(env: Env): WorkerServing {
+ if (e2eServingMode(env) === 'dev-server') return { kind: 'dev-server' };
+ const outputDir = env[PREBUILT_APP_ENV];
+ if (outputDir === undefined || outputDir === '') {
+ throw new Error(
+ `${E2E_SERVING_ENV}=prebuilt serves the prebuilt DEV app its globalSetup records in ${PREBUILT_APP_ENV}, and this invocation has none; run it through a Playwright config that lists the prebuilt setup`,
+ );
+ }
+ return { kind: 'prebuilt', outputDir };
+}
+
+function stringRecord(value: unknown): Readonly<Record<string, string>> | undefined {
+ if (typeof value !== 'object' || value === null || Array.isArray(value)) return undefined;
+ const entries = Object.entries(value);
+ const strings = entries.filter(
+ (entry): entry is [string, string] => typeof entry[1] === 'string',
+ );
+ return strings.length === entries.length ? Object.fromEntries(strings) : undefined;
+}
+
+export function parsePrebuiltManifest(value: unknown): PrebuiltManifest | undefined {
+ if (typeof value !== 'object' || value === null) return undefined;
+ const { digest, inputs, options } = value as Record<string, unknown>;
+ const inputDigests = stringRecord(inputs);
+ const optionValues = stringRecord(options);
+ if (typeof digest !== 'string' || inputDigests === undefined || optionValues === undefined) {
+ return undefined;
+ }
+ return { digest, inputs: inputDigests, options: optionValues };
+}
+
+export function readPrebuiltManifest(path: string): PrebuiltManifest | undefined {
+ let text: string;
+ try {
+ text = readFileSync(path, 'utf8');
+ } catch (error) {
+ if ((error as NodeJS.ErrnoException).code === 'ENOENT') return undefined;
+ throw error;
+ }
+ let value: unknown;
+ try {
+ value = JSON.parse(text);
+ } catch {
+ return undefined;
+ }
+ return parsePrebuiltManifest(value);
+}
+
+type UnservedDecision = Exclude<PrebuiltServingDecision, { serve: true }>;
+
+export function describeChanges(decision: Extract<UnservedDecision, { reason: 'stale' }>): string {
+ const changed = [
+ ...decision.changedInputs,
+ ...decision.changedOptions.map((option) => `build option ${option}`),
+ ];
+ const listed = changed.slice(0, LISTED_CHANGES).join(', ');
+ return changed.length > LISTED_CHANGES
+ ? `${listed} and ${changed.length - LISTED_CHANGES} more`
+ : listed;
+}
+
+export function describePrebuiltRefusal(outputDir: string, decision: UnservedDecision): string {
+ return decision.reason === 'missing'
+ ? `the prebuilt DEV app at ${outputDir} has no build manifest, so it is not served`
+ : `the prebuilt DEV app at ${outputDir} is stale, because ${describeChanges(decision)} changed since it was built, so it is not served; run again to rebuild it`;
+}
+
+export function prebuiltServerCommand(
+ host: string,
+ serving: PrebuiltServing,
+ okRoot: string,
+): PrebuiltServerCommand {
+ const decision = prebuiltServingDecision(
+ readPrebuiltManifest(join(serving.outputDir, PREBUILT_MANIFEST_FILE)),
+ prebuiltManifest(okRoot, PREBUILT_BUILD_OPTIONS),
+ );
+ if (!decision.serve) throw new Error(describePrebuiltRefusal(serving.outputDir, decision));
+ return {
+ args: ['exec', 'vite', 'preview', '--config', PREBUILT_BUILD_OPTIONS.config, '--host', host],
+ env: { [PREBUILT_OUT_DIR_ENV]: join(serving.outputDir, 'dist') },
+ };
+}
diff --git a/packages/app/tests/stress/_helpers/prebuilt/served-modules.ts b/packages/app/tests/stress/_helpers/prebuilt/served-modules.ts
new file mode 100644
index 000000000..fef362d40
--- /dev/null
+++ b/packages/app/tests/stress/_helpers/prebuilt/served-modules.ts
@@ -0,0 +1,194 @@
+import { readdirSync, readFileSync, statSync } from 'node:fs';
+import { extname, join } from 'node:path';
+import {
+ PREBUILT_MODULE_CHUNKS,
+ PREBUILT_MODULE_MANIFEST,
+} from '../../../../src/build/prebuilt-dev-contract.ts';
+import { APP_PACKAGE_ROOT } from '../seed-key.ts';
+import { type WorkerServing, workerServingFromEnv } from './prebuilt-serving.ts';
+
+type ModuleLocator = {
+ readonly moduleUrl: (source: string) => string;
+ readonly entryScriptUrl: () => string;
+ readonly requestPaths: (sources: readonly string[]) => ReadonlySet<string>;
+};
+
+export type ServedModules = ModuleLocator & {
+ readonly moduleRoute: (...sources: readonly string[]) => (url: URL) => boolean;
+ readonly entryScriptRoute: () => (url: URL) => boolean;
+};
+
+const SHELL_HTML = 'index.html';
+const BROWSER_MODULE_EXTENSIONS = new Set(['.ts', '.tsx', '.mts', '.js', '.jsx', '.mjs']);
+const MODULE_SCRIPT = /<script\b(?=[^>]*\btype="module")[^>]*\bsrc="([^"]+)"[^>]*>/g;
+const URL_BASE = 'http://served-module.invalid';
+
+type ModuleManifestEntry = {
+ readonly file: string;
+ readonly loadedOnItsOwn: boolean;
+};
+
+export function parseModuleManifest(value: unknown): ReadonlyMap<string, ModuleManifestEntry> {
+ if (typeof value !== 'object' || value === null || Array.isArray(value)) {
+ throw new Error('the prebuilt module manifest is not a JSON object');
+ }
+ const entries = new Map<string, ModuleManifestEntry>();
+ for (const [source, raw] of Object.entries(value)) {
+ if (typeof raw !== 'object' || raw === null) {
+ throw new Error(`the prebuilt module manifest entry for ${source} is not an object`);
+ }
+ const { file, isEntry, isDynamicEntry } = raw as Record<string, unknown>;
+ if (typeof file !== 'string' || file === '') {
+ throw new Error(`the prebuilt module manifest entry for ${source} names no file`);
+ }
+ entries.set(source, { file, loadedOnItsOwn: isEntry === true || isDynamicEntry === true });
+ }
+ return entries;
+}
+
+export function parseModuleChunks(value: unknown): ReadonlyMap<string, string> {
+ if (typeof value !== 'object' || value === null || Array.isArray(value)) {
+ throw new Error('the prebuilt module chunk map is not a JSON object');
+ }
+ const chunks = new Map<string, string>();
+ for (const [source, chunk] of Object.entries(value)) {
+ if (typeof chunk !== 'string' || chunk === '') {
+ throw new Error(`the prebuilt module chunk map names no chunk for ${source}`);
+ }
+ chunks.set(source, chunk);
+ }
+ return chunks;
+}
+
+function pathnameOf(url: string): string {
+ return new URL(url, URL_BASE).pathname;
+}
+
+function devModuleLocator(appRoot: string): ModuleLocator {
+ const moduleUrl = (source: string) => {
+ if (statSync(join(appRoot, source), { throwIfNoEntry: false })?.isFile() !== true) {
+ throw new Error(
+ `${source} is not a source file of the app, so the dev server serves no module for it`,
+ );
+ }
+ return BROWSER_MODULE_EXTENSIONS.has(extname(source)) ? `/${source}` : `/${source}?import`;
+ };
+ return {
+ moduleUrl,
+ entryScriptUrl() {
+ const sources = [
+ ...readFileSync(join(appRoot, SHELL_HTML), 'utf8').matchAll(MODULE_SCRIPT),
+ ].map((match) => match[1]);
+ const [source] = sources;
+ if (sources.length !== 1 || source === undefined) {
+ throw new Error(
+ `${SHELL_HTML} has ${sources.length} module scripts, so it names no single entry script`,
+ );
+ }
+ return source;
+ },
+ requestPaths(sources) {
+ return new Set(sources.map((source) => pathnameOf(moduleUrl(source))));
+ },
+ };
+}
+
+function readJson(path: string): unknown {
+ return JSON.parse(readFileSync(path, 'utf8'));
+}
+
+function prebuiltModuleLocator(outputDir: string): ModuleLocator {
+ const distDir = join(outputDir, 'dist');
+ const manifest = parseModuleManifest(readJson(join(distDir, PREBUILT_MODULE_MANIFEST)));
+ const carriers = parseModuleChunks(readJson(join(distDir, PREBUILT_MODULE_CHUNKS)));
+ const ownerOf = new Map(
+ [...manifest]
+ .filter(([, entry]) => entry.loadedOnItsOwn)
+ .map(([source, entry]) => [entry.file, source]),
+ );
+ const chunkOf = (source: string) => {
+ const file = manifest.get(source)?.file;
+ if (file === undefined) {
+ throw new Error(
+ `${source} names no chunk of the prebuilt DEV app (${distDir}), so no request serves it alone`,
+ );
+ }
+ return `/${file}`;
+ };
+ return {
+ moduleUrl: chunkOf,
+ entryScriptUrl: () => chunkOf(SHELL_HTML),
+ requestPaths(sources) {
+ const requested = new Set(sources);
+ const paths = new Set<string>();
+ for (const source of sources) {
+ const chunk = carriers.get(source);
+ if (chunk === undefined) {
+ throw new Error(`${source} is in no chunk of the prebuilt DEV app (${distDir})`);
+ }
+ const owner = ownerOf.get(chunk);
+ if (owner === undefined) {
+ throw new Error(
+ `${source} is bundled into ${chunk}, which no single module loads on its own, so delaying it would delay more than ${source}`,
+ );
+ }
+ if (!requested.has(owner)) {
+ throw new Error(
+ `${source} is bundled into ${chunk}, the chunk of ${owner}, which this route does not delay, so delaying it would delay more than ${source}`,
+ );
+ }
+ paths.add(`/${chunk}`);
+ }
+ return paths;
+ },
+ };
+}
+
+export function resolveServedModules(
+ serving: WorkerServing,
+ appRoot: string = APP_PACKAGE_ROOT,
+): ServedModules {
+ const locator =
+ serving.kind === 'dev-server'
+ ? devModuleLocator(appRoot)
+ : prebuiltModuleLocator(serving.outputDir);
+ return {
+ ...locator,
+ moduleRoute(...sources) {
+ if (sources.length === 0)
+ throw new Error('a module route needs at least one source to match');
+ const paths = locator.requestPaths(sources);
+ return (url) => paths.has(url.pathname);
+ },
+ entryScriptRoute() {
+ const path = pathnameOf(locator.entryScriptUrl());
+ return (url) => url.pathname === path;
+ },
+ };
+}
+
+let workerModules: ServedModules | undefined;
+
+function servedModules(): ServedModules {
+ workerModules ??= resolveServedModules(workerServingFromEnv(process.env));
+ return workerModules;
+}
+
+export function servedModuleUrl(source: string): string {
+ return servedModules().moduleUrl(source);
+}
+
+export function servedModuleRoute(...sources: readonly string[]): (url: URL) => boolean {
+ return servedModules().moduleRoute(...sources);
+}
+
+export function servedEntryScriptRoute(): (url: URL) => boolean {
+ return servedModules().entryScriptRoute();
+}
+
+export function appSourcesIn(directory: string, name: RegExp): string[] {
+ return readdirSync(join(APP_PACKAGE_ROOT, directory), { withFileTypes: true })
+ .filter((entry) => entry.isFile() && name.test(entry.name))
+ .map((entry) => `${directory}/${entry.name}`)
+ .sort();
+}
diff --git a/packages/app/tests/stress/_helpers/reuse/fresh-slot.ts b/packages/app/tests/stress/_helpers/reuse/fresh-slot.ts
new file mode 100644
index 000000000..56663183e
--- /dev/null
+++ b/packages/app/tests/stress/_helpers/reuse/fresh-slot.ts
@@ -0,0 +1,169 @@
+import { lstatSync, readdirSync } from 'node:fs';
+import { join } from 'node:path';
+
+type SlotEntry =
+ | { readonly kind: 'empty-file'; readonly required: boolean }
+ | { readonly kind: 'directory'; readonly required: boolean; readonly entries: SlotTree }
+ | { readonly kind: 'server-owned' };
+
+type SlotTree = Readonly<Record<string, SlotEntry>>;
+
+const SERVER_OWNED: SlotEntry = { kind: 'server-owned' };
+
+function directory(required: boolean, entries: SlotTree = {}): SlotEntry {
+ return { kind: 'directory', required, entries };
+}
+
+const SERVER_WRITTEN_TREE: SlotTree = {
+ '.git': SERVER_OWNED,
+ '.okignore': { kind: 'empty-file', required: false },
+ '.claude': directory(true, { skills: directory(true) }),
+ '.ok': directory(false, {
+ skills: directory(false),
+ local: directory(false, {
+ 'acp-registry-cache.json': SERVER_OWNED,
+ cache: SERVER_OWNED,
+ 'principal.json': SERVER_OWNED,
+ 'removed-docs.json': SERVER_OWNED,
+ 'server.lock': SERVER_OWNED,
+ 'stale-external-writes.json': SERVER_OWNED,
+ 'state.json': SERVER_OWNED,
+ }),
+ }),
+};
+
+function withFixtureDoc(tree: SlotTree, segments: readonly string[]): SlotTree {
+ const [head, ...rest] = segments;
+ if (head === undefined) return tree;
+ if (rest.length === 0) return { ...tree, [`${head}.md`]: { kind: 'empty-file', required: true } };
+ const existing = tree[head];
+ const entries = existing?.kind === 'directory' ? existing.entries : {};
+ return { ...tree, [head]: directory(true, withFixtureDoc(entries, rest)) };
+}
+
+function freshSlotTree(fixtureDocNames: readonly string[]): SlotTree {
+ return fixtureDocNames.reduce(
+ (tree, docName) => withFixtureDoc(tree, docName.split('/')),
+ SERVER_WRITTEN_TREE,
+ );
+}
+
+function compare(root: string, relative: string, tree: SlotTree, problems: string[]): void {
+ const present = new Set<string>();
+ for (const name of readdirSync(join(root, relative))) {
+ present.add(name);
+ const path = relative === '' ? name : `${relative}/${name}`;
+ const expected = tree[name];
+ if (expected === undefined) {
+ problems.push(`${path} is not in a freshly booted slot`);
+ continue;
+ }
+ if (expected.kind === 'server-owned') continue;
+ const stat = lstatSync(join(root, path));
+ if (expected.kind === 'empty-file') {
+ if (!stat.isFile()) problems.push(`${path} is not a file`);
+ else if (stat.size > 0) problems.push(`${path} is not empty`);
+ continue;
+ }
+ if (!stat.isDirectory()) {
+ problems.push(`${path} is not a directory`);
+ continue;
+ }
+ compare(root, path, expected.entries, problems);
+ }
+ for (const [name, expected] of Object.entries(tree)) {
+ if (expected.kind === 'server-owned' || !expected.required || present.has(name)) continue;
+ problems.push(`${relative === '' ? name : `${relative}/${name}`} is missing`);
+ }
+}
+
+export function freshSlotResidue(
+ contentDir: string,
+ fixtureDocNames: readonly string[],
+): readonly string[] {
+ const problems: string[] = [];
+ compare(contentDir, '', freshSlotTree(fixtureDocNames), problems);
+ return problems.sort();
+}
+
+export interface ServerResetResidue {
+ readonly loadedDocuments: readonly string[];
+ readonly clientConnections: number;
+ readonly agentSessions: number;
+ readonly agentPresence: readonly string[];
+ readonly staleExternalWrites: readonly string[];
+ readonly removedDocuments: readonly string[];
+}
+
+type Fields = Record<string, unknown>;
+
+function isFields(value: unknown): value is Fields {
+ return typeof value === 'object' && value !== null && !Array.isArray(value);
+}
+
+function stringList(fields: Fields, key: string): readonly string[] {
+ const value = fields[key];
+ if (!Array.isArray(value) || !value.every((item) => typeof item === 'string')) {
+ throw new TypeError(`the test-reset response residue.${key} is not a list of strings`);
+ }
+ return value;
+}
+
+function count(fields: Fields, key: string): number {
+ const value = fields[key];
+ if (typeof value !== 'number' || !Number.isInteger(value) || value < 0) {
+ throw new TypeError(`the test-reset response residue.${key} is not a non-negative integer`);
+ }
+ return value;
+}
+
+const RESIDUE_FIELDS: ReadonlySet<string> = new Set([
+ 'loadedDocuments',
+ 'clientConnections',
+ 'agentSessions',
+ 'agentPresence',
+ 'staleExternalWrites',
+ 'removedDocuments',
+] satisfies readonly (keyof ServerResetResidue)[]);
+
+export function parseServerResetResidue(body: unknown): ServerResetResidue {
+ if (!isFields(body) || body.scope !== 'server' || !isFields(body.residue)) {
+ throw new TypeError('the test-reset response is not a server-scope report');
+ }
+ const { residue } = body;
+ const uninspected = Object.keys(residue).filter((key) => !RESIDUE_FIELDS.has(key));
+ if (uninspected.length > 0) {
+ throw new TypeError(
+ `the test-reset response residue reports ${uninspected.join(', ')}, which this check does not inspect`,
+ );
+ }
+ return {
+ loadedDocuments: stringList(residue, 'loadedDocuments'),
+ clientConnections: count(residue, 'clientConnections'),
+ agentSessions: count(residue, 'agentSessions'),
+ agentPresence: stringList(residue, 'agentPresence'),
+ staleExternalWrites: stringList(residue, 'staleExternalWrites'),
+ removedDocuments: stringList(residue, 'removedDocuments'),
+ };
+}
+
+export function serverResetResidueProblems(residue: ServerResetResidue): readonly string[] {
+ const problems: string[] = [];
+ if (residue.loadedDocuments.length > 0) {
+ problems.push(`loaded documents ${residue.loadedDocuments.join(', ')}`);
+ }
+ if (residue.clientConnections > 0) {
+ problems.push(`${residue.clientConnections} client connections`);
+ }
+ if (residue.agentSessions > 0) problems.push(`${residue.agentSessions} agent sessions`);
+ if (residue.agentPresence.length > 0) {
+ problems.push(`agent presence for ${residue.agentPresence.join(', ')}`);
+ }
+ if (residue.staleExternalWrites.length > 0) {
+ problems.push(`stale external writes for ${residue.staleExternalWrites.join(', ')}`);
+ }
+ if (residue.removedDocuments.length > 0) {
+ problems.push(`removal records for ${residue.removedDocuments.join(', ')}`);
+ }
+ return problems;
+}
diff --git a/packages/app/tests/stress/_helpers/reuse/lease-fixture.ts b/packages/app/tests/stress/_helpers/reuse/lease-fixture.ts
new file mode 100644
index 000000000..467a6f296
--- /dev/null
+++ b/packages/app/tests/stress/_helpers/reuse/lease-fixture.ts
@@ -0,0 +1,254 @@
+import { realpathSync } from 'node:fs';
+import { join } from 'node:path';
+import { type Browser, type TestInfo, test, type WorkerInfo } from '@playwright/test';
+import { resetContentToFixtureBaseline, resetFixtureFolders } from '../content-reset.ts';
+import {
+ checkApiConfig,
+ openBudgetPhase,
+ REQUIRED_FIXTURE_DOC_NAMES,
+ WORKER_SERVER_BUDGET_RESERVES,
+ WORKER_SERVER_SETUP_STARVATION_LINE_MS,
+ type WorkerServer,
+ warmupAppFirstLoad,
+} from '../fixtures.ts';
+import { e2eServingMode } from '../prebuilt/prebuilt-serving.ts';
+import { APP_PACKAGE_ROOT } from '../seed-key.ts';
+import { checkCollabSync } from '../server-process.ts';
+import { declareSetupNonResult } from '../setup-non-result.ts';
+import {
+ freshSlotResidue,
+ parseServerResetResidue,
+ serverResetResidueProblems,
+} from './fresh-slot.ts';
+import {
+ type LeaseOutcome,
+ type LeaseRequest,
+ type LeaseVerdict,
+ REUSE_RUN_ENV,
+ REUSE_SOCKET_ENV,
+} from './lease-protocol.ts';
+import { recordRunAbort, runAbortReason } from './reuse-run.ts';
+import { connectLeaseClient, type LeaseClient } from './supervisor-socket.ts';
+
+export const SERVER_LEASE_ANNOTATION = 'ok:server-lease';
+
+const SUPERVISOR_CONNECT_MS = 5_000;
+const SUPERVISOR_CALL_MS = 10_000;
+const LEASE_RESET_MS = 30_000;
+
+export type ServerLease =
+ | {
+ readonly kind: 'leased';
+ readonly run: string;
+ readonly server: WorkerServer;
+ readonly note: string;
+ verify(): Promise<LeaseVerdict>;
+ }
+ | { readonly kind: 'fresh'; readonly note: string };
+
+let currentLease: ServerLease | undefined;
+
+function describe(error: unknown): string {
+ return error instanceof Error ? error.message : String(error);
+}
+
+function remainingMs(deadlineAt: number): number {
+ return Math.max(1, deadlineAt - Date.now());
+}
+
+async function requestTestReset(server: WorkerServer, query: string): Promise<unknown> {
+ const response = await fetch(`${server.baseURL}/api/test-reset?${query}`, {
+ method: 'POST',
+ signal: AbortSignal.timeout(LEASE_RESET_MS),
+ });
+ if (!response.ok) {
+ throw new Error(
+ `test-reset?${query} returned status ${response.status}: ${await response.text()}`,
+ );
+ }
+ return response.json();
+}
+
+async function resetLeasedSlot(server: WorkerServer): Promise<void> {
+ await resetContentToFixtureBaseline(server.baseURL, server.contentDir);
+ await resetFixtureFolders(server.baseURL, server.contentDir);
+ for (const docName of REQUIRED_FIXTURE_DOC_NAMES) {
+ await requestTestReset(server, `docName=${encodeURIComponent(docName)}`);
+ }
+ const kept = serverResetResidueProblems(
+ parseServerResetResidue(await requestTestReset(server, 'scope=server')),
+ );
+ if (kept.length > 0) throw new Error(`the server kept ${kept.join('; ')} after its reset`);
+ const left = freshSlotResidue(server.contentDir, REQUIRED_FIXTURE_DOC_NAMES);
+ if (left.length > 0) throw new Error(`the reset left ${left.join('; ')}`);
+ await checkApiConfig(server.baseURL, WORKER_SERVER_BUDGET_RESERVES.apiConfig);
+ await checkCollabSync(server.port, WORKER_SERVER_BUDGET_RESERVES.collabSync);
+}
+
+function abortedRunRefusal(slot: number, detail: string): Error {
+ const reason = `server reuse refused slot ${slot}: ${detail}, so this invocation gives no further verdicts`;
+ declareSetupNonResult(test.info(), reason);
+ return new Error(reason);
+}
+
+function invocationConfig(workerInfo: WorkerInfo): string {
+ return `${workerInfo.config.configFile ?? '(inline config)'}#${workerInfo.project.name}`;
+}
+
+export async function leaseWorkerServer(
+ { workerServerEnv, browser }: { workerServerEnv: Record<string, string>; browser: Browser },
+ use: (lease: ServerLease | undefined) => Promise<void>,
+ workerInfo: WorkerInfo,
+): Promise<void> {
+ const socketPath = process.env[REUSE_SOCKET_ENV];
+ if (socketPath === undefined || socketPath === '') {
+ await use(undefined);
+ return;
+ }
+ const slot = workerInfo.parallelIndex;
+ const startFresh = async (note: string) => {
+ console.warn(`[server reuse w${workerInfo.workerIndex}] ${note}`);
+ currentLease = { kind: 'fresh', note };
+ try {
+ await use(currentLease);
+ } finally {
+ currentLease = undefined;
+ }
+ };
+ if (e2eServingMode(process.env) === 'prebuilt') {
+ await startFresh(
+ `slot ${slot}: server reuse leases dev servers, and this invocation serves the prebuilt DEV app, so this worker starts its own server`,
+ );
+ return;
+ }
+ const run = process.env[REUSE_RUN_ENV];
+ if (run === undefined || run === '') {
+ await startFresh(
+ `slot ${slot}: ${REUSE_SOCKET_ENV} asks for server reuse, but this invocation has no ${REUSE_RUN_ENV} because its config does not run the reuse globalSetup, so this worker starts its own server`,
+ );
+ return;
+ }
+ const abortedBecause = runAbortReason(run);
+ if (abortedBecause !== undefined) {
+ throw abortedRunRefusal(slot, `run ${run} lost an earlier lease: ${abortedBecause}`);
+ }
+ const deadlineAt = Date.now() + WORKER_SERVER_SETUP_STARVATION_LINE_MS;
+ let client: LeaseClient;
+ try {
+ client = await connectLeaseClient(socketPath, SUPERVISOR_CONNECT_MS);
+ } catch (error) {
+ await startFresh(`slot ${slot}: ${describe(error)}, so this worker starts its own server`);
+ return;
+ }
+ try {
+ const request: LeaseRequest = {
+ worktree: realpathSync(join(APP_PACKAGE_ROOT, '..', '..')),
+ config: invocationConfig(workerInfo),
+ run,
+ slot,
+ workerServerEnv,
+ serverMode: 'dev',
+ };
+ let outcome: LeaseOutcome;
+ try {
+ outcome = await client.acquire(request, remainingMs(deadlineAt));
+ } catch (error) {
+ client.close();
+ await startFresh(
+ `slot ${slot}: the lease request failed (${describe(error)}), so this worker starts its own server`,
+ );
+ return;
+ }
+ if (outcome.kind === 'refused') {
+ if (outcome.reason === 'run-aborted') throw abortedRunRefusal(slot, outcome.detail);
+ client.close();
+ await startFresh(
+ `slot ${slot} was refused (${outcome.reason}: ${outcome.detail}), so this worker starts its own server`,
+ );
+ return;
+ }
+ const server: WorkerServer = {
+ port: outcome.port,
+ baseURL: outcome.baseURL,
+ contentDir: outcome.contentDir,
+ };
+ try {
+ if (outcome.launched) {
+ const overhead = openBudgetPhase(
+ 'leased server warmup',
+ WORKER_SERVER_BUDGET_RESERVES.setupOverhead,
+ );
+ await warmupAppFirstLoad(browser, server.baseURL, overhead, deadlineAt);
+ } else {
+ await resetLeasedSlot(server);
+ }
+ } catch (error) {
+ const discarded = await client.release(outcome.token, true, SUPERVISOR_CALL_MS).then(
+ () => 'so it was discarded and this worker starts its own server',
+ (releaseError: unknown) =>
+ `so this worker starts its own server; the supervisor did not confirm discarding the leased server (${describe(releaseError)})`,
+ );
+ client.close();
+ await startFresh(
+ `slot ${slot}: the leased server could not be ${outcome.launched ? 'warmed up' : 'reset'} (${describe(error)}), ${discarded}`,
+ );
+ return;
+ }
+ const note = outcome.launched
+ ? `slot ${slot}: started an owned server at ${server.baseURL}${outcome.relaunchedBecause === null ? '' : ` because ${outcome.relaunchedBecause}`}`
+ : `slot ${slot}: reused the owned server at ${server.baseURL}`;
+ currentLease = {
+ kind: 'leased',
+ run,
+ server,
+ note,
+ verify: () =>
+ client.verify(outcome.token, SUPERVISOR_CALL_MS).catch(
+ (error: unknown): LeaseVerdict => ({
+ held: false,
+ reason: `the supervisor could not confirm it: ${describe(error)}`,
+ }),
+ ),
+ };
+ try {
+ await use(currentLease);
+ } finally {
+ currentLease = undefined;
+ }
+ await client.release(outcome.token, false, SUPERVISOR_CALL_MS).catch((error: unknown) => {
+ console.warn(
+ `[server reuse w${workerInfo.workerIndex}] slot ${slot}: releasing the lease failed (${describe(error)}); if the supervisor never received the release, closing its connection makes it discard the server`,
+ );
+ });
+ } finally {
+ client.close();
+ }
+}
+
+async function requireHeldLease(
+ lease: Extract<ServerLease, { kind: 'leased' }>,
+ testInfo: TestInfo,
+ when: string,
+): Promise<void> {
+ const verdict = await lease.verify();
+ if (verdict.held) return;
+ const reason = `the reused server's lease was lost ${when}: ${verdict.reason}`;
+ declareSetupNonResult(testInfo, reason);
+ recordRunAbort(lease.run, verdict.reason);
+ throw new Error(reason);
+}
+
+export async function checkServerLease(
+ // biome-ignore lint/correctness/noEmptyPattern: Playwright fixture contract
+ {}: object,
+ use: (value: undefined) => Promise<void>,
+ testInfo: TestInfo,
+): Promise<void> {
+ const before = currentLease;
+ if (before?.kind === 'leased') await requireHeldLease(before, testInfo, 'before the test');
+ await use(undefined);
+ const after = currentLease;
+ if (after === undefined) return;
+ testInfo.annotations.push({ type: SERVER_LEASE_ANNOTATION, description: after.note });
+ if (after.kind === 'leased') await requireHeldLease(after, testInfo, 'during the test');
+}
diff --git a/packages/app/tests/stress/_helpers/reuse/lease-protocol.ts b/packages/app/tests/stress/_helpers/reuse/lease-protocol.ts
new file mode 100644
index 000000000..b1e934e7f
--- /dev/null
+++ b/packages/app/tests/stress/_helpers/reuse/lease-protocol.ts
@@ -0,0 +1,231 @@
+export const REUSE_SOCKET_ENV = 'OK_TEST_E2E_REUSE_SOCKET';
+export const REUSE_RUN_ENV = 'OK_TEST_E2E_REUSE_RUN';
+
+export const SERVER_MODES = ['dev'] as const;
+export type ServerMode = (typeof SERVER_MODES)[number];
+
+export type WorkerServerEnv = Readonly<Record<string, string>>;
+
+export interface SlotCompatibility {
+ readonly config: string;
+ readonly workerServerEnv: WorkerServerEnv;
+ readonly serverMode: ServerMode;
+}
+
+export interface LeaseRequest extends SlotCompatibility {
+ readonly worktree: string;
+ readonly run: string;
+ readonly slot: number;
+}
+
+export const REFUSAL_REASONS = [
+ 'supervisor-closed',
+ 'run-aborted',
+ 'busy',
+ 'incompatible',
+ 'launch-failed',
+] as const;
+export type RefusalReason = (typeof REFUSAL_REASONS)[number];
+
+export type LeaseOutcome =
+ | {
+ readonly kind: 'granted';
+ readonly token: string;
+ readonly slot: number;
+ readonly baseURL: string;
+ readonly port: number;
+ readonly contentDir: string;
+ readonly launched: boolean;
+ readonly relaunchedBecause: string | null;
+ }
+ | { readonly kind: 'refused'; readonly reason: RefusalReason; readonly detail: string };
+
+export type LeaseVerdict =
+ | { readonly held: true }
+ | { readonly held: false; readonly reason: string };
+
+export interface SlotStatus {
+ readonly slot: number;
+ readonly baseURL: string | null;
+ readonly config: string | null;
+ readonly leasedBy: string | null;
+}
+
+export type ClientMessage =
+ | { readonly id: number; readonly op: 'acquire'; readonly request: LeaseRequest }
+ | { readonly id: number; readonly op: 'verify'; readonly token: string }
+ | {
+ readonly id: number;
+ readonly op: 'release';
+ readonly token: string;
+ readonly discard: boolean;
+ }
+ | { readonly id: number; readonly op: 'status' }
+ | { readonly id: number; readonly op: 'shutdown' };
+
+export type ServerMessage =
+ | { readonly id: number; readonly ok: true; readonly result: unknown }
+ | { readonly id: number; readonly ok: false; readonly error: string };
+
+type Fields = Record<string, unknown>;
+
+function fieldsOf(value: unknown, what: string): Fields {
+ if (typeof value !== 'object' || value === null || Array.isArray(value)) {
+ throw new TypeError(`${what} is not a JSON object`);
+ }
+ return value as Fields;
+}
+
+function stringField(fields: Fields, key: string, what: string): string {
+ const value = fields[key];
+ if (typeof value !== 'string' || value.length === 0) {
+ throw new TypeError(`${what}.${key} is not a non-empty string`);
+ }
+ return value;
+}
+
+function integerField(fields: Fields, key: string, what: string): number {
+ const value = fields[key];
+ if (typeof value !== 'number' || !Number.isInteger(value) || value < 0) {
+ throw new TypeError(`${what}.${key} is not a non-negative integer`);
+ }
+ return value;
+}
+
+function workerServerEnvField(fields: Fields, what: string): WorkerServerEnv {
+ const env = fieldsOf(fields.workerServerEnv, `${what}.workerServerEnv`);
+ for (const [key, value] of Object.entries(env)) {
+ if (typeof value !== 'string') {
+ throw new TypeError(`${what}.workerServerEnv.${key} is not a string`);
+ }
+ }
+ return env as WorkerServerEnv;
+}
+
+function isServerMode(value: unknown): value is ServerMode {
+ return (SERVER_MODES as readonly unknown[]).includes(value);
+}
+
+export function parseLeaseRequest(value: unknown): LeaseRequest {
+ const what = 'lease request';
+ const fields = fieldsOf(value, what);
+ if (!isServerMode(fields.serverMode)) {
+ throw new TypeError(`${what}.serverMode is not one of ${SERVER_MODES.join(', ')}`);
+ }
+ return {
+ worktree: stringField(fields, 'worktree', what),
+ config: stringField(fields, 'config', what),
+ run: stringField(fields, 'run', what),
+ slot: integerField(fields, 'slot', what),
+ workerServerEnv: workerServerEnvField(fields, what),
+ serverMode: fields.serverMode,
+ };
+}
+
+export function parseClientMessage(line: string): ClientMessage {
+ const fields = fieldsOf(JSON.parse(line), 'supervisor request');
+ const id = integerField(fields, 'id', 'supervisor request');
+ switch (fields.op) {
+ case 'acquire':
+ return { id, op: 'acquire', request: parseLeaseRequest(fields.request) };
+ case 'verify':
+ return { id, op: 'verify', token: stringField(fields, 'token', 'verify request') };
+ case 'release':
+ if (typeof fields.discard !== 'boolean') {
+ throw new TypeError('release request.discard is not a boolean');
+ }
+ return {
+ id,
+ op: 'release',
+ token: stringField(fields, 'token', 'release request'),
+ discard: fields.discard,
+ };
+ case 'status':
+ return { id, op: 'status' };
+ case 'shutdown':
+ return { id, op: 'shutdown' };
+ default:
+ throw new TypeError(`supervisor request.op ${JSON.stringify(fields.op)} is not supported`);
+ }
+}
+
+export function parseServerMessage(line: string): ServerMessage {
+ const fields = fieldsOf(JSON.parse(line), 'supervisor response');
+ const id = integerField(fields, 'id', 'supervisor response');
+ if (fields.ok === true) return { id, ok: true, result: fields.result };
+ if (fields.ok === false) {
+ return { id, ok: false, error: stringField(fields, 'error', 'supervisor response') };
+ }
+ throw new TypeError('supervisor response.ok is not a boolean');
+}
+
+function isRefusalReason(value: unknown): value is RefusalReason {
+ return (REFUSAL_REASONS as readonly unknown[]).includes(value);
+}
+
+export function parseLeaseOutcome(value: unknown): LeaseOutcome {
+ const what = 'lease outcome';
+ const fields = fieldsOf(value, what);
+ if (fields.kind === 'refused') {
+ if (!isRefusalReason(fields.reason)) {
+ throw new TypeError(`${what}.reason ${JSON.stringify(fields.reason)} is not supported`);
+ }
+ return { kind: 'refused', reason: fields.reason, detail: stringField(fields, 'detail', what) };
+ }
+ if (fields.kind !== 'granted') throw new TypeError(`${what}.kind is not granted or refused`);
+ if (typeof fields.launched !== 'boolean') {
+ throw new TypeError(`${what}.launched is not a boolean`);
+ }
+ const relaunchedBecause = fields.relaunchedBecause;
+ if (relaunchedBecause !== null && typeof relaunchedBecause !== 'string') {
+ throw new TypeError(`${what}.relaunchedBecause is not a string or null`);
+ }
+ return {
+ kind: 'granted',
+ token: stringField(fields, 'token', what),
+ slot: integerField(fields, 'slot', what),
+ baseURL: stringField(fields, 'baseURL', what),
+ port: integerField(fields, 'port', what),
+ contentDir: stringField(fields, 'contentDir', what),
+ launched: fields.launched,
+ relaunchedBecause,
+ };
+}
+
+export function parseLeaseVerdict(value: unknown): LeaseVerdict {
+ const fields = fieldsOf(value, 'lease verdict');
+ if (fields.held === true) return { held: true };
+ if (fields.held === false) {
+ return { held: false, reason: stringField(fields, 'reason', 'lease verdict') };
+ }
+ throw new TypeError('lease verdict.held is not a boolean');
+}
+
+function changedEnvKeys(recorded: WorkerServerEnv, requested: WorkerServerEnv): string[] {
+ const keys = new Set([...Object.keys(recorded), ...Object.keys(requested)]);
+ return [...keys].filter((key) => recorded[key] !== requested[key]).sort();
+}
+
+export function workerServerEnvChange(
+ recorded: SlotCompatibility,
+ requested: SlotCompatibility,
+): string | undefined {
+ const changedEnv = changedEnvKeys(recorded.workerServerEnv, requested.workerServerEnv);
+ return changedEnv.length === 0
+ ? undefined
+ : `its workerServerEnv differs in ${changedEnv.join(', ')}`;
+}
+
+export function compatibilityMismatch(
+ recorded: SlotCompatibility,
+ requested: SlotCompatibility,
+): string | undefined {
+ const differences: string[] = [];
+ if (recorded.config !== requested.config) {
+ differences.push(`invocation config ${requested.config} is not ${recorded.config}`);
+ }
+ if (recorded.serverMode !== requested.serverMode) {
+ differences.push(`server mode ${requested.serverMode} is not ${recorded.serverMode}`);
+ }
+ return differences.length === 0 ? undefined : differences.join('; ');
+}
diff --git a/packages/app/tests/stress/_helpers/reuse/leased-server.reuse-case.ts b/packages/app/tests/stress/_helpers/reuse/leased-server.reuse-case.ts
new file mode 100644
index 000000000..fa03c5c32
--- /dev/null
+++ b/packages/app/tests/stress/_helpers/reuse/leased-server.reuse-case.ts
@@ -0,0 +1,159 @@
+import { existsSync, writeFileSync } from 'node:fs';
+import { join } from 'node:path';
+import { HocuspocusProvider } from '@hocuspocus/provider';
+import {
+ bindConfigDoc,
+ CONFIG_DOC_NAME_PROJECT,
+ CONFIG_DOC_NAME_PROJECT_LOCAL,
+ CONFIG_DOC_NAME_USER,
+ type ConfigBinding,
+ ConfigSchema,
+ type WriteScope,
+} from '@inkeep/open-knowledge-core';
+import { expect } from '@playwright/test';
+import * as Y from 'yjs';
+import { test, type WorkerServer } from '../fixtures.ts';
+import { REUSE_SOCKET_ENV } from './lease-protocol.ts';
+import { CASE_SERVER_ANNOTATION } from './reuse-run.test-helper.ts';
+import { connectLeaseClient } from './supervisor-socket.ts';
+
+const STORAGE_PROBE_KEY = 'ok-reuse-probe';
+const CREATED_PAGE = 'reuse-created';
+const CREATED_FOLDER_PAGE = 'sidebar-folder/reuse-created-inside';
+const NESTED_FIXTURE = 'sidebar-folder/nested-doc';
+const SETTING_SCOPES = [
+ { docName: CONFIG_DOC_NAME_PROJECT, scope: 'project', file: ['.ok', 'config.yml'] },
+ {
+ docName: CONFIG_DOC_NAME_PROJECT_LOCAL,
+ scope: 'project-local',
+ file: ['.ok', 'local', 'config.yml'],
+ },
+ { docName: CONFIG_DOC_NAME_USER, scope: 'user', file: ['.ok', 'global.yml'] },
+] as const satisfies ReadonlyArray<{ docName: string; scope: WriteScope; file: readonly string[] }>;
+
+async function withConfigBinding<T>(
+ server: WorkerServer,
+ docName: string,
+ scope: WriteScope,
+ use: (binding: ConfigBinding) => Promise<T>,
+): Promise<T> {
+ const document = new Y.Doc();
+ const provider = new HocuspocusProvider({
+ url: `ws://127.0.0.1:${server.port}/collab`,
+ name: docName,
+ document,
+ });
+ const binding = bindConfigDoc(provider, scope);
+ try {
+ await expect.poll(() => binding.hasSynced()).toBe(true);
+ return await use(binding);
+ } finally {
+ binding.dispose();
+ provider.destroy();
+ document.destroy();
+ }
+}
+
+async function pageDocNames(server: WorkerServer): Promise<string[]> {
+ const response = await fetch(`${server.baseURL}/api/pages`);
+ const body = (await response.json()) as { pages: Array<{ docName: string }> };
+ return body.pages.map((page) => page.docName);
+}
+
+test('a leased worker server serves the app', async ({ page, workerServer }) => {
+ test.info().annotations.push({ type: CASE_SERVER_ANNOTATION, description: workerServer.baseURL });
+ await page.goto(`${workerServer.baseURL}/`);
+ await expect(page.getByRole('treeitem', { name: 'test-doc.md', exact: true })).toBeVisible();
+});
+
+test('a run leaves settings, content, agent state and browser storage behind', async ({
+ page,
+ workerServer,
+ api,
+}) => {
+ test.info().annotations.push({ type: CASE_SERVER_ANNOTATION, description: workerServer.baseURL });
+ const patches = {
+ project: { content: { attachmentFolderPath: 'assets' } },
+ 'project-local': { search: { semantic: { enabled: true } } },
+ user: { appearance: { theme: 'dark' } },
+ } as const;
+ for (const { docName, scope, file } of SETTING_SCOPES) {
+ await withConfigBinding(workerServer, docName, scope, async (binding) => {
+ expect(binding.patch(patches[scope]).ok).toBe(true);
+ await expect.poll(() => existsSync(join(workerServer.contentDir, ...file))).toBe(true);
+ });
+ }
+ await api.createPage(`${CREATED_PAGE}.md`);
+ await api.writeAsAgent(CREATED_PAGE, '# left by an earlier run\n', {
+ agentId: 'reuse-probe',
+ agentName: 'Reuse probe',
+ clientName: 'reuse-probe',
+ colorSeed: 'reuse-probe',
+ });
+ await api.replaceDoc('test-doc', 'edited by an earlier run\n');
+ await api.createPage(`${CREATED_FOLDER_PAGE}.md`);
+ await api.replaceDoc(NESTED_FIXTURE, 'edited by an earlier run\n');
+ await page.goto(`${workerServer.baseURL}/`);
+ await page.evaluate((key) => localStorage.setItem(key, 'from an earlier run'), STORAGE_PROBE_KEY);
+ expect(await pageDocNames(workerServer)).toEqual(
+ expect.arrayContaining([CREATED_PAGE, CREATED_FOLDER_PAGE]),
+ );
+});
+
+test('a later run finds none of an earlier run’s settings, content or browser storage', async ({
+ page,
+ workerServer,
+}) => {
+ test.info().annotations.push({ type: CASE_SERVER_ANNOTATION, description: workerServer.baseURL });
+ const defaults = ConfigSchema.parse({});
+ for (const { docName, scope, file } of SETTING_SCOPES) {
+ expect(existsSync(join(workerServer.contentDir, ...file)), file.join('/')).toBe(false);
+ await withConfigBinding(workerServer, docName, scope, async (binding) => {
+ expect(binding.current(), docName).toEqual(defaults);
+ });
+ }
+ const pages = await pageDocNames(workerServer);
+ expect(pages).toContain(NESTED_FIXTURE);
+ expect(pages).not.toContain(CREATED_PAGE);
+ expect(pages).not.toContain(CREATED_FOLDER_PAGE);
+ await page.goto(`${workerServer.baseURL}/`);
+ await expect(page.getByRole('treeitem', { name: 'test-doc.md', exact: true })).toBeVisible();
+ await expect(page.getByRole('treeitem', { name: `${CREATED_PAGE}.md` })).toHaveCount(0);
+ expect(await page.evaluate((key) => localStorage.getItem(key), STORAGE_PROBE_KEY)).toBeNull();
+ for (const docName of ['test-doc', NESTED_FIXTURE]) {
+ const response = await fetch(
+ `${workerServer.baseURL}/api/document?docName=${encodeURIComponent(docName)}`,
+ );
+ expect(((await response.json()) as { content: string }).content, docName).toBe('');
+ }
+});
+
+test('a run leaves server state that no reset can account for', async ({ workerServer }) => {
+ test.info().annotations.push({ type: CASE_SERVER_ANNOTATION, description: workerServer.baseURL });
+ writeFileSync(join(workerServer.contentDir, '.ok', 'local', 'unrecognised-state.json'), '{}\n');
+});
+
+test('the supervisor shuts down while a test holds its lease', async ({ workerServer }) => {
+ const socketPath = process.env[REUSE_SOCKET_ENV];
+ if (socketPath === undefined) throw new Error(`${REUSE_SOCKET_ENV} is required`);
+ expect((await fetch(`${workerServer.baseURL}/api/config`)).status).toBe(200);
+ const client = await connectLeaseClient(socketPath, 5_000);
+ try {
+ await client.shutdown(60_000);
+ } finally {
+ client.close();
+ }
+ const supervisorState = async () => {
+ try {
+ (await connectLeaseClient(socketPath, 1_000)).close();
+ return 'serving';
+ } catch {
+ return 'gone';
+ }
+ };
+ await expect.poll(supervisorState).toBe('gone');
+});
+
+test('a later test of the run whose supervisor shut down', async ({ workerServer }) => {
+ test.info().annotations.push({ type: CASE_SERVER_ANNOTATION, description: workerServer.baseURL });
+});
diff --git a/packages/app/tests/stress/_helpers/reuse/owned-dev-server.ts b/packages/app/tests/stress/_helpers/reuse/owned-dev-server.ts
new file mode 100644
index 000000000..8879c0a21
--- /dev/null
+++ b/packages/app/tests/stress/_helpers/reuse/owned-dev-server.ts
@@ -0,0 +1,120 @@
+import { mkdtempSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { TEST_SERVER_HOST_FAMILY } from '../../../../src/build/test-server-startup-contract.ts';
+import {
+ seedRequiredFixtureFiles,
+ spawnWorkerDevServer,
+ WORKER_SERVER_BUDGET_RESERVES,
+ waitForServerReady,
+} from '../fixtures.ts';
+import type { WorkerServing } from '../prebuilt/prebuilt-serving.ts';
+import {
+ beginViteStartup,
+ closeServerLog,
+ createViteStartupRequest,
+ openServerLog,
+ prepareViteCacheDir,
+ type ServerLog,
+ tailServerLog,
+} from '../server-process.ts';
+import { removeAllDuringTeardown, runTeardownPhases } from '../teardown-fs.ts';
+import { completeWorkerServerRelease, killGracefully } from '../worker-server-lifetime.ts';
+import type { WorkerServerEnv } from './lease-protocol.ts';
+import type { LaunchOwnedServer, OwnedServer } from './server-supervisor.ts';
+
+export async function launchOwnedWorkerServer({
+ label,
+ workerServerEnv,
+ serving,
+}: {
+ readonly label: string;
+ readonly workerServerEnv: WorkerServerEnv;
+ readonly serving: WorkerServing;
+}): Promise<OwnedServer> {
+ const teardown: Array<() => void | Promise<void>> = [];
+ let ownedProc: import('node:child_process').ChildProcess | undefined;
+ const releaseAll = () =>
+ runTeardownPhases(
+ () => runTeardownPhases(...[...teardown].reverse()),
+ () => (ownedProc === undefined ? undefined : completeWorkerServerRelease(ownedProc)),
+ );
+ const request = createViteStartupRequest('127.0.0.1');
+ teardown.push(() => request.dispose());
+ let openedServerLog: ServerLog | undefined;
+ try {
+ const serverLog = openServerLog(label);
+ openedServerLog = serverLog;
+ teardown.push(() => removeAllDuringTeardown(serverLog.path));
+ teardown.push(() => closeServerLog(serverLog));
+ const contentDir = mkdtempSync(join(tmpdir(), `ok-${label}-`));
+ teardown.push(() => removeAllDuringTeardown(contentDir));
+ const viteCacheDir = prepareViteCacheDir(label);
+ teardown.push(() => removeAllDuringTeardown(viteCacheDir));
+ seedRequiredFixtureFiles(contentDir);
+ const proc = spawnWorkerDevServer(
+ request,
+ workerServerEnv,
+ contentDir,
+ viteCacheDir,
+ serverLog,
+ serving,
+ );
+ ownedProc = proc;
+ const lifetimeLost = new Promise<never>((_, reject) => {
+ proc.on('error', reject);
+ });
+ teardown.push(() => killGracefully(proc, WORKER_SERVER_BUDGET_RESERVES.devServerReap));
+ const endpoint = await Promise.race([
+ waitForServerReady({
+ pending: beginViteStartup(request, proc),
+ contentDir,
+ viteCacheDir,
+ serverLog,
+ proc,
+ }),
+ lifetimeLost,
+ ]);
+ let released: Promise<void> | undefined;
+ return {
+ proc,
+ identity: {
+ nonce: request.nonce,
+ address: request.host,
+ family: TEST_SERVER_HOST_FAMILY[request.host],
+ port: endpoint.port,
+ },
+ baseURL: endpoint.baseURL,
+ contentDir,
+ release: () => {
+ released ??= releaseAll();
+ return released;
+ },
+ };
+ } catch (error) {
+ const tail =
+ openedServerLog === undefined
+ ? '--- no dev server log was open when this failure ended the launch ---'
+ : `--- dev server log tail ---\n${tailServerLog(openedServerLog)}`;
+ let cleanupFailure: string | undefined;
+ try {
+ await releaseAll();
+ } catch (cleanupError) {
+ cleanupFailure = `--- cleanup after this failure did not complete: ${describe(cleanupError)} ---`;
+ }
+ throw new Error(
+ [describe(error), tail, cleanupFailure].filter((part) => part !== undefined).join('\n'),
+ );
+ }
+}
+
+function describe(error: unknown): string {
+ return error instanceof Error ? error.message : String(error);
+}
+
+export const launchOwnedDevServer: LaunchOwnedServer = (slot, workerServerEnv) =>
+ launchOwnedWorkerServer({
+ label: `reuse-s${slot}`,
+ workerServerEnv,
+ serving: { kind: 'dev-server' },
+ });
diff --git a/packages/app/tests/stress/_helpers/reuse/reuse-run.config.ts b/packages/app/tests/stress/_helpers/reuse/reuse-run.config.ts
new file mode 100644
index 000000000..937c0ff38
--- /dev/null
+++ b/packages/app/tests/stress/_helpers/reuse/reuse-run.config.ts
@@ -0,0 +1,19 @@
+import { join } from 'node:path';
+import { fileURLToPath } from 'node:url';
+import { defineConfig } from '@playwright/test';
+import stock from '../../../../playwright.config.ts';
+
+const runDir = process.env.OK_REUSE_CASE_RUN_DIR;
+if (!runDir) throw new Error('OK_REUSE_CASE_RUN_DIR is required');
+
+export default defineConfig({
+ ...stock,
+ testDir: fileURLToPath(new URL('.', import.meta.url)),
+ testMatch: /.*\.reuse-case\.ts$/,
+ globalSetup: [fileURLToPath(new URL('./reuse-run.ts', import.meta.url))],
+ outputDir: join(runDir, 'test-results'),
+ reporter: [['json', { outputFile: join(runDir, 'results.json') }]],
+ retries: 0,
+ workers: 1,
+ use: { ...stock.use, headless: true },
+});
diff --git a/packages/app/tests/stress/_helpers/reuse/reuse-run.test-helper.ts b/packages/app/tests/stress/_helpers/reuse/reuse-run.test-helper.ts
new file mode 100644
index 000000000..266342c80
--- /dev/null
+++ b/packages/app/tests/stress/_helpers/reuse/reuse-run.test-helper.ts
@@ -0,0 +1,125 @@
+import { type ChildProcess, spawn } from 'node:child_process';
+import { join } from 'node:path';
+import { createInterface } from 'node:readline';
+import { fileURLToPath } from 'node:url';
+import type { JSONReportSpec } from '@playwright/test/reporter';
+import { createNestedPlaywrightEnv } from '../port-ownership/nested-playwright-env.test-helper.ts';
+import {
+ runNestedPlaywright,
+ specsOf,
+} from '../port-ownership/nested-playwright-run.test-helper.ts';
+import { APP_PACKAGE_ROOT, killGracefully } from '../server-process.ts';
+import { REUSE_SOCKET_ENV } from './lease-protocol.ts';
+
+const SUPERVISOR_CLI = fileURLToPath(new URL('./reuse-supervisor.ts', import.meta.url));
+const RUN_CONFIG = fileURLToPath(new URL('./reuse-run.config.ts', import.meta.url));
+const CASE_FILE = fileURLToPath(new URL('./leased-server.reuse-case.ts', import.meta.url));
+const SUPERVISOR_READY_MS = 30_000;
+
+export const CASE_SERVER_ANNOTATION = 'ok:reuse-case-server';
+const SUPERVISOR_EXIT_MS = 60_000;
+
+type SupervisorExit = { code: number | null; signal: NodeJS.Signals | null };
+
+export interface RunningSupervisor {
+ readonly socketPath: string;
+ readonly proc: ChildProcess;
+ readonly transcript: () => string;
+ exitWithin(ms: number): Promise<SupervisorExit | 'still running'>;
+ stop(): Promise<void>;
+}
+
+export async function startSupervisor(scratch: string): Promise<RunningSupervisor> {
+ const socketPath = join(scratch, 'supervisor.sock');
+ let transcript = '';
+ const proc = spawn(
+ process.execPath,
+ [SUPERVISOR_CLI, 'serve', '--socket', socketPath, '--until-stdin-closes'],
+ { cwd: APP_PACKAGE_ROOT, stdio: ['pipe', 'pipe', 'pipe'] },
+ );
+ proc.stderr?.on('data', (chunk: Buffer) => {
+ transcript += chunk.toString('utf8');
+ });
+ const exited = new Promise<SupervisorExit>((resolve) => {
+ proc.once('exit', (code, signal) => resolve({ code, signal }));
+ });
+ const exitWithin = async (ms: number): Promise<SupervisorExit | 'still running'> => {
+ let bound: ReturnType<typeof setTimeout> | undefined;
+ return Promise.race([
+ exited,
+ new Promise<'still running'>((resolve) => {
+ bound = setTimeout(() => resolve('still running'), ms);
+ }),
+ ]).finally(() => clearTimeout(bound));
+ };
+ const stop = async () => {
+ proc.stdin?.end();
+ if ((await exitWithin(SUPERVISOR_EXIT_MS)) === 'still running') await killGracefully(proc);
+ };
+ const ready = await new Promise<boolean>((resolve) => {
+ const timer = setTimeout(() => resolve(false), SUPERVISOR_READY_MS);
+ const lines = createInterface({ input: proc.stdout as NodeJS.ReadableStream });
+ lines.on('line', (line) => {
+ transcript += `${line}\n`;
+ if (line === `${REUSE_SOCKET_ENV}=${socketPath}`) {
+ clearTimeout(timer);
+ resolve(true);
+ }
+ });
+ void exited.then(() => {
+ clearTimeout(timer);
+ resolve(false);
+ });
+ });
+ if (!ready) {
+ await stop();
+ throw new Error(`the reuse supervisor did not start serving ${socketPath}:\n${transcript}`);
+ }
+ return { socketPath, proc, transcript: () => transcript, exitWithin, stop };
+}
+
+export interface ReuseCaseRun {
+ readonly exitCode: number | null;
+ readonly transcript: string;
+ readonly specs: JSONReportSpec[];
+}
+
+function grepFor(titles: readonly string[]): string {
+ return titles.map((title) => title.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')).join('|');
+}
+
+export async function runReuseCase(socketPath: string, ...titles: string[]): Promise<ReuseCaseRun> {
+ const { exitCode, transcript, report } = await runNestedPlaywright({
+ args: ['--config', RUN_CONFIG, CASE_FILE, '--grep', grepFor(titles)],
+ runDirPrefix: 'ok-reuse-case-',
+ env: (runDir) =>
+ createNestedPlaywrightEnv({
+ [REUSE_SOCKET_ENV]: socketPath,
+ OK_REUSE_CASE_RUN_DIR: runDir,
+ PLAYWRIGHT_HTML_OPEN: 'never',
+ }),
+ });
+ return {
+ exitCode,
+ transcript,
+ specs: specsOf(report?.suites ?? []).filter((spec) => titles.includes(spec.title)),
+ };
+}
+
+export function annotationsOf(run: ReuseCaseRun, type: string): Array<string | undefined> {
+ return run.specs.flatMap((spec) =>
+ spec.tests.flatMap((test) =>
+ test.results.flatMap((result) =>
+ result.annotations
+ .filter((annotation) => annotation.type === type)
+ .map((annotation) => annotation.description),
+ ),
+ ),
+ );
+}
+
+export function statusesOf(run: ReuseCaseRun): Array<string | undefined> {
+ return run.specs.flatMap((spec) =>
+ spec.tests.flatMap((test) => test.results.map((result) => result.status)),
+ );
+}
diff --git a/packages/app/tests/stress/_helpers/reuse/reuse-run.ts b/packages/app/tests/stress/_helpers/reuse/reuse-run.ts
new file mode 100644
index 000000000..153fd9b50
--- /dev/null
+++ b/packages/app/tests/stress/_helpers/reuse/reuse-run.ts
@@ -0,0 +1,45 @@
+import { randomUUID } from 'node:crypto';
+import { mkdirSync, readFileSync, rmSync, writeFileSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { REUSE_RUN_ENV, REUSE_SOCKET_ENV } from './lease-protocol.ts';
+
+function runDirectory(run: string): string {
+ return join(tmpdir(), `ok-e2e-reuse-run-${run}`);
+}
+
+function runAbortRecord(run: string): string {
+ return join(runDirectory(run), 'aborted');
+}
+
+function errorCode(error: unknown): string | undefined {
+ return (error as NodeJS.ErrnoException | undefined)?.code;
+}
+
+export function recordRunAbort(run: string, reason: string): void {
+ try {
+ writeFileSync(runAbortRecord(run), reason, { flag: 'wx' });
+ } catch (error) {
+ if (errorCode(error) !== 'EEXIST') throw error;
+ }
+}
+
+export function runAbortReason(run: string): string | undefined {
+ try {
+ return readFileSync(runAbortRecord(run), 'utf8');
+ } catch (error) {
+ if (errorCode(error) === 'ENOENT') return undefined;
+ throw error;
+ }
+}
+
+export default function assignReuseRun(): (() => void) | undefined {
+ if (process.env[REUSE_SOCKET_ENV] === undefined || process.env[REUSE_SOCKET_ENV] === '') {
+ return undefined;
+ }
+ const run = randomUUID();
+ const directory = runDirectory(run);
+ mkdirSync(directory, { mode: 0o700 });
+ process.env[REUSE_RUN_ENV] = run;
+ return () => rmSync(directory, { recursive: true, force: true });
+}
diff --git a/packages/app/tests/stress/_helpers/reuse/reuse-supervisor.ts b/packages/app/tests/stress/_helpers/reuse/reuse-supervisor.ts
new file mode 100644
index 000000000..ac905e11d
--- /dev/null
+++ b/packages/app/tests/stress/_helpers/reuse/reuse-supervisor.ts
@@ -0,0 +1,97 @@
+import { createHash } from 'node:crypto';
+import { realpathSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { parseArgs } from 'node:util';
+import { APP_PACKAGE_ROOT } from '../seed-key.ts';
+import { REUSE_SOCKET_ENV } from './lease-protocol.ts';
+import { launchOwnedDevServer } from './owned-dev-server.ts';
+import { WorkerServerSupervisor } from './server-supervisor.ts';
+import { connectLeaseClient, serveSupervisor } from './supervisor-socket.ts';
+
+const OK_ROOT = realpathSync(join(APP_PACKAGE_ROOT, '..', '..'));
+const SUPERVISOR_CALL_MS = 60_000;
+const IDENTITY_TIMEOUT_MS = 5_000;
+
+export const DEFAULT_SUPERVISOR_SOCKET = join(
+ tmpdir(),
+ `ok-e2e-reuse-${createHash('sha256').update(OK_ROOT).digest('hex').slice(0, 12)}`,
+ 'supervisor.sock',
+);
+
+const USAGE = `usage: node tests/stress/_helpers/reuse/reuse-supervisor.ts <serve|status|shutdown> [--socket <path>] [--until-stdin-closes]
+
+serve owns one dev server per Playwright worker slot and leases it to successive runs
+ that set ${REUSE_SOCKET_ENV} to its socket; runs until shutdown, SIGINT/SIGTERM or,
+ with --until-stdin-closes, until its owner closes its stdin; exits 0 once it has
+ released every owned server, or 1 when a release failed, naming each on stderr
+status prints the slots, their servers and current holders
+shutdown asks the serving supervisor to release every owned server and stop; exits 0 once
+ the supervisor accepts the request, before any release runs, so whether the
+ releases succeeded is the serving supervisor's own exit status
+
+The default socket is ${DEFAULT_SUPERVISOR_SOCKET}`;
+
+async function main(argv: readonly string[]): Promise<number> {
+ const { positionals, values } = parseArgs({
+ args: [...argv],
+ allowPositionals: true,
+ options: { socket: { type: 'string' }, 'until-stdin-closes': { type: 'boolean' } },
+ });
+ const socketPath = values.socket ?? DEFAULT_SUPERVISOR_SOCKET;
+ switch (positionals[0]) {
+ case 'serve': {
+ const supervisor = new WorkerServerSupervisor({
+ okRoot: OK_ROOT,
+ launch: launchOwnedDevServer,
+ identityTimeoutMs: IDENTITY_TIMEOUT_MS,
+ });
+ const endpoint = await serveSupervisor(supervisor, socketPath);
+ process.once('SIGINT', () => void endpoint.close());
+ process.once('SIGTERM', () => void endpoint.close());
+ if (values['until-stdin-closes'] === true) {
+ process.stdin.once('end', () => void endpoint.close());
+ process.stdin.resume();
+ }
+ process.stdout.write(`${REUSE_SOCKET_ENV}=${socketPath}\n`);
+ const closed = await endpoint.closed;
+ process.stdin.destroy();
+ return closed.released ? 0 : 1;
+ }
+ case 'status': {
+ const client = await connectLeaseClient(socketPath, IDENTITY_TIMEOUT_MS);
+ try {
+ process.stdout.write(
+ `${JSON.stringify(await client.status(SUPERVISOR_CALL_MS), null, 2)}\n`,
+ );
+ } finally {
+ client.close();
+ }
+ return 0;
+ }
+ case 'shutdown': {
+ const client = await connectLeaseClient(socketPath, IDENTITY_TIMEOUT_MS);
+ try {
+ await client.shutdown(SUPERVISOR_CALL_MS);
+ } finally {
+ client.close();
+ }
+ return 0;
+ }
+ default:
+ process.stderr.write(`${USAGE}\n`);
+ return 2;
+ }
+}
+
+if (import.meta.main) {
+ main(process.argv.slice(2)).then(
+ (code) => {
+ process.exitCode = code;
+ },
+ (error: unknown) => {
+ process.stderr.write(`${error instanceof Error ? error.message : String(error)}\n`);
+ process.exitCode = 1;
+ },
+ );
+}
diff --git a/packages/app/tests/stress/_helpers/reuse/server-supervisor.ts b/packages/app/tests/stress/_helpers/reuse/server-supervisor.ts
new file mode 100644
index 000000000..33cd76e97
--- /dev/null
+++ b/packages/app/tests/stress/_helpers/reuse/server-supervisor.ts
@@ -0,0 +1,463 @@
+import type { ChildProcess } from 'node:child_process';
+import { randomUUID } from 'node:crypto';
+import { existsSync } from 'node:fs';
+import { isAbsolute, relative, sep } from 'node:path';
+import {
+ changedLoadedInputs,
+ parseTestServerLoadedInputs,
+} from '../../../../src/build/test-server-loaded-inputs.ts';
+import {
+ TEST_SERVER_IDENTITY_PATH,
+ TEST_SERVER_LOADED_INPUTS_PATH,
+ type TestServerStartupReceipt,
+} from '../../../../src/build/test-server-startup-contract.ts';
+import {
+ reuseDecision,
+ type ServerFingerprint,
+ serverFingerprint,
+} from '../../../foundation/test-server-inputs.ts';
+import {
+ compatibilityMismatch,
+ type LeaseOutcome,
+ type LeaseRequest,
+ type LeaseVerdict,
+ type RefusalReason,
+ type SlotCompatibility,
+ type SlotStatus,
+ type WorkerServerEnv,
+ workerServerEnvChange,
+} from './lease-protocol.ts';
+
+export interface OwnedServer {
+ readonly proc: ChildProcess;
+ readonly identity: TestServerStartupReceipt;
+ readonly baseURL: string;
+ readonly contentDir: string;
+ release(): Promise<void>;
+}
+
+export type LaunchOwnedServer = (
+ slot: number,
+ workerServerEnv: WorkerServerEnv,
+) => Promise<OwnedServer>;
+
+export interface SupervisorOptions {
+ readonly okRoot: string;
+ readonly launch: LaunchOwnedServer;
+ readonly identityTimeoutMs: number;
+}
+
+interface Lease {
+ readonly token: string;
+ readonly run: string;
+ granted: boolean;
+ lost: string | undefined;
+}
+
+interface OwnedSlotServer {
+ readonly slot: number;
+ readonly server: OwnedServer;
+ readonly compatibility: SlotCompatibility;
+ readonly fingerprint: ServerFingerprint;
+}
+
+interface SlotEntry {
+ readonly slot: number;
+ owned: OwnedSlotServer | undefined;
+ lease: Lease | undefined;
+ discardedBecause: string | undefined;
+}
+
+function isRunning(proc: ChildProcess): boolean {
+ return proc.exitCode === null && proc.signalCode === null;
+}
+
+function describeExit(proc: ChildProcess): string {
+ return proc.signalCode === null ? `with code ${proc.exitCode}` : `on ${proc.signalCode}`;
+}
+
+function sameIdentity(observed: unknown, expected: TestServerStartupReceipt): boolean {
+ if (typeof observed !== 'object' || observed === null) return false;
+ const fields = observed as Record<string, unknown>;
+ return (
+ fields.nonce === expected.nonce &&
+ fields.address === expected.address &&
+ fields.family === expected.family &&
+ fields.port === expected.port
+ );
+}
+
+export async function identityProblem(
+ server: OwnedServer,
+ timeoutMs: number,
+): Promise<string | undefined> {
+ let response: Response;
+ try {
+ response = await fetch(`${server.baseURL}${TEST_SERVER_IDENTITY_PATH}`, {
+ signal: AbortSignal.timeout(timeoutMs),
+ });
+ } catch (error) {
+ return `its identity endpoint did not answer: ${error instanceof Error ? error.message : String(error)}`;
+ }
+ if (response.status !== 200) return `its identity endpoint returned status ${response.status}`;
+ let observed: unknown;
+ try {
+ observed = await response.json();
+ } catch {
+ return 'its identity endpoint did not return JSON';
+ }
+ return sameIdentity(observed, server.identity)
+ ? undefined
+ : 'its identity endpoint answers for a different server';
+}
+
+function displayPath(okRoot: string, path: string): string {
+ const fromRoot = relative(okRoot, path);
+ const outside = fromRoot === '..' || fromRoot.startsWith(`..${sep}`) || isAbsolute(fromRoot);
+ return outside ? path : fromRoot.split(sep).join('/');
+}
+
+async function loadedInputsProblem(
+ server: OwnedServer,
+ okRoot: string,
+ timeoutMs: number,
+): Promise<string | undefined> {
+ let response: Response;
+ try {
+ response = await fetch(`${server.baseURL}${TEST_SERVER_LOADED_INPUTS_PATH}`, {
+ signal: AbortSignal.timeout(timeoutMs),
+ });
+ } catch (error) {
+ return `its loaded-inputs endpoint did not answer: ${error instanceof Error ? error.message : String(error)}`;
+ }
+ if (response.status !== 200)
+ return `its loaded-inputs endpoint returned status ${response.status}`;
+ let reported: unknown;
+ try {
+ reported = await response.json();
+ } catch {
+ return 'its loaded-inputs endpoint did not return JSON';
+ }
+ const loaded = parseTestServerLoadedInputs(reported);
+ if (loaded === undefined) {
+ return 'its loaded-inputs endpoint did not report the inputs its configuration loaded';
+ }
+ const changed = changedLoadedInputs(loaded);
+ return changed.length === 0
+ ? undefined
+ : `its loaded configuration inputs changed: ${changed.map((path) => displayPath(okRoot, path)).join(', ')}`;
+}
+
+function refused(reason: RefusalReason, detail: string): LeaseOutcome {
+ return { kind: 'refused', reason, detail };
+}
+
+function describe(error: unknown): string {
+ return error instanceof Error ? error.message : String(error);
+}
+
+export class WorkerServerSupervisor {
+ readonly #options: SupervisorOptions;
+ readonly #slots = new Map<number, SlotEntry>();
+ readonly #abortedRuns = new Map<string, string>();
+ readonly #releases = new Map<OwnedSlotServer, Promise<void>>();
+ readonly #admissions = new Set<Promise<LeaseOutcome>>();
+ #closed = false;
+
+ constructor(options: SupervisorOptions) {
+ this.#options = options;
+ }
+
+ async acquire(request: LeaseRequest): Promise<LeaseOutcome> {
+ const admission = this.#admit(request);
+ this.#admissions.add(admission);
+ try {
+ return await admission;
+ } finally {
+ this.#admissions.delete(admission);
+ }
+ }
+
+ async #admit(request: LeaseRequest): Promise<LeaseOutcome> {
+ if (this.#closed) return refused('supervisor-closed', 'the supervisor is shutting down');
+ const aborted = this.#abortedRuns.get(request.run);
+ if (aborted !== undefined) {
+ return refused('run-aborted', `run ${request.run} lost an earlier lease: ${aborted}`);
+ }
+ if (request.worktree !== this.#options.okRoot) {
+ return refused(
+ 'incompatible',
+ `worktree ${request.worktree} is not this supervisor's worktree ${this.#options.okRoot}; it would test that checkout's code`,
+ );
+ }
+ let entry = this.#slots.get(request.slot);
+ if (entry?.lease !== undefined) {
+ return refused('busy', `slot ${request.slot} is leased to run ${entry.lease.run}`);
+ }
+ if (entry?.owned !== undefined) {
+ const mismatch = compatibilityMismatch(entry.owned.compatibility, request);
+ if (mismatch !== undefined) return refused('incompatible', mismatch);
+ }
+ if (entry === undefined) {
+ entry = {
+ slot: request.slot,
+ owned: undefined,
+ lease: undefined,
+ discardedBecause: undefined,
+ };
+ this.#slots.set(request.slot, entry);
+ }
+ const lease: Lease = {
+ token: randomUUID(),
+ run: request.run,
+ granted: false,
+ lost: undefined,
+ };
+ entry.lease = lease;
+
+ let relaunchedBecause: string | null = entry.discardedBecause ?? null;
+ const owned = entry.owned;
+ if (owned !== undefined) {
+ try {
+ const unusable =
+ workerServerEnvChange(owned.compatibility, request) ??
+ (await this.#unusableReason(owned));
+ if (unusable === undefined && entry.owned === owned) {
+ return this.#granted(entry, lease, owned, false, null);
+ }
+ relaunchedBecause = unusable ?? 'its server was discarded while admission checked it';
+ await this.#discard(entry, owned);
+ } catch (error) {
+ entry.lease = undefined;
+ return refused(
+ 'launch-failed',
+ `slot ${request.slot} could not check or discard its previous server: ${describe(error)}`,
+ );
+ }
+ }
+ if (this.#closed) {
+ entry.lease = undefined;
+ return refused(
+ 'supervisor-closed',
+ 'the supervisor shut down while admission checked the slot',
+ );
+ }
+
+ let launched: OwnedSlotServer;
+ try {
+ launched = await this.#launch(request);
+ } catch (error) {
+ entry.lease = undefined;
+ return refused(
+ 'launch-failed',
+ `slot ${request.slot} could not start an owned server: ${describe(error)}`,
+ );
+ }
+ if (this.#closed) {
+ entry.lease = undefined;
+ await this.#discard(entry, launched);
+ return refused('supervisor-closed', 'the supervisor shut down while the slot was starting');
+ }
+ entry.owned = launched;
+ entry.discardedBecause = undefined;
+ this.#watchOwner(entry, launched);
+ return this.#granted(entry, lease, launched, true, relaunchedBecause);
+ }
+
+ async verify(token: string): Promise<LeaseVerdict> {
+ const entry = this.#entryLeasedTo(token);
+ if (entry?.lease === undefined) return { held: false, reason: 'the lease is not held' };
+ const { lease } = entry;
+ if (lease.lost !== undefined) return { held: false, reason: lease.lost };
+ const owned = entry.owned;
+ if (owned === undefined) {
+ return this.#lose(entry, lease, 'its server is no longer owned by the supervisor');
+ }
+ if (!isRunning(owned.server.proc)) {
+ return this.#lose(
+ entry,
+ lease,
+ `its server process exited ${describeExit(owned.server.proc)} during the lease`,
+ );
+ }
+ const problem = await identityProblem(owned.server, this.#options.identityTimeoutMs);
+ if (problem !== undefined) {
+ return this.#lose(entry, lease, `its server identity changed during the lease: ${problem}`);
+ }
+ return lease.lost === undefined ? { held: true } : { held: false, reason: lease.lost };
+ }
+
+ release(token: string, discard: boolean): Promise<void> {
+ return this.#endLease(token, discard ? 'its last holder discarded it' : undefined);
+ }
+
+ async abandon(token: string): Promise<void> {
+ const entry = this.#entryLeasedTo(token);
+ if (entry?.lease === undefined) return;
+ const { run } = entry.lease;
+ try {
+ await this.#endLease(token, 'its holder disconnected without releasing it');
+ } catch (error) {
+ console.warn(
+ `[server reuse] slot ${entry.slot}: discarding the server of run ${run}, whose lease connection closed without a release, failed: ${describe(error)}`,
+ );
+ }
+ }
+
+ status(): SlotStatus[] {
+ return [...this.#slots.values()]
+ .sort((left, right) => left.slot - right.slot)
+ .map((entry) => ({
+ slot: entry.slot,
+ baseURL: entry.owned?.server.baseURL ?? null,
+ config: entry.owned?.compatibility.config ?? null,
+ leasedBy: entry.lease?.run ?? null,
+ }));
+ }
+
+ async shutdown(): Promise<void> {
+ this.#closed = true;
+ const discarding: Promise<void>[] = [];
+ for (const entry of this.#slots.values()) {
+ if (entry.lease !== undefined && entry.lease.lost === undefined) {
+ entry.lease.lost = 'the supervisor shut down during the lease';
+ this.#abortedRuns.set(entry.lease.run, entry.lease.lost);
+ }
+ if (entry.owned !== undefined) discarding.push(this.#discard(entry, entry.owned));
+ }
+ const discarded = Promise.allSettled(discarding);
+ await Promise.allSettled(this.#admissions);
+ await discarded;
+ const failures = (
+ await Promise.all(
+ [...this.#releases].map(([owned, releasing]) =>
+ releasing.then(
+ () => [],
+ (reason: unknown) => [{ slot: owned.slot, reason }],
+ ),
+ ),
+ )
+ ).flat();
+ if (failures.length > 0) {
+ throw new AggregateError(
+ failures.map(({ reason }) => reason),
+ `releasing the owned servers failed: ${failures.map(({ slot, reason }) => `slot ${slot}: ${describe(reason)}`).join('; ')}`,
+ );
+ }
+ }
+
+ async #endLease(token: string, discardBecause: string | undefined): Promise<void> {
+ const entry = this.#entryLeasedTo(token);
+ if (entry?.lease === undefined) return;
+ const reason = entry.lease.lost ?? discardBecause;
+ entry.lease = undefined;
+ const owned = entry.owned;
+ if (owned === undefined || reason === undefined) return;
+ entry.discardedBecause = reason;
+ await this.#discard(entry, owned);
+ }
+
+ #entryLeasedTo(token: string): SlotEntry | undefined {
+ for (const entry of this.#slots.values()) {
+ if (entry.lease?.token === token) return entry;
+ }
+ return undefined;
+ }
+
+ async #unusableReason(owned: OwnedSlotServer): Promise<string | undefined> {
+ const { server } = owned;
+ if (!isRunning(server.proc)) return `its server process exited ${describeExit(server.proc)}`;
+ if (!existsSync(server.contentDir)) return `its content root ${server.contentDir} is gone`;
+ const problem = await identityProblem(server, this.#options.identityTimeoutMs);
+ if (problem !== undefined) return `its server identity changed: ${problem}`;
+ const decision = reuseDecision(
+ owned.fingerprint,
+ serverFingerprint(this.#options.okRoot, owned.compatibility.workerServerEnv),
+ );
+ if (!decision.reuse) {
+ return `its consumed inputs changed: ${[...decision.changedInputs, ...decision.changedEnv].join(', ')}`;
+ }
+ return loadedInputsProblem(server, this.#options.okRoot, this.#options.identityTimeoutMs);
+ }
+
+ async #launch(request: LeaseRequest): Promise<OwnedSlotServer> {
+ const compatibility: SlotCompatibility = {
+ config: request.config,
+ workerServerEnv: { ...request.workerServerEnv },
+ serverMode: request.serverMode,
+ };
+ const fingerprint = serverFingerprint(this.#options.okRoot, compatibility.workerServerEnv);
+ const server = await this.#options.launch(request.slot, compatibility.workerServerEnv);
+ return { slot: request.slot, server, compatibility, fingerprint };
+ }
+
+ #granted(
+ entry: SlotEntry,
+ lease: Lease,
+ owned: OwnedSlotServer,
+ launched: boolean,
+ relaunchedBecause: string | null,
+ ): LeaseOutcome {
+ lease.granted = true;
+ return {
+ kind: 'granted',
+ token: lease.token,
+ slot: entry.slot,
+ baseURL: owned.server.baseURL,
+ port: owned.server.identity.port,
+ contentDir: owned.server.contentDir,
+ launched,
+ relaunchedBecause,
+ };
+ }
+
+ #watchOwner(entry: SlotEntry, owned: OwnedSlotServer): void {
+ owned.server.proc.on('error', (error: Error) => {
+ if (entry.owned !== owned) return;
+ const lease = entry.lease;
+ let ending: Promise<unknown>;
+ if (lease?.granted === true) {
+ ending = this.#lose(
+ entry,
+ lease,
+ `its server lifetime failed during the lease: ${describe(error)}`,
+ );
+ } else {
+ entry.discardedBecause = `its server lifetime failed: ${describe(error)}`;
+ ending = this.#discard(entry, owned);
+ }
+ ending.catch((releaseError: unknown) => {
+ console.warn(
+ `[server reuse] slot ${entry.slot}: releasing the server whose lifetime failed did not complete: ${describe(releaseError)}`,
+ );
+ });
+ });
+ owned.server.proc.once('exit', () => {
+ const lease = entry.lease;
+ if (entry.owned !== owned || lease?.granted !== true || lease.lost !== undefined) return;
+ const reason = `its server process exited ${describeExit(owned.server.proc)} during the lease`;
+ this.#lose(entry, lease, reason).catch((error: unknown) => {
+ console.warn(
+ `[server reuse] slot ${entry.slot}: releasing its exited server failed: ${describe(error)}`,
+ );
+ });
+ });
+ }
+
+ async #lose(entry: SlotEntry, lease: Lease, reason: string): Promise<LeaseVerdict> {
+ lease.lost ??= reason;
+ this.#abortedRuns.set(lease.run, lease.lost);
+ entry.discardedBecause = lease.lost;
+ if (entry.owned !== undefined) await this.#discard(entry, entry.owned);
+ return { held: false, reason: lease.lost };
+ }
+
+ #discard(entry: SlotEntry, owned: OwnedSlotServer): Promise<void> {
+ if (entry.owned === owned) entry.owned = undefined;
+ const pending = this.#releases.get(owned);
+ if (pending !== undefined) return pending;
+ const releasing = owned.server.release();
+ this.#releases.set(owned, releasing);
+ return releasing;
+ }
+}
diff --git a/packages/app/tests/stress/_helpers/reuse/supervisor-socket.ts b/packages/app/tests/stress/_helpers/reuse/supervisor-socket.ts
new file mode 100644
index 000000000..834b4a8d4
--- /dev/null
+++ b/packages/app/tests/stress/_helpers/reuse/supervisor-socket.ts
@@ -0,0 +1,273 @@
+import { lstatSync, mkdirSync, type Stats, unlinkSync } from 'node:fs';
+import { connect, createServer, type Socket } from 'node:net';
+import { dirname } from 'node:path';
+import { createInterface } from 'node:readline';
+import {
+ type ClientMessage,
+ type LeaseOutcome,
+ type LeaseRequest,
+ type LeaseVerdict,
+ parseClientMessage,
+ parseLeaseOutcome,
+ parseLeaseVerdict,
+ parseServerMessage,
+ type ServerMessage,
+} from './lease-protocol.ts';
+import type { WorkerServerSupervisor } from './server-supervisor.ts';
+
+export type EndpointClosed =
+ | { readonly released: true }
+ | { readonly released: false; readonly failure: string };
+
+export interface SupervisorEndpoint {
+ readonly socketPath: string;
+ readonly closed: Promise<EndpointClosed>;
+ close(): Promise<void>;
+}
+
+type SocketPathState = 'live' | 'stale' | 'gone';
+
+function probeSocketPath(socketPath: string): Promise<SocketPathState> {
+ return new Promise((resolve, reject) => {
+ const probe = connect(socketPath);
+ probe.once('connect', () => {
+ probe.destroy();
+ resolve('live');
+ });
+ probe.once('error', (error: NodeJS.ErrnoException) => {
+ if (error.code === 'ECONNREFUSED') resolve('stale');
+ else if (error.code === 'ENOENT') resolve('gone');
+ else {
+ reject(
+ new Error(
+ `cannot claim ${socketPath}: connecting to it failed with ${error.code ?? error.message}, so it is neither a live supervisor nor a stale socket`,
+ ),
+ );
+ }
+ });
+ });
+}
+
+function describeEntry(entry: Stats): string {
+ if (entry.isFile()) return 'a regular file';
+ if (entry.isDirectory()) return 'a directory';
+ if (entry.isSymbolicLink()) return 'a symbolic link';
+ if (entry.isFIFO()) return 'a FIFO';
+ if (entry.isCharacterDevice()) return 'a character device';
+ if (entry.isBlockDevice()) return 'a block device';
+ return 'an entry of another kind';
+}
+
+async function claimSocketPath(socketPath: string): Promise<void> {
+ mkdirSync(dirname(socketPath), { recursive: true, mode: 0o700 });
+ const entry = lstatSync(socketPath, { throwIfNoEntry: false });
+ if (entry === undefined) return;
+ if (!entry.isSocket()) {
+ throw new Error(`cannot claim ${socketPath}: it is ${describeEntry(entry)}, not a socket`);
+ }
+ const state = await probeSocketPath(socketPath);
+ if (state === 'live') throw new Error(`another supervisor already serves ${socketPath}`);
+ if (state === 'stale') unlinkSync(socketPath);
+}
+
+function describe(error: unknown): string {
+ return error instanceof Error ? error.message : String(error);
+}
+
+export async function serveSupervisor(
+ supervisor: WorkerServerSupervisor,
+ socketPath: string,
+): Promise<SupervisorEndpoint> {
+ await claimSocketPath(socketPath);
+ const connections = new Set<Socket>();
+ let resolveClosed: (outcome: EndpointClosed) => void = () => {};
+ const closed = new Promise<EndpointClosed>((resolve) => {
+ resolveClosed = resolve;
+ });
+ let closing: Promise<void> | undefined;
+
+ const server = createServer((socket) => {
+ connections.add(socket);
+ const held = new Set<string>();
+ let open = true;
+ const send = (message: ServerMessage) => {
+ if (open) socket.write(`${JSON.stringify(message)}\n`);
+ };
+ const answer = async (message: ClientMessage): Promise<unknown> => {
+ switch (message.op) {
+ case 'acquire': {
+ const outcome = await supervisor.acquire(message.request);
+ if (outcome.kind !== 'granted') return outcome;
+ if (!open) {
+ await supervisor.abandon(outcome.token);
+ return outcome;
+ }
+ held.add(outcome.token);
+ return outcome;
+ }
+ case 'verify':
+ return held.has(message.token)
+ ? supervisor.verify(message.token)
+ : { held: false, reason: 'the lease is not held by this connection' };
+ case 'release':
+ if (held.delete(message.token)) await supervisor.release(message.token, message.discard);
+ return null;
+ case 'status':
+ return supervisor.status();
+ case 'shutdown':
+ return null;
+ }
+ };
+ const lines = createInterface({ input: socket, crlfDelay: Number.POSITIVE_INFINITY });
+ lines.on('error', () => {});
+ lines.on('line', (line) => {
+ let message: ClientMessage;
+ try {
+ message = parseClientMessage(line);
+ } catch (error) {
+ send({ id: 0, ok: false, error: `malformed request: ${describe(error)}` });
+ return;
+ }
+ answer(message).then(
+ (result) => {
+ send({ id: message.id, ok: true, result });
+ if (message.op === 'shutdown') void endpoint.close();
+ },
+ (error: unknown) => send({ id: message.id, ok: false, error: describe(error) }),
+ );
+ });
+ socket.on('error', () => {});
+ socket.once('close', () => {
+ open = false;
+ connections.delete(socket);
+ for (const token of held) void supervisor.abandon(token);
+ held.clear();
+ });
+ });
+
+ await new Promise<void>((resolve, reject) => {
+ server.once('error', reject);
+ server.listen({ path: socketPath }, () => {
+ server.off('error', reject);
+ resolve();
+ });
+ });
+
+ const endpoint: SupervisorEndpoint = {
+ socketPath,
+ closed,
+ close: () => {
+ closing ??= (async () => {
+ await new Promise<void>((resolve) => {
+ server.close(() => resolve());
+ for (const socket of connections) socket.end();
+ });
+ try {
+ await supervisor.shutdown();
+ resolveClosed({ released: true });
+ } catch (error) {
+ const failure = describe(error);
+ console.warn(`[server reuse] ${failure}`);
+ resolveClosed({ released: false, failure });
+ }
+ })();
+ return closing;
+ },
+ };
+ return endpoint;
+}
+
+export interface LeaseClient {
+ acquire(request: LeaseRequest, timeoutMs: number): Promise<LeaseOutcome>;
+ verify(token: string, timeoutMs: number): Promise<LeaseVerdict>;
+ release(token: string, discard: boolean, timeoutMs: number): Promise<void>;
+ status(timeoutMs: number): Promise<unknown>;
+ shutdown(timeoutMs: number): Promise<void>;
+ close(): void;
+}
+
+type Pending = {
+ resolve: (result: unknown) => void;
+ reject: (error: Error) => void;
+ timer: ReturnType<typeof setTimeout>;
+};
+
+export function connectLeaseClient(socketPath: string, timeoutMs: number): Promise<LeaseClient> {
+ return new Promise((resolve, reject) => {
+ const socket = connect(socketPath);
+ const pending = new Map<number, Pending>();
+ let nextId = 1;
+ let failure: Error | undefined;
+ const failAll = (error: Error) => {
+ failure ??= error;
+ for (const [id, waiter] of pending) {
+ clearTimeout(waiter.timer);
+ waiter.reject(error);
+ pending.delete(id);
+ }
+ };
+ const connectTimer = setTimeout(() => {
+ socket.destroy();
+ reject(new Error(`the supervisor at ${socketPath} did not accept within ${timeoutMs}ms`));
+ }, timeoutMs);
+
+ socket.once('error', (error) => {
+ clearTimeout(connectTimer);
+ failAll(error);
+ reject(new Error(`the supervisor at ${socketPath} is not reachable: ${error.message}`));
+ });
+ socket.once('close', () =>
+ failAll(new Error(`the supervisor at ${socketPath} closed the lease connection`)),
+ );
+ const lines = createInterface({ input: socket, crlfDelay: Number.POSITIVE_INFINITY });
+ lines.on('error', () => {});
+ lines.on('line', (line) => {
+ let message: ServerMessage;
+ try {
+ message = parseServerMessage(line);
+ } catch (error) {
+ failAll(new Error(`the supervisor sent a malformed response: ${describe(error)}`));
+ socket.destroy();
+ return;
+ }
+ const waiter = pending.get(message.id);
+ if (waiter === undefined) return;
+ pending.delete(message.id);
+ clearTimeout(waiter.timer);
+ if (message.ok) waiter.resolve(message.result);
+ else waiter.reject(new Error(`the supervisor refused the request: ${message.error}`));
+ });
+
+ const call = (body: Record<string, unknown>, callTimeoutMs: number): Promise<unknown> => {
+ if (failure !== undefined) return Promise.reject(failure);
+ const id = nextId++;
+ return new Promise((resolveCall, rejectCall) => {
+ const timer = setTimeout(() => {
+ pending.delete(id);
+ rejectCall(
+ new Error(`the supervisor did not answer ${String(body.op)} within ${callTimeoutMs}ms`),
+ );
+ }, callTimeoutMs);
+ pending.set(id, { resolve: resolveCall, reject: rejectCall, timer });
+ socket.write(`${JSON.stringify({ id, ...body })}\n`);
+ });
+ };
+
+ socket.once('connect', () => {
+ clearTimeout(connectTimer);
+ resolve({
+ acquire: async (request, ms) =>
+ parseLeaseOutcome(await call({ op: 'acquire', request }, ms)),
+ verify: async (token, ms) => parseLeaseVerdict(await call({ op: 'verify', token }, ms)),
+ release: async (token, discard, ms) => {
+ await call({ op: 'release', token, discard }, ms);
+ },
+ status: (ms) => call({ op: 'status' }, ms),
+ shutdown: async (ms) => {
+ await call({ op: 'shutdown' }, ms);
+ },
+ close: () => socket.end(),
+ });
+ });
+ });
+}
diff --git a/packages/app/tests/stress/_helpers/server-process.ts b/packages/app/tests/stress/_helpers/server-process.ts
index 3795ca95e..ada73d0e7 100644
--- a/packages/app/tests/stress/_helpers/server-process.ts
+++ b/packages/app/tests/stress/_helpers/server-process.ts
@@ -226,6 +226,8 @@ function parseStartupReceipt(pending: PendingViteStartup): TestServerStartupRece
};
}
+const RECEIPT_POLL_INTERVAL_MS = 100;
+
function waitForStartupReceipt(
pending: PendingViteStartup,
deadlineAt: number,
@@ -233,11 +235,13 @@ function waitForStartupReceipt(
return new Promise((resolve, reject) => {
let watcher: ReturnType<typeof watch> | undefined;
let timer: ReturnType<typeof setTimeout> | undefined;
+ let poll: ReturnType<typeof setInterval> | undefined;
let settled = false;
const cleanup = () => {
watcher?.off('error', onWatchError);
watcher?.close();
if (timer !== undefined) clearTimeout(timer);
+ if (poll !== undefined) clearInterval(poll);
pending.proc.off('error', onError);
pending.proc.off('exit', onExit);
};
@@ -294,6 +298,7 @@ function waitForStartupReceipt(
),
remainingMs,
);
+ poll = setInterval(readReceipt, RECEIPT_POLL_INTERVAL_MS);
readReceipt();
if (pending.proc.exitCode !== null || pending.proc.signalCode !== null) onExit();
} catch (err) {
diff --git a/packages/app/tests/stress/_helpers/worker-server-command.ts b/packages/app/tests/stress/_helpers/worker-server-command.ts
new file mode 100644
index 000000000..1465b03b0
--- /dev/null
+++ b/packages/app/tests/stress/_helpers/worker-server-command.ts
@@ -0,0 +1,60 @@
+import { type ChildProcess, spawn } from 'node:child_process';
+import type { PrebuiltServerCommand, WorkerServing } from './prebuilt/prebuilt-serving.ts';
+import { APP_PACKAGE_ROOT, type ServerLog, type ViteStartupRequest } from './server-process.ts';
+
+export interface WorkerServerLaunch {
+ readonly request: Omit<ViteStartupRequest, 'dispose'>;
+ readonly workerServerEnv: Readonly<Record<string, string>>;
+ readonly contentDir: string;
+ readonly viteCacheDir: string;
+ readonly serverLog: ServerLog;
+ readonly serving: WorkerServing;
+ readonly command: PrebuiltServerCommand | null;
+ readonly reapMs: number;
+}
+
+export function spawnWorkerServerCommand({
+ request,
+ workerServerEnv,
+ contentDir,
+ viteCacheDir,
+ serverLog,
+ serving,
+ command,
+}: WorkerServerLaunch): ChildProcess {
+ if (serving.kind === 'prebuilt') {
+ if (command === null) throw new Error('prebuilt worker server has no admitted command');
+ return spawn('pnpm', [...command.args], {
+ cwd: APP_PACKAGE_ROOT,
+ detached: true,
+ env: {
+ ...process.env,
+ ...workerServerEnv,
+ ...request.environment,
+ ...command.env,
+ OK_TEST_CONTENT_DIR: contentDir,
+ OK_TEST_VITE_CACHE_DIR: viteCacheDir,
+ OK_TEST_SKIP_I18N_COMPILE: '1',
+ OK_TEST_GIT_ENABLED: '1',
+ NO_COLOR: process.env.NO_COLOR ?? '1',
+ },
+ stdio: ['ignore', serverLog.fd, 'inherit'],
+ });
+ }
+ const proc = spawn('pnpm', ['run', 'dev', '--host', request.host], {
+ cwd: APP_PACKAGE_ROOT,
+ detached: true,
+ env: {
+ ...process.env,
+ ...workerServerEnv,
+ ...request.environment,
+ OK_TEST_CONTENT_DIR: contentDir,
+ OK_TEST_VITE_CACHE_DIR: viteCacheDir,
+ OK_TEST_SKIP_I18N_COMPILE: '1',
+ OK_TEST_GIT_ENABLED: '1',
+ NO_COLOR: process.env.NO_COLOR ?? '1',
+ },
+ stdio: ['ignore', serverLog.fd, 'inherit'],
+ });
+ return proc;
+}
diff --git a/packages/app/tests/stress/_helpers/worker-server-lifetime.cjs b/packages/app/tests/stress/_helpers/worker-server-lifetime.cjs
new file mode 100644
index 000000000..253f59df3
--- /dev/null
+++ b/packages/app/tests/stress/_helpers/worker-server-lifetime.cjs
@@ -0,0 +1,118 @@
+const { removeAllDuringTeardown, runTeardownPhases } = require('./teardown-fs.ts');
+const { killGracefully } = require('./server-process.ts');
+const { spawnWorkerServerCommand } = require('./worker-server-command.ts');
+
+const launch = JSON.parse(process.argv[2]);
+let proc;
+let releasing;
+let finishing;
+let ownerLost = !process.connected;
+let terminal;
+
+function send(message) {
+ if (process.connected) process.send(message, () => {});
+}
+
+function describe(error) {
+ return error instanceof Error ? error.message : String(error);
+}
+
+function commandStatus() {
+ return { code: proc.exitCode, signal: proc.signalCode };
+}
+
+function commandError(error) {
+ return {
+ name: error.name,
+ message: error.message,
+ stack: error.stack,
+ errno: error.errno,
+ code: error.code,
+ syscall: error.syscall,
+ path: error.path,
+ spawnargs: error.spawnargs,
+ };
+}
+
+async function stop() {
+ releasing ??= (async () => {
+ try {
+ if (proc) await killGracefully(proc, launch.reapMs);
+ send({ type: 'stopped', terminal, error: null });
+ } catch (error) {
+ send({ type: 'stopped', terminal, error: describe(error) });
+ throw error;
+ }
+ })();
+ return releasing;
+}
+
+function finish() {
+ finishing ??= (async () => {
+ try {
+ await runTeardownPhases(
+ () => stop(),
+ () =>
+ removeAllDuringTeardown(
+ launch.contentDir,
+ launch.viteCacheDir,
+ launch.request.receiptDir,
+ launch.serverLog.path,
+ ),
+ );
+ send({ type: 'released', error: null });
+ process.exitCode = 0;
+ } catch (error) {
+ process.stderr.write(`worker server lifetime release failed: ${describe(error)}\n`);
+ send({ type: 'released', error: describe(error) });
+ process.exitCode = 1;
+ } finally {
+ if (process.connected) process.disconnect();
+ }
+ })();
+ return finishing;
+}
+
+process.once('disconnect', () => {
+ ownerLost = true;
+ void finish();
+});
+process.on('message', (message) => {
+ if (message?.type === 'stop') void stop().catch(() => finish());
+ else if (message?.type === 'release') void finish();
+ else {
+ process.stderr.write('worker server lifetime received an invalid control message\n');
+ void finish();
+ }
+});
+for (const signal of ['SIGTERM', 'SIGINT']) {
+ process.once(signal, () => {
+ ownerLost = true;
+ void finish();
+ });
+}
+
+if (ownerLost) void finish();
+else {
+ try {
+ proc = spawnWorkerServerCommand(launch);
+ proc.once('error', (error) => {
+ terminal = commandStatus();
+ send({ type: 'command-error', status: terminal, error: commandError(error) });
+ void stop().catch(() => finish());
+ });
+ proc.once('exit', (code, signal) => {
+ terminal = { code, signal };
+ send({ type: 'command-exit', code, signal });
+ void stop().catch(() => finish());
+ });
+ proc.once('close', (code, signal) => {
+ terminal = { code, signal };
+ send({ type: 'command-close', code, signal });
+ });
+ send({ type: 'started' });
+ } catch (error) {
+ send({ type: 'launch-error', message: describe(error) });
+ void stop().catch(() => finish());
+ }
+}
diff --git a/packages/app/tests/stress/_helpers/worker-server-lifetime.ts b/packages/app/tests/stress/_helpers/worker-server-lifetime.ts
new file mode 100644
index 000000000..516d3dbc5
--- /dev/null
+++ b/packages/app/tests/stress/_helpers/worker-server-lifetime.ts
@@ -0,0 +1,275 @@
+import { type ChildProcess, spawn } from 'node:child_process';
+import { constants } from 'node:os';
+import { join } from 'node:path';
+import { z } from 'zod';
+import { prebuiltServerCommand, type WorkerServing } from './prebuilt/prebuilt-serving.ts';
+import {
+ APP_PACKAGE_ROOT,
+ killGracefully as killCommandGracefully,
+ type ServerLog,
+ type ViteStartupRequest,
+} from './server-process.ts';
+import { runTeardownPhases } from './teardown-fs.ts';
+import type { WorkerServerLaunch } from './worker-server-command.ts';
+
+interface Lifetime {
+ readonly stopped: Promise<void>;
+ readonly released: Promise<void>;
+ readonly exited: Promise<void>;
+ readonly ownerExited: () => boolean;
+ readonly reapMs: number;
+}
+
+const lifetimes = new WeakMap<ChildProcess, Lifetime>();
+
+const commandStatusSchema = z.object({
+ code: z.number().int().nullable(),
+ signal: z
+ .custom<NodeJS.Signals>(
+ (value) => typeof value === 'string' && Object.hasOwn(constants.signals, value),
+ )
+ .nullable(),
+});
+
+const commandErrorSchema = z.object({
+ name: z.string(),
+ message: z.string(),
+ stack: z.string().optional(),
+ errno: z.number().optional(),
+ code: z.string().optional(),
+ syscall: z.string().optional(),
+ path: z.string().optional(),
+ spawnargs: z.array(z.string()).optional(),
+});
+
+const lifetimeMessageSchema = z.discriminatedUnion('type', [
+ z.object({ type: z.literal('started') }),
+ z.object({ type: z.literal('command-exit'), ...commandStatusSchema.shape }),
+ z.object({ type: z.literal('command-close'), ...commandStatusSchema.shape }),
+ z.object({
+ type: z.literal('command-error'),
+ status: commandStatusSchema,
+ error: commandErrorSchema,
+ }),
+ z.object({ type: z.literal('launch-error'), message: z.string() }),
+ z.object({ type: z.literal('stopped'), error: z.string().nullable() }),
+ z.object({ type: z.literal('released'), error: z.string().nullable() }),
+]);
+
+export function acquireWorkerServer(
+ request: ViteStartupRequest,
+ workerServerEnv: Readonly<Record<string, string>>,
+ contentDir: string,
+ viteCacheDir: string,
+ serverLog: ServerLog,
+ serving: WorkerServing,
+ reapMs: number,
+): ChildProcess {
+ const command =
+ serving.kind === 'prebuilt'
+ ? prebuiltServerCommand(request.host, serving, join(APP_PACKAGE_ROOT, '..', '..'))
+ : null;
+ const launch: WorkerServerLaunch = {
+ request,
+ workerServerEnv,
+ contentDir,
+ viteCacheDir,
+ serverLog: { path: serverLog.path, fd: 1 },
+ serving,
+ command,
+ reapMs,
+ };
+ const proc = spawn(
+ process.execPath,
+ [join(import.meta.dirname, 'worker-server-lifetime.cjs'), JSON.stringify(launch)],
+ {
+ cwd: APP_PACKAGE_ROOT,
+ detached: true,
+ env: process.env,
+ stdio: ['ignore', serverLog.fd, 'inherit', 'ipc'],
+ },
+ );
+ let commandStatus: z.infer<typeof commandStatusSchema> | undefined;
+ let nativeExitCode = proc.exitCode;
+ let nativeSignalCode = proc.signalCode;
+ Object.defineProperties(proc, {
+ exitCode: {
+ get: () => (commandStatus === undefined ? null : commandStatus.code),
+ set: (code: number | null) => {
+ nativeExitCode = code;
+ },
+ },
+ signalCode: {
+ get: () => (commandStatus === undefined ? null : commandStatus.signal),
+ set: (signal: NodeJS.Signals | null) => {
+ nativeSignalCode = signal;
+ },
+ },
+ });
+ let commandExitEmitted = false;
+ let commandCloseEmitted = false;
+ let launchFailed = false;
+ let releaseCompleted = false;
+ let transportFailure: Error | undefined;
+ const nativeEmit = proc.emit.bind(proc);
+ let resolveStopped: () => void;
+ let rejectStopped: (error: Error) => void;
+ const stopped = new Promise<void>((resolve, reject) => {
+ resolveStopped = resolve;
+ rejectStopped = reject;
+ });
+ let resolveReleased: () => void;
+ let rejectReleased: (error: Error) => void;
+ const released = new Promise<void>((resolve, reject) => {
+ resolveReleased = resolve;
+ rejectReleased = reject;
+ });
+ let resolveExited: () => void;
+ let rejectExited: (error: Error) => void;
+ const exited = new Promise<void>((resolve, reject) => {
+ resolveExited = resolve;
+ rejectExited = reject;
+ });
+ void stopped.catch(() => undefined);
+ void released.catch(() => undefined);
+ void exited.catch(() => undefined);
+ proc.emit = (event: string | symbol, ...args: unknown[]): boolean => {
+ if (event === 'exit') return false;
+ if (event === 'close') {
+ rejectStopped(new Error('worker server lifetime closed before stop completed'));
+ rejectReleased(new Error('worker server lifetime closed before release completed'));
+ if (
+ nativeExitCode === 0 &&
+ nativeSignalCode === null &&
+ releaseCompleted &&
+ (commandCloseEmitted || launchFailed) &&
+ transportFailure === undefined
+ ) {
+ resolveExited();
+ } else {
+ const error =
+ transportFailure ??
+ new Error(
+ `worker server lifetime closed without completed release (code=${nativeExitCode}, signal=${nativeSignalCode})`,
+ );
+ rejectExited(error);
+ if (!commandCloseEmitted && !launchFailed) nativeEmit('error', error);
+ }
+ return false;
+ }
+ if (event === 'error') {
+ const [error] = args;
+ transportFailure =
+ error instanceof Error ? error : new Error('worker server lifetime failed');
+ rejectStopped(transportFailure);
+ rejectReleased(transportFailure);
+ }
+ return nativeEmit(event, ...args);
+ };
+ proc.on('message', (input: unknown) => {
+ const result = lifetimeMessageSchema.safeParse(input);
+ if (!result.success) {
+ transportFailure = new Error('worker server lifetime sent an invalid message');
+ rejectStopped(transportFailure);
+ rejectReleased(transportFailure);
+ nativeEmit('error', transportFailure);
+ return;
+ }
+ const message = result.data;
+ switch (message.type) {
+ case 'started':
+ return;
+ case 'command-exit':
+ commandStatus = { code: message.code, signal: message.signal };
+ if (!commandExitEmitted) {
+ commandExitEmitted = true;
+ nativeEmit('exit', message.code, message.signal);
+ }
+ return;
+ case 'command-close':
+ commandStatus = { code: message.code, signal: message.signal };
+ if (!commandCloseEmitted) {
+ commandCloseEmitted = true;
+ nativeEmit('close', message.code, message.signal);
+ }
+ return;
+ case 'command-error':
+ commandStatus = message.status;
+ nativeEmit('error', Object.assign(new Error(message.error.message), message.error));
+ return;
+ case 'launch-error':
+ launchFailed = true;
+ nativeEmit('error', new Error(message.message));
+ return;
+ case 'stopped':
+ if (message.error === null) resolveStopped();
+ else rejectStopped(new Error(message.error));
+ return;
+ case 'released':
+ if (message.error === null) {
+ releaseCompleted = true;
+ resolveReleased();
+ } else rejectReleased(new Error(message.error));
+ return;
+ default:
+ message satisfies never;
+ }
+ });
+ lifetimes.set(proc, {
+ stopped,
+ released,
+ exited,
+ ownerExited: () => nativeExitCode !== null || nativeSignalCode !== null,
+ reapMs,
+ });
+ return proc;
+}
+
+export async function killGracefully(proc: ChildProcess, timeoutMs: number): Promise<void> {
+ const lifetime = lifetimes.get(proc);
+ if (lifetime === undefined) return killCommandGracefully(proc, timeoutMs);
+ if (lifetime.ownerExited()) return lifetime.exited;
+ if (proc.connected) proc.send({ type: 'stop' });
+ let timer: ReturnType<typeof setTimeout> | undefined;
+ try {
+ await Promise.race([
+ lifetime.stopped,
+ new Promise<never>((_, reject) => {
+ timer = setTimeout(
+ () => reject(new Error(`worker server lifetime stop exceeded ${timeoutMs}ms`)),
+ timeoutMs,
+ );
+ }),
+ ]);
+ } finally {
+ clearTimeout(timer);
+ }
+}
+
+export async function completeWorkerServerRelease(
+ proc: ChildProcess,
+ cleanup: () => void | Promise<void> = () => undefined,
+): Promise<void> {
+ await runTeardownPhases(cleanup, async () => {
+ const lifetime = lifetimes.get(proc);
+ if (lifetime === undefined) return;
+ if (proc.connected) proc.send({ type: 'release' });
+ let timer: ReturnType<typeof setTimeout> | undefined;
+ try {
+ await Promise.race([
+ runTeardownPhases(
+ () => lifetime.released,
+ () => lifetime.exited,
+ ),
+ new Promise<never>((_, reject) => {
+ timer = setTimeout(() => {
+ if (proc.connected) proc.disconnect();
+ reject(new Error(`worker server lifetime release exceeded ${lifetime.reapMs}ms`));
+ }, lifetime.reapMs);
+ }),
+ ]);
+ } finally {
+ clearTimeout(timer);
+ }
+ });
+}
diff --git a/packages/app/tests/stress/agents-panel-reload-visibility.e2e.ts b/packages/app/tests/stress/agents-panel-reload-visibility.e2e.ts
index 4869d9e0a..8734514d1 100644
--- a/packages/app/tests/stress/agents-panel-reload-visibility.e2e.ts
+++ b/packages/app/tests/stress/agents-panel-reload-visibility.e2e.ts
@@ -4,12 +4,12 @@ import AxeBuilder from '@axe-core/playwright';
import type { Locator, Page } from '@playwright/test';
import { HISTORY_PANEL_WIDTH_PX } from '../../src/hooks/use-history-presentation-mode';
import { AGENTS_PANEL_WIDTH_KEY } from '../../src/lib/agents-panel-width-store';
-import { expect, switchColorThemeAndSettleFade, test } from './_helpers';
+import { expect, servedModuleRoute, switchColorThemeAndSettleFade, test } from './_helpers';
const AGENTS_PANEL_MOUNT = '[data-agents-panel-mount]';
const AGENTS_REVEAL_TAB = '[data-terminal-reveal="right"]';
const COLLAPSE_AGENT_PANEL_BUTTON = 'Collapse agent panel';
-const SESSIONS_CHUNK_GLOB = '**/src/components/SessionsHost*';
+const SESSIONS_HOST_SOURCE = 'src/components/SessionsHost.tsx';
const CONFIG_RESPONSE_GLOB = '**/api/config';
const AGENTS_ORDER_STORAGE_KEY = 'ok-agent-session-order-v1';
const SEED_AGENT_ID = 'panel-level-fixture-agent';
@@ -1109,7 +1109,7 @@ test.describe('agents panel reload visibility level', () => {
await expectPersistedPanelLevel(page, false);
const marks: RouteMark[] = [];
- await page.route(SESSIONS_CHUNK_GLOB, async (route) => {
+ await page.route(servedModuleRoute(SESSIONS_HOST_SOURCE), async (route) => {
await route.continue();
marks.push({ route: 'sessions-chunk-response', at: Date.now() });
});
@@ -1150,7 +1150,7 @@ test.describe('agents panel reload visibility level', () => {
await expectPersistedPanelLevel(page, false);
const marks: RouteMark[] = [];
- await page.route(SESSIONS_CHUNK_GLOB, async (route) => {
+ await page.route(servedModuleRoute(SESSIONS_HOST_SOURCE), async (route) => {
await waitForRouteMark(
marks,
'roster-frame',
@@ -1214,7 +1214,7 @@ test.describe('agents panel reload visibility level', () => {
await expectPersistedPanelLevel(page, true);
const marks: RouteMark[] = [];
- await page.route(SESSIONS_CHUNK_GLOB, async (route) => {
+ await page.route(servedModuleRoute(SESSIONS_HOST_SOURCE), async (route) => {
await waitForRouteMark(
marks,
'roster-frame',
diff --git a/packages/app/tests/stress/assertion-timeout.e2e.ts b/packages/app/tests/stress/assertion-timeout.e2e.ts
new file mode 100644
index 000000000..d59064ab1
--- /dev/null
+++ b/packages/app/tests/stress/assertion-timeout.e2e.ts
@@ -0,0 +1,49 @@
+import { expect, test } from '@playwright/test';
+import {
+ ASSERTION_ELAPSED_ANNOTATION,
+ MET_CASE,
+ UNMET_CASE,
+} from './_helpers/assertion-timeout/cases.ts';
+import {
+ type ConfigurationMode,
+ runAssertionTimeoutCases,
+} from './_helpers/assertion-timeout/run-assertion-timeout.test-helper.ts';
+
+const ASSERTION_TIMEOUT_MS = 15_000;
+
+test.describe.configure({ retries: 0 });
+
+async function expectDefaultAssertionTimeout(mode: ConfigurationMode): Promise<void> {
+ const run = await runAssertionTimeoutCases(mode);
+ const context = `nested ${mode} run exited ${run.exitCode}:\n${run.transcript}`;
+
+ expect(run.forbidOnly, context).toBe(mode === 'ci');
+ expect(
+ run.cases.get(MET_CASE)?.map((r) => r.status),
+ context,
+ ).toEqual(['passed']);
+
+ const unmet = run.cases.get(UNMET_CASE) ?? [];
+ expect(
+ unmet.map((r) => r.status),
+ context,
+ ).toEqual(['failed']);
+ const [failure] = unmet;
+ expect(failure.message, context).toContain('toBeVisible() failed');
+ expect(failure.message, context).toContain(`Timeout: ${ASSERTION_TIMEOUT_MS}ms`);
+
+ const elapsed = Number(
+ failure.annotations.find((a) => a.type === ASSERTION_ELAPSED_ANNOTATION)?.description,
+ );
+ test.info().annotations.push({
+ type: `ok:unmet-assertion-elapsed-ms:${mode}`,
+ description: String(elapsed),
+ });
+ expect(elapsed, context).toBeGreaterThanOrEqual(ASSERTION_TIMEOUT_MS);
+}
+
+test('locally an unmet web-first assertion fails at the 15 s default while a met one passes', () =>
+ expectDefaultAssertionTimeout('local'));
+
+test('under CI an unmet web-first assertion fails at the 15 s default while a met one passes', () =>
+ expectDefaultAssertionTimeout('ci'));
diff --git a/packages/app/tests/stress/language-first-paint.e2e.ts b/packages/app/tests/stress/language-first-paint.e2e.ts
index 9b1d71557..727fef30f 100644
--- a/packages/app/tests/stress/language-first-paint.e2e.ts
+++ b/packages/app/tests/stress/language-first-paint.e2e.ts
@@ -1,5 +1,5 @@
import type { Page } from '@playwright/test';
-import { expect, test } from './_helpers';
+import { expect, servedEntryScriptRoute, test } from './_helpers';
const STORAGE_KEY = 'ok-language-v1';
@@ -19,7 +19,7 @@ async function seedLanguageCache(page: Page, entry: CachedLanguage): Promise<voi
}
async function gotoWithoutApp(page: Page): Promise<void> {
- await page.route('**/src/main.tsx', (route) => route.abort());
+ await page.route(servedEntryScriptRoute(), (route) => route.abort());
await page.goto('/', { waitUntil: 'commit' });
}
diff --git a/packages/app/tests/stress/language-load-failure.e2e.ts b/packages/app/tests/stress/language-load-failure.e2e.ts
index 8e5b3adb3..393d3a6da 100644
--- a/packages/app/tests/stress/language-load-failure.e2e.ts
+++ b/packages/app/tests/stress/language-load-failure.e2e.ts
@@ -1,12 +1,12 @@
import type { Page } from '@playwright/test';
-import { expect, test } from './_helpers';
+import { expect, servedModuleRoute, servedModuleUrl, test } from './_helpers';
const REQUESTED = 'es';
-const CATALOG_ROUTE = `**/locales/${REQUESTED}/messages.json*`;
+const CATALOG_SOURCE = `src/locales/${REQUESTED}/messages.json`;
async function withholdCatalog(page: Page): Promise<{ release: () => Promise<void> }> {
- await page.route(CATALOG_ROUTE, (route) => route.abort('failed'));
+ await page.route(servedModuleRoute(CATALOG_SOURCE), (route) => route.abort('failed'));
return {
release: async () => {
await page.unrouteAll({ behavior: 'ignoreErrors' });
@@ -47,16 +47,15 @@ test.describe('a locale catalog that cannot be fetched', () => {
await expect(reloadAction(page)).toBeVisible();
await withheld.release();
- const outcome = await page.evaluate(async () => {
- const url = '/src/locales/es/messages.json?import';
+ const outcome = await page.evaluate(async (url) => {
const refetched = await fetch(url);
try {
- await import(/* @vite-ignore */ url);
+ await import(url);
return { refetched: refetched.status, reimported: 'resolved' };
} catch {
return { refetched: refetched.status, reimported: 'rejected' };
}
- });
+ }, servedModuleUrl(CATALOG_SOURCE));
expect(outcome).toEqual({ refetched: 200, reimported: 'rejected' });
});
diff --git a/packages/app/tests/stress/loop-markers.e2e.ts b/packages/app/tests/stress/loop-markers.e2e.ts
new file mode 100644
index 000000000..6f7075cf4
--- /dev/null
+++ b/packages/app/tests/stress/loop-markers.e2e.ts
@@ -0,0 +1,43 @@
+import { E2E_PAGE_MODULES_SOURCE } from '../../src/build/prebuilt-dev-contract.ts';
+import { TEST_LOOP_MARKERS_PATH } from '../../src/build/test-loop-marker-contract.ts';
+import type * as PageModules from '../../src/test-utils/e2e-page-modules.ts';
+import { expect, servedModuleUrl, test } from './_helpers';
+import { expectedLoopMarker, type LoopMarkerClass } from './_helpers/loop-markers.ts';
+
+function expectedMarker(markerClass: LoopMarkerClass): string {
+ const { onDisk, instrument } = expectedLoopMarker(markerClass);
+ if (instrument !== undefined) {
+ expect(onDisk, `the ${markerClass} marker source holds the instrument's edit`).toBe(instrument);
+ }
+ return onDisk;
+}
+
+async function servedTestServerMarkers(baseURL: string | undefined): Promise<unknown> {
+ const response = await fetch(new URL(TEST_LOOP_MARKERS_PATH, baseURL));
+ expect(response.status).toBe(200);
+ return response.json();
+}
+
+test('the served app module carries the app marker its source holds', async ({ page }) => {
+ const expected = expectedMarker('app');
+ const pageModules = servedModuleUrl(E2E_PAGE_MODULES_SOURCE);
+ await expect
+ .poll(async () => {
+ await page.goto('/');
+ return page.evaluate(async (moduleUrl) => {
+ const { TEST_LOOP_APP_MARKER }: typeof PageModules = await import(moduleUrl);
+ return TEST_LOOP_APP_MARKER;
+ }, pageModules);
+ })
+ .toBe(expected);
+});
+
+test('the test server reports the server marker its source holds', async ({ baseURL }) => {
+ const expected = expectedMarker('server');
+ expect(await servedTestServerMarkers(baseURL)).toMatchObject({ server: expected });
+});
+
+test('the test server reports the core marker its source holds', async ({ baseURL }) => {
+ const expected = expectedMarker('core');
+ expect(await servedTestServerMarkers(baseURL)).toMatchObject({ core: expected });
+});
diff --git a/packages/app/tests/stress/okignore-settings.e2e.ts b/packages/app/tests/stress/okignore-settings.e2e.ts
index 14858a001..94c40e0e9 100644
--- a/packages/app/tests/stress/okignore-settings.e2e.ts
+++ b/packages/app/tests/stress/okignore-settings.e2e.ts
@@ -1,4 +1,8 @@
-import { expect, test } from './_helpers';
+import { appSourcesIn, expect, servedModuleRoute, test } from './_helpers';
+
+const SETTINGS_SECTIONS = 'src/components/settings';
+const SETTINGS_SECTION_FILE = /^[A-Za-z]+Section\.tsx$/;
+const HOTKEYS_SECTION = `${SETTINGS_SECTIONS}/HotkeysSection.tsx`;
test.describe('Settings — Ignore patterns section (US-007 / US-008 / US-009 / US-010 / US-011 / US-012 / US-013)', () => {
test.beforeEach(async ({ api }) => {
@@ -466,13 +470,15 @@ test.describe('Settings selected section loading', () => {
}, testInfo) => {
await api.testReset();
- const themesModule = /\/src\/components\/settings\/ThemePluginSection\.tsx(?:\?|$)/;
+ const themesModule = servedModuleRoute(`${SETTINGS_SECTIONS}/ThemePluginSection.tsx`);
const releaseThemes = Promise.withResolvers<void>();
await page.route(themesModule, async (route) => {
await releaseThemes.promise;
await route.continue();
});
- const themesRequested = page.waitForRequest(themesModule);
+ const themesRequested = page.waitForRequest((request) =>
+ themesModule(new URL(request.url())),
+ );
const section = page.getByTestId(panel);
try {
@@ -503,9 +509,7 @@ test.describe('Settings selected section loading', () => {
api,
}) => {
await api.testReset();
- await page.route(/\/src\/components\/settings\/HotkeysSection\.tsx(?:\?|$)/, (route) =>
- route.abort(),
- );
+ await page.route(servedModuleRoute(HOTKEYS_SECTION), (route) => route.abort());
const dialog = page.getByTestId('settings-dialog');
await page.goto('/#settings');
@@ -522,8 +526,11 @@ test.describe('Settings selected section loading', () => {
}) => {
await api.testReset();
- const otherSectionModules =
- /\/src\/components\/settings\/(?!HotkeysSection\.tsx)[A-Za-z]+Section\.tsx(?:\?|$)/;
+ const otherSectionModules = servedModuleRoute(
+ ...appSourcesIn(SETTINGS_SECTIONS, SETTINGS_SECTION_FILE).filter(
+ (source) => source !== HOTKEYS_SECTION,
+ ),
+ );
const releaseOtherSections = Promise.withResolvers<void>();
await page.route(otherSectionModules, async (route) => {
await releaseOtherSections.promise;
diff --git a/packages/app/tests/stress/prebuilt-dev-serving.e2e.ts b/packages/app/tests/stress/prebuilt-dev-serving.e2e.ts
new file mode 100644
index 000000000..58747acad
--- /dev/null
+++ b/packages/app/tests/stress/prebuilt-dev-serving.e2e.ts
@@ -0,0 +1,137 @@
+import { spawnSync } from 'node:child_process';
+import { readdirSync, writeFileSync } from 'node:fs';
+import { join, relative, sep } from 'node:path';
+import { test as base, expect } from '@playwright/test';
+import { DEV_HARNESS_SENTINEL } from '../../scripts/check-dev-harness-absent.mjs';
+import {
+ REQUIRED_FIXTURE_ENTRY_NAMES,
+ WORKER_SERVER_BUDGET_TOTAL_MS,
+} from './_helpers/fixtures.ts';
+import type { PrebuiltDevApp } from './_helpers/prebuilt/prebuilt-dev-app.ts';
+import { preparePrebuiltDevApp } from './_helpers/prebuilt/prebuilt-dev-build.ts';
+import { launchOwnedWorkerServer } from './_helpers/reuse/owned-dev-server.ts';
+import type { OwnedServer } from './_helpers/reuse/server-supervisor.ts';
+import { APP_PACKAGE_ROOT } from './_helpers/seed-key.ts';
+
+type PrebuiltServing = { readonly app: PrebuiltDevApp; readonly server: OwnedServer };
+
+const PNG_1X1 = Buffer.from(
+ 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg==',
+ 'base64',
+);
+
+const test = base.extend<object, { prebuiltApp: PrebuiltDevApp; prebuilt: PrebuiltServing }>({
+ prebuiltApp: [
+ // biome-ignore lint/correctness/noEmptyPattern: Playwright fixture contract
+ async ({}, use) => {
+ await use(await preparePrebuiltDevApp());
+ },
+ { scope: 'worker', timeout: WORKER_SERVER_BUDGET_TOTAL_MS },
+ ],
+ prebuilt: [
+ async ({ prebuiltApp: app }, use) => {
+ const server = await launchOwnedWorkerServer({
+ label: 'prebuilt-serving',
+ workerServerEnv: {},
+ serving: { kind: 'prebuilt', outputDir: app.outputDir },
+ });
+ try {
+ await use({ app, server });
+ } finally {
+ await server.release();
+ }
+ },
+ { scope: 'worker', timeout: WORKER_SERVER_BUDGET_TOTAL_MS },
+ ],
+});
+
+test.describe.configure({ mode: 'default' });
+
+function firstFile(dir: string, extension: string): string {
+ const found = readdirSync(dir, { recursive: true, encoding: 'utf8' }).find((name) =>
+ name.endsWith(extension),
+ );
+ if (found === undefined) throw new Error(`no ${extension} under ${dir}`);
+ return found.split(sep).join('/');
+}
+
+test('the prebuilt DEV app is built outside the shipped output and keeps the DEV hooks the production gate rejects', ({
+ prebuilt,
+}) => {
+ expect(
+ relative(join(APP_PACKAGE_ROOT, 'node_modules', '.cache'), prebuilt.app.distDir),
+ ).not.toMatch(/^\.\./);
+
+ const gate = spawnSync(
+ process.execPath,
+ [
+ join(APP_PACKAGE_ROOT, 'scripts', 'check-dev-harness-absent.mjs'),
+ '--dist',
+ prebuilt.app.distDir,
+ ],
+ { encoding: 'utf8' },
+ );
+
+ expect(gate.status).toBe(1);
+ expect(gate.stderr).toContain(`'${DEV_HARNESS_SENTINEL}' reached the production build`);
+});
+
+test('the prebuilt DEV app serves its own fonts and Excalidraw assets and renders the editor', async ({
+ prebuilt,
+ page,
+ request,
+}) => {
+ const { app, server } = prebuilt;
+ const font = await request.get(
+ `${server.baseURL}/assets/${firstFile(join(app.distDir, 'assets'), '.woff2')}`,
+ );
+ expect(font.status()).toBe(200);
+ expect(font.headers()['content-type']).toBe('font/woff2');
+ const excalidraw = await request.get(
+ `${server.baseURL}/excalidraw-assets/${firstFile(join(app.distDir, 'excalidraw-assets'), '.woff2')}`,
+ );
+ expect(excalidraw.status()).toBe(200);
+ const icon = await request.get(`${server.baseURL}/favicon.svg`);
+ expect(icon.status()).toBe(200);
+ expect(icon.headers()['content-type']).toBe('image/svg+xml');
+
+ const failed: string[] = [];
+ page.on('response', (response) => {
+ if (response.status() >= 400) failed.push(`${response.status()} ${response.url()}`);
+ });
+ await page.goto(`${server.baseURL}/`);
+ await expect(
+ page.getByRole('treeitem', { name: REQUIRED_FIXTURE_ENTRY_NAMES[0], exact: true }),
+ ).toBeVisible();
+ expect(failed).toEqual([]);
+});
+
+test('user assets keep their disposition and a missing asset is a 404 rather than the app shell', async ({
+ prebuilt,
+ request,
+}) => {
+ const { server } = prebuilt;
+ writeFileSync(join(server.contentDir, 'serving-smoke.png'), PNG_1X1);
+ writeFileSync(join(server.contentDir, 'serving-smoke.zip'), Buffer.from('PK\u0005\u0006'));
+
+ const inline = await request.get(`${server.baseURL}/serving-smoke.png`);
+ expect(inline.status()).toBe(200);
+ expect(inline.headers()['content-disposition']).toBe('inline');
+ const attachment = await request.get(`${server.baseURL}/serving-smoke.zip`);
+ expect(attachment.status()).toBe(200);
+ expect(attachment.headers()['content-disposition']).toBe('attachment');
+
+ for (const missing of [
+ '/missing-asset.png',
+ '/assets/missing-font.woff2',
+ '/assets/missing-chunk.js',
+ ]) {
+ const response = await request.get(`${server.baseURL}${missing}`);
+ expect(response.status(), missing).toBe(404);
+ expect(await response.text(), missing).not.toContain('<div id="root">');
+ }
+
+ const route = await request.get(`${server.baseURL}/some/folder/page`);
+ expect(route.status()).toBe(200);
+ expect(await route.text()).toContain('<div id="root">');
+});
diff --git a/packages/app/tests/stress/server-reuse-admission.e2e.ts b/packages/app/tests/stress/server-reuse-admission.e2e.ts
new file mode 100644
index 000000000..2080f2213
--- /dev/null
+++ b/packages/app/tests/stress/server-reuse-admission.e2e.ts
@@ -0,0 +1,121 @@
+import { mkdtempSync, realpathSync, rmSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { expect, test } from '@playwright/test';
+import { SERVER_LEASE_ANNOTATION } from './_helpers/reuse/lease-fixture.ts';
+import {
+ annotationsOf,
+ CASE_SERVER_ANNOTATION,
+ type RunningSupervisor,
+ runReuseCase,
+ startSupervisor,
+ statusesOf,
+} from './_helpers/reuse/reuse-run.test-helper.ts';
+import { connectLeaseClient } from './_helpers/reuse/supervisor-socket.ts';
+import { APP_PACKAGE_ROOT } from './_helpers/server-process.ts';
+import { SETUP_NON_RESULT_ANNOTATION } from './_helpers/setup-non-result.ts';
+
+test.describe.configure({ retries: 0 });
+
+const SERVES_THE_APP = 'a leased worker server serves the app';
+const SHUTS_DOWN_MID_TEST = 'the supervisor shuts down while a test holds its lease';
+const LATER_IN_THE_RUN = 'a later test of the run whose supervisor shut down';
+const CALL_MS = 60_000;
+const SUPERVISOR_EXIT_MS = 60_000;
+
+let scratch: string;
+let supervisor: RunningSupervisor | undefined;
+
+test.beforeEach(async () => {
+ test.setTimeout(480_000);
+ scratch = mkdtempSync(join(tmpdir(), 'ok-reuse-e2e-'));
+ supervisor = await startSupervisor(scratch);
+});
+
+test.afterEach(async () => {
+ await supervisor?.stop();
+ supervisor = undefined;
+ rmSync(scratch, { recursive: true, force: true });
+});
+
+test('successive opted-in runs reuse the supervisor slot and a run finding it busy starts its own server', async () => {
+ if (supervisor === undefined) throw new Error('the supervisor did not start');
+ const { socketPath } = supervisor;
+
+ const first = await runReuseCase(socketPath, SERVES_THE_APP);
+ expect(statusesOf(first), first.transcript).toEqual(['passed']);
+ const [started] = annotationsOf(first, SERVER_LEASE_ANNOTATION);
+ expect(started, first.transcript).toMatch(
+ /^slot 0: started an owned server at http:\/\/127\.0\.0\.1:\d+$/,
+ );
+ const leasedURL = started?.split(' at ')[1];
+ expect(annotationsOf(first, CASE_SERVER_ANNOTATION)).toEqual([leasedURL]);
+
+ const second = await runReuseCase(socketPath, SERVES_THE_APP);
+ expect(statusesOf(second), second.transcript).toEqual(['passed']);
+ expect(annotationsOf(second, SERVER_LEASE_ANNOTATION)).toEqual([
+ `slot 0: reused the owned server at ${leasedURL}`,
+ ]);
+ expect(annotationsOf(second, CASE_SERVER_ANNOTATION)).toEqual([leasedURL]);
+
+ const holder = await connectLeaseClient(socketPath, 5_000);
+ try {
+ const status = (await holder.status(CALL_MS)) as Array<{ config: string }>;
+ const held = await holder.acquire(
+ {
+ worktree: realpathSync(join(APP_PACKAGE_ROOT, '..', '..')),
+ config: status[0]?.config ?? '',
+ run: 'outer-holder',
+ slot: 0,
+ workerServerEnv: {},
+ serverMode: 'dev',
+ },
+ CALL_MS,
+ );
+ expect(held.kind).toBe('granted');
+
+ const busy = await runReuseCase(socketPath, SERVES_THE_APP);
+ expect(statusesOf(busy), busy.transcript).toEqual(['passed']);
+ expect(annotationsOf(busy, SERVER_LEASE_ANNOTATION)).toEqual([
+ 'slot 0 was refused (busy: slot 0 is leased to run outer-holder), so this worker starts its own server',
+ ]);
+ const [busyServer] = annotationsOf(busy, CASE_SERVER_ANNOTATION);
+ expect(busyServer).toMatch(/^http:\/\/127\.0\.0\.1:\d+$/);
+ expect(busyServer).not.toBe(leasedURL);
+ } finally {
+ holder.close();
+ }
+});
+
+test('a supervisor that stops during a leased test turns that test into a declared non-result', async () => {
+ if (supervisor === undefined) throw new Error('the supervisor did not start');
+ const run = await runReuseCase(supervisor.socketPath, SHUTS_DOWN_MID_TEST);
+ expect(statusesOf(run), run.transcript).toEqual(['failed']);
+ expect(annotationsOf(run, SETUP_NON_RESULT_ANNOTATION)).toEqual([
+ expect.stringMatching(/^the reused server's lease was lost during the test: /),
+ ]);
+ expect(await supervisor.exitWithin(SUPERVISOR_EXIT_MS), supervisor.transcript()).toEqual({
+ code: 0,
+ signal: null,
+ });
+});
+
+test('a supervisor that stops during a leased test aborts its run, so a later test of that run is refused as a declared non-result', async () => {
+ if (supervisor === undefined) throw new Error('the supervisor did not start');
+ const run = await runReuseCase(supervisor.socketPath, SHUTS_DOWN_MID_TEST, LATER_IN_THE_RUN);
+ expect(statusesOf(run), run.transcript).toEqual(['failed', 'failed']);
+ const nonResults = annotationsOf(run, SETUP_NON_RESULT_ANNOTATION);
+ expect(nonResults, run.transcript).toHaveLength(2);
+ const [lost, refused] = nonResults;
+ const lossPrefix = "the reused server's lease was lost during the test: ";
+ expect(lost, run.transcript).toMatch(new RegExp(`^${lossPrefix}.`));
+ const lossReason = lost?.slice(lossPrefix.length);
+ expect(refused, run.transcript).toMatch(
+ /^server reuse refused slot 0: run \S+ lost an earlier lease: .+, so this invocation gives no further verdicts$/,
+ );
+ expect(refused).toContain(`lost an earlier lease: ${lossReason}, so`);
+ expect(annotationsOf(run, SERVER_LEASE_ANNOTATION), run.transcript).toEqual([
+ expect.stringMatching(/^slot 0: started an owned server at http:\/\/127\.0\.0\.1:\d+$/),
+ ]);
+ expect(annotationsOf(run, CASE_SERVER_ANNOTATION)).toEqual([]);
+});
diff --git a/packages/app/tests/stress/server-reuse-reset.e2e.ts b/packages/app/tests/stress/server-reuse-reset.e2e.ts
new file mode 100644
index 000000000..d8870f4ee
--- /dev/null
+++ b/packages/app/tests/stress/server-reuse-reset.e2e.ts
@@ -0,0 +1,70 @@
+import { mkdtempSync, rmSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { expect, test } from '@playwright/test';
+import { SERVER_LEASE_ANNOTATION } from './_helpers/reuse/lease-fixture.ts';
+import {
+ annotationsOf,
+ CASE_SERVER_ANNOTATION,
+ type RunningSupervisor,
+ runReuseCase,
+ startSupervisor,
+ statusesOf,
+} from './_helpers/reuse/reuse-run.test-helper.ts';
+
+test.describe.configure({ retries: 0 });
+
+const LEAVES_STATE = 'a run leaves settings, content, agent state and browser storage behind';
+const FINDS_NO_STATE =
+ 'a later run finds none of an earlier run’s settings, content or browser storage';
+const LEAVES_UNACCOUNTED_STATE = 'a run leaves server state that no reset can account for';
+
+let scratch: string;
+let supervisor: RunningSupervisor | undefined;
+
+test.beforeEach(async () => {
+ test.setTimeout(600_000);
+ scratch = mkdtempSync(join(tmpdir(), 'ok-reuse-reset-e2e-'));
+ supervisor = await startSupervisor(scratch);
+});
+
+test.afterEach(async () => {
+ await supervisor?.stop();
+ supervisor = undefined;
+ rmSync(scratch, { recursive: true, force: true });
+});
+
+test('a reused server is reset to its booted state, and one it cannot prove reset is discarded for a fresh server', async () => {
+ if (supervisor === undefined) throw new Error('the supervisor did not start');
+ const { socketPath } = supervisor;
+
+ const leaving = await runReuseCase(socketPath, LEAVES_STATE);
+ expect(statusesOf(leaving), leaving.transcript).toEqual(['passed']);
+ const [leasedURL] = annotationsOf(leaving, CASE_SERVER_ANNOTATION);
+ expect(annotationsOf(leaving, SERVER_LEASE_ANNOTATION)).toEqual([
+ `slot 0: started an owned server at ${leasedURL}`,
+ ]);
+
+ const finding = await runReuseCase(socketPath, FINDS_NO_STATE);
+ expect(statusesOf(finding), finding.transcript).toEqual(['passed']);
+ expect(annotationsOf(finding, SERVER_LEASE_ANNOTATION)).toEqual([
+ `slot 0: reused the owned server at ${leasedURL}`,
+ ]);
+ expect(annotationsOf(finding, CASE_SERVER_ANNOTATION)).toEqual([leasedURL]);
+
+ const unaccounted = await runReuseCase(socketPath, LEAVES_UNACCOUNTED_STATE);
+ expect(statusesOf(unaccounted), unaccounted.transcript).toEqual(['passed']);
+ expect(annotationsOf(unaccounted, SERVER_LEASE_ANNOTATION)).toEqual([
+ `slot 0: reused the owned server at ${leasedURL}`,
+ ]);
+ expect(annotationsOf(unaccounted, CASE_SERVER_ANNOTATION)).toEqual([leasedURL]);
+
+ const afterUnaccounted = await runReuseCase(socketPath, FINDS_NO_STATE);
+ expect(statusesOf(afterUnaccounted), afterUnaccounted.transcript).toEqual(['passed']);
+ expect(annotationsOf(afterUnaccounted, SERVER_LEASE_ANNOTATION)).toEqual([
+ 'slot 0: the leased server could not be reset (the reset left .ok/local/unrecognised-state.json is not in a freshly booted slot), so it was discarded and this worker starts its own server',
+ ]);
+ const [freshURL] = annotationsOf(afterUnaccounted, CASE_SERVER_ANNOTATION);
+ expect(freshURL).toMatch(/^http:\/\/127\.0\.0\.1:\d+$/);
+ expect(freshURL).not.toBe(leasedURL);
+});
diff --git a/packages/app/tests/stress/theme-color-conversion.e2e.ts b/packages/app/tests/stress/theme-color-conversion.e2e.ts
index 68783caba..b25329317 100644
--- a/packages/app/tests/stress/theme-color-conversion.e2e.ts
+++ b/packages/app/tests/stress/theme-color-conversion.e2e.ts
@@ -1,9 +1,12 @@
import { PREVIEW_THEME_TOKENS } from '@inkeep/open-knowledge-core';
+import { E2E_PAGE_MODULES_SOURCE } from '../../src/build/prebuilt-dev-contract.ts';
+import type * as PageModules from '../../src/test-utils/e2e-page-modules.ts';
import {
expect,
FADE_DURATION_MS,
installThemeFadeProbe,
openColorThemes,
+ servedModuleUrl,
type ThemeFadeProbeWindow,
test,
} from './_helpers';
@@ -13,9 +16,8 @@ test('painted comment underlines retain their sRGB contrast in both default mode
}) => {
await page.emulateMedia({ reducedMotion: 'reduce' });
await page.goto('/');
- const samples = await page.evaluate(async () => {
- const modulePath = '/src/comments/anchor-layers.ts';
- const { buildAnchorSegments, commentColor } = await import(modulePath);
+ const samples = await page.evaluate(async (pageModules) => {
+ const { buildAnchorSegments, commentColor }: typeof PageModules = await import(pageModules);
const root = document.documentElement;
const originalClass = root.className;
const surface = document.createElement('div');
@@ -59,7 +61,7 @@ test('painted comment underlines retain their sRGB contrast in both default mode
root.className = originalClass;
surface.remove();
}
- });
+ }, servedModuleUrl(E2E_PAGE_MODULES_SOURCE));
const luminance = (rgb: number[]) =>
rgb.reduce((sum, value, index) => {
const channel = value / 255;
@@ -80,9 +82,9 @@ test('converts CSS Color 4 tokens for live xterm themes without losing alpha cha
page,
}) => {
await openColorThemes(page);
- const converted = await page.evaluate(async () => {
- const modulePath = '/src/lib/css-color-to-hex.ts';
- const { cssColorToHex } = await import(modulePath);
+ const pageModules = servedModuleUrl(E2E_PAGE_MODULES_SOURCE);
+ const converted = await page.evaluate(async (pageModules) => {
+ const { cssColorToHex }: typeof PageModules = await import(pageModules);
return {
transparentRgb: cssColorToHex('rgb(80 120 255 / 0)', { alpha: true }),
translucentRgb: cssColorToHex('rgb(80 120 255 / 0.18)', { alpha: true }),
@@ -100,7 +102,7 @@ test('converts CSS Color 4 tokens for live xterm themes without losing alpha cha
).join('')}`;
})(),
};
- });
+ }, pageModules);
expect(converted.transparentRgb).toBe('#5078ff00');
expect(converted.translucentRgb).toBe('#5078ff2e');
@@ -108,17 +110,19 @@ test('converts CSS Color 4 tokens for live xterm themes without losing alpha cha
expect(converted.color4Black).toBe('#000000ff');
expect(converted.color4Translucent).toBe(converted.color4Reference);
- const invalidRegisteredTokens = await page.evaluate(async (tokens) => {
- const modulePath = '/src/lib/theme-color-properties.ts';
- const { THEME_COLOR_PROPERTIES } = await import(modulePath);
- return tokens
- .filter(
- (token) =>
- THEME_COLOR_PROPERTIES.includes(token.name) &&
- (!CSS.supports('color', token.light) || !CSS.supports('color', token.dark)),
- )
- .map((token) => token.name);
- }, PREVIEW_THEME_TOKENS);
+ const invalidRegisteredTokens = await page.evaluate(
+ async ({ tokens, pageModules }) => {
+ const { THEME_COLOR_PROPERTIES }: typeof PageModules = await import(pageModules);
+ return tokens
+ .filter(
+ (token) =>
+ THEME_COLOR_PROPERTIES.includes(token.name) &&
+ (!CSS.supports('color', token.light) || !CSS.supports('color', token.dark)),
+ )
+ .map((token) => token.name);
+ },
+ { tokens: PREVIEW_THEME_TOKENS, pageModules },
+ );
expect(invalidRegisteredTokens).toEqual([]);
const dracula = page.getByRole('button', { name: 'Use Dracula for the active light mode' });
@@ -131,10 +135,9 @@ test('converts CSS Color 4 tokens for live xterm themes without losing alpha cha
const monokai = page.getByRole('button', { name: 'Use Monokai for the active light mode' });
await installThemeFadeProbe(page);
const fade = await monokai.evaluate(
- async (element, { token }) => {
+ async (element, { token, pageModules }) => {
if (!(element instanceof HTMLButtonElement)) throw new Error('Expected a palette button');
- const modulePath = '/src/components/terminal-theme.ts';
- const { computeLiveXtermTheme } = await import(modulePath);
+ const { computeLiveXtermTheme }: typeof PageModules = await import(pageModules);
const { armThemeFade } = window as typeof window & ThemeFadeProbeWindow;
if (!armThemeFade) {
throw new Error(
@@ -150,7 +153,7 @@ test('converts CSS Color 4 tokens for live xterm themes without losing alpha cha
element.click();
return armed.report();
},
- { token: '--selection-soft' },
+ { token: '--selection-soft', pageModules },
);
const detail = `\n${fade.token} fade report:\n${JSON.stringify(fade)}\n`;
diff --git a/packages/app/tests/stress/theme-fade.e2e.ts b/packages/app/tests/stress/theme-fade.e2e.ts
index 1fc025668..09d4482d6 100644
--- a/packages/app/tests/stress/theme-fade.e2e.ts
+++ b/packages/app/tests/stress/theme-fade.e2e.ts
@@ -1,4 +1,6 @@
import type { Page } from '@playwright/test';
+import { E2E_PAGE_MODULES_SOURCE } from '../../src/build/prebuilt-dev-contract.ts';
+import type * as PageModules from '../../src/test-utils/e2e-page-modules.ts';
import {
expect,
FADE_DURATION_MS,
@@ -6,6 +8,7 @@ import {
installThemeFadeProbe,
openColorThemes,
runningRootAnimations,
+ servedModuleUrl,
switchColorThemeAndSettleFade,
type ThemeFadeProbeWindow,
test,
@@ -156,10 +159,13 @@ test('default light and dark modes fade surfaces without losing selection', asyn
await page.getByTestId('settings-sidebar-item-preferences').click();
await page.getByTestId('theme-picker-light').click();
expect(
- await page
- .locator('html')
- .evaluate((root) => getComputedStyle(root).getPropertyValue('--radius').trim()),
- ).toBe('0.625rem');
+ await page.locator('html').evaluate((root) => {
+ const radius = CSSNumericValue.parse(getComputedStyle(root).getPropertyValue('--radius'));
+ return radius instanceof CSSUnitValue
+ ? { value: radius.value, unit: radius.unit }
+ : radius.toString();
+ }),
+ ).toEqual({ value: 0.625, unit: 'rem' });
expect(
await page
.getByTestId('theme-picker-light')
@@ -328,11 +334,10 @@ test('default light and dark modes fade surfaces without losing selection', asyn
).toEqual([]);
await expect(page.locator('html')).toHaveClass(/dark/);
await expect(page.locator('html')).not.toHaveAttribute('data-theme-color-fading');
- const selectionColor = await page.evaluate(async () => {
- const modulePath = '/src/components/terminal-theme.ts';
- const { computeLiveXtermTheme } = await import(modulePath);
+ const selectionColor = await page.evaluate(async (pageModules) => {
+ const { computeLiveXtermTheme }: typeof PageModules = await import(pageModules);
return computeLiveXtermTheme('dark').selectionBackground;
- });
+ }, servedModuleUrl(E2E_PAGE_MODULES_SOURCE));
expect(selectionColor).toMatch(/^#[0-9a-f]{6}52$/);
await search.fill('');
diff --git a/packages/app/tsconfig.json b/packages/app/tsconfig.json
index 097b0bc1e..24e89a8be 100644
--- a/packages/app/tsconfig.json
+++ b/packages/app/tsconfig.json
@@ -89,9 +89,13 @@
"tests/half-open-forced-close.test-helper.ts",
"tests/integration/audit-barrier.test-helper.ts",
"tests/stress/_helpers/acp-catalog.ts",
+ "tests/stress/_helpers/assertion-timeout",
"tests/stress/_helpers/boot-verdict",
"tests/stress/_helpers/editor-state.ts",
"tests/stress/_helpers/image.ts",
+ "tests/stress/_helpers/loop-markers.ts",
+ "tests/stress/_helpers/prebuilt",
+ "tests/stress/_helpers/reuse",
"tests/stress/_helpers/theme-fade.ts"
],
"exclude": ["node_modules", "dist", "**/*.test.ts", "**/*.test.tsx"]
diff --git a/packages/app/vite.config.ts b/packages/app/vite.config.ts
index 31badf881..764281ae0 100644
--- a/packages/app/vite.config.ts
+++ b/packages/app/vite.config.ts
@@ -4,6 +4,7 @@ import { defineConfig } from 'vite';
import { injectAppVersionEnv } from './src/build/app-version';
import { chromeTokensVitePlugin } from './src/build/chrome-tokens-vite-plugin';
import { rejectionLoopGuardPlugin } from './src/build/rejection-loop-guard-plugin';
+import { testLoopMarkerPlugin } from './src/build/test-loop-marker-plugin';
import { TEST_SERVER_STARTUP_ENV } from './src/build/test-server-startup-contract';
import {
parseTestServerStartupRequest,
@@ -74,7 +75,9 @@ export default defineConfig({
// options are shared with the Electron renderer build.
babel(RENDERER_BABEL_OPTIONS),
hocuspocusPlugin(),
- ...(startupRequest === undefined ? [] : [testServerStartupPlugin(startupRequest)]),
+ ...(startupRequest === undefined
+ ? []
+ : [testServerStartupPlugin(startupRequest), testLoopMarkerPlugin()]),
],
resolve: {
tsconfigPaths: true,
diff --git a/packages/app/vite.prebuilt-dev.ts b/packages/app/vite.prebuilt-dev.ts
new file mode 100644
index 000000000..df573a2eb
--- /dev/null
+++ b/packages/app/vite.prebuilt-dev.ts
@@ -0,0 +1,39 @@
+import { isAbsolute, resolve } from 'node:path';
+import { defineConfig, mergeConfig } from 'vite';
+import {
+ E2E_PAGE_MODULES_CHUNK,
+ E2E_PAGE_MODULES_SOURCE,
+ PREBUILT_MODULE_MANIFEST,
+ PREBUILT_OUT_DIR_ENV,
+ PREBUILT_OUTPUT_OPTIONS,
+} from './src/build/prebuilt-dev-contract';
+import { prebuiltModuleChunksPlugin } from './src/build/prebuilt-module-chunks-plugin';
+import { prebuiltHocuspocusPlugin } from './src/server/hocuspocus-plugin';
+import base from './vite.config';
+import { rendererHtmlInput } from './vite.entries';
+
+const outDir = process.env[PREBUILT_OUT_DIR_ENV];
+if (outDir === undefined || !isAbsolute(outDir)) {
+ throw new Error(`${PREBUILT_OUT_DIR_ENV} must name the prebuilt DEV app's absolute directory`);
+}
+
+export default mergeConfig(
+ base,
+ defineConfig({
+ plugins: [prebuiltHocuspocusPlugin(), prebuiltModuleChunksPlugin(import.meta.dirname)],
+ build: {
+ outDir,
+ emptyOutDir: true,
+ manifest: PREBUILT_MODULE_MANIFEST,
+ rolldownOptions: {
+ input: {
+ ...rendererHtmlInput(import.meta.dirname),
+ [E2E_PAGE_MODULES_CHUNK]: resolve(import.meta.dirname, E2E_PAGE_MODULES_SOURCE),
+ },
+ preserveEntrySignatures: 'exports-only',
+ output: PREBUILT_OUTPUT_OPTIONS,
+ },
+ },
+ preview: { port: base.server?.port, strictPort: base.server?.strictPort },
+ }),
+);
diff --git a/packages/app/vitest.integration.config.ts b/packages/app/vitest.integration.config.ts
index d732e8634..1277a27eb 100644
--- a/packages/app/vitest.integration.config.ts
+++ b/packages/app/vitest.integration.config.ts
@@ -1,6 +1,11 @@
import { defineConfig, type ViteUserConfig } from 'vitest/config';
import { appVitestConfig } from './vitest.config';
+export const WORKER_LIFETIME_TESTS = [
+ 'tests/meta/worker-server-lifetime.test.ts',
+ 'tests/meta/standalone-prebuilt-worker-lifetime.test.ts',
+];
+
// Integration tier: multi-client CRDT convergence over the real Node test
// harness (tests/integration) plus the meta suite. Reuses the
// app base config (lingui macro shim, single-instance dedupe, development-
@@ -18,7 +23,11 @@ export const appIntegrationVitestConfig = {
test: {
...appVitestConfig.test,
include: ['tests/integration/**/*.test.ts?(x)', 'tests/meta/**/*.test.ts?(x)'],
- exclude: [...appVitestConfig.test.exclude, '**/per-session-um-perf.test.ts'],
+ exclude: [
+ ...appVitestConfig.test.exclude,
+ '**/per-session-um-perf.test.ts',
+ ...WORKER_LIFETIME_TESTS,
+ ],
// Per-test / per-hook budget for the CRDT convergence suite, carried over
// from the bun `--timeout 30000`. Declared here (not only inherited) so the
// CI test-coverage meta-guard reads it directly off this config.
diff --git a/packages/app/vitest.worker-lifetime.config.ts b/packages/app/vitest.worker-lifetime.config.ts
new file mode 100644
index 000000000..36001eeb9
--- /dev/null
+++ b/packages/app/vitest.worker-lifetime.config.ts
@@ -0,0 +1,14 @@
+import { defineConfig, type ViteUserConfig } from 'vitest/config';
+import { appVitestConfig } from './vitest.config';
+import { appIntegrationVitestConfig, WORKER_LIFETIME_TESTS } from './vitest.integration.config';
+
+export default defineConfig({
+ ...appIntegrationVitestConfig,
+ test: {
+ ...appIntegrationVitestConfig.test,
+ include: [...WORKER_LIFETIME_TESTS],
+ exclude: [...appVitestConfig.test.exclude],
+ testTimeout: 30_000,
+ hookTimeout: 30_000,
+ },
+} satisfies ViteUserConfig);
diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts
index 08b9dc92b..fff3536a1 100644
--- a/packages/core/src/index.ts
+++ b/packages/core/src/index.ts
@@ -2021,6 +2021,7 @@ export {
TERMINAL_PLACEMENTS,
type TerminalPlacement,
} from './terminal-layout.ts';
+export { TEST_LOOP_CORE_MARKER } from './test-loop-marker.ts';
export type {
AnsiSlotName,
Base16Palette,
diff --git a/packages/core/src/test-loop-marker.ts b/packages/core/src/test-loop-marker.ts
new file mode 100644
index 000000000..48006ddf6
--- /dev/null
+++ b/packages/core/src/test-loop-marker.ts
@@ -0,0 +1 @@
+export const TEST_LOOP_CORE_MARKER = 'core-base';
diff --git a/packages/server/src/api-extension.ts b/packages/server/src/api-extension.ts
index 6127ff30a..571b0a1f9 100644
--- a/packages/server/src/api-extension.ts
+++ b/packages/server/src/api-extension.ts
@@ -1341,6 +1341,8 @@ interface ApiExtensionBaseOptions {
getBridgeLossReporter?: () => BridgeDeriveLossReporter | undefined;
getPrincipal?: () => Principal | null;
homeDirOverride?: string;
+ resetSettingsForTest?: () => void;
+ settleDiskEventsForTest?: () => Promise<void>;
agentIntegrations?: AgentRegistryHostSeam;
savedThemeLockTimeoutMs?: number;
acpRegistry?: AcpRegistry;
@@ -1573,6 +1575,8 @@ export function createApiExtension(options: ApiExtensionOptions): Extension & {
getBridgeLossReporter,
getPrincipal,
homeDirOverride,
+ resetSettingsForTest,
+ settleDiskEventsForTest,
agentIntegrations,
savedThemeLockTimeoutMs,
acpRegistry,
@@ -5526,6 +5530,10 @@ export function createApiExtension(options: ApiExtensionOptions): Extension & {
renameAttributionCounter,
});
const testRoutes = createTestRoutes({
+ agentPresence: agentPresenceBroadcaster,
+ resetSettingsForTest,
+ settleDiskEvents: settleDiskEventsForTest,
+ recentlyRemovedDocs,
conflicts,
durabilityState,
resolveAlias,
diff --git a/packages/server/src/config-file-watcher.ts b/packages/server/src/config-file-watcher.ts
index 73dd2654e..bd70b8898 100644
--- a/packages/server/src/config-file-watcher.ts
+++ b/packages/server/src/config-file-watcher.ts
@@ -7,10 +7,25 @@ import { startPolledPathWatcher } from './polled-path-watcher.ts';
export type ConfigFileWatcherUnsubscribe = () => Promise<void>;
+export type ConfigFileWatcher = ConfigFileWatcherUnsubscribe & {
+ readonly resync: () => void;
+};
+
+function readOrNull(path: string, log: ReturnType<typeof getLogger>): string | null {
+ try {
+ return readFileSync(path, 'utf-8');
+ } catch (err) {
+ if (errnoCode(err) !== 'ENOENT') {
+ log.warn({ err, path }, 'config file baseline read failed; treating it as absent');
+ }
+ return null;
+ }
+}
+
export async function startConfigFileWatcher(
absPath: string,
onChange: (content: string) => void,
-): Promise<ConfigFileWatcherUnsubscribe> {
+): Promise<ConfigFileWatcher> {
const log = getLogger('config-file-watcher');
const watchDir = dirname(absPath);
@@ -23,10 +38,7 @@ export async function startConfigFileWatcher(
}
}
- let lastContent: string | null = null;
- try {
- lastContent = readFileSync(absPath, 'utf-8');
- } catch {}
+ let lastContent = readOrNull(absPath, log);
const handlePath = (path: string, logMissing = true): void => {
if (path !== absPath) return;
let content: string;
@@ -64,13 +76,17 @@ export async function startConfigFileWatcher(
},
});
handlePath(absPath, false);
- return stop;
+ return Object.assign(() => stop(), {
+ resync: () => {
+ lastContent = readOrNull(absPath, log);
+ },
+ });
}
export async function startMultiPathConfigFileWatcher(
absPaths: ReadonlyArray<string>,
onChange: (path: string, content: string) => void,
-): Promise<ConfigFileWatcherUnsubscribe> {
+): Promise<ConfigFileWatcher> {
if (absPaths.length === 0) {
throw new Error('startMultiPathConfigFileWatcher requires at least one absolute path');
}
@@ -91,13 +107,10 @@ export async function startMultiPathConfigFileWatcher(
}
const lastContent = new Map<string, string | null>();
- for (const path of watchedPaths) {
- try {
- lastContent.set(path, readFileSync(path, 'utf-8'));
- } catch {
- lastContent.set(path, null);
- }
- }
+ const resync = (): void => {
+ for (const path of watchedPaths) lastContent.set(path, readOrNull(path, log));
+ };
+ resync();
const handlePath = (path: string, logMissing = true): void => {
if (!watchedPaths.has(path)) return;
@@ -136,5 +149,5 @@ export async function startMultiPathConfigFileWatcher(
},
});
for (const path of watchedPaths) handlePath(path, false);
- return stop;
+ return Object.assign(() => stop(), { resync });
}
diff --git a/packages/server/src/config-persistence.ts b/packages/server/src/config-persistence.ts
index 4cba06649..050b27429 100644
--- a/packages/server/src/config-persistence.ts
+++ b/packages/server/src/config-persistence.ts
@@ -403,6 +403,27 @@ async function storeConfigDocInner(
}
}
+export function resetConfigDocForTest(
+ document: Y.Doc | null,
+ documentName: string,
+ ctx: ConfigPersistenceCtx,
+): void {
+ try {
+ tracedUnlinkSync(configDocAbsPath(documentName, ctx));
+ } catch (e) {
+ if ((e as NodeJS.ErrnoException).code !== 'ENOENT') throw e;
+ }
+ ctx.lkgCache.delete(documentName);
+ if (document === null) return;
+ const ytext = document.getText('source');
+ if (ytext.length > 0) {
+ document.transact(() => {
+ ytext.delete(0, ytext.length);
+ }, CONFIG_VALIDATION_REVERT_ORIGIN);
+ }
+ loadConfigDoc(document, documentName, ctx);
+}
+
type ApplyExternalConfigChangeOutcome = 'applied' | 'rejected' | 'no-op';
export function isConfigEcho(
diff --git a/packages/server/src/file-watcher-settle.test.ts b/packages/server/src/file-watcher-settle.test.ts
new file mode 100644
index 000000000..aa87b2e28
--- /dev/null
+++ b/packages/server/src/file-watcher-settle.test.ts
@@ -0,0 +1,419 @@
+import {
+ mkdirSync,
+ mkdtempSync,
+ readdirSync,
+ readFileSync,
+ realpathSync,
+ rmSync,
+ unlinkSync,
+ writeFileSync,
+} from 'node:fs';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { afterEach, beforeEach, expect, test, vi } from 'vitest';
+import { createContentFilter, isShareableOkArtifact } from './content-filter.ts';
+import {
+ type DiskEvent,
+ getWatcherDecisionRingSnapshot,
+ INTERIM_SETTLE_NO_PROGRESS_MS,
+ startWatcher,
+ type WatcherHandle,
+} from './file-watcher.ts';
+import {
+ forgetNativeSubscriptions,
+ nativeSubscriptionOn,
+} from './parcel-watcher-double.test-helper.ts';
+
+vi.mock('@parcel/watcher', async () => {
+ const { parcelWatcherModule } = await import('./parcel-watcher-double.test-helper.ts');
+ return parcelWatcherModule;
+});
+
+const DROP_NOTICE = 'Events were dropped by the kernel. File system must be re-scanned.';
+
+let contentDir: string;
+const handles: WatcherHandle[] = [];
+const latches: Array<() => void> = [];
+
+beforeEach(() => {
+ contentDir = realpathSync(mkdtempSync(join(tmpdir(), 'ok-watcher-settle-')));
+ mkdirSync(join(contentDir, '.ok'));
+ writeFileSync(join(contentDir, 'left.md'), '# Left\n');
+});
+
+afterEach(async () => {
+ vi.useRealTimers();
+ for (const release of latches.splice(0)) release();
+ for (const handle of handles.splice(0)) await handle.unsubscribe();
+ forgetNativeSubscriptions();
+ rmSync(contentDir, { recursive: true, force: true });
+});
+
+async function watchContent(
+ backend: 'parcel' | 'chokidar',
+ onEvent: (event: DiskEvent) => Promise<void> | undefined = () => undefined,
+): Promise<{ watcher: WatcherHandle; events: DiskEvent[] }> {
+ const events: DiskEvent[] = [];
+ const watcher = await startWatcher(
+ contentDir,
+ async (event) => {
+ events.push(event);
+ await onEvent(event);
+ },
+ createContentFilter({ projectDir: contentDir, contentDir }),
+ { forceBackend: backend, platform: 'linux' },
+ );
+ handles.push(watcher);
+ return { watcher, events };
+}
+
+function fencesOnDisk(): string[] {
+ return readdirSync(join(contentDir, '.ok')).filter((name) =>
+ name.startsWith('watcher-settle-fence-'),
+ );
+}
+
+function decisionsFor(name: string): Array<{ decision: string; kind: string }> {
+ return getWatcherDecisionRingSnapshot()
+ .filter((record) => record.path.endsWith(name))
+ .map(({ decision, kind }) => ({ decision, kind }));
+}
+
+function indexOf(watcher: WatcherHandle): { files: string[]; folders: string[] } {
+ return {
+ files: [...watcher.getAllFilesIndex()].map(([name]) => name),
+ folders: [...watcher.getFolderIndex().keys()],
+ };
+}
+
+function settling(watcher: WatcherHandle): {
+ readonly done: () => boolean;
+ readonly promise: Promise<void>;
+} {
+ let done = false;
+ const promise = watcher.settle().then(() => {
+ done = true;
+ });
+ return { done: () => done, promise };
+}
+
+type SettleState = 'pending' | 'resolved' | { readonly rejected: string };
+
+function observed(settle: Promise<void>): () => SettleState {
+ let state: SettleState = 'pending';
+ settle.then(
+ () => {
+ state = 'resolved';
+ },
+ (err: unknown) => {
+ state = { rejected: err instanceof Error ? err.message : String(err) };
+ },
+ );
+ return () => state;
+}
+
+test('settle resolves only after the watcher has finished handling a deletion that reached the disk before it', async () => {
+ const left = join(contentDir, 'left.md');
+ const handling = Promise.withResolvers<void>();
+ latches.push(handling.resolve);
+ const { watcher, events } = await watchContent('parcel', (event) =>
+ event.kind === 'delete' ? handling.promise : undefined,
+ );
+ unlinkSync(left);
+
+ const settle = settling(watcher);
+ const [fence] = fencesOnDisk();
+ expect(fence).toMatch(/^watcher-settle-fence-/);
+ const delivered = nativeSubscriptionOn(contentDir).deliver([
+ { type: 'delete', path: left },
+ { type: 'create', path: join(contentDir, '.ok', fence) },
+ ]);
+ await expect.poll(() => events).toEqual([{ kind: 'delete', path: left, docName: 'left' }]);
+ await new Promise<void>((flushed) => setImmediate(flushed));
+
+ expect(settle.done()).toBe(false);
+ handling.resolve();
+ await delivered;
+ await settle.promise;
+ expect(events).toEqual([{ kind: 'delete', path: left, docName: 'left' }]);
+ expect(fencesOnDisk()).toEqual([]);
+ expect([...watcher.getAllFilesIndex()].map(([name]) => name)).toEqual([]);
+});
+
+test('settle waits for its own fence, not for whichever batch arrives next', async () => {
+ const left = join(contentDir, 'left.md');
+ const { watcher, events } = await watchContent('parcel');
+ unlinkSync(left);
+
+ const settle = settling(watcher);
+ const [fence] = fencesOnDisk();
+ await nativeSubscriptionOn(contentDir).deliver([{ type: 'delete', path: left }]);
+
+ expect(events).toEqual([{ kind: 'delete', path: left, docName: 'left' }]);
+ expect(settle.done()).toBe(false);
+ await nativeSubscriptionOn(contentDir).deliver([
+ { type: 'create', path: join(contentDir, '.ok', fence) },
+ ]);
+ await settle.promise;
+ expect(settle.done()).toBe(true);
+});
+
+test('a dropped-events notice settles a fence the watcher never reported, once recovery has read the disk', async () => {
+ const left = join(contentDir, 'left.md');
+ const { watcher, events } = await watchContent('parcel');
+ unlinkSync(left);
+
+ const settle = settling(watcher);
+ await nativeSubscriptionOn(contentDir).deliver([], new Error(DROP_NOTICE));
+ await settle.promise;
+
+ expect(events).toEqual([{ kind: 'delete', path: left, docName: 'left' }]);
+ expect(fencesOnDisk()).toEqual([]);
+});
+
+test('settle rejects when the watcher stops before its fence arrives, and leaves no fence behind', async () => {
+ const { watcher } = await watchContent('parcel');
+
+ const settle = watcher.settle();
+ expect(fencesOnDisk()).toHaveLength(1);
+ handles.splice(handles.indexOf(watcher), 1);
+ await watcher.unsubscribe();
+
+ await expect(settle).rejects.toThrow('the file watcher stopped before it saw its settle fence');
+ expect(fencesOnDisk()).toEqual([]);
+});
+
+test('a watcher error that ends the subscription fails a pending settle and every later one, and leaves no fence behind', async () => {
+ const { watcher } = await watchContent('parcel');
+ const pending = observed(watcher.settle());
+ expect(fencesOnDisk()).toHaveLength(1);
+
+ const failure = `inotify_add_watch on '${join(contentDir, 'guides')}' failed: No space left on device`;
+ await nativeSubscriptionOn(contentDir).deliver([], new Error(failure));
+
+ const refusal = `the file watcher failed before it saw its settle fence: ${failure}`;
+ expect(pending()).toEqual({ rejected: refusal });
+ expect(fencesOnDisk()).toEqual([]);
+ const later = observed(watcher.settle());
+ await new Promise<void>((flushed) => setImmediate(flushed));
+ expect(later()).toEqual({ rejected: refusal });
+ expect(fencesOnDisk()).toEqual([]);
+});
+
+const NO_PROGRESS = `the file watcher handled no batch for ${INTERIM_SETTLE_NO_PROGRESS_MS} ms before it saw its settle fence`;
+
+function flushed(): Promise<void> {
+ return new Promise<void>((done) => setImmediate(done));
+}
+
+test('a settle whose fence the watcher never reports ends after the no-progress window with that cause, and a later settle still works', async () => {
+ const { watcher } = await watchContent('parcel');
+ vi.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout'] });
+
+ const lost = observed(watcher.settle());
+ expect(fencesOnDisk()).toHaveLength(1);
+ await vi.advanceTimersByTimeAsync(INTERIM_SETTLE_NO_PROGRESS_MS - 1);
+ await flushed();
+ expect(lost()).toBe('pending');
+ await vi.advanceTimersByTimeAsync(1);
+ await flushed();
+
+ expect(lost()).toEqual({ rejected: NO_PROGRESS });
+ expect(fencesOnDisk()).toEqual([]);
+ const later = observed(watcher.settle());
+ const [fence] = fencesOnDisk();
+ await nativeSubscriptionOn(contentDir).deliver([
+ { type: 'create', path: join(contentDir, '.ok', fence) },
+ ]);
+ expect(later()).toBe('resolved');
+ expect(fencesOnDisk()).toEqual([]);
+});
+
+test('a watcher that keeps handling batches is not ended by the no-progress window, and its settle resolves once the fence arrives', async () => {
+ const { watcher, events } = await watchContent('parcel');
+ vi.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout'] });
+
+ const settle = observed(watcher.settle());
+ const [fence] = fencesOnDisk();
+ for (const name of ['first', 'second', 'third']) {
+ await vi.advanceTimersByTimeAsync(INTERIM_SETTLE_NO_PROGRESS_MS - 1);
+ const page = join(contentDir, `${name}.md`);
+ writeFileSync(page, `# ${name}\n`);
+ await nativeSubscriptionOn(contentDir).deliver([{ type: 'create', path: page }]);
+ }
+ await vi.advanceTimersByTimeAsync(INTERIM_SETTLE_NO_PROGRESS_MS - 1);
+ await flushed();
+
+ expect(settle()).toBe('pending');
+ expect(events).toEqual(
+ ['first', 'second', 'third'].map((name) => ({
+ kind: 'create',
+ path: join(contentDir, `${name}.md`),
+ docName: name,
+ content: `# ${name}\n`,
+ })),
+ );
+ await nativeSubscriptionOn(contentDir).deliver([
+ { type: 'create', path: join(contentDir, '.ok', fence) },
+ ]);
+ expect(settle()).toBe('resolved');
+});
+
+test('a batch whose handling never finishes ends a pending settle at the no-progress window, though new batches keep arriving behind it', async () => {
+ const left = join(contentDir, 'left.md');
+ const reached = Promise.withResolvers<void>();
+ const handling = Promise.withResolvers<void>();
+ latches.push(handling.resolve);
+ const { watcher } = await watchContent('parcel', (event) => {
+ if (event.kind !== 'delete') return undefined;
+ reached.resolve();
+ return handling.promise;
+ });
+ vi.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout'] });
+ unlinkSync(left);
+
+ const settle = observed(watcher.settle());
+ const [fence] = fencesOnDisk();
+ void nativeSubscriptionOn(contentDir).deliver([
+ { type: 'delete', path: left },
+ { type: 'create', path: join(contentDir, '.ok', fence) },
+ ]);
+ await reached.promise;
+ for (const name of ['first', 'second']) {
+ await vi.advanceTimersByTimeAsync(INTERIM_SETTLE_NO_PROGRESS_MS / 2 - 1);
+ const page = join(contentDir, `${name}.md`);
+ writeFileSync(page, `# ${name}\n`);
+ void nativeSubscriptionOn(contentDir).deliver([{ type: 'create', path: page }]);
+ }
+ await flushed();
+ expect(settle()).toBe('pending');
+ await vi.advanceTimersByTimeAsync(2);
+ await flushed();
+
+ expect(settle()).toEqual({ rejected: NO_PROGRESS });
+ expect(fencesOnDisk()).toEqual([]);
+});
+
+test('a settle whose fence cannot be written fails with that error and leaves nothing behind', async () => {
+ rmSync(join(contentDir, '.ok'), { recursive: true });
+ writeFileSync(join(contentDir, '.ok'), 'not a directory\n');
+ const { watcher } = await watchContent('parcel');
+
+ await expect(watcher.settle()).rejects.toMatchObject({ code: 'EEXIST' });
+ expect(readFileSync(join(contentDir, '.ok'), 'utf8')).toBe('not a directory\n');
+});
+
+test('with no settle pending, a batch and a dropped-events recovery are handled as before settling existed, and nothing is written under .ok', async () => {
+ const fresh = join(contentDir, 'fresh.md');
+ const left = join(contentDir, 'left.md');
+ const guides = join(contentDir, 'guides');
+ const start = join(guides, 'start.md');
+ const notes = join(contentDir, 'notes.txt');
+ const strayName = 'watcher-settle-fence-left-by-an-earlier-run';
+ const stray = join(contentDir, '.ok', strayName);
+ const { watcher, events } = await watchContent('parcel');
+ writeFileSync(fresh, '# Fresh\n');
+ writeFileSync(left, '# Left again\n');
+ mkdirSync(guides);
+ writeFileSync(start, '# Start\n');
+ writeFileSync(notes, 'notes\n');
+ writeFileSync(stray, '');
+
+ await nativeSubscriptionOn(contentDir).deliver([
+ { type: 'create', path: fresh },
+ { type: 'update', path: left },
+ { type: 'create', path: guides },
+ { type: 'create', path: start },
+ { type: 'create', path: notes },
+ { type: 'create', path: stray },
+ ]);
+ unlinkSync(notes);
+ await nativeSubscriptionOn(contentDir).deliver([], new Error(DROP_NOTICE));
+
+ expect(events).toEqual([
+ { kind: 'create', path: fresh, docName: 'fresh', content: '# Fresh\n' },
+ { kind: 'create', path: start, docName: 'guides/start', content: '# Start\n' },
+ {
+ kind: 'update',
+ path: left,
+ docName: 'left',
+ content: '# Left again\n',
+ previousIndexedFields: { title: 'Left' },
+ },
+ { kind: 'folder-create', path: guides, relativePath: 'guides' },
+ { kind: 'asset-create', path: notes, relativePath: 'notes.txt' },
+ {
+ kind: 'file-create',
+ path: notes,
+ relativePath: 'notes.txt',
+ size: 6,
+ modifiedTs: expect.any(Number),
+ inode: expect.any(Number),
+ },
+ { kind: 'asset-delete', path: notes, relativePath: 'notes.txt' },
+ { kind: 'file-delete', path: notes, relativePath: 'notes.txt' },
+ ]);
+ expect(decisionsFor(strayName)).toEqual([{ decision: 'drop-filter-excluded', kind: 'create' }]);
+ expect(indexOf(watcher)).toEqual({
+ files: ['left', 'fresh', 'guides/start'],
+ folders: ['.ok', 'guides'],
+ });
+ expect(readdirSync(join(contentDir, '.ok'))).toEqual([strayName]);
+});
+
+test('a settle fence reaches no consumer of .ok files, while a template edit in the same batch is still dispatched', async () => {
+ const template = join(contentDir, '.ok', 'templates', 'note.md');
+ mkdirSync(join(contentDir, '.ok', 'templates'));
+ mkdirSync(join(contentDir, '.ok', 'schemas'));
+ mkdirSync(join(contentDir, 'docs', '.ok'), { recursive: true });
+ writeFileSync(join(contentDir, '.ok', 'config.yml'), 'content:\n dir: .\n');
+ writeFileSync(join(contentDir, '.ok', 'schemas', 'frontmatter.json'), '{}\n');
+ writeFileSync(join(contentDir, 'docs', '.ok', 'frontmatter.yml'), 'type: doc\n');
+ writeFileSync(template, '# Note template\n');
+ const { watcher, events } = await watchContent('parcel');
+ const indexBefore = indexOf(watcher);
+
+ const settle = observed(watcher.settle());
+ const [fence] = fencesOnDisk();
+ writeFileSync(template, '# Note template, revised\n');
+ await nativeSubscriptionOn(contentDir).deliver([
+ { type: 'update', path: template },
+ { type: 'create', path: join(contentDir, '.ok', fence) },
+ ]);
+ expect(settle()).toBe('resolved');
+ expect(fencesOnDisk()).toEqual([]);
+ await nativeSubscriptionOn(contentDir).deliver([
+ { type: 'delete', path: join(contentDir, '.ok', fence) },
+ ]);
+
+ expect(events).toEqual([
+ {
+ kind: 'update',
+ path: template,
+ docName: '.ok/templates/note',
+ content: '# Note template, revised\n',
+ previousIndexedFields: { title: 'Note template' },
+ },
+ ]);
+ expect(decisionsFor(fence)).toEqual([
+ { decision: 'drop-filter-excluded', kind: 'create' },
+ { decision: 'drop-filter-excluded', kind: 'delete' },
+ ]);
+ expect(indexOf(watcher)).toEqual(indexBefore);
+ expect(readdirSync(join(contentDir, '.ok')).sort()).toEqual([
+ 'config.yml',
+ 'schemas',
+ 'templates',
+ ]);
+ expect(isShareableOkArtifact(`.ok/${fence}`)).toBe(false);
+ expect(isShareableOkArtifact('.ok/config.yml')).toBe(true);
+});
+
+test('a chokidar watcher refuses to settle, since it cannot order its batches', async () => {
+ const { watcher } = await watchContent('chokidar');
+
+ await expect(watcher.settle()).rejects.toThrow(
+ 'settling disk events needs the parcel watcher backend, and this watcher runs on chokidar',
+ );
+ expect(fencesOnDisk()).toEqual([]);
+});
diff --git a/packages/server/src/file-watcher.ts b/packages/server/src/file-watcher.ts
index 2130e15b5..e26b6cc70 100644
--- a/packages/server/src/file-watcher.ts
+++ b/packages/server/src/file-watcher.ts
@@ -1,3 +1,4 @@
+import { randomUUID } from 'node:crypto';
import {
type Dirent,
existsSync,
@@ -26,7 +27,13 @@ import {
stripDocExtension,
} from './doc-extensions.ts';
import { canonicalContentPathIsRefused, resolvesIntoPrivateState } from './fs-safety.ts';
-import { classifyFsPath, normalizeFsPath } from './fs-traced.ts';
+import {
+ classifyFsPath,
+ normalizeFsPath,
+ tracedMkdirSync,
+ tracedRmSync,
+ tracedWriteFileSync,
+} from './fs-traced.ts';
import { errnoCode } from './http/handler-utils.ts';
import { getLogger } from './logger.ts';
import {
@@ -252,6 +259,7 @@ export interface WatcherHandle {
pruneFileIndexNowExcluded: () => number;
pruneFolderIndexNowExcluded: () => number;
rescanFromDisk: () => Promise<void>;
+ settle: () => Promise<void>;
}
export const writeTracker = new Map<string, Array<{ hash: string; timestamp: number }>>();
@@ -3438,6 +3446,9 @@ type RecoveryPublicMutations = GeneralMutationRecord & {
hashEffects: Map<string, string | null>;
};
+/* WARN: LEASE_RESET_MS in packages/app/tests/stress/_helpers/reuse/lease-fixture.ts must exceed this bound plus the rest of a slot reset. */
+export const INTERIM_SETTLE_NO_PROGRESS_MS = 20_000;
+
async function startParcelWatcher(
contentDir: string,
contentFilter: ContentFilter | undefined,
@@ -3455,7 +3466,13 @@ async function startParcelWatcher(
onRawBatch?: (absPaths: readonly string[]) => void,
discoveredStructuralDirs?: ReadonlySet<string>,
onRecoveryComplete?: () => Promise<void>,
-): Promise<(AsyncSubscription & { runOrdered(work: () => Promise<void>): Promise<void> }) | null> {
+): Promise<
+ | (AsyncSubscription & {
+ runOrdered(work: () => Promise<void>): Promise<void>;
+ settle(): Promise<void>;
+ })
+ | null
+> {
let parcel: typeof import('@parcel/watcher');
try {
parcel = await import('@parcel/watcher');
@@ -3478,13 +3495,20 @@ async function startParcelWatcher(
: undefined;
let closed = false;
+ let failure: Error | undefined;
+ const settleFences = new Map<string, PromiseWithResolvers<void>>();
+ const settleProgress = new Set<() => void>();
let workTail = Promise.resolve();
const runOrdered = (work: () => Promise<void>): Promise<void> => {
if (closed) return Promise.resolve();
const next = workTail.then(work);
- workTail = next.catch((err) => {
- log.error({ err }, 'parcel batch error');
- });
+ workTail = next
+ .catch((err) => {
+ log.error({ err }, 'parcel batch error');
+ })
+ .then(() => {
+ for (const progressed of settleProgress) progressed();
+ });
return next;
};
let recoveryRunning = false;
@@ -3563,6 +3587,10 @@ async function startParcelWatcher(
const notification = classifyParcelNotification(err, events);
if (notification.kind === 'error') {
log.error({ err: notification.error }, 'parcel watcher callback error');
+ failure = new Error(
+ `the file watcher failed before it saw its settle fence: ${notification.error.message}`,
+ );
+ for (const fence of settleFences.values()) fence.reject(failure);
return;
}
if (notification.kind === 'rescan') {
@@ -3591,9 +3619,17 @@ async function startParcelWatcher(
} catch (handleErr) {
log.error({ err: handleErr }, 'parcel batch error');
}
+ for (const event of notification.events)
+ settleFences.get(basename(event.path))?.resolve();
});
if (notification.kind !== 'rescan') return records;
- return Promise.all([records, requestRecovery()]);
+ const fencesBeforeRescan = [...settleFences.values()];
+ return Promise.all([
+ records,
+ requestRecovery().then(() => {
+ for (const fence of fencesBeforeRescan) fence.resolve();
+ }),
+ ]);
},
subscribeOpts,
platform,
@@ -3601,8 +3637,44 @@ async function startParcelWatcher(
return {
runOrdered,
+ async settle() {
+ if (closed) return;
+ if (failure) throw failure;
+ const name = `watcher-settle-fence-${randomUUID()}`;
+ const fencePath = join(contentDir, '.ok', name);
+ const reached = Promise.withResolvers<void>();
+ let quiet: ReturnType<typeof setTimeout> | undefined;
+ const progressed = (): void => {
+ clearTimeout(quiet);
+ quiet = setTimeout(() => {
+ reached.reject(
+ new Error(
+ `the file watcher handled no batch for ${INTERIM_SETTLE_NO_PROGRESS_MS} ms before it saw its settle fence`,
+ ),
+ );
+ }, INTERIM_SETTLE_NO_PROGRESS_MS);
+ };
+ settleFences.set(name, reached);
+ let fenceWritten = false;
+ try {
+ tracedMkdirSync(dirname(fencePath), { recursive: true });
+ tracedWriteFileSync(fencePath, '');
+ fenceWritten = true;
+ settleProgress.add(progressed);
+ progressed();
+ await reached.promise;
+ } finally {
+ clearTimeout(quiet);
+ settleProgress.delete(progressed);
+ settleFences.delete(name);
+ if (fenceWritten) tracedRmSync(fencePath, { force: true });
+ }
+ },
async unsubscribe() {
closed = true;
+ for (const fence of settleFences.values()) {
+ fence.reject(new Error('the file watcher stopped before it saw its settle fence'));
+ }
await subscription.unsubscribe();
await workTail;
},
@@ -3804,6 +3876,7 @@ export async function startWatcher(
let subscription: AsyncSubscription;
let runOrderedRescan: ((work: () => Promise<void>) => Promise<void>) | undefined;
+ let settleSubscription: (() => Promise<void>) | undefined;
let backend: WatcherBackend;
const forceChokidar = process.env.OK_FILE_WATCHER_BACKEND === 'chokidar';
const platform = opts.platform ?? process.platform;
@@ -3834,6 +3907,7 @@ export async function startWatcher(
if (parcelSub) {
subscription = parcelSub;
runOrderedRescan = parcelSub.runOrdered;
+ settleSubscription = () => parcelSub.settle();
backend = 'parcel';
} else {
if (opts.forceBackend === 'parcel') {
@@ -4112,6 +4186,14 @@ export async function startWatcher(
if (pruned > 0 || aliasesChanged) bumpFileIndexGeneration();
return pruned;
},
+ async settle() {
+ if (settleSubscription === undefined) {
+ throw new Error(
+ 'settling disk events needs the parcel watcher backend, and this watcher runs on chokidar',
+ );
+ }
+ await settleSubscription();
+ },
async rescanFromDisk() {
const reseed = async () => {
const mutations: GeneralMutationRecord = {
diff --git a/packages/server/src/http/test-reset-home-isolation-process.test-helper.ts b/packages/server/src/http/test-reset-home-isolation-process.test-helper.ts
new file mode 100644
index 000000000..a4b56de75
--- /dev/null
+++ b/packages/server/src/http/test-reset-home-isolation-process.test-helper.ts
@@ -0,0 +1,42 @@
+import { homedir } from 'node:os';
+import { bootServer } from '../boot.ts';
+import { ConfigSchema } from '../config/schema.ts';
+
+process.once('disconnect', () => process.exit(0));
+setTimeout(() => process.exit(2), 30_000).unref();
+
+const [contentDir, home, homeMode] = process.argv.slice(2);
+if (
+ contentDir === undefined ||
+ home === undefined ||
+ (homeMode !== 'isolated' && homeMode !== 'process-home')
+) {
+ throw new Error(
+ 'The test-reset home fixture needs a content dir, a home dir and isolated or process-home.',
+ );
+}
+if (homedir() !== home) {
+ throw new Error(
+ `The test-reset home fixture refuses to start: its HOME is ${homedir()}, not the test home ${home}.`,
+ );
+}
+const server = await bootServer({
+ host: '127.0.0.1',
+ config: ConfigSchema.parse({}),
+ contentDir,
+ port: 0,
+ quiet: true,
+ gitEnabled: false,
+ idleShutdownMs: null,
+ enableTestRoutes: true,
+ acpRegistryFetchImpl: (async () => {
+ throw new Error('the test-reset home fixture makes no network requests');
+ }) as typeof fetch,
+ ...(homeMode === 'isolated' ? { configHomedirOverride: home } : {}),
+});
+await server.ready;
+process.send?.({ port: server.port });
+process.on('message', async () => {
+ await server.destroy();
+ process.exit(0);
+});
diff --git a/packages/server/src/http/test-reset-home-isolation.test.ts b/packages/server/src/http/test-reset-home-isolation.test.ts
new file mode 100644
index 000000000..9d4cf03b4
--- /dev/null
+++ b/packages/server/src/http/test-reset-home-isolation.test.ts
@@ -0,0 +1,151 @@
+import { fork } from 'node:child_process';
+import {
+ existsSync,
+ mkdirSync,
+ mkdtempSync,
+ readFileSync,
+ realpathSync,
+ rmSync,
+ writeFileSync,
+} from 'node:fs';
+import { tmpdir } from 'node:os';
+import { dirname, join } from 'node:path';
+import { fileURLToPath } from 'node:url';
+import { resolveConfigPath } from '@inkeep/open-knowledge-core/server';
+import { afterEach, describe, expect, test } from 'vitest';
+
+const FIXTURE = fileURLToPath(
+ new URL('./test-reset-home-isolation-process.test-helper.ts', import.meta.url),
+);
+const SAVED_SETTINGS = '{}\n';
+
+type HomeMode = 'isolated' | 'process-home';
+
+interface ResetServer {
+ readonly port: number;
+ readonly settingsFiles: readonly string[];
+ readonly userSettingsFile: string;
+ stop(): Promise<void>;
+}
+
+const scratchDirs: string[] = [];
+const runningServers: ResetServer[] = [];
+
+afterEach(async () => {
+ for (const server of runningServers.splice(0)) await server.stop();
+ for (const dir of scratchDirs.splice(0)) rmSync(dir, { recursive: true, force: true });
+});
+
+function scratchDir(prefix: string): string {
+ const dir = realpathSync(mkdtempSync(join(tmpdir(), prefix)));
+ scratchDirs.push(dir);
+ return dir;
+}
+
+function seedSavedSettings(
+ contentDir: string,
+ home: string,
+): Pick<ResetServer, 'settingsFiles' | 'userSettingsFile'> {
+ const userSettingsFile = resolveConfigPath('user', contentDir, home);
+ const settingsFiles = [
+ resolveConfigPath('project', contentDir),
+ resolveConfigPath('project-local', contentDir),
+ userSettingsFile,
+ ];
+ for (const file of settingsFiles) {
+ mkdirSync(dirname(file), { recursive: true });
+ writeFileSync(file, SAVED_SETTINGS, 'utf-8');
+ }
+ writeFileSync(join(contentDir, '.ok', '.gitignore'), '', 'utf-8');
+ return { settingsFiles, userSettingsFile };
+}
+
+async function startResetServer(homeMode: HomeMode): Promise<ResetServer> {
+ const contentDir = scratchDir('ok-reset-home-content-');
+ const home = scratchDir('ok-reset-home-');
+ const seeded = seedSavedSettings(contentDir, home);
+ const child = fork(FIXTURE, [contentDir, home, homeMode], {
+ execArgv: ['--import', 'tsx', '--conditions=@inkeep/source'],
+ silent: true,
+ env: {
+ ...process.env,
+ HOME: home,
+ XDG_CONFIG_HOME: join(home, 'config'),
+ TMPDIR: home,
+ TMP: home,
+ TEMP: home,
+ },
+ });
+ let stderr = '';
+ child.stderr?.on('data', (chunk) => {
+ stderr += String(chunk);
+ });
+ child.stdout?.resume();
+ const exited = new Promise<number | null>((resolve) => child.once('exit', resolve));
+ const stop = async () => {
+ if (child.connected) child.send('stop');
+ await exited;
+ };
+ const port = await new Promise<number>((resolve, reject) => {
+ child.once('message', (message) => {
+ if (
+ typeof message !== 'object' ||
+ message === null ||
+ !('port' in message) ||
+ typeof message.port !== 'number'
+ ) {
+ reject(new Error(`The test-reset home fixture sent no port: ${stderr}`));
+ return;
+ }
+ resolve(message.port);
+ });
+ child.once('error', reject);
+ child.once('exit', (code) =>
+ reject(new Error(`The test-reset home fixture exited ${code}: ${stderr}`)),
+ );
+ });
+ const server: ResetServer = { port, ...seeded, stop };
+ runningServers.push(server);
+ return server;
+}
+
+function savedBytes(files: readonly string[]): Record<string, string | null> {
+ return Object.fromEntries(
+ files.map((file) => [file, existsSync(file) ? readFileSync(file, 'utf-8') : null]),
+ );
+}
+
+function postServerReset(server: ResetServer): Promise<Response> {
+ return fetch(`http://127.0.0.1:${server.port}/api/test-reset?scope=server`, {
+ method: 'POST',
+ signal: AbortSignal.timeout(20_000),
+ });
+}
+
+describe('a server-scope test reset and the server’s config home', () => {
+ test('a server started on the process home refuses a server-scope reset and keeps every saved settings file', async () => {
+ const server = await startResetServer('process-home');
+ const before = savedBytes(server.settingsFiles);
+
+ const response = await postServerReset(server);
+
+ expect(response.status).toBe(400);
+ expect(await response.json()).toMatchObject({
+ type: 'urn:ok:error:invalid-request',
+ title: 'scope=server needs a server started with an isolated config home.',
+ });
+ expect(savedBytes(server.settingsFiles)).toEqual(before);
+ expect(before[server.userSettingsFile]).toBe(SAVED_SETTINGS);
+ }, 60_000);
+
+ test('a server started with an isolated config home admits a server-scope reset and removes the saved user settings', async () => {
+ const server = await startResetServer('isolated');
+ expect(readFileSync(server.userSettingsFile, 'utf-8')).toBe(SAVED_SETTINGS);
+
+ const response = await postServerReset(server);
+
+ expect(response.status).toBe(200);
+ expect(await response.json()).toMatchObject({ scope: 'server' });
+ expect(existsSync(server.userSettingsFile)).toBe(false);
+ }, 60_000);
+});
diff --git a/packages/server/src/http/test-reset-watcher-settle.test.ts b/packages/server/src/http/test-reset-watcher-settle.test.ts
new file mode 100644
index 000000000..c5c5baad0
--- /dev/null
+++ b/packages/server/src/http/test-reset-watcher-settle.test.ts
@@ -0,0 +1,91 @@
+import {
+ mkdtempSync,
+ readdirSync,
+ readFileSync,
+ realpathSync,
+ rmSync,
+ writeFileSync,
+} from 'node:fs';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { afterEach, expect, test, vi } from 'vitest';
+import type { BootedServer } from '../boot.ts';
+import { bootCompositionRig } from '../composition-rig.test-helper.ts';
+import {
+ forgetNativeSubscriptions,
+ nativeSubscriptionOn,
+} from '../parcel-watcher-double.test-helper.ts';
+
+vi.mock('@parcel/watcher', async () => {
+ const { parcelWatcherModule } = await import('../parcel-watcher-double.test-helper.ts');
+ return parcelWatcherModule;
+});
+
+const KEPT = '# Kept\n';
+
+const servers: BootedServer[] = [];
+const scratch: string[] = [];
+
+afterEach(async () => {
+ for (const server of servers.splice(0)) await server.destroy();
+ forgetNativeSubscriptions();
+ for (const dir of scratch.splice(0)) rmSync(dir, { recursive: true, force: true });
+});
+
+async function serverWithTestDoc(): Promise<{ server: BootedServer; contentDir: string }> {
+ const contentDir = realpathSync(mkdtempSync(join(tmpdir(), 'ok-reset-settle-')));
+ scratch.push(contentDir);
+ writeFileSync(join(contentDir, 'test-doc.md'), KEPT);
+ const server = await bootCompositionRig(contentDir, { enableTestRoutes: true });
+ servers.push(server);
+ await server.ready;
+ return { server, contentDir };
+}
+
+function postServerReset(server: BootedServer): Promise<Response> {
+ return fetch(`http://127.0.0.1:${server.port}/api/test-reset?scope=server`, {
+ method: 'POST',
+ signal: AbortSignal.timeout(20_000),
+ });
+}
+
+function fencesIn(contentDir: string): string[] {
+ return readdirSync(join(contentDir, '.ok')).filter((name) =>
+ name.startsWith('watcher-settle-fence-'),
+ );
+}
+
+test('a server-scope reset runs once the watcher has handled its settle fence', async () => {
+ const { server, contentDir } = await serverWithTestDoc();
+
+ const reset = postServerReset(server);
+ await expect.poll(() => fencesIn(contentDir)).toHaveLength(1);
+ const [fence] = fencesIn(contentDir);
+ expect(readFileSync(join(contentDir, 'test-doc.md'), 'utf-8')).toBe(KEPT);
+ await nativeSubscriptionOn(contentDir).deliver([
+ { type: 'create', path: join(contentDir, '.ok', fence) },
+ ]);
+ const response = await reset;
+
+ expect(response.status).toBe(200);
+ expect(await response.json()).toMatchObject({ scope: 'server' });
+ expect(readFileSync(join(contentDir, 'test-doc.md'), 'utf-8')).toBe('');
+ expect(fencesIn(contentDir)).toEqual([]);
+});
+
+test('after a watcher error ends its subscription, a server-scope reset answers with that error and resets nothing', async () => {
+ const { server, contentDir } = await serverWithTestDoc();
+ const failure = `inotify_add_watch on '${join(contentDir, 'guides')}' failed: No space left on device`;
+ await nativeSubscriptionOn(contentDir).deliver([], new Error(failure));
+
+ const response = await postServerReset(server);
+
+ expect(response.status).toBe(500);
+ expect(await response.json()).toMatchObject({
+ type: 'urn:ok:error:internal-server-error',
+ title: 'The server-scope reset did not run, because the file watcher could not settle.',
+ detail: `the file watcher failed before it saw its settle fence: ${failure}`,
+ });
+ expect(readFileSync(join(contentDir, 'test-doc.md'), 'utf-8')).toBe(KEPT);
+ expect(fencesIn(contentDir)).toEqual([]);
+});
diff --git a/packages/server/src/http/test-routes.test.ts b/packages/server/src/http/test-routes.test.ts
index b35e86ed4..27331690c 100644
--- a/packages/server/src/http/test-routes.test.ts
+++ b/packages/server/src/http/test-routes.test.ts
@@ -1,14 +1,28 @@
+import { mkdtempSync, readFileSync, realpathSync, rmSync, writeFileSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
import type { Hocuspocus } from '@hocuspocus/server';
-import { describe, expect, test } from 'vitest';
+import { afterEach, describe, expect, test } from 'vitest';
import type { AgentSessionManager } from '../agent-sessions.ts';
+import { makeCaptureRes, makeSyntheticReq } from '../composition-rig.test-helper.ts';
+import type { ConflictAuthority } from '../conflict-authority.ts';
import { loggerFactory } from '../logger.ts';
-import { createTestRoutes } from './test-routes.ts';
+import { createTestRoutes, type TestRouteDeps } from './test-routes.ts';
-type Deps = Parameters<typeof createTestRoutes>[0];
+const contentDirs: string[] = [];
-function buildGroup() {
+afterEach(() => {
+ for (const dir of contentDirs.splice(0)) rmSync(dir, { recursive: true, force: true });
+});
+
+function buildGroup(overrides: Partial<TestRouteDeps> = {}) {
const deps = {
- resetDocumentDurability: undefined,
+ agentPresence: undefined,
+ resetSettingsForTest: undefined,
+ settleDiskEvents: undefined,
+ recentlyRemovedDocs: undefined,
+ conflicts: {} as ConflictAuthority,
+ durabilityState: {} as TestRouteDeps['durabilityState'],
resolveAlias: (docName) => docName,
contentDir: '/nonexistent-content',
log: loggerFactory.getLogger('test'),
@@ -21,10 +35,29 @@ function buildGroup() {
signalChannel: undefined,
flushGitCommit: undefined,
rescanFiles: undefined,
- } satisfies Deps;
+ ...overrides,
+ } satisfies TestRouteDeps;
return createTestRoutes(deps);
}
+async function postTestReset(group: ReturnType<typeof buildGroup>, query: string) {
+ const resolved = group.table.resolve('/api/test-reset');
+ if (resolved === null) throw new Error('/api/test-reset is not registered');
+ const { res, captured } = makeCaptureRes();
+ await resolved.dispatch(
+ makeSyntheticReq({ method: 'POST', url: `/api/test-reset${query}` }),
+ res,
+ );
+ return captured;
+}
+
+function contentDirWithTestDoc(text: string): string {
+ const dir = realpathSync(mkdtempSync(join(tmpdir(), 'test-routes-')));
+ contentDirs.push(dir);
+ writeFileSync(join(dir, 'test-doc.md'), text, 'utf-8');
+ return dir;
+}
+
const FAMILY_PATHS = [
'/api/test-reset',
'/api/test-flush-git',
@@ -42,3 +75,74 @@ describe('createTestRoutes table', () => {
for (const path of FAMILY_PATHS) expect(table.isMutating(path), path).toBe(true);
});
});
+
+describe('POST /api/test-reset scope', () => {
+ test('a server-scope reset on a server without an isolated config home is refused before anything is reset', async () => {
+ const contentDir = contentDirWithTestDoc('the developer’s own notes\n');
+ const closedSessions: Array<string | undefined> = [];
+ const group = buildGroup({
+ contentDir,
+ sessionManager: {
+ closeAll: async (docName?: string) => {
+ closedSessions.push(docName);
+ },
+ } as AgentSessionManager,
+ });
+
+ const captured = await postTestReset(group, '?scope=server');
+
+ expect(captured.status).toBe(400);
+ expect(JSON.parse(captured.body)).toMatchObject({
+ type: 'urn:ok:error:invalid-request',
+ title: 'scope=server needs a server started with an isolated config home.',
+ });
+ expect(closedSessions).toEqual([]);
+ expect(readFileSync(join(contentDir, 'test-doc.md'), 'utf-8')).toBe(
+ 'the developer’s own notes\n',
+ );
+ });
+
+ test.each(['false', 'true', ''])(
+ 'a server-scope reset naming reset-okignore=%s is refused before anything is reset',
+ async (value) => {
+ const contentDir = contentDirWithTestDoc('kept\n');
+ writeFileSync(join(contentDir, '.okignore'), 'kept-by-request/\n', 'utf-8');
+ const closedSessions: Array<string | undefined> = [];
+ const group = buildGroup({
+ contentDir,
+ resetSettingsForTest: () => {},
+ sessionManager: {
+ closeAll: async (docName?: string) => {
+ closedSessions.push(docName);
+ },
+ } as AgentSessionManager,
+ });
+
+ const captured = await postTestReset(group, `?scope=server&reset-okignore=${value}`);
+
+ expect(captured.status).toBe(400);
+ expect(JSON.parse(captured.body)).toMatchObject({
+ type: 'urn:ok:error:invalid-request',
+ title:
+ 'reset-okignore cannot be combined with scope=server: a server-scope reset always restores .okignore.',
+ });
+ expect(closedSessions).toEqual([]);
+ expect(readFileSync(join(contentDir, 'test-doc.md'), 'utf-8')).toBe('kept\n');
+ expect(readFileSync(join(contentDir, '.okignore'), 'utf-8')).toBe('kept-by-request/\n');
+ },
+ );
+
+ test('an unknown scope is rejected as an invalid request before anything is reset', async () => {
+ const contentDir = contentDirWithTestDoc('kept\n');
+ const group = buildGroup({ contentDir, resetSettingsForTest: () => {} });
+
+ const captured = await postTestReset(group, '?scope=everything');
+
+ expect(captured.status).toBe(400);
+ expect(JSON.parse(captured.body)).toMatchObject({
+ type: 'urn:ok:error:invalid-request',
+ title: 'scope must be one of document, server.',
+ });
+ expect(readFileSync(join(contentDir, 'test-doc.md'), 'utf-8')).toBe('kept\n');
+ });
+});
diff --git a/packages/server/src/http/test-routes.ts b/packages/server/src/http/test-routes.ts
index 3156e62e4..89267b530 100644
--- a/packages/server/src/http/test-routes.ts
+++ b/packages/server/src/http/test-routes.ts
@@ -9,7 +9,9 @@ import {
TestRescanFilesSuccessSchema,
TestResetSuccessSchema,
} from '@inkeep/open-knowledge-core';
+import type { AgentPresenceBroadcaster } from '../agent-presence.ts';
import type { AgentSessionManager } from '../agent-sessions.ts';
+import { isConfigDoc, isSystemDoc } from '../cc1-broadcast.ts';
import { CONFIG_VALIDATION_REVERT_ORIGIN } from '../config-edit-origin.ts';
import type { ConflictAuthority } from '../conflict-authority.ts';
import type { ContentFilter } from '../content-filter.ts';
@@ -17,13 +19,34 @@ import { safeContentPath } from '../content-path.ts';
import type { DerivedDocumentIndexApiPort } from '../derived-document-index.ts';
import { canonicalDocName } from '../doc-extensions.ts';
import type { DocumentDurabilityState } from '../document-durability-state.ts';
+import type { RecentlyRemovedDocs } from '../recently-removed-docs.ts';
import { type ApiRouteGroup, createApiRouteGroup } from './api-pipeline.ts';
import { errorResponse } from './error-response.ts';
import { getRequestId } from './request-id.ts';
import { withValidation } from './request-validation.ts';
import { successResponse } from './success-response.ts';
+const TEST_RESET_SCOPES = ['document', 'server'] as const;
+type TestResetScope = (typeof TEST_RESET_SCOPES)[number];
+
+interface TestServerResetResidue {
+ readonly loadedDocuments: readonly string[];
+ readonly clientConnections: number;
+ readonly agentSessions: number;
+ readonly agentPresence: readonly string[];
+ readonly staleExternalWrites: readonly string[];
+ readonly removedDocuments: readonly string[];
+}
+
+function isTestResetScope(value: string): value is TestResetScope {
+ return (TEST_RESET_SCOPES as readonly string[]).includes(value);
+}
+
export interface TestRouteDeps {
+ agentPresence: Pick<AgentPresenceBroadcaster, 'getPresenceMap' | 'clearPresence'> | undefined;
+ resetSettingsForTest: (() => void) | undefined;
+ settleDiskEvents: (() => Promise<void>) | undefined;
+ recentlyRemovedDocs: Pick<RecentlyRemovedDocs, 'entries' | 'delete'> | undefined;
conflicts: ConflictAuthority;
durabilityState: DocumentDurabilityState;
resolveAlias: (docName: string) => string;
@@ -42,6 +65,10 @@ export interface TestRouteDeps {
export function createTestRoutes(deps: TestRouteDeps): ApiRouteGroup {
const {
+ agentPresence,
+ resetSettingsForTest,
+ settleDiskEvents,
+ recentlyRemovedDocs,
conflicts,
durabilityState,
resolveAlias,
@@ -57,11 +84,90 @@ export function createTestRoutes(deps: TestRouteDeps): ApiRouteGroup {
flushGitCommit,
rescanFiles,
} = deps;
+ async function unloadDocumentForTest(docName: string): Promise<void> {
+ const debounceId = `onStoreDocument-${docName}`;
+ if (hocuspocus.debouncer.isDebounced(debounceId)) {
+ await hocuspocus.debouncer.executeNow(debounceId);
+ }
+ const doc = hocuspocus.documents.get(docName);
+ if (doc) await (forceUnloadDocument ?? hocuspocus.unloadDocument.bind(hocuspocus))(doc);
+ durabilityState.deleteReconciledBase(docName);
+ conflicts.dissolveReconcile(docName);
+ }
+
+ function isServerStateDoc(docName: string): boolean {
+ return isSystemDoc(docName) || isConfigDoc(docName);
+ }
+
+ async function resetLoadedState(keep: string): Promise<void> {
+ for (const docName of hocuspocus.documents.keys()) {
+ if (docName === keep || isServerStateDoc(docName)) continue;
+ await unloadDocumentForTest(docName);
+ }
+ for (const agentId of Object.keys(agentPresence?.getPresenceMap() ?? {})) {
+ agentPresence?.clearPresence(agentId);
+ }
+ for (const [docName] of recentlyRemovedDocs?.entries() ?? []) {
+ recentlyRemovedDocs?.delete(docName);
+ }
+ }
+
+ function residualServerState(): TestServerResetResidue {
+ let clientConnections = 0;
+ for (const document of hocuspocus.documents.values()) {
+ clientConnections += document.getConnections().length;
+ }
+ return {
+ loadedDocuments: [...hocuspocus.documents.keys()]
+ .filter((docName) => !isServerStateDoc(docName))
+ .sort(),
+ clientConnections,
+ agentSessions: sessionManager.liveSessionCount,
+ agentPresence: Object.keys(agentPresence?.getPresenceMap() ?? {}).sort(),
+ staleExternalWrites: durabilityState
+ .listStaleExternalWrites()
+ .map((conflict) => conflict.docName)
+ .sort(),
+ removedDocuments: (recentlyRemovedDocs?.entries() ?? []).map(([docName]) => docName).sort(),
+ };
+ }
+
const handleTestReset = withValidation(
EmptyRequestSchema,
async (req, res) => {
try {
const url = new URL(req.url ?? '/', `http://${req.headers.host ?? 'localhost'}`);
+ const scope = url.searchParams.get('scope') ?? 'document';
+ if (!isTestResetScope(scope)) {
+ errorResponse(
+ res,
+ 400,
+ 'urn:ok:error:invalid-request',
+ `scope must be one of ${TEST_RESET_SCOPES.join(', ')}.`,
+ { handler: 'test-reset' },
+ );
+ return;
+ }
+ if (scope === 'server' && url.searchParams.has('reset-okignore')) {
+ errorResponse(
+ res,
+ 400,
+ 'urn:ok:error:invalid-request',
+ 'reset-okignore cannot be combined with scope=server: a server-scope reset always restores .okignore.',
+ { handler: 'test-reset' },
+ );
+ return;
+ }
+ if (scope === 'server' && resetSettingsForTest === undefined) {
+ errorResponse(
+ res,
+ 400,
+ 'urn:ok:error:invalid-request',
+ 'scope=server needs a server started with an isolated config home.',
+ { handler: 'test-reset' },
+ );
+ return;
+ }
const docName = canonicalDocName(
resolveAlias(url.searchParams.get('docName') ?? 'test-doc'),
);
@@ -76,18 +182,33 @@ export function createTestRoutes(deps: TestRouteDeps): ApiRouteGroup {
});
return;
}
- await sessionManager.closeAll(docName);
- hocuspocus.closeConnections(docName);
- const debounceId = `onStoreDocument-${docName}`;
- if (hocuspocus.debouncer.isDebounced(debounceId)) {
- await hocuspocus.debouncer.executeNow(debounceId);
+ if (scope === 'server') {
+ try {
+ await settleDiskEvents?.();
+ } catch (err) {
+ errorResponse(
+ res,
+ 500,
+ 'urn:ok:error:internal-server-error',
+ 'The server-scope reset did not run, because the file watcher could not settle.',
+ {
+ handler: 'test-reset',
+ detail: err instanceof Error ? err.message : String(err),
+ cause: err,
+ },
+ );
+ return;
+ }
}
- const doc = hocuspocus.documents.get(docName);
- if (doc) await (forceUnloadDocument ?? hocuspocus.unloadDocument.bind(hocuspocus))(doc);
- durabilityState.deleteReconciledBase(docName);
- conflicts.dissolveReconcile(docName);
+ await sessionManager.closeAll(scope === 'server' ? undefined : docName);
+ hocuspocus.closeConnections(scope === 'server' ? undefined : docName);
+ await unloadDocumentForTest(docName);
writeFileSync(filePath, '', 'utf-8');
await derivedDocumentIndex?.testOnly?.resetDocumentForTest(docName);
+ if (scope === 'server') {
+ await resetLoadedState(docName);
+ resetSettingsForTest?.();
+ }
const resetOkignoreParam = url.searchParams.get('reset-okignore');
const resetOkignore = resetOkignoreParam !== 'false';
if (resetOkignore) {
@@ -114,7 +235,13 @@ export function createTestRoutes(deps: TestRouteDeps): ApiRouteGroup {
}
}
signalChannel?.('files');
- successResponse(res, 200, TestResetSuccessSchema, {}, { handler: 'test-reset' });
+ successResponse(
+ res,
+ 200,
+ TestResetSuccessSchema,
+ scope === 'server' ? { scope, residue: residualServerState() } : {},
+ { handler: 'test-reset' },
+ );
} catch (e) {
errorResponse(res, 500, 'urn:ok:error:internal-server-error', 'Internal server error.', {
handler: 'test-reset',
diff --git a/packages/server/src/index.ts b/packages/server/src/index.ts
index c0191ac22..8fa1ee430 100644
--- a/packages/server/src/index.ts
+++ b/packages/server/src/index.ts
@@ -1,4 +1,5 @@
export type { Principal } from '@inkeep/open-knowledge-core';
+export { TEST_LOOP_CORE_MARKER } from '@inkeep/open-knowledge-core';
export {
GitDirAccessError,
MalformedGitPointerError,
@@ -684,6 +685,7 @@ export {
spansCurrentPath,
spansPreviousPath,
} from './telemetry-file-sink.ts';
+export { TEST_LOOP_SERVER_MARKER } from './test-loop-marker.ts';
export {
initToleranceTelemetryWriter,
isToleranceTelemetryEnabled,
diff --git a/packages/server/src/server-factory.ts b/packages/server/src/server-factory.ts
index 473e1f4ca..7556a78f9 100644
--- a/packages/server/src/server-factory.ts
+++ b/packages/server/src/server-factory.ts
@@ -13,6 +13,7 @@ import {
CONFIG_DOC_NAME_USER,
CONFIG_DOC_NAMES,
type ConfigDiagnosticsReport,
+ type ConfigDocName,
type ConfigIssue,
createBasenameIndex,
DEFAULT_ATTACHMENT_FOLDER_PATH,
@@ -85,7 +86,11 @@ import {
startConfigFileWatcher,
startMultiPathConfigFileWatcher,
} from './config-file-watcher.ts';
-import { applyExternalConfigChange, isConfigEcho } from './config-persistence.ts';
+import {
+ applyExternalConfigChange,
+ isConfigEcho,
+ resetConfigDocForTest,
+} from './config-persistence.ts';
import { bindConflictAuthority, ConflictAuthority } from './conflict-authority.ts';
import { createGeneratedIndexWarningScope } from './content/generate-index.ts';
import { type GeneratedArtifactEnv, writeGeneratedArtifact } from './content/generated-artifact.ts';
@@ -2583,6 +2588,9 @@ export function createServer(options: ServerOptions): ServerInstance {
getFolderAliasIndex: () => (watcher ? watcher.getFolderAliasIndex() : new Map()),
resolveTrackedFile,
rescanFiles: () => watcher?.rescanFromDisk(),
+ settleDiskEventsForTest: async () => {
+ await watcher?.settle();
+ },
enableTestRoutes,
shadowRef,
flushGitCommit: () => persistence.flushPendingGitCommit(),
@@ -2610,6 +2618,20 @@ export function createServer(options: ServerOptions): ServerInstance {
acpRegistry,
loadAcpCustomAgents: () => loadCustomAgents(lockDir, getLogger('acp-registry')),
homeDirOverride: configHomedirOverride,
+ resetSettingsForTest:
+ configHomedirOverride === undefined
+ ? undefined
+ : () => {
+ for (const configDocName of CONFIG_DOC_NAMES) {
+ resetConfigDocForTest(
+ hocuspocus.documents.get(configDocName) ?? null,
+ configDocName,
+ persistence.configPersistenceCtx,
+ );
+ configFileWatcherResyncs.get(configDocName)?.();
+ applyPersistedConfigToConsumers(configDocName);
+ }
+ },
forceUnloadDocument,
ready,
recentlyRemovedDocs,
@@ -2713,6 +2735,7 @@ export function createServer(options: ServerOptions): ServerInstance {
docName: string;
cleanup: ConfigFileWatcherUnsubscribe;
}> = [];
+ const configFileWatcherResyncs = new Map<ConfigDocName, () => void>();
function safeRescuePath(shadowGitDir: string, docName: string): string | null {
const rescueBase = resolve(shadowGitDir, 'rescue');
@@ -3907,6 +3930,7 @@ export function createServer(options: ServerOptions): ServerInstance {
}
}
configFileWatcherCleanups.length = 0;
+ configFileWatcherResyncs.clear();
} catch (err) {
phaseErrors.push({
phase: 'watcher-unsubscribe',
@@ -4365,6 +4389,7 @@ export function createServer(options: ServerOptions): ServerInstance {
}
});
configFileWatcherCleanups.push({ docName: configDocName, cleanup });
+ configFileWatcherResyncs.set(configDocName, cleanup.resync);
log.info({ docName: configDocName, path: absPath }, '[config-file-watcher] started');
} catch (err) {
log.warn(
@@ -4520,6 +4545,7 @@ export function createServer(options: ServerOptions): ServerInstance {
});
if (ignoreCleanup) {
configFileWatcherCleanups.push({ docName: '__ignore-files__', cleanup: ignoreCleanup });
+ configFileWatcherResyncs.set(CONFIG_DOC_NAME_OKIGNORE, ignoreCleanup.resync);
ignoreLog.info(
{ okignorePath, gitignorePath },
'[ignore-watcher] multi-path watcher started',
diff --git a/packages/server/src/shadow-fixture-caller.test-helper.mjs b/packages/server/src/shadow-fixture-caller.test-helper.mjs
new file mode 100644
index 000000000..9ca02be1c
--- /dev/null
+++ b/packages/server/src/shadow-fixture-caller.test-helper.mjs
@@ -0,0 +1,19 @@
+import { fork } from 'node:child_process';
+
+const owner = fork(new URL('./shadow-fixture-owner.test-helper.mjs', import.meta.url), [], {
+ execArgv: [],
+ stdio: ['ignore', 'inherit', 'inherit', 'ipc'],
+});
+owner.on('message', (message) => {
+ if (process.connected) process.send(message);
+});
+owner.once('close', (code) => {
+ process.exitCode = code ?? 1;
+ if (process.connected) process.disconnect();
+});
+process.on('message', (message) => {
+ if (owner.connected) owner.send(message);
+});
+process.once('disconnect', () => {
+ if (owner.connected) owner.disconnect();
+});
diff --git a/packages/server/src/shadow-fixture-launch.test-helper.mjs b/packages/server/src/shadow-fixture-launch.test-helper.mjs
new file mode 100644
index 000000000..bcf424410
--- /dev/null
+++ b/packages/server/src/shadow-fixture-launch.test-helper.mjs
@@ -0,0 +1,53 @@
+import { fork } from 'node:child_process';
+import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs';
+import { resolve } from 'node:path';
+import { fileURLToPath } from 'node:url';
+
+const [resultDirectory, packageDirectory, ...command] = process.argv.slice(2);
+if (!resultDirectory || !packageDirectory || command.length === 0)
+ throw new Error('fixture launch requires results, cwd, and node arguments');
+const resultDir = resolve(resultDirectory);
+mkdirSync(resultDir, { recursive: true });
+const owner = fork(new URL('./shadow-fixture-owner.test-helper.mjs', import.meta.url), [], {
+ execArgv: [],
+ stdio: ['ignore', 'inherit', 'inherit', 'ipc'],
+});
+writeFileSync(
+ resolve(resultDir, 'launcher.json'),
+ `${JSON.stringify({ pid: process.pid, ownerPid: owner.pid, node: process.version, cwd: process.cwd(), command, packageDirectory: resolve(packageDirectory) })}\n`,
+);
+owner.on('message', (message) => {
+ if (message.type === 'cleaned')
+ process.stdout.write(
+ `${JSON.stringify({ ownerCleanup: message.receipt.reason, nativeRunnerExit: message.receipt.runnerExit, normalResidue: message.receipt.normalResidue })}\n`,
+ );
+});
+owner.once('exit', (code, signal) => {
+ const path = resolve(resultDir, 'cleanup.json');
+ const receipt = existsSync(path) ? JSON.parse(readFileSync(path, 'utf8')) : undefined;
+ writeFileSync(
+ resolve(resultDir, 'launcher-exit.json'),
+ `${JSON.stringify({ ownerExit: { code, signal }, runnerExit: receipt?.runnerExit, workDirAbsent: receipt?.workDirAbsent })}\n`,
+ );
+ process.exitCode =
+ code !== 0 || !receipt?.workDirAbsent || (receipt.normalResidue?.length ?? 0) > 0
+ ? 1
+ : (receipt.runnerExit?.code ?? 1);
+});
+owner.send({
+ type: 'start',
+ config: {
+ resultDir,
+ cwd: resolve(packageDirectory),
+ args: command,
+ scratchRoot: process.env.OK_SHADOW_SCRATCH_ROOT,
+ maxLifetimeMs: Number(process.env.OK_SHADOW_MAX_LIFETIME_MS ?? 540_000),
+ },
+});
+for (const signal of ['SIGINT', 'SIGTERM']) {
+ process.once(signal, () => {
+ if (owner.connected) owner.send({ type: 'release' });
+ });
+}
+if (process.argv[1] !== fileURLToPath(import.meta.url))
+ throw new Error('fixture launch must run as its entrypoint');
diff --git a/packages/server/src/shadow-fixture-owner.test-helper.mjs b/packages/server/src/shadow-fixture-owner.test-helper.mjs
new file mode 100644
index 000000000..38f5ad3b0
--- /dev/null
+++ b/packages/server/src/shadow-fixture-owner.test-helper.mjs
@@ -0,0 +1,377 @@
+import { spawn } from 'node:child_process';
+import { randomUUID } from 'node:crypto';
+import {
+ appendFileSync,
+ existsSync,
+ mkdirSync,
+ mkdtempSync,
+ readdirSync,
+ readFileSync,
+ realpathSync,
+ renameSync,
+ rmSync,
+ watch,
+ writeFileSync,
+} from 'node:fs';
+import { tmpdir } from 'node:os';
+import { basename, join } from 'node:path';
+import { fileURLToPath } from 'node:url';
+
+const preload = fileURLToPath(new URL('./shadow-fixture-preload.test-helper.cjs', import.meta.url));
+const children = new Map();
+const consumed = new Set();
+const events = [];
+const environments = new Map();
+let config;
+let workDir;
+let transport;
+let subscription;
+let scanTimer;
+let deadline;
+let cleanupPromise;
+let runner;
+let runnerExit;
+let stopping = false;
+
+function bindCompilerOwner(owner, child, terminate) {
+ let spawned = false;
+ let disconnected = !owner.connected;
+ const disconnect = () => {
+ disconnected = true;
+ if (spawned) terminate();
+ };
+ owner.once('disconnect', disconnect);
+ child.once('spawn', () => {
+ spawned = true;
+ if (disconnected) terminate();
+ });
+ child.once('exit', () => {
+ spawned = false;
+ owner.off('disconnect', disconnect);
+ });
+ child.once('close', () => owner.off('disconnect', disconnect));
+}
+
+function write(path, value) {
+ writeFileSync(`${path}.pending`, `${JSON.stringify(value)}\n`);
+ renameSync(`${path}.pending`, path);
+}
+
+function notify(message) {
+ if (process.connected) process.send(message, () => {});
+}
+
+function record(event) {
+ const row = { at: Date.now(), ownerPid: process.pid, ...event };
+ events.push(row);
+ if (config) appendFileSync(join(config.resultDir, 'ownership.jsonl'), `${JSON.stringify(row)}\n`);
+ notify({ type: 'event', row });
+}
+
+function signal(entry, value) {
+ if (entry.child.exitCode !== null || entry.child.signalCode !== null) return;
+ record({ type: 'signal', id: entry.id, pid: entry.child.pid, signal: value });
+ entry.child.kill(value);
+}
+
+function environmentFor(value) {
+ const base = environments.get(value.basePid);
+ if (!base) throw new Error('fixture environment was not registered');
+ const env = { ...base, ...value.set };
+ for (const name of value.unset) delete env[name];
+ return env;
+}
+
+function pendingFences() {
+ return readdirSync(workDir, { withFileTypes: true }).flatMap((entry) => {
+ if (!entry.isDirectory() || !entry.name.startsWith('ok-shadow-settle-fence-')) return [];
+ const dir = join(workDir, entry.name, '.ok');
+ return existsSync(dir)
+ ? readdirSync(dir)
+ .filter((name) => name.startsWith('watcher-settle-fence-'))
+ .map((name) => join(dir, name))
+ : [];
+ });
+}
+
+function acquire(id, executable, args, options, onEvent) {
+ if (stopping) throw new Error('fixture owner is closing');
+ const child = spawn(executable, args, options);
+ const closed = Promise.withResolvers();
+ const entry = { id, child, closed: closed.promise, close: undefined };
+ children.set(id, entry);
+ child.on('message', (message) => {
+ if (message.type === 'runtime-environment') {
+ environments.set(child.pid, message.env);
+ record({ type: 'runtime-environment-registered', id, pid: child.pid });
+ write(join(transport, `environment-${child.pid}.ready`), { ready: true });
+ }
+ });
+ child.once('spawn', () => {
+ record({ type: 'spawn', id, pid: child.pid, executable, args, cwd: options.cwd });
+ onEvent({ type: 'spawn', pid: child.pid });
+ });
+ child.on('error', (error) =>
+ onEvent({ type: 'error', code: error.code, message: error.message }),
+ );
+ child.once('exit', (code, value) => {
+ record({ type: 'exit', id, pid: child.pid, code, signal: value });
+ onEvent({ type: 'exit', code, signal: value });
+ });
+ child.once('close', (code, value) => {
+ entry.close = { code, signal: value };
+ record({ type: 'close', id, pid: child.pid, code, signal: value });
+ onEvent({ type: 'close', code, signal: value });
+ closed.resolve();
+ });
+ if (config.autoCleanup) bindCompilerOwner(process, child, () => signal(entry, 'SIGTERM'));
+ return entry;
+}
+
+function request(job) {
+ if (basename(job.executable) === 'node' && !job.args.includes(preload)) {
+ record({ type: 'scope-gap', id: job.id, executable: job.executable, args: job.args });
+ throw new Error('fixture transport has no envelope for this node acquisition');
+ }
+ let sequence = 0;
+ const outputPath = join(transport, `${job.id}.events`);
+ const stdout = [];
+ const stderr = [];
+ let bytes = 0;
+ let error;
+ let timeout;
+ const held = job.kind === 'async' && config.holdCapture && pendingFences().length > 0;
+ if (held) record({ type: 'capture-held', id: job.id });
+ const emit = (event) => {
+ if (!existsSync(transport)) return;
+ if (!held)
+ appendFileSync(outputPath, `${JSON.stringify({ sequence: sequence++, ...event })}\n`);
+ if (event.type === 'error') error = { code: event.code, message: event.message };
+ if (event.type !== 'close') return;
+ clearTimeout(timeout);
+ if (job.kind === 'sync') {
+ write(join(transport, `${job.id}.result`), {
+ pid: entry.child.pid,
+ status: event.code,
+ signal: event.signal,
+ stdout: Buffer.concat(stdout).toString('base64'),
+ stderr: Buffer.concat(stderr).toString('base64'),
+ error,
+ });
+ }
+ };
+ const entry = acquire(
+ job.id,
+ job.executable,
+ job.args,
+ {
+ cwd: job.options.cwd,
+ env: environmentFor(job.options.env),
+ windowsHide: job.options.windowsHide,
+ stdio: ['pipe', 'pipe', 'pipe'],
+ },
+ emit,
+ );
+ for (const [name, stream, chunks] of [
+ ['stdout', entry.child.stdout, stdout],
+ ['stderr', entry.child.stderr, stderr],
+ ]) {
+ stream.on('data', (data) => {
+ bytes += data.length;
+ if (job.kind === 'sync') chunks.push(data);
+ emit({ type: name, data: data.toString('base64') });
+ if (job.kind === 'sync' && bytes > job.options.maxBuffer) {
+ error = { code: 'ENOBUFS', message: `spawnSync ${job.executable} ENOBUFS` };
+ signal(entry, job.options.killSignal ?? 'SIGTERM');
+ }
+ });
+ }
+ entry.child.stdin.on('error', (failure) => {
+ record({ type: 'stdin-error', id: job.id, code: failure.code });
+ });
+ if (job.kind === 'sync' && !config.holdSyncInput) {
+ entry.child.stdin.end(Buffer.from(job.input ?? '', 'base64'));
+ }
+ if (job.options.timeout > 0) {
+ timeout = setTimeout(() => {
+ error = { code: 'ETIMEDOUT', message: `spawnSync ${job.executable} ETIMEDOUT` };
+ signal(entry, job.options.killSignal ?? 'SIGTERM');
+ }, job.options.timeout);
+ }
+}
+
+function scan() {
+ if (stopping) return;
+ for (const name of readdirSync(transport).sort()) {
+ if (!name.endsWith('.request') && !name.endsWith('.action')) continue;
+ if (consumed.has(name)) continue;
+ const job = JSON.parse(readFileSync(join(transport, name), 'utf8'));
+ if (job.runId !== config.runId) throw new Error('fixture request has another run identity');
+ if (name.endsWith('.request')) {
+ consumed.add(name);
+ record({ type: 'request', id: job.id, kind: job.kind, executable: job.executable });
+ request(job);
+ } else {
+ const entry = children.get(job.id);
+ if (!entry) continue;
+ consumed.add(name);
+ if (entry.close) continue;
+ if (job.type === 'stdin') entry.child.stdin.write(Buffer.from(job.data, 'base64'));
+ else if (job.type === 'end') entry.child.stdin.end();
+ else if (job.type === 'kill') signal(entry, job.signal);
+ else throw new Error('unknown fixture transport action');
+ }
+ }
+}
+
+async function cleanup(reason) {
+ if (cleanupPromise) return cleanupPromise;
+ cleanupPromise = (async () => {
+ stopping = true;
+ clearTimeout(deadline);
+ clearInterval(scanTimer);
+ subscription?.close();
+ const before = [...children.values()].filter((entry) => !entry.close).map((entry) => entry.id);
+ for (const entry of children.values()) signal(entry, 'SIGTERM');
+ const escalation = setTimeout(() => {
+ for (const entry of children.values()) signal(entry, 'SIGKILL');
+ }, 1_000);
+ await Promise.all([...children.values()].map((entry) => entry.closed));
+ clearTimeout(escalation);
+ const normalResidue =
+ reason === 'runner closed'
+ ? readdirSync(workDir).filter((name) => name !== 'transport')
+ : null;
+ const runtimePath = join(transport, 'runtimes.jsonl');
+ const runtimes = existsSync(runtimePath)
+ ? readFileSync(runtimePath, 'utf8')
+ .split('\n')
+ .filter(Boolean)
+ .map((line) => JSON.parse(line))
+ : [];
+ rmSync(workDir, { recursive: true, force: true });
+ const receipt = {
+ reason:
+ reason === 'runner closed' &&
+ runtimes.some((row) => row.workerExit !== undefined && row.code !== 0) &&
+ runtimes.some((row) => row.terminateRequested !== undefined)
+ ? 'test worker lost'
+ : reason,
+ trigger: reason,
+ before,
+ runnerExit,
+ normalResidue,
+ children: [...children.values()].map((entry) => ({
+ id: entry.id,
+ pid: entry.child.pid,
+ close: entry.close,
+ })),
+ workDir,
+ workDirAbsent: !existsSync(workDir),
+ events,
+ runtimes,
+ finishedAt: Date.now(),
+ };
+ write(join(config.resultDir, 'cleanup.json'), receipt);
+ notify({ type: 'cleaned', receipt });
+ if (process.connected) process.disconnect();
+ process.exitCode = 0;
+ })();
+ return cleanupPromise;
+}
+
+function start(value) {
+ if (config) throw new Error('fixture owner already started');
+ config = { runId: randomUUID(), autoCleanup: true, maxLifetimeMs: 540_000, ...value };
+ mkdirSync(config.resultDir, { recursive: true });
+ workDir = realpathSync(mkdtempSync(join(config.scratchRoot ?? tmpdir(), 'ok-shadow-owned-')));
+ transport = join(workDir, 'transport');
+ mkdirSync(transport);
+ write(join(config.resultDir, 'started.json'), {
+ ownerPid: process.pid,
+ workDir,
+ runId: config.runId,
+ });
+ subscription = watch(transport, () => scan());
+ scanTimer = setInterval(scan, 20);
+ deadline = setTimeout(() => void cleanup('maximum lifetime reached'), config.maxLifetimeMs);
+ const log = config.logFile ?? join(config.resultDir, 'runner.log');
+ const entry = acquire(
+ 'runner',
+ process.execPath,
+ ['--require', preload, ...config.args],
+ {
+ cwd: config.cwd,
+ env: {
+ ...process.env,
+ ...config.env,
+ TMPDIR: workDir,
+ TMP: workDir,
+ TEMP: workDir,
+ OK_SHADOW_OWNER_TRANSPORT: transport,
+ OK_SHADOW_OWNER_RUN_ID: config.runId,
+ },
+ stdio: ['ignore', 'pipe', 'pipe', 'ipc'],
+ },
+ (event) => {
+ if (event.type === 'close') {
+ runnerExit = { code: event.code, signal: event.signal };
+ void cleanup('runner closed');
+ }
+ },
+ );
+ runner = entry;
+ entry.child.stdout.on('data', (data) => appendFileSync(log, data));
+ entry.child.stderr.on('data', (data) => appendFileSync(log, data));
+ entry.child.on('message', (message) => {
+ if (message.type === 'runtime-environment') return;
+ record(message);
+ if (message.type === 'worker-loss') {
+ if (stopping) return;
+ const activeGit = [...children.values()].some(
+ (child) => child.id !== 'runner' && !child.close,
+ );
+ const scratch = readdirSync(workDir).some((name) => name !== 'transport');
+ if (activeGit || scratch) void cleanup('test worker lost');
+ }
+ });
+ notify({ type: 'started', workDir, runId: config.runId });
+}
+
+process.on('message', (message) => {
+ if (message.type === 'start') start(message.config);
+ else if (message.type === 'release') void cleanup('normal release');
+ else if (message.type === 'lose-runner') signal(runner, 'SIGKILL');
+ else if (message.type === 'lose-worker') runner.child.send({ type: 'lose-worker' });
+ else if (message.type === 'hold-git')
+ acquire(
+ 'held-native-git',
+ 'git',
+ ['hash-object', '--stdin'],
+ {
+ cwd: workDir,
+ env: Object.fromEntries(
+ Object.entries(process.env).filter(([name]) => !name.startsWith('GIT_')),
+ ),
+ stdio: ['pipe', 'pipe', 'pipe'],
+ },
+ () => {},
+ );
+ else if (message.type === 'end-input') {
+ for (const entry of children.values()) if (entry.id !== 'runner') entry.child.stdin.end();
+ } else throw new Error('unknown fixture owner action');
+});
+process.once('disconnect', () => {
+ if (config?.autoCleanup) void cleanup('caller disconnected');
+});
+process.once('uncaughtException', (error) => {
+ if (!config) {
+ process.exitCode = 1;
+ if (process.connected) process.disconnect();
+ return;
+ }
+ record({ type: 'owner-error', name: error.name });
+ void cleanup('owner error').then(() => {
+ process.exitCode = 1;
+ });
+});
diff --git a/packages/server/src/shadow-fixture-preload.test-helper.cjs b/packages/server/src/shadow-fixture-preload.test-helper.cjs
new file mode 100644
index 000000000..90e9e0e12
--- /dev/null
+++ b/packages/server/src/shadow-fixture-preload.test-helper.cjs
@@ -0,0 +1,219 @@
+const cp = require('node:child_process');
+const { randomUUID } = require('node:crypto');
+const {
+ appendFileSync,
+ existsSync,
+ readFileSync,
+ renameSync,
+ watch,
+ writeFileSync,
+} = require('node:fs');
+const { syncBuiltinESMExports } = require('node:module');
+const { join } = require('node:path');
+const { PassThrough, Writable } = require('node:stream');
+const threads = require('node:worker_threads');
+
+const transport = process.env.OK_SHADOW_OWNER_TRANSPORT;
+const runId = process.env.OK_SHADOW_OWNER_RUN_ID;
+const clock = new Int32Array(new SharedArrayBuffer(4));
+const NativeChildProcess = cp.ChildProcess;
+const environmentKey = 'ok-shadow-fixture-environment';
+const baseline = threads.isMainThread
+ ? { ...process.env }
+ : threads.getEnvironmentData(environmentKey);
+
+function environmentFor(env) {
+ return {
+ basePid: process.pid,
+ set: Object.fromEntries(
+ Object.entries(env).filter(([name, value]) => baseline[name] !== value),
+ ),
+ unset: Object.keys(baseline).filter((name) => !(name in env)),
+ };
+}
+
+function submit(name, value) {
+ const path = join(transport, name);
+ writeFileSync(`${path}.pending`, JSON.stringify({ runId, ...value }));
+ renameSync(`${path}.pending`, path);
+}
+
+function optionsFor(options) {
+ if (options.shell || options.detached)
+ throw new Error('fixture transport does not support this launch shape');
+ return {
+ cwd: options.cwd ?? process.cwd(),
+ env: environmentFor(options.env ?? process.env),
+ windowsHide: options.windowsHide,
+ timeout: options.timeout ?? 0,
+ maxBuffer: options.maxBuffer ?? 1_048_576,
+ killSignal: options.killSignal ?? 'SIGTERM',
+ };
+}
+
+function sync(executable, args = [], options = {}) {
+ const id = randomUUID();
+ const selected = optionsFor(options);
+ submit(`${id}.request`, {
+ id,
+ kind: 'sync',
+ executable,
+ args,
+ options: selected,
+ input: options.input === undefined ? undefined : Buffer.from(options.input).toString('base64'),
+ });
+ const resultPath = join(transport, `${id}.result`);
+ const end = Date.now() + Math.max(selected.timeout + 10_000, 540_000);
+ while (!existsSync(resultPath)) {
+ if (!existsSync(transport) || Date.now() > end)
+ throw new Error('fixture transport ended before synchronous native result');
+ Atomics.wait(clock, 0, 0, 10);
+ }
+ const result = JSON.parse(readFileSync(resultPath, 'utf8'));
+ const decode = (data) => {
+ const buffer = Buffer.from(data, 'base64');
+ return options.encoding ? buffer.toString(options.encoding) : buffer;
+ };
+ result.stdout = decode(result.stdout);
+ result.stderr = decode(result.stderr);
+ result.output = [null, result.stdout, result.stderr];
+ result.error &&= Object.assign(new Error(result.error.message), result.error);
+ return result;
+}
+
+class FixtureChild extends NativeChildProcess {
+ constructor(executable, args, options) {
+ super();
+ const id = randomUUID();
+ let counter = 0;
+ let consumed = 0;
+ let complete = false;
+ const eventPath = join(transport, `${id}.events`);
+ const action = (value) =>
+ submit(`${id}-${String(counter++).padStart(8, '0')}.action`, { id, ...value });
+ this.stdout = new PassThrough();
+ this.stderr = new PassThrough();
+ this.stdin = new Writable({
+ write(data, _encoding, done) {
+ action({ type: 'stdin', data: Buffer.from(data).toString('base64') });
+ done();
+ },
+ final(done) {
+ action({ type: 'end' });
+ done();
+ },
+ });
+ this.stdio = [this.stdin, this.stdout, this.stderr];
+ this.spawnfile = executable;
+ this.spawnargs = [executable, ...args];
+ this.kill = (value = 'SIGTERM') => {
+ if (complete || this.exitCode !== null || this.signalCode !== null) return false;
+ this.killed = true;
+ action({ type: 'kill', signal: value });
+ return true;
+ };
+ const read = () => {
+ if (complete || !existsSync(eventPath)) return;
+ const rows = readFileSync(eventPath, 'utf8')
+ .split('\n')
+ .filter(Boolean)
+ .map((line) => JSON.parse(line));
+ for (const row of rows.slice(consumed)) {
+ consumed++;
+ if (row.type === 'spawn') {
+ this.pid = row.pid;
+ this.emit('spawn');
+ } else if (row.type === 'stdout' || row.type === 'stderr') {
+ this[row.type].write(Buffer.from(row.data, 'base64'));
+ } else if (row.type === 'error') {
+ this.emit('error', Object.assign(new Error(row.message), { code: row.code }));
+ } else if (row.type === 'exit') {
+ this.exitCode = row.code;
+ this.signalCode = row.signal;
+ this.emit('exit', row.code, row.signal);
+ } else if (row.type === 'close') {
+ appendFileSync(
+ join(transport, 'runtimes.jsonl'),
+ `${JSON.stringify({ at: Date.now(), pid: process.pid, threadId: threads.threadId, facadeClosed: id, code: row.code, signal: row.signal })}\n`,
+ );
+ complete = true;
+ clearInterval(timer);
+ subscription.close();
+ this.stdin.destroy();
+ this.stdout.end();
+ this.stderr.end();
+ this.emit('close', row.code, row.signal);
+ }
+ }
+ };
+ const subscription = watch(transport, read);
+ const timer = setInterval(read, 20);
+ submit(`${id}.request`, { id, kind: 'async', executable, args, options: optionsFor(options) });
+ }
+}
+
+if (transport) {
+ appendFileSync(
+ join(transport, 'runtimes.jsonl'),
+ `${JSON.stringify({ at: Date.now(), pid: process.pid, threadId: threads.threadId, execArgv: process.execArgv, node: process.version })}\n`,
+ );
+ cp.spawnSync = sync;
+ cp.execFileSync = (executable, args, options = {}) => {
+ const result = sync(executable, args, options);
+ if (result.error || result.status !== 0) {
+ const error =
+ result.error ??
+ new Error(`Command failed: ${executable} ${args.join(' ')}\n${result.stderr}`);
+ throw Object.assign(error, result);
+ }
+ return result.stdout;
+ };
+ cp.spawn = (executable, args = [], options = {}) => new FixtureChild(executable, args, options);
+ if (threads.isMainThread) {
+ threads.setEnvironmentData(environmentKey, baseline);
+ if (process.connected) process.send({ type: 'runtime-environment', env: baseline });
+ const environmentReady = join(transport, `environment-${process.pid}.ready`);
+ const acknowledgementDeadline = Date.now() + 10_000;
+ while (!existsSync(environmentReady)) {
+ if (!existsSync(transport) || Date.now() > acknowledgementDeadline)
+ throw new Error('fixture environment acknowledgement did not arrive');
+ Atomics.wait(clock, 0, 0, 10);
+ }
+ const NativeWorker = threads.Worker;
+ const workers = new Set();
+ threads.Worker = class extends NativeWorker {
+ constructor(filename, options) {
+ const execArgv = options?.execArgv ?? process.execArgv;
+ super(filename, {
+ ...options,
+ execArgv: execArgv.includes(__filename)
+ ? execArgv
+ : ['--require', __filename, ...execArgv],
+ });
+ workers.add(this);
+ const id = this.threadId;
+ if (process.connected) process.send({ type: 'worker-start', threadId: id });
+ this.once('exit', (code) => {
+ workers.delete(this);
+ appendFileSync(
+ join(transport, 'runtimes.jsonl'),
+ `${JSON.stringify({ at: Date.now(), workerExit: id, code })}\n`,
+ );
+ if (code !== 0 && process.connected) process.send({ type: 'worker-loss', code });
+ });
+ }
+ };
+ process.on('message', (message) => {
+ if (message.type !== 'lose-worker') return;
+ for (const worker of workers) {
+ appendFileSync(
+ join(transport, 'runtimes.jsonl'),
+ `${JSON.stringify({ at: Date.now(), terminateRequested: worker.threadId })}\n`,
+ );
+ void worker.terminate();
+ }
+ });
+ process.channel?.unref();
+ }
+ syncBuiltinESMExports();
+}
diff --git a/packages/server/src/shadow-fixture-safety.test.ts b/packages/server/src/shadow-fixture-safety.test.ts
new file mode 100644
index 000000000..a5928dd80
--- /dev/null
+++ b/packages/server/src/shadow-fixture-safety.test.ts
@@ -0,0 +1,431 @@
+import { fork } from 'node:child_process';
+import { createHash } from 'node:crypto';
+import {
+ appendFileSync,
+ existsSync,
+ mkdirSync,
+ mkdtempSync,
+ readdirSync,
+ readFileSync,
+ rmSync,
+ writeFileSync,
+} from 'node:fs';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { fileURLToPath } from 'node:url';
+import { expect, type TestContext, test } from 'vitest';
+
+const OWNER = fileURLToPath(new URL('./shadow-fixture-owner.test-helper.mjs', import.meta.url));
+const SUBJECT = fileURLToPath(new URL('./shadow-fixture-subject.test-helper.mjs', import.meta.url));
+const CALLER = fileURLToPath(new URL('./shadow-fixture-caller.test-helper.mjs', import.meta.url));
+const VITEST = fileURLToPath(new URL('vitest.mjs', import.meta.resolve('vitest/package.json')));
+
+function createOwner(context: TestContext, args = [SUBJECT], configuration = {}, caller = false) {
+ const evidenceRoot = process.env.OK_SHADOW_SAFETY_EVIDENCE_DIR ?? tmpdir();
+ mkdirSync(evidenceRoot, { recursive: true });
+ const resultDir = mkdtempSync(join(evidenceRoot, 'ok-shadow-safety-result-'));
+ const owner = fork(caller ? CALLER : OWNER, [], {
+ execArgv: [],
+ stdio: ['ignore', 'pipe', 'pipe', 'ipc'],
+ });
+ writeFileSync(join(resultDir, 'sibling.txt'), 'Outside the owned scratch root.\n');
+ appendFileSync(
+ join(resultDir, 'parent-events.jsonl'),
+ `${JSON.stringify({ type: 'spawn', pid: owner.pid, at: Date.now() })}\n`,
+ );
+ owner.on('exit', (code, signal) =>
+ appendFileSync(
+ join(resultDir, 'parent-events.jsonl'),
+ `${JSON.stringify({ type: 'exit', code, signal, at: Date.now() })}\n`,
+ ),
+ );
+ owner.on('close', (code, signal) =>
+ appendFileSync(
+ join(resultDir, 'parent-events.jsonl'),
+ `${JSON.stringify({ type: 'close', code, signal, at: Date.now() })}\n`,
+ ),
+ );
+ const exited = new Promise<void>((resolve) => owner.once('exit', () => resolve()));
+ const stdoutClosed = new Promise<void>((resolve) => owner.stdout?.once('close', () => resolve()));
+ const stderrClosed = new Promise<void>((resolve) => owner.stderr?.once('close', () => resolve()));
+ const disconnected = new Promise<void>((resolve) => owner.once('disconnect', () => resolve()));
+ const closed = Promise.all([exited, stdoutClosed, stderrClosed, disconnected]).then(() => {
+ appendFileSync(
+ join(resultDir, 'parent-events.jsonl'),
+ `${JSON.stringify({ type: 'exit-and-streams-closed', at: Date.now() })}\n`,
+ );
+ });
+ owner.stderr?.on('data', (data) => appendFileSync(join(resultDir, 'owner.stderr'), data));
+ owner.stdout?.on('data', (data) => appendFileSync(join(resultDir, 'owner.stdout'), data));
+ context.onTestFinished(async () => {
+ if (owner.connected) owner.send({ type: 'release' });
+ await closed;
+ if (!process.env.OK_SHADOW_SAFETY_EVIDENCE_DIR)
+ rmSync(resultDir, { recursive: true, force: true });
+ });
+ owner.send({
+ type: 'start',
+ config: {
+ resultDir,
+ args,
+ cwd: process.cwd(),
+ maxLifetimeMs: 20_000,
+ env: { OK_SHADOW_SUBJECT_RESULT: join(resultDir, 'result.json') },
+ ...configuration,
+ },
+ });
+ return { owner, closed, resultDir };
+}
+
+function events(resultDir: string): { type: string; args?: string[]; id: string }[] {
+ const path = join(resultDir, 'ownership.jsonl');
+ return existsSync(path)
+ ? readFileSync(path, 'utf8')
+ .split('\n')
+ .filter(Boolean)
+ .map((line) => JSON.parse(line))
+ : [];
+}
+
+function pendingFence(resultDir: string): string | undefined {
+ const startedPath = join(resultDir, 'started.json');
+ if (!existsSync(startedPath)) return undefined;
+ const { workDir } = JSON.parse(readFileSync(startedPath, 'utf8'));
+ if (!existsSync(workDir)) return undefined;
+ for (const entry of readdirSync(workDir, { withFileTypes: true })) {
+ if (!entry.isDirectory() || !entry.name.startsWith('ok-shadow-settle-fence-')) continue;
+ const ok = join(workDir, entry.name, '.ok');
+ if (!existsSync(ok)) continue;
+ const name = readdirSync(ok).find((file) => file.startsWith('watcher-settle-fence-'));
+ if (name) return join(ok, name);
+ }
+ return undefined;
+}
+
+test('native Git acquisitions preserve synchronous results under a live owner', async (context) => {
+ const { resultDir, closed } = createOwner(context);
+ await expect
+ .poll(() => existsSync(join(resultDir, 'cleanup.json')), { timeout: 15_000 })
+ .toBe(true);
+ await closed;
+ const result = JSON.parse(readFileSync(join(resultDir, 'result.json'), 'utf8'));
+ const receipt = JSON.parse(readFileSync(join(resultDir, 'cleanup.json'), 'utf8'));
+ expect(result.output).toMatch(/^git version 2\./);
+ expect(receipt.runnerExit).toEqual({ code: 0, signal: null });
+ expect(receipt.children.filter((child: { id: string }) => child.id !== 'runner')).toHaveLength(1);
+ expect(receipt.children.every((child: { close: unknown }) => child.close)).toBe(true);
+ expect(receipt.normalResidue).toEqual([]);
+ expect(receipt.workDirAbsent).toBe(true);
+});
+
+test('native Git acquisitions and scratch are reclaimed after caller loss', async (context) => {
+ const { resultDir, closed, owner } = createOwner(context, [SUBJECT, 'async-hold'], {
+ maxLifetimeMs: 6_000,
+ });
+ await expect
+ .poll(
+ () => events(resultDir).some((row) => row.type === 'spawn' && row.args?.includes('--stdin')),
+ { timeout: 3_000 },
+ )
+ .toBe(true);
+ const started = JSON.parse(readFileSync(join(resultDir, 'started.json'), 'utf8'));
+ expect(existsSync(started.workDir)).toBe(true);
+ owner.disconnect();
+ await expect
+ .poll(() => existsSync(join(resultDir, 'cleanup.json')), { timeout: 1_000 })
+ .toBe(true);
+ await closed;
+ const receipt = JSON.parse(readFileSync(join(resultDir, 'cleanup.json'), 'utf8'));
+ expect(receipt.reason).toBe('caller disconnected');
+ expect(receipt.before.length).toBeGreaterThan(0);
+ expect(receipt.children.every((child: { close: unknown }) => child.close)).toBe(true);
+ expect(receipt.workDirAbsent).toBe(true);
+});
+
+test('native Git transport preserves byte streams and native failures', async (context) => {
+ const { resultDir, closed } = createOwner(context, [SUBJECT, 'conformance'], {
+ maxLifetimeMs: 8_000,
+ });
+ await expect
+ .poll(() => existsSync(join(resultDir, 'result.json')), { timeout: 6_000 })
+ .toBe(true);
+ await closed;
+ const result = JSON.parse(readFileSync(join(resultDir, 'result.json'), 'utf8'));
+ const receipt = JSON.parse(readFileSync(join(resultDir, 'cleanup.json'), 'utf8'));
+ const input = Buffer.from([65, 0, 66, 10, 255]);
+ const expectedBlob = createHash('sha1')
+ .update(Buffer.from(`blob ${input.length}\0`))
+ .update(input)
+ .digest('hex');
+ expect(Buffer.from(result.listed, 'base64')).toEqual(Buffer.from('line\nname.md\0note.md\0'));
+ expect(result.output).toBe(`${expectedBlob}\n`);
+ expect(result.closed).toEqual({ code: 0, signal: null });
+ const missing = receipt.events.find(
+ (row: { type: string; args?: string[] }) =>
+ row.type === 'spawn' && row.args?.includes('refs/heads/missing'),
+ );
+ expect(missing).toBeDefined();
+ const nativeFailure = receipt.children.find(
+ (child: { id: string }) => child.id === missing.id,
+ ).close;
+ expect(result.rejected.status).toBe(nativeFailure.code);
+ expect(result.rejected.signal).toBe(nativeFailure.signal);
+ expect(result.rejected.stderr.length).toBeGreaterThan(0);
+ expect(receipt.runnerExit).toEqual({ code: 0, signal: null });
+ expect(receipt.normalResidue).toEqual([]);
+ expect(receipt.workDirAbsent).toBe(true);
+});
+
+test('native Git transport preserves synchronous timeouts and asynchronous signals', async (context) => {
+ const { resultDir, closed } = createOwner(context, [SUBJECT, 'termination'], {
+ holdSyncInput: true,
+ });
+ await expect
+ .poll(() => existsSync(join(resultDir, 'result.json')), { timeout: 5_000 })
+ .toBe(true);
+ await closed;
+ const result = JSON.parse(readFileSync(join(resultDir, 'result.json'), 'utf8'));
+ const receipt = JSON.parse(readFileSync(join(resultDir, 'cleanup.json'), 'utf8'));
+ expect(result.timeout).toEqual({ code: 'ETIMEDOUT', status: null, signal: 'SIGTERM' });
+ expect(result.signaled).toBe(true);
+ expect(result.closed).toEqual({ code: null, signal: 'SIGTERM' });
+ const git = receipt.children.filter((child: { id: string }) => child.id !== 'runner');
+ expect(git).toHaveLength(2);
+ expect(
+ git.every((child: { close: { signal: string } }) => child.close.signal === 'SIGTERM'),
+ ).toBe(true);
+ expect(receipt.normalResidue).toEqual([]);
+ expect(receipt.workDirAbsent).toBe(true);
+});
+
+test('synchronous native Git is reclaimed after abrupt caller loss', async (context) => {
+ const { resultDir, closed, owner } = createOwner(
+ context,
+ [SUBJECT, 'sync-hold'],
+ { holdSyncInput: true, maxLifetimeMs: 6_000 },
+ true,
+ );
+ await expect
+ .poll(
+ () => events(resultDir).some((row) => row.type === 'spawn' && row.args?.includes('--stdin')),
+ { timeout: 3_000 },
+ )
+ .toBe(true);
+ expect(owner.exitCode).toBeNull();
+ const lostAt = Date.now();
+ owner.kill('SIGKILL');
+ await expect
+ .poll(() => existsSync(join(resultDir, 'cleanup.json')), { timeout: 3_000 })
+ .toBe(true);
+ await closed;
+ const receipt = JSON.parse(readFileSync(join(resultDir, 'cleanup.json'), 'utf8'));
+ expect(owner.signalCode).toBe('SIGKILL');
+ expect(receipt.reason).toBe('caller disconnected');
+ expect(receipt.finishedAt - lostAt).toBeLessThan(3_000);
+ expect(receipt.before.length).toBeGreaterThan(1);
+ expect(receipt.children.every((child: { close: unknown }) => child.close)).toBe(true);
+ expect(receipt.workDirAbsent).toBe(true);
+ expect(readFileSync(join(resultDir, 'sibling.txt'), 'utf8')).toBe(
+ 'Outside the owned scratch root.\n',
+ );
+});
+
+test('the lifetime oracle rejects disabled owner-loss cleanup before bounded fallback', async (context) => {
+ const { resultDir, closed, owner } = createOwner(
+ context,
+ [SUBJECT, 'async-hold'],
+ { autoCleanup: false, maxLifetimeMs: 1_500 },
+ true,
+ );
+ await expect
+ .poll(
+ () => events(resultDir).some((row) => row.type === 'spawn' && row.args?.includes('--stdin')),
+ { timeout: 1_000 },
+ )
+ .toBe(true);
+ const started = JSON.parse(readFileSync(join(resultDir, 'started.json'), 'utf8'));
+ expect(existsSync(started.workDir)).toBe(true);
+ owner.kill('SIGKILL');
+ let rejection: unknown;
+ try {
+ await expect
+ .poll(() => existsSync(join(resultDir, 'cleanup.json')), { timeout: 300 })
+ .toBe(true);
+ } catch (error) {
+ rejection = error;
+ }
+ expect(rejection).toBeInstanceOf(Error);
+ expect(existsSync(started.workDir)).toBe(true);
+ const activeBeforeFallback = events(resultDir);
+ const git = activeBeforeFallback.find(
+ (row) => row.type === 'spawn' && row.args?.includes('--stdin'),
+ );
+ expect(git).toBeDefined();
+ expect(activeBeforeFallback.some((row) => row.type === 'close' && row.id === git?.id)).toBe(
+ false,
+ );
+ writeFileSync(
+ join(resultDir, 'negative-oracle.json'),
+ JSON.stringify({
+ rejection: String(rejection),
+ scratchPresent: true,
+ nativeGitCloseObserved: false,
+ }),
+ );
+ await expect
+ .poll(() => existsSync(join(resultDir, 'cleanup.json')), { timeout: 3_000 })
+ .toBe(true);
+ await closed;
+ const receipt = JSON.parse(readFileSync(join(resultDir, 'cleanup.json'), 'utf8'));
+ expect(receipt.reason).toBe('maximum lifetime reached');
+ expect(receipt.children.every((child: { close: unknown }) => child.close)).toBe(true);
+ expect(receipt.workDirAbsent).toBe(true);
+ expect(readFileSync(join(resultDir, 'sibling.txt'), 'utf8')).toBe(
+ 'Outside the owned scratch root.\n',
+ );
+});
+
+test('the frozen shadow fixture is reclaimed after abrupt test-worker loss', async (context) => {
+ const { resultDir, closed, owner } = createOwner(
+ context,
+ [
+ VITEST,
+ 'run',
+ 'src/shadow-settle-fence.test.ts',
+ '--pool=threads',
+ '--maxWorkers=1',
+ '--no-file-parallelism',
+ '--reporter=verbose',
+ ],
+ { holdCapture: true },
+ );
+ await expect.poll(() => pendingFence(resultDir), { timeout: 10_000, interval: 10 }).toBeDefined();
+ const fence = pendingFence(resultDir);
+ expect(fence).toBeDefined();
+ if (!fence) throw new Error('pending fence disappeared before the owner-loss observation');
+ expect(readFileSync(fence).length).toBe(0);
+ const before = events(resultDir);
+ expect(before.some((row) => row.type === 'worker-start')).toBe(true);
+ expect(before.some((row) => row.type === 'spawn' && row.args?.includes('write-tree'))).toBe(true);
+ await expect
+ .poll(() => events(resultDir).some((row) => row.type === 'capture-held'), { timeout: 3_000 })
+ .toBe(true);
+ owner.send({ type: 'lose-worker' });
+ await expect
+ .poll(() => existsSync(join(resultDir, 'cleanup.json')), { timeout: 5_000 })
+ .toBe(true);
+ await closed;
+ const receipt = JSON.parse(readFileSync(join(resultDir, 'cleanup.json'), 'utf8'));
+ expect(receipt.reason).toBe('test worker lost');
+ expect(
+ receipt.runtimes.some(
+ (row: { workerExit?: number; code: number }) =>
+ row.workerExit !== undefined && row.code !== 0,
+ ),
+ ).toBe(true);
+ expect(receipt.children.every((child: { close: unknown }) => child.close)).toBe(true);
+ expect(receipt.workDirAbsent).toBe(true);
+ expect(existsSync(fence)).toBe(false);
+});
+
+test('the frozen shadow fixture and live native Git are reclaimed after abrupt runner loss', async (context) => {
+ const { resultDir, closed, owner } = createOwner(
+ context,
+ [
+ VITEST,
+ 'run',
+ 'src/shadow-settle-fence.test.ts',
+ '--pool=threads',
+ '--maxWorkers=1',
+ '--no-file-parallelism',
+ '--reporter=verbose',
+ ],
+ { holdCapture: true },
+ );
+ await expect.poll(() => pendingFence(resultDir), { timeout: 10_000, interval: 10 }).toBeDefined();
+ const fence = pendingFence(resultDir);
+ expect(fence).toBeDefined();
+ if (!fence) throw new Error('pending fence disappeared before the runner-loss observation');
+ expect(readFileSync(fence).length).toBe(0);
+ owner.send({ type: 'hold-git' });
+ await expect
+ .poll(
+ () => events(resultDir).some((row) => row.type === 'spawn' && row.id === 'held-native-git'),
+ { timeout: 3_000 },
+ )
+ .toBe(true);
+ expect(
+ events(resultDir).some((row) => row.type === 'close' && row.id === 'held-native-git'),
+ ).toBe(false);
+ owner.send({ type: 'lose-runner' });
+ await expect
+ .poll(() => existsSync(join(resultDir, 'cleanup.json')), { timeout: 5_000 })
+ .toBe(true);
+ await closed;
+ const receipt = JSON.parse(readFileSync(join(resultDir, 'cleanup.json'), 'utf8'));
+ expect(receipt.runnerExit).toEqual({ code: null, signal: 'SIGKILL' });
+ expect(receipt.before).toContain('held-native-git');
+ expect(
+ receipt.children.find((child: { id: string }) => child.id === 'held-native-git').close.signal,
+ ).toBe('SIGTERM');
+ expect(receipt.children.every((child: { close: unknown }) => child.close)).toBe(true);
+ expect(receipt.workDirAbsent).toBe(true);
+ expect(existsSync(fence)).toBe(false);
+ expect(readFileSync(join(resultDir, 'sibling.txt'), 'utf8')).toBe(
+ 'Outside the owned scratch root.\n',
+ );
+});
+
+test('caller loss during acquisition leaves no untethered fixture', async (context) => {
+ const { resultDir, closed, owner } = createOwner(
+ context,
+ [SUBJECT, 'async-hold'],
+ { maxLifetimeMs: 6_000 },
+ true,
+ );
+ await expect
+ .poll(() => existsSync(join(resultDir, 'started.json')), { timeout: 3_000, interval: 10 })
+ .toBe(true);
+ const started = JSON.parse(readFileSync(join(resultDir, 'started.json'), 'utf8'));
+ expect(existsSync(started.workDir)).toBe(true);
+ owner.kill('SIGKILL');
+ await expect
+ .poll(() => existsSync(join(resultDir, 'cleanup.json')), { timeout: 20_000 })
+ .toBe(true);
+ await closed;
+ const receipt = JSON.parse(readFileSync(join(resultDir, 'cleanup.json'), 'utf8'));
+ expect(owner.signalCode).toBe('SIGKILL');
+ expect(receipt.trigger).toBe('caller disconnected');
+ expect(receipt.children.some((child: { id: string }) => child.id === 'runner')).toBe(true);
+ expect(receipt.children.every((child: { close: unknown }) => child.close)).toBe(true);
+ expect(receipt.workDirAbsent).toBe(true);
+ expect(readFileSync(join(resultDir, 'sibling.txt'), 'utf8')).toBe(
+ 'Outside the owned scratch root.\n',
+ );
+});
+
+test('an unsupported node acquisition is rejected and existing resources are reclaimed', async (context) => {
+ const { resultDir, closed } = createOwner(context, [SUBJECT, 'unsupported-node']);
+ await expect
+ .poll(() => existsSync(join(resultDir, 'cleanup.json')), { timeout: 5_000 })
+ .toBe(true);
+ await closed;
+ const receipt = JSON.parse(readFileSync(join(resultDir, 'cleanup.json'), 'utf8'));
+ expect(
+ receipt.events.some(
+ (row: { type: string; args?: string[] }) =>
+ row.type === 'spawn' && row.args?.includes('--stdin'),
+ ),
+ ).toBe(true);
+ expect(receipt.events.some((row: { type: string }) => row.type === 'scope-gap')).toBe(true);
+ expect(receipt.reason).toBe('owner error');
+ expect(receipt.children.some((child: { id: string }) => child.id !== 'runner')).toBe(true);
+ expect(receipt.children.every((child: { close: unknown }) => child.close)).toBe(true);
+ expect(
+ receipt.events.filter(
+ (row: { type: string; executable?: string }) =>
+ row.type === 'spawn' && row.executable === process.execPath,
+ ),
+ ).toHaveLength(1);
+ expect(receipt.workDirAbsent).toBe(true);
+});
diff --git a/packages/server/src/shadow-fixture-subject.test-helper.mjs b/packages/server/src/shadow-fixture-subject.test-helper.mjs
new file mode 100644
index 000000000..05dabfba9
--- /dev/null
+++ b/packages/server/src/shadow-fixture-subject.test-helper.mjs
@@ -0,0 +1,77 @@
+import { Worker } from 'node:worker_threads';
+
+const worker = new Worker(
+ `
+ const { execFileSync, spawn } = require('node:child_process');
+ const { appendFileSync, mkdtempSync, rmSync, writeFileSync } = require('node:fs');
+ const { tmpdir } = require('node:os');
+ const { join } = require('node:path');
+ if (${JSON.stringify(process.argv[2])} === 'async-hold') {
+ const child = spawn('git', ['hash-object', '--stdin']);
+ child.on('error', (error) => { throw error; });
+ } else if (${JSON.stringify(process.argv[2])} === 'unsupported-node') {
+ void (async () => {
+ const child = spawn('git', ['hash-object', '--stdin']);
+ child.on('error', (error) => { throw error; });
+ await new Promise((resolve) => child.on('spawn', resolve));
+ spawn(process.execPath, ['--eval', 'process.exit(0)']);
+ })().catch((error) => { throw error; });
+ } else if (${JSON.stringify(process.argv[2])} === 'sync-hold') {
+ execFileSync('git', ['hash-object', '--stdin'], { timeout: 60_000 });
+ } else if (${JSON.stringify(process.argv[2])} === 'termination') {
+ void (async () => {
+ let timeout;
+ try { execFileSync('git', ['hash-object', '--stdin'], { timeout: 80, killSignal: 'SIGTERM' }); }
+ catch (error) { timeout = { code: error.code, status: error.status, signal: error.signal }; }
+ appendFileSync(process.env.OK_SHADOW_SUBJECT_RESULT + '.stages', 'timeout observed\\n');
+ const child = spawn('git', ['hash-object', '--stdin']);
+ child.on('error', (error) => { throw error; });
+ const closed = new Promise((resolve) => child.on('close', (code, signal) => {
+ appendFileSync(process.env.OK_SHADOW_SUBJECT_RESULT + '.stages', 'close observed\\n');
+ resolve({ code, signal });
+ }));
+ await new Promise((resolve) => child.on('spawn', resolve));
+ appendFileSync(process.env.OK_SHADOW_SUBJECT_RESULT + '.stages', 'spawn observed\\n');
+ const signaled = child.kill('SIGTERM');
+ writeFileSync(process.env.OK_SHADOW_SUBJECT_RESULT, JSON.stringify({ timeout, signaled, closed: await closed }));
+ })().catch((error) => { throw error; });
+ } else if (${JSON.stringify(process.argv[2])} === 'conformance') {
+ void (async () => {
+ const root = mkdtempSync(join(tmpdir(), 'native-git-'));
+ try {
+ const git = (args) => execFileSync('git', args, { cwd: root, timeout: 10_000 });
+ git(['init', '--initial-branch=main', '--object-format=sha1']);
+ git(['config', '--local', 'maintenance.auto', 'false']);
+ writeFileSync(join(root, 'note.md'), '# Note\\n');
+ writeFileSync(join(root, 'line\\nname.md'), '# Line\\n');
+ git(['add', '-A']);
+ const listed = git(['ls-files', '-z']);
+ const input = Buffer.from([65, 0, 66, 10, 255]);
+ const child = spawn('git', ['hash-object', '--stdin'], { cwd: root });
+ const output = [];
+ child.stdout.on('data', (data) => output.push(data));
+ const completed = new Promise((resolve, reject) => {
+ child.on('error', reject);
+ child.on('close', (code, signal) => resolve({ code, signal }));
+ });
+ child.stdin.end(input);
+ const closed = await completed;
+ let rejected;
+ try { git(['rev-parse', '--verify', 'refs/heads/missing']); }
+ catch (error) { rejected = { status: error.status, signal: error.signal, stderr: error.stderr.toString() }; }
+ writeFileSync(process.env.OK_SHADOW_SUBJECT_RESULT, JSON.stringify({
+ listed: listed.toString('base64'), output: Buffer.concat(output).toString(), closed, rejected,
+ }));
+ } finally { rmSync(root, { recursive: true, force: true }); }
+ })().catch((error) => { throw error; });
+ } else {
+ const output = execFileSync('git', ['--version'], { encoding: 'utf8', timeout: 10_000 });
+ writeFileSync(process.env.OK_SHADOW_SUBJECT_RESULT, JSON.stringify({ output }));
+ }
+`,
+ { eval: true },
+);
+worker.on('error', (error) => {
+ process.stderr.write(String(error));
+ process.exitCode = 1;
+});
diff --git a/packages/server/src/shadow-repo.ts b/packages/server/src/shadow-repo.ts
index c99ade2ad..f10dee01f 100644
--- a/packages/server/src/shadow-repo.ts
+++ b/packages/server/src/shadow-repo.ts
@@ -89,6 +89,7 @@ const MACHINE_LOCAL_FILES = OK_MACHINE_LOCAL_ROOT_FILES.map((name) => `${OK_DIR}
const SHADOW_EXCLUDE_PATTERNS: readonly string[] = [
...MACHINE_LOCAL_DIRS.map((dir) => `**/${dir}`),
...MACHINE_LOCAL_FILES.map((file) => `**/${file}`),
+ `**/${OK_DIR}/watcher-settle-fence-*`,
ATOMIC_TEMP_GLOB,
];
diff --git a/packages/server/src/shadow-settle-fence-nested.test.ts b/packages/server/src/shadow-settle-fence-nested.test.ts
new file mode 100644
index 000000000..7bbf9b820
--- /dev/null
+++ b/packages/server/src/shadow-settle-fence-nested.test.ts
@@ -0,0 +1,174 @@
+import { execFileSync } from 'node:child_process';
+import {
+ existsSync,
+ mkdirSync,
+ mkdtempSync,
+ readdirSync,
+ readFileSync,
+ realpathSync,
+ rmSync,
+ writeFileSync,
+} from 'node:fs';
+import { tmpdir } from 'node:os';
+import { dirname, join } from 'node:path';
+import { expect, test, vi } from 'vitest';
+import { configureTestGitRepository } from '../../../test-support/configure-git-fixture.test-helper.ts';
+import { createContentFilter } from './content-filter.ts';
+import { startWatcher, type WatcherHandle } from './file-watcher.ts';
+import {
+ destroyShadowRepo,
+ initShadowRepo,
+ type ShadowHandle,
+ saveVersion,
+ shadowGit,
+} from './shadow-repo.ts';
+
+const INITIAL_DOCUMENT = '# Initial\n';
+const CHANGED_DOCUMENT = '# Changed\n';
+const CONFIGURATION = 'content:\n dir: .\n';
+const TEMPLATE = '# Template\n';
+const ORDINARY_NEIGHBOR = '# Ordinary prefix neighbor\n';
+const TEMPLATE_NEIGHBOR = '# Template prefix neighbor\n';
+
+function diskFiles(root: string, prefix = ''): string[] {
+ return readdirSync(join(root, prefix), { withFileTypes: true }).flatMap((entry) => {
+ if (entry.name === '.git') return [];
+ const path = prefix ? `${prefix}/${entry.name}` : entry.name;
+ return entry.isDirectory() ? diskFiles(root, path) : [path];
+ });
+}
+
+test('checkpoints omit a pending settle fence in a nested content directory', async (context) => {
+ const root = realpathSync(mkdtempSync(join(tmpdir(), 'ok-shadow-settle-fence-nested-')));
+ const content = join(root, 'docs');
+ const release = Promise.withResolvers<void>();
+ let shadow: ShadowHandle | undefined;
+ let watcher: WatcherHandle | undefined;
+ let entered = false;
+ let settleState = 'not-started';
+ let pending: Promise<{ status: string; error?: unknown }> | undefined;
+ context.onTestFinished(async () => {
+ release.resolve();
+ try {
+ const outcome = await pending;
+ if (outcome?.status === 'rejected') throw outcome.error;
+ } finally {
+ try {
+ await watcher?.unsubscribe();
+ } finally {
+ if (shadow) destroyShadowRepo(shadow);
+ rmSync(root, { recursive: true, force: true });
+ }
+ }
+ });
+ context.onTestFinished(() => {
+ vi.unstubAllEnvs();
+ });
+ vi.stubEnv('OK_FILE_WATCHER_BACKEND', 'parcel');
+ mkdirSync(join(content, '.ok/templates'), { recursive: true });
+ mkdirSync(join(content, '.ok/local'), { recursive: true });
+ const expected = (document: string) => [
+ ['docs/note.md', document],
+ ['docs/.ok/config.yml', CONFIGURATION],
+ ['docs/.ok/templates/note.md', TEMPLATE],
+ ['docs/watcher-settle-fence-note.md', ORDINARY_NEIGHBOR],
+ ['docs/.ok/templates/watcher-settle-fence-note.md', TEMPLATE_NEIGHBOR],
+ ];
+ for (const [path, bytes] of expected(INITIAL_DOCUMENT)) writeFileSync(join(root, path), bytes);
+ writeFileSync(join(content, '.ok/local/fixture-state.json'), '{}\n');
+ execFileSync('git', ['init', '--initial-branch=main', root], {
+ env: Object.fromEntries(
+ Object.entries(process.env).filter(([name]) => !name.startsWith('GIT_')),
+ ),
+ stdio: 'pipe',
+ timeout: 10_000,
+ });
+ configureTestGitRepository(root);
+ const handle = await initShadowRepo(root);
+ shadow = handle;
+ const subscription = await startWatcher(
+ content,
+ async (event) => {
+ if (
+ event.kind === 'update' &&
+ event.docName === 'note' &&
+ event.content === CHANGED_DOCUMENT
+ ) {
+ entered = true;
+ await release.promise;
+ }
+ },
+ createContentFilter({ projectDir: root, contentDir: content }),
+ { forceBackend: 'parcel' },
+ );
+ watcher = subscription;
+ expect(subscription.getFileIndex().has('note')).toBe(true);
+ const git = shadowGit(handle);
+ const capture = async () => {
+ const result = await saveVersion(handle, 'docs', [], 'main', 'Nested settle fence capture');
+ const commit = (await git.raw('rev-parse', result.checkpointRef)).trim();
+ expect((await git.raw('cat-file', '-t', commit)).trim()).toBe('commit');
+ return [commit, result.checkpointRef];
+ };
+ const inspect = async (objects: string[], document: string) => {
+ const snapshots: { object: string; paths: string[] }[] = [];
+ for (const object of objects) {
+ const paths = (await git.raw('ls-tree', '-r', '-z', '--name-only', object))
+ .split('\0')
+ .filter(Boolean);
+ for (const [path, bytes] of expected(document)) {
+ expect(paths).toContain(path);
+ expect(await git.raw('show', `${object}:${path}`)).toBe(bytes);
+ }
+ expect(paths).not.toContain('docs/.ok/local/fixture-state.json');
+ snapshots.push({ object, paths });
+ }
+ return snapshots;
+ };
+ const baseline = await inspect(await capture(), INITIAL_DOCUMENT);
+ for (const snapshot of baseline)
+ expect(snapshot.paths.sort()).toEqual(
+ expected(INITIAL_DOCUMENT)
+ .map(([path]) => path)
+ .sort(),
+ );
+ writeFileSync(join(content, 'note.md'), CHANGED_DOCUMENT);
+ await expect.poll(() => entered, { timeout: 10_000 }).toBe(true);
+ const before = new Set(diskFiles(root));
+ settleState = 'pending';
+ pending = subscription.settle().then(
+ () => {
+ settleState = 'resolved';
+ return { status: 'resolved' };
+ },
+ (error: unknown) => {
+ settleState = 'rejected';
+ return { status: 'rejected', error };
+ },
+ );
+ const minted = () => diskFiles(root).filter((path) => !before.has(path));
+ await expect.poll(minted, { timeout: 10_000 }).toHaveLength(1);
+ const fences = minted();
+ for (const fence of fences) {
+ expect(dirname(join(root, fence))).toBe(join(content, '.ok'));
+ expect(readFileSync(join(root, fence)).length).toBe(0);
+ }
+ expect(settleState).toBe('pending');
+ const objects = await capture();
+ const whilePending = await inspect(objects, CHANGED_DOCUMENT);
+ expect(settleState).toBe('pending');
+ for (const fence of fences) expect(existsSync(join(root, fence))).toBe(true);
+ release.resolve();
+ expect(await pending).toEqual({ status: 'resolved' });
+ for (const fence of fences) expect(existsSync(join(root, fence))).toBe(false);
+ const afterCleanup = await inspect(objects, CHANGED_DOCUMENT);
+ const admitted = (snapshots: typeof whilePending) =>
+ snapshots.flatMap(({ object, paths }) => {
+ const captured = paths.filter((path) => fences.includes(path));
+ return captured.length ? [{ object, fences: captured }] : [];
+ });
+ expect({ whilePending: admitted(whilePending), afterCleanup: admitted(afterCleanup) }).toEqual({
+ whilePending: [],
+ afterCleanup: [],
+ });
+});
diff --git a/packages/server/src/shadow-settle-fence.test.ts b/packages/server/src/shadow-settle-fence.test.ts
new file mode 100644
index 000000000..1ffb837a1
--- /dev/null
+++ b/packages/server/src/shadow-settle-fence.test.ts
@@ -0,0 +1,252 @@
+import { execFileSync } from 'node:child_process';
+import {
+ existsSync,
+ mkdirSync,
+ mkdtempSync,
+ readdirSync,
+ readFileSync,
+ realpathSync,
+ rmSync,
+ writeFileSync,
+} from 'node:fs';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { expect, type TestContext, test, vi } from 'vitest';
+import { configureTestGitRepository } from '../../../test-support/configure-git-fixture.test-helper.ts';
+import { createContentFilter } from './content-filter.ts';
+import { startWatcher, type WatcherHandle } from './file-watcher.ts';
+import {
+ buildWipTree,
+ commitWip,
+ commitWipFromTree,
+ destroyShadowRepo,
+ initShadowRepo,
+ SERVICE_WRITER,
+ type ShadowHandle,
+ saveVersion,
+ shadowGit,
+} from './shadow-repo.ts';
+
+const INITIAL_DOCUMENT = '# Initial\n';
+const CHANGED_DOCUMENT = '# Changed\n';
+const CONFIGURATION = 'content:\n dir: .\n';
+const TEMPLATE = '# Template\n';
+const CONTRIBUTOR = {
+ id: 'agent-settle-fence-test',
+ name: 'Settle fence contributor',
+ email: '[email protected]',
+};
+
+type Producer = 'service' | 'contributor' | 'checkpoint';
+type SettleOutcome = { status: 'resolved' } | { status: 'rejected'; error: unknown };
+
+function diskFiles(root: string, prefix = ''): string[] {
+ return readdirSync(join(root, prefix), { withFileTypes: true }).flatMap((entry) => {
+ if (entry.name === '.git') return [];
+ const path = prefix ? `${prefix}/${entry.name}` : entry.name;
+ return entry.isDirectory() ? diskFiles(root, path) : [path];
+ });
+}
+
+async function createFixture(context: TestContext, holdConsumer: boolean) {
+ const root = realpathSync(mkdtempSync(join(tmpdir(), 'ok-shadow-settle-fence-')));
+ const entered = { value: false };
+ const release = Promise.withResolvers<void>();
+ let shadow: ShadowHandle | undefined;
+ let watcher: WatcherHandle | undefined;
+ let pending: Promise<SettleOutcome> | undefined;
+ let settleState = 'not-started';
+ context.onTestFinished(async () => {
+ release.resolve();
+ try {
+ const outcome = await pending;
+ if (outcome?.status === 'rejected') throw outcome.error;
+ } finally {
+ try {
+ await watcher?.unsubscribe();
+ } finally {
+ if (shadow) destroyShadowRepo(shadow);
+ rmSync(root, { recursive: true, force: true });
+ }
+ }
+ });
+ context.onTestFinished(() => {
+ vi.unstubAllEnvs();
+ });
+ vi.stubEnv('OK_FILE_WATCHER_BACKEND', 'parcel');
+ mkdirSync(join(root, '.ok/templates'), { recursive: true });
+ mkdirSync(join(root, '.ok/local'), { recursive: true });
+ writeFileSync(join(root, 'doc.md'), INITIAL_DOCUMENT);
+ writeFileSync(join(root, '.ok/config.yml'), CONFIGURATION);
+ writeFileSync(join(root, '.ok/templates/note.md'), TEMPLATE);
+ writeFileSync(join(root, '.ok/local/fixture-state.json'), '{}\n');
+ execFileSync('git', ['init', '--initial-branch=main', root], {
+ env: Object.fromEntries(
+ Object.entries(process.env).filter(([name]) => !name.startsWith('GIT_')),
+ ),
+ stdio: 'pipe',
+ timeout: 10_000,
+ });
+ configureTestGitRepository(root);
+ const handle = await initShadowRepo(root);
+ shadow = handle;
+ const subscription = await startWatcher(
+ root,
+ async (event) => {
+ if (
+ event.kind === 'update' &&
+ event.docName === 'doc' &&
+ event.content === CHANGED_DOCUMENT
+ ) {
+ entered.value = true;
+ if (holdConsumer) await release.promise;
+ }
+ },
+ createContentFilter({ projectDir: root, contentDir: root }),
+ { forceBackend: 'parcel' },
+ );
+ watcher = subscription;
+ expect(subscription.getFileIndex().has('doc')).toBe(true);
+ return {
+ root,
+ shadow: handle,
+ entered,
+ releaseConsumer: () => release.resolve(),
+ beginSettle() {
+ settleState = 'pending';
+ pending = subscription.settle().then(
+ (): SettleOutcome => {
+ settleState = 'resolved';
+ return { status: 'resolved' };
+ },
+ (error: unknown): SettleOutcome => {
+ settleState = 'rejected';
+ return { status: 'rejected', error };
+ },
+ );
+ return pending;
+ },
+ settleState: () => settleState,
+ };
+}
+
+type Fixture = Awaited<ReturnType<typeof createFixture>>;
+
+async function capture(fixture: Fixture, producer: Producer): Promise<string[]> {
+ const git = shadowGit(fixture.shadow);
+ if (producer === 'checkpoint') {
+ const result = await saveVersion(fixture.shadow, '.', [], 'main', 'Settle fence capture');
+ const commit = (await git.raw('rev-parse', result.checkpointRef)).trim();
+ expect((await git.raw('cat-file', '-t', commit)).trim()).toBe('commit');
+ return [commit, result.checkpointRef];
+ }
+ const writer = producer === 'service' ? SERVICE_WRITER : CONTRIBUTOR;
+ let tree: string | undefined;
+ let commit: string;
+ if (producer === 'service') {
+ commit = await commitWip(fixture.shadow, writer, '.', 'Settle fence capture');
+ } else {
+ tree = await buildWipTree(fixture.shadow, '.');
+ expect((await git.raw('cat-file', '-t', tree)).trim()).toBe('tree');
+ commit = await commitWipFromTree(fixture.shadow, writer, tree, 'Settle fence capture');
+ expect((await git.raw('rev-parse', `${commit}^{tree}`)).trim()).toBe(tree);
+ }
+ const ref = `refs/wip/main/${writer.id}`;
+ expect((await git.raw('rev-parse', ref)).trim()).toBe(commit);
+ expect((await git.raw('cat-file', '-t', commit)).trim()).toBe('commit');
+ return tree ? [tree, commit, ref] : [commit, ref];
+}
+
+async function inspectCapture(fixture: Fixture, objects: readonly string[], document: string) {
+ const git = shadowGit(fixture.shadow);
+ const snapshots: { object: string; paths: string[] }[] = [];
+ for (const object of objects) {
+ const paths = (await git.raw('ls-tree', '-r', '-z', '--name-only', object))
+ .split('\0')
+ .filter(Boolean);
+ for (const [path, content] of [
+ ['doc.md', document],
+ ['.ok/config.yml', CONFIGURATION],
+ ['.ok/templates/note.md', TEMPLATE],
+ ]) {
+ expect(paths).toContain(path);
+ expect(await git.raw('show', `${object}:${path}`)).toBe(content);
+ }
+ expect(paths).not.toContain('.ok/local/fixture-state.json');
+ snapshots.push({ object, paths });
+ }
+ return snapshots;
+}
+
+async function assertPendingFenceOmitted(context: TestContext, producer: Producer) {
+ const fixture = await createFixture(context, true);
+ await inspectCapture(fixture, await capture(fixture, producer), INITIAL_DOCUMENT);
+ writeFileSync(join(fixture.root, 'doc.md'), CHANGED_DOCUMENT);
+ await expect.poll(() => fixture.entered.value, { timeout: 10_000 }).toBe(true);
+ const before = new Set(diskFiles(fixture.root));
+ const settled = fixture.beginSettle();
+ const minted = () => diskFiles(fixture.root).filter((path) => !before.has(path));
+ await expect.poll(minted, { timeout: 10_000 }).toHaveLength(1);
+ const fences = minted();
+ for (const fence of fences) expect(readFileSync(join(fixture.root, fence)).length).toBe(0);
+ expect(fixture.settleState()).toBe('pending');
+ const objects = await capture(fixture, producer);
+ const whilePending = await inspectCapture(fixture, objects, CHANGED_DOCUMENT);
+ expect(fixture.settleState()).toBe('pending');
+ for (const fence of fences) expect(existsSync(join(fixture.root, fence))).toBe(true);
+ fixture.releaseConsumer();
+ expect(await settled).toEqual({ status: 'resolved' });
+ for (const fence of fences) expect(existsSync(join(fixture.root, fence))).toBe(false);
+ const afterCleanup = await inspectCapture(fixture, objects, CHANGED_DOCUMENT);
+ const admitted = (snapshots: typeof whilePending) =>
+ snapshots.flatMap(({ object, paths }) => {
+ const capturedFences = paths.filter((path) => fences.includes(path));
+ return capturedFences.length ? [{ object, fences: capturedFences }] : [];
+ });
+ expect({ whilePending: admitted(whilePending), afterCleanup: admitted(afterCleanup) }).toEqual({
+ whilePending: [],
+ afterCleanup: [],
+ });
+}
+
+test('service WIP snapshots omit a pending settle fence', async (context) => {
+ await assertPendingFenceOmitted(context, 'service');
+});
+
+test('contributor WIP snapshots omit a pending settle fence', async (context) => {
+ await assertPendingFenceOmitted(context, 'contributor');
+});
+
+test('checkpoints omit a pending settle fence', async (context) => {
+ await assertPendingFenceOmitted(context, 'checkpoint');
+});
+
+async function assertNoPendingContentPreserved(context: TestContext, producer: Producer) {
+ const fixture = await createFixture(context, false);
+ await inspectCapture(fixture, await capture(fixture, producer), INITIAL_DOCUMENT);
+ const before = diskFiles(fixture.root).sort();
+ writeFileSync(join(fixture.root, 'doc.md'), CHANGED_DOCUMENT);
+ await expect.poll(() => fixture.entered.value, { timeout: 10_000 }).toBe(true);
+ const snapshots = await inspectCapture(
+ fixture,
+ await capture(fixture, producer),
+ CHANGED_DOCUMENT,
+ );
+ for (const { paths } of snapshots) {
+ expect(paths.sort()).toEqual(['.ok/config.yml', '.ok/templates/note.md', 'doc.md']);
+ }
+ expect(diskFiles(fixture.root).sort()).toEqual(before);
+ expect(fixture.settleState()).toBe('not-started');
+}
+
+test('service WIP snapshots preserve content when no settle is pending', async (context) => {
+ await assertNoPendingContentPreserved(context, 'service');
+});
+
+test('contributor WIP snapshots preserve content when no settle is pending', async (context) => {
+ await assertNoPendingContentPreserved(context, 'contributor');
+});
+
+test('checkpoints preserve content when no settle is pending', async (context) => {
+ await assertNoPendingContentPreserved(context, 'checkpoint');
+});
diff --git a/packages/server/src/test-loop-marker.ts b/packages/server/src/test-loop-marker.ts
new file mode 100644
index 000000000..81fa8eab9
--- /dev/null
+++ b/packages/server/src/test-loop-marker.ts
@@ -0,0 +1 @@
+export const TEST_LOOP_SERVER_MARKER = 'server-base';
diff --git a/test-support/setup-files-contract.test.ts b/test-support/setup-files-contract.test.ts
index ce6469e11..54a9dca0d 100644
--- a/test-support/setup-files-contract.test.ts
+++ b/test-support/setup-files-contract.test.ts
@@ -30,6 +30,7 @@ const KNOWN_TEST_PROJECTS = [
'packages/app/vitest.fidelity.config.ts',
'packages/app/vitest.integration.config.ts',
'packages/app/vitest.node.config.ts',
+ 'packages/app/vitest.worker-lifetime.config.ts',
'packages/cli/vitest.config.ts',
'packages/cli/vitest.e2e.config.ts',
'packages/core/vitest.config.ts',
diff --git a/turbo.json b/turbo.json
index e0b5882b9..223ba40bc 100644
--- a/turbo.json
+++ b/turbo.json
@@ -297,7 +297,8 @@
},
"test:e2e": {
"dependsOn": ["^build", "build"],
- "cache": false
+ "cache": false,
+ "passThroughEnv": ["OK_TEST_E2E_SERVING"]
},
"test:visual": {
"dependsOn": ["^build"],
@@ -324,11 +325,13 @@
},
"test:a11y": {
"dependsOn": ["^build"],
- "cache": false
+ "cache": false,
+ "passThroughEnv": ["OK_TEST_E2E_SERVING"]
},
"@inkeep/open-knowledge-app#test:a11y": {
"dependsOn": ["^build", "build"],
- "cache": false
+ "cache": false,
+ "passThroughEnv": ["OK_TEST_E2E_SERVING"]
},
"test:perf:bench": {
"dependsOn": [],