Multi-Agent Collaboration for Team and Friends
TypeScript agent-automationagent-orchestrationagent-workflowai-agentsai-collaborationai-memoryai-workforceautonomous-agentsclaude-codeclaude-code-skillscodexcoding-agentmulti-agentmultiple-agentone-person-businessone-person-companyone-person-teamopencodesolopreneur
fix(e2e): initialize cold services after the actual health decision (#937)
Generf4265de142d2
diff --git a/src/web/src/test/e2e-service-startup.test.ts b/src/web/src/test/e2e-service-startup.test.tsindex 66bf0fe8e..cc307c01c 100644--- a/src/web/src/test/e2e-service-startup.test.ts+++ b/src/web/src/test/e2e-service-startup.test.ts@@ -1,6 +1,7 @@import { EventEmitter } from "node:events"-import { rmSync } from "node:fs"-import { resolve } from "node:path"+import { existsSync, mkdirSync, readFileSync, rmSync, symlinkSync, writeFileSync } from "node:fs"+import { createRequire } from "node:module"+import { dirname, resolve } from "node:path"import { afterAll, afterEach, beforeEach, describe, expect, it, vi } from "vitest"const transport = vi.hoisted(() => ({ spawn: vi.fn(), spawnSync: vi.fn(), root: "" }))@@ -22,10 +23,16 @@ vi.mock("./e2e-ui/_setup/paths", async () => {})vi.stubEnv("CI", "")-vi.stubEnv("ALOOK_E2E_FORCE_FRESH", "1")-const { startServices, stopServicesAndRestore, stopHealthSupervisor } = await import("./e2e-ui/_setup/services")+vi.stubEnv("ALOOK_E2E_FORCE_FRESH", "")+let runtime = await import("./e2e-ui/_setup/services")const { createLifecycleState, readLifecycleState, writeLifecycleState } = await import("./e2e-ui/_setup/service-lifecycle")+const statePath = resolve(transport.root, "state.json")+const stateDir = resolve(transport.root, "src/web/.wrangler/state")+const backupDir = `${stateDir}.e2e-backup`+const priorFile = resolve(stateDir, "prior-state")+const initializedFile = resolve(stateDir, "initialized-state")+type SimulatedChild = EventEmitter & { pid: number; exitCode: number | null; signalCode: NodeJS.Signals | null }const children: SimulatedChild[] = []const health = () => new Response('{"status":"ok"}', { headers: { "content-type": "application/json" } })@@ -36,15 +43,49 @@ function heldHealth() {return { promise, release }}-beforeEach(() => {+async function loadRuntime() {+ vi.resetModules()+ runtime = await import("./e2e-ui/_setup/services")+}++function priorState() {+ mkdirSync(stateDir, { recursive: true })+ writeFileSync(priorFile, "original local state")+}++function coldHealth() {+ vi.stubGlobal("fetch", vi.fn((url: string) => Promise.resolve(+ url.endsWith("/health") && children.length === 0+ ? new Response('{"status":"unavailable"}', { status: 503 })+ : health(),+ )))+}++function resetCalls() {+ return transport.spawnSync.mock.calls.filter(([command, args]) => command === "pnpm" && args[1] === "db:reset")+}++beforeEach(async () => {+ vi.stubEnv("CI", "")+ vi.stubEnv("ALOOK_E2E_FORCE_FRESH", "")+ await loadRuntime()+ rmSync(stateDir, { recursive: true, force: true })+ rmSync(backupDir, { recursive: true, force: true })children.length = 0transport.spawn.mockReset().mockImplementation(() => {const child = Object.assign(new EventEmitter(), { pid: 2_000_000_000 + children.length, exitCode: null, signalCode: null })children.push(child)return child})- transport.spawnSync.mockReset().mockReturnValue({ status: 0, stdout: "", stderr: "" })- writeLifecycleState(resolve(transport.root, "state.json"), createLifecycleState())+ transport.spawnSync.mockReset().mockImplementation((command, args) => {+ if (command === "pnpm" && args[1] === "db:reset") {+ rmSync(stateDir, { recursive: true, force: true })+ mkdirSync(stateDir, { recursive: true })+ writeFileSync(initializedFile, "initialized")+ }+ return { status: 0, stdout: "", stderr: "" }+ })+ writeLifecycleState(statePath, createLifecycleState())vi.spyOn(process, "kill").mockImplementation((_pid, signal) => {if (signal === 0) throw Object.assign(new Error("simulated process has stopped"), { code: "ESRCH" })return true@@ -53,8 +94,8 @@ beforeEach(() => {})afterEach(async () => {- stopHealthSupervisor()- await stopServicesAndRestore()+ runtime.stopHealthSupervisor()+ await runtime.stopServicesAndRestore()vi.useRealTimers()vi.restoreAllMocks()vi.unstubAllGlobals()@@ -67,6 +108,8 @@ afterAll(() => {describe("normal UI service startup", () => {it("waits for each exact health before launching the next shared-state runtime", async () => {+ vi.stubEnv("ALOOK_E2E_FORCE_FRESH", "1")+ await loadRuntime()const web = heldHealth(), ws = heldHealth()vi.stubGlobal("fetch", vi.fn((url: string) => {if (url === "http://localhost:3000/api/health") return web.promise.then(health)@@ -74,7 +117,7 @@ describe("normal UI service startup", () => {return Promise.resolve(health())}))- const starting = startServices()+ const starting = runtime.startServices()await vi.advanceTimersByTimeAsync(1_000)expect(transport.spawn).toHaveBeenCalledTimes(1)expect(readLifecycleState(resolve(transport.root, "state.json"))?.services.map((service) => service.name)).toEqual(["web"])@@ -90,16 +133,18 @@ describe("normal UI service startup", () => {expect(transport.spawn).toHaveBeenCalledTimes(3)expect(readLifecycleState(resolve(transport.root, "state.json"))?.status).toBe("healthy")expect(vi.mocked(fetch).mock.calls.some(([url]) => url === "http://localhost:8790/health")).toBe(true)- await stopServicesAndRestore()+ await runtime.stopServicesAndRestore()expect(readLifecycleState(resolve(transport.root, "state.json"))?.status).toBe("stopped")expect(process.kill).toHaveBeenCalledWith(-children[0].pid, "SIGTERM")})it("rejects an exit during readiness and never launches later services", async () => {+ vi.stubEnv("ALOOK_E2E_FORCE_FRESH", "1")+ await loadRuntime()const ws = heldHealth()vi.stubGlobal("fetch", vi.fn((url: string) => url === "http://localhost:8789/health" ? ws.promise.then(health) : Promise.resolve(health())))- const starting = startServices()+ const starting = runtime.startServices()const failed = expect(starting).rejects.toThrow("ws-do exited before readiness (code 1, signal null)")await vi.advanceTimersByTimeAsync(1_000)expect(transport.spawn).toHaveBeenCalledTimes(2)@@ -109,8 +154,170 @@ describe("normal UI service startup", () => {expect(transport.spawn).toHaveBeenCalledTimes(2)expect(readLifecycleState(resolve(transport.root, "state.json"))?.failure).toMatchObject({ service: "ws-do", kind: "child-close", code: 1 })- await stopServicesAndRestore()+ await runtime.stopServicesAndRestore()expect(readLifecycleState(resolve(transport.root, "state.json"))?.status).toBe("stopped")expect(process.kill).toHaveBeenCalledWith(-children[1].pid, "SIGTERM")})++ it("initializes an absent database before the first spawn under default local cold startup", async () => {+ coldHealth()+ const spawnChild = transport.spawn.getMockImplementation()!+ const initializedAtSpawn: boolean[] = []+ transport.spawn.mockImplementation(() => {+ initializedAtSpawn.push(existsSync(initializedFile))+ return spawnChild()+ })++ const starting = runtime.startServices()+ await vi.advanceTimersByTimeAsync(1_000)+ await starting++ expect(resetCalls()).toHaveLength(1)+ expect(resetCalls()[0]).toEqual(["pnpm", ["run", "db:reset"], {+ cwd: transport.root, encoding: "utf8", stdio: ["ignore", "ignore", "pipe"],+ }])+ expect(initializedAtSpawn).toEqual([true, true, true])+ expect(readLifecycleState(statePath)?.restorePending).toBe(false)+ expect(existsSync(backupDir)).toBe(false)+ })++ it("backs up prior state and persists restoration before a failed reset, then restores once", async () => {+ priorState()+ coldHealth()+ const observed: Array<{ backup: string; restorePending: boolean | undefined }> = []+ transport.spawnSync.mockImplementation((command, args) => {+ if (command !== "pnpm" || args[1] !== "db:reset") return { status: 0, stdout: "", stderr: "" }+ observed.push({+ backup: readFileSync(resolve(backupDir, "prior-state"), "utf8"),+ restorePending: readLifecycleState(statePath)?.restorePending,+ })+ rmSync(stateDir, { recursive: true, force: true })+ mkdirSync(stateDir, { recursive: true })+ writeFileSync(initializedFile, "partial reset")+ return { status: 1, signal: null, stdout: "", stderr: "migration failed\n" }+ })+ const stderr = vi.spyOn(process.stderr, "write").mockReturnValue(true)+ const starting = runtime.startServices()+ const failed = starting.then(() => null, (error: unknown) => error)+ await vi.advanceTimersByTimeAsync(1_000)+ expect(await failed).toMatchObject({ message: "db:reset failed (exit 1, signal null)" })++ expect(observed).toEqual([{ backup: "original local state", restorePending: true }])+ expect(stderr).toHaveBeenCalledWith("migration failed\n")+ expect(transport.spawn).not.toHaveBeenCalled()+ await runtime.stopServicesAndRestore()+ expect(readFileSync(priorFile, "utf8")).toBe("original local state")+ expect(existsSync(initializedFile)).toBe(false)+ expect(existsSync(backupDir)).toBe(false)+ expect(readLifecycleState(statePath)).toMatchObject({ status: "stopped", restorePending: false })+ writeFileSync(priorFile, "after first restore")+ await runtime.stopServicesAndRestore()+ expect(readFileSync(priorFile, "utf8")).toBe("after first restore")+ expect(resetCalls()).toHaveLength(1)+ })++ it("reuses all exact healthy services once without owning processes or changing prior state", async () => {+ priorState()+ vi.stubGlobal("fetch", vi.fn(() => Promise.resolve(health())))++ expect(await runtime.startServices()).toEqual([])+ for (const url of ["http://localhost:3000/api/health", "http://localhost:8789/health", "http://localhost:8790/health"]) {+ expect(vi.mocked(fetch).mock.calls.filter(([called]) => called === url)).toHaveLength(1)+ }+ expect(transport.spawn).not.toHaveBeenCalled()+ expect(transport.spawnSync).not.toHaveBeenCalled()+ expect(existsSync(backupDir)).toBe(false)+ expect(readLifecycleState(statePath)).toMatchObject({ status: "healthy", restorePending: false, services: [] })+ await runtime.stopServicesAndRestore()+ expect(process.kill).not.toHaveBeenCalled()+ expect(readFileSync(priorFile, "utf8")).toBe("original local state")+ })++ it("initializes all services when only part of the required health batch matches", async () => {+ vi.stubGlobal("fetch", vi.fn((url: string) => Promise.resolve(+ url === "http://localhost:8789/health" && children.length === 0+ ? new Response('{"status":"unavailable"}', { status: 503 })+ : health(),+ )))+ const starting = runtime.startServices()+ await vi.advanceTimersByTimeAsync(1_000)++ expect((await starting).map(({ name }) => name)).toEqual(["web", "ws-do", "queue-worker"])+ expect(resetCalls()).toHaveLength(1)+ expect(readFileSync(initializedFile, "utf8")).toBe("initialized")+ })++ it.each([+ ["ALOOK_E2E_FORCE_FRESH", 3],+ ["CI", 1],+ ] as const)("starts fresh under %s even if existing health would match", async (key, count) => {+ if (key === "CI") {+ const modules = resolve(transport.root, "src/web/node_modules")+ mkdirSync(modules, { recursive: true })+ symlinkSync(dirname(createRequire(import.meta.url).resolve("wrangler/package.json")), resolve(modules, "wrangler"), "dir")+ }+ vi.stubEnv(key, "1")+ await loadRuntime()+ vi.stubGlobal("fetch", vi.fn(() => Promise.resolve(health())))+ const spawnChild = transport.spawn.getMockImplementation()!+ const initialProbeCounts: number[] = []+ transport.spawn.mockImplementation(() => {+ initialProbeCounts.push(vi.mocked(fetch).mock.calls.length)+ return spawnChild()+ })++ const starting = runtime.startServices()+ await vi.advanceTimersByTimeAsync(1_000)+ expect(await starting).toHaveLength(count)+ expect(initialProbeCounts[0]).toBe(0)+ expect(resetCalls()).toHaveLength(1)+ expect(existsSync(initializedFile)).toBe(true)+ expect(readLifecycleState(statePath)?.restorePending).toBe(false)+ })++ it.each([false, true])("keeps a warmup500 fatal with reuse=%s and preserves the correct state owner", async (reuse) => {+ priorState()+ vi.stubGlobal("fetch", vi.fn((url: string) => Promise.resolve(+ url === "http://localhost:3000/c/channels/warmup"+ ? new Response("original warmup failure", { status: 500 })+ : !reuse && url.endsWith("/health") && children.length === 0+ ? new Response('{"status":"unavailable"}', { status: 503 })+ : health(),+ )))+ const starting = runtime.startServices()+ const failed = starting.then(() => null, (error: unknown) => error)+ await vi.advanceTimersByTimeAsync(1_000)+ expect(await failed).toMatchObject({ message: "Route warm-up failed (500 /c/channels/warmup)" })+ expect(resetCalls()).toHaveLength(reuse ? 0 : 1)+ expect(transport.spawn).toHaveBeenCalledTimes(reuse ? 0 : 3)+ expect(readLifecycleState(statePath)?.restorePending).toBe(!reuse)++ const originalAtTermination: boolean[] = []+ vi.mocked(process.kill).mockImplementation((_pid, signal) => {+ if (signal === 0) throw Object.assign(new Error("simulated process has stopped"), { code: "ESRCH" })+ originalAtTermination.push(existsSync(priorFile))+ return true+ })+ await runtime.stopServicesAndRestore()+ expect(originalAtTermination).toEqual(reuse ? [] : [false, false, false])+ expect(readFileSync(priorFile, "utf8")).toBe("original local state")+ expect(existsSync(backupDir)).toBe(false)+ expect(readLifecycleState(statePath)).toMatchObject({ status: "stopped", restorePending: false })+ expect(vi.mocked(fetch).mock.calls.filter(([url]) => url === "http://localhost:3000/c/channels/warmup")).toHaveLength(1)+ })++ it("rejects an uninitialized lifecycle before changing the database or spawning", async () => {+ priorState()+ coldHealth()+ rmSync(statePath)+ const starting = runtime.startServices()+ const failed = starting.then(() => null, (error: unknown) => error)+ await vi.advanceTimersByTimeAsync(1_000)+ expect(await failed).toMatchObject({ message: "service lifecycle state was not initialized" })++ expect(resetCalls()).toHaveLength(0)+ expect(transport.spawn).not.toHaveBeenCalled()+ expect(existsSync(backupDir)).toBe(false)+ expect(readFileSync(priorFile, "utf8")).toBe("original local state")+ })})diff --git a/src/web/src/test/e2e-ui/_setup/global-setup.ts b/src/web/src/test/e2e-ui/_setup/global-setup.tsindex dcfefac38..ba3b8b08b 100644--- a/src/web/src/test/e2e-ui/_setup/global-setup.ts+++ b/src/web/src/test/e2e-ui/_setup/global-setup.ts@@ -5,10 +5,7 @@ import { AUTH_DIR, MANIFEST_PATH, SERVICE_STATE_PATH } from "./paths"import { loginAndSaveState } from "./auth"import {prepareServices,- resetDb,startServices,- backupState,- REUSE_EXISTING,stopServicesAndRestore,} from "./services"import { createLifecycleState, writeLifecycleState } from "./service-lifecycle"@@ -22,14 +19,6 @@ export default async function globalSetup(): Promise<void> {try {prepareServices()- // Local runs: back up the developer's D1/DO state, wipe to a clean DB, and- // restore it on teardown (see global-teardown). CI has no prior state.- let backedUp = false- if (!REUSE_EXISTING) {- backedUp = backupState()- writeLifecycleState(SERVICE_STATE_PATH, createLifecycleState(backedUp))- resetDb()- }await startServices()// Unique-per-run stamp so re-runs against a non-reset DB don't collide.diff --git a/src/web/src/test/e2e-ui/_setup/services.ts b/src/web/src/test/e2e-ui/_setup/services.tsindex 21214a155..09cf94505 100644--- a/src/web/src/test/e2e-ui/_setup/services.ts+++ b/src/web/src/test/e2e-ui/_setup/services.ts@@ -82,10 +82,6 @@ export function readinessExitMessage(: `${service} exited before readiness (code ${exitCode}, signal ${signalCode})`}-// Reuse a server the developer already has running (local iteration). CI-// always starts fresh. Migration-sensitive focused journeys may opt into the-// same fresh path locally; backupState/restoreState preserve developer data.-export const REUSE_EXISTING = !process.env.CI && !process.env.ALOOK_E2E_FORCE_FRESHexport const SINGLE_RUNTIME = !!process.env.CI// Local D1/DO state that `db:reset` wipes. Backing it up to a sibling path@@ -96,7 +92,7 @@ const STATE_DIR = resolve(REPO_ROOT, "src/web/.wrangler/state")const STATE_BACKUP_DIR = resolve(REPO_ROOT, "src/web/.wrangler/state.e2e-backup")// Returns true if a backup was taken (i.e. there was existing state to save).-export function backupState(): boolean {+function backupState(): boolean {if (process.env.CI || !existsSync(STATE_DIR)) return falsermSync(STATE_BACKUP_DIR, { recursive: true, force: true })cpSync(STATE_DIR, STATE_BACKUP_DIR, { recursive: true })@@ -276,7 +272,7 @@ export function prepareServices(): void {}}-export function resetDb(): void {+function resetDb(): void {// `wrangler d1 migrations apply` prints the full migrations table (twice) on// every run — noise in CI when it succeeds. DROP stdout entirely (that table// is the noise) and capture only stderr so a real failure is still legible.@@ -495,8 +491,9 @@ async function warmUpRoutes(): Promise<void> {export async function startServices(): Promise<ManagedService[]> {const definitions = serviceDefinitions(SINGLE_RUNTIME)+ const reuseExisting = !process.env.CI && !process.env.ALOOK_E2E_FORCE_FRESH- if (REUSE_EXISTING && (await Promise.all(definitions.map(({ healthUrl }) => hasExactHealth(healthUrl)))).every((health) => health.ok)) {+ if (reuseExisting && (await Promise.all(definitions.map(({ healthUrl }) => hasExactHealth(healthUrl)))).every((health) => health.ok)) {const state = readLifecycleState(SERVICE_STATE_PATH)if (state) writeLifecycleState(SERVICE_STATE_PATH, { ...state, status: "healthy" })await warmUpRoutes()@@ -512,6 +509,11 @@ export async function startServices(): Promise<ManagedService[]> {killPortOrphans(ports)// Give the OS a moment to release the sockets before we bind them.await new Promise((r) => setTimeout(r, 1000))+ const initialState = readLifecycleState(SERVICE_STATE_PATH)+ if (!initialState) throw new Error("service lifecycle state was not initialized")+ const restorePending = backupState()+ writeLifecycleState(SERVICE_STATE_PATH, { ...initialState, restorePending })+ resetDb()rmSync(SERVICE_LOG_DIR, { recursive: true, force: true })mkdirSync(SERVICE_LOG_DIR, { recursive: true })