Multiplayer agent harness for work.
Renew verified background ownership during slow startup (#1469)
Regan Bellb388454abe03
diff --git a/src/runs/background-controller.ts b/src/runs/background-controller.tsindex 897bb14b1..4df7e3dc3 100644--- a/src/runs/background-controller.ts+++ b/src/runs/background-controller.ts@@ -11,6 +11,7 @@ export interface BackgroundControllerDeps {drained(): Promise<void>;onError(error: unknown): void;validityMs?: number;+ startupTimeoutMs?: number;pollMs?: number;}@@ -24,6 +25,8 @@ export function createBackgroundController(deps: BackgroundControllerDeps) {let watchdog: ReturnType<typeof setTimeout> | null = null;let poller: ReturnType<typeof setInterval> | null = null;let pending: Promise<void> | null = null;+ let starting = false;+ let refreshing: Promise<void> | null = null;let draining: Promise<void> = Promise.resolve();const validityMs = deps.validityMs ?? 10_000;const fence = (): void => {@@ -54,8 +57,44 @@ export function createBackgroundController(deps: BackgroundControllerDeps) {watchdog = setTimeout(fence, validityMs);watchdog.unref?.();};+ const refreshStartup = (): Promise<void> => {+ if (!starting || !running || !activation || activation.signal.aborted) return Promise.resolve();+ if (refreshing) return refreshing;+ const currentActivation = activation;+ const generation = admission;+ refreshing = (async () => {+ try {+ const state = await deps.store.get();+ if (!starting || !running || activation !== currentActivation || currentActivation?.signal.aborted) return;+ const member = state.members.find((entry) => entry.instanceId === deps.identity.instanceId);+ if (+ state.generation !== generation ||+ !member ||+ member.retired ||+ member.state !== "admitted" ||+ member.generation !== generation ||+ !(state.enabled ? state.desiredDeploymentId === deps.identity.deploymentId : deps.legacyEnabled)+ ) {+ fence();+ return;+ }+ renew();+ } catch (error) {+ if (starting && activation === currentActivation) {+ fence();+ deps.onError(error);+ }+ }+ })().finally(() => {+ refreshing = null;+ });+ return refreshing;+ };const reconcile = (): Promise<void> => {- if (pending) return pending;+ if (pending) {+ if (starting) void refreshStartup();+ return pending;+ }pending = (async () => {try {if (!registered) {@@ -82,7 +121,18 @@ export function createBackgroundController(deps: BackgroundControllerDeps) {await release();return;}- await deps.start(activation.signal);+ starting = true;+ const startupDeadline = setTimeout(() => {+ fence();+ deps.onError(new Error("Background startup timed out; waiting for activation cleanup before retrying"));+ }, deps.startupTimeoutMs ?? 120_000);+ startupDeadline.unref?.();+ try {+ await deps.start(activation.signal);+ } finally {+ clearTimeout(startupDeadline);+ starting = false;+ }if (activation.signal.aborted || !running) await release();else await deps.store.markReady(deps.identity.instanceId, state.generation);} else {diff --git a/test/background-controller.test.ts b/test/background-controller.test.tsindex 7e2e53871..f1be31399 100644--- a/test/background-controller.test.ts+++ b/test/background-controller.test.ts@@ -207,3 +207,112 @@ test("activation is not ready until startup completes and expiry fences a late sassert.equal((await store.get()).members[0]?.ready, false);await controller.stop();});++for (const outcome of ["complete", "transition", "read-failure", "read-stall", "stop", "startup-timeout"] as const) {+ test(`pending startup renews verified ownership and fences on ${outcome}`, async (t) => {+ t.mock.timers.enable({ apis: ["Date", "setTimeout", "setInterval"] });+ const { store } = await setup();+ const started = Promise.withResolvers<void>();+ const finish = Promise.withResolvers<void>();+ const read = Promise.withResolvers<void>();+ let readMode = "normal";+ let signal: AbortSignal | undefined;+ let starts = 0;+ let starting = false;+ let stops = 0;+ let reads = 0;+ let maxReads = 0;+ const errors: unknown[] = [];+ const controller = createBackgroundController({+ store: {+ ...store,+ get: async () => {+ reads++;+ maxReads = Math.max(maxReads, reads);+ try {+ if (readMode === "fail") throw new Error("offline");+ if (readMode === "stall") await read.promise;+ return await store.get();+ } finally {+ reads--;+ }+ },+ },+ identity: { instanceId: "a", deploymentId: "a", taskArn: "task:a" },+ legacyEnabled: true,+ start: async (value) => {+ starts++;+ starting = true;+ signal = value;+ started.resolve();+ await finish.promise;+ starting = false;+ },+ fence() {},+ relinquish: async () => {+ assert.equal(starting, false);+ stops++;+ },+ drained: async () => {},+ onError: (error) => errors.push(error),+ pollMs: 10,+ validityMs: 100,+ startupTimeoutMs: 500,+ });+ controller.start();+ await started.promise;+ const flush = () => new Promise<void>((resolve) => setImmediate(resolve));+ for (let tick = 0; tick < 30; tick++) {+ t.mock.timers.tick(10);+ await flush();+ assert.equal(controller.canClaim(), true);+ assert.equal(signal?.aborted, false);+ assert.equal((await store.get()).members[0]?.ready, false);+ }+ assert.equal(starts, 1);+ assert.equal(stops, 0);+ if (outcome === "transition") {+ await store.transition({+ expectedGeneration: 0,+ requestId: "switch",+ desiredDeploymentId: null,+ bootstrapTaskArns: ["task:a"],+ });+ }+ if (outcome === "read-failure") readMode = "fail";+ if (outcome === "read-stall" || outcome === "stop") readMode = "stall";+ t.mock.timers.tick(10);+ await flush();+ const stopping = outcome === "stop" ? controller.stop() : undefined;+ if (outcome === "startup-timeout") {+ for (let tick = 0; tick < 30; tick++) {+ t.mock.timers.tick(10);+ await flush();+ }+ assert.equal(starts, 1);+ assert.equal(stops, 0);+ assert.equal((await store.get()).members[0]?.ready, false);+ }+ if (outcome === "read-stall" || outcome === "stop") {+ t.mock.timers.tick(110);+ await flush();+ assert.equal(signal?.aborted, true);+ readMode = "normal";+ read.resolve();+ await flush();+ }+ assert.equal(controller.canClaim(), outcome === "complete");+ assert.equal(signal?.aborted, outcome !== "complete");+ const reconciled = controller.reconcile();+ finish.resolve();+ await reconciled;+ await stopping;+ assert.equal((await store.get()).members[0]?.ready, outcome === "complete");+ assert.equal(starts, 1);+ assert.equal(maxReads, 1);+ assert.equal(errors.length, outcome === "read-failure" || outcome === "startup-timeout" ? 1 : 0);+ await controller.stop();+ await controller.drained();+ assert.equal(stops, 1);+ });+}