Skip to content

armgit / qm

Public Imported from GitHub
0

Multiplayer agent harness for work.

Renew verified background ownership during slow startup (#1469)

Regan Bellb388454abe03
diff --git a/src/runs/background-controller.ts b/src/runs/background-controller.ts
index 897bb14b1..4df7e3dc3 100644
--- a/src/runs/background-controller.ts
+++ b/src/runs/background-controller.ts
@@ -11,6 +11,7 @@ export interface BackgroundControllerDeps {
drained(): Promise<void>;
onError(error: unknown): void;
validityMs?: number;
+ startupTimeoutMs?: number;
pollMs?: number;
}
@@ -24,6 +25,8 @@ export function createBackgroundController(deps: BackgroundControllerDeps) {
let watchdog: ReturnType<typeof setTimeout> | null = null;
let poller: ReturnType<typeof setInterval> | null = null;
let pending: Promise<void> | null = null;
+ let starting = false;
+ let refreshing: Promise<void> | null = null;
let draining: Promise<void> = Promise.resolve();
const validityMs = deps.validityMs ?? 10_000;
const fence = (): void => {
@@ -54,8 +57,44 @@ export function createBackgroundController(deps: BackgroundControllerDeps) {
watchdog = setTimeout(fence, validityMs);
watchdog.unref?.();
};
+ const refreshStartup = (): Promise<void> => {
+ if (!starting || !running || !activation || activation.signal.aborted) return Promise.resolve();
+ if (refreshing) return refreshing;
+ const currentActivation = activation;
+ const generation = admission;
+ refreshing = (async () => {
+ try {
+ const state = await deps.store.get();
+ if (!starting || !running || activation !== currentActivation || currentActivation?.signal.aborted) return;
+ const member = state.members.find((entry) => entry.instanceId === deps.identity.instanceId);
+ if (
+ state.generation !== generation ||
+ !member ||
+ member.retired ||
+ member.state !== "admitted" ||
+ member.generation !== generation ||
+ !(state.enabled ? state.desiredDeploymentId === deps.identity.deploymentId : deps.legacyEnabled)
+ ) {
+ fence();
+ return;
+ }
+ renew();
+ } catch (error) {
+ if (starting && activation === currentActivation) {
+ fence();
+ deps.onError(error);
+ }
+ }
+ })().finally(() => {
+ refreshing = null;
+ });
+ return refreshing;
+ };
const reconcile = (): Promise<void> => {
- if (pending) return pending;
+ if (pending) {
+ if (starting) void refreshStartup();
+ return pending;
+ }
pending = (async () => {
try {
if (!registered) {
@@ -82,7 +121,18 @@ export function createBackgroundController(deps: BackgroundControllerDeps) {
await release();
return;
}
- await deps.start(activation.signal);
+ starting = true;
+ const startupDeadline = setTimeout(() => {
+ fence();
+ deps.onError(new Error("Background startup timed out; waiting for activation cleanup before retrying"));
+ }, deps.startupTimeoutMs ?? 120_000);
+ startupDeadline.unref?.();
+ try {
+ await deps.start(activation.signal);
+ } finally {
+ clearTimeout(startupDeadline);
+ starting = false;
+ }
if (activation.signal.aborted || !running) await release();
else await deps.store.markReady(deps.identity.instanceId, state.generation);
} else {
diff --git a/test/background-controller.test.ts b/test/background-controller.test.ts
index 7e2e53871..f1be31399 100644
--- a/test/background-controller.test.ts
+++ b/test/background-controller.test.ts
@@ -207,3 +207,112 @@ test("activation is not ready until startup completes and expiry fences a late s
assert.equal((await store.get()).members[0]?.ready, false);
await controller.stop();
});
+
+for (const outcome of ["complete", "transition", "read-failure", "read-stall", "stop", "startup-timeout"] as const) {
+ test(`pending startup renews verified ownership and fences on ${outcome}`, async (t) => {
+ t.mock.timers.enable({ apis: ["Date", "setTimeout", "setInterval"] });
+ const { store } = await setup();
+ const started = Promise.withResolvers<void>();
+ const finish = Promise.withResolvers<void>();
+ const read = Promise.withResolvers<void>();
+ let readMode = "normal";
+ let signal: AbortSignal | undefined;
+ let starts = 0;
+ let starting = false;
+ let stops = 0;
+ let reads = 0;
+ let maxReads = 0;
+ const errors: unknown[] = [];
+ const controller = createBackgroundController({
+ store: {
+ ...store,
+ get: async () => {
+ reads++;
+ maxReads = Math.max(maxReads, reads);
+ try {
+ if (readMode === "fail") throw new Error("offline");
+ if (readMode === "stall") await read.promise;
+ return await store.get();
+ } finally {
+ reads--;
+ }
+ },
+ },
+ identity: { instanceId: "a", deploymentId: "a", taskArn: "task:a" },
+ legacyEnabled: true,
+ start: async (value) => {
+ starts++;
+ starting = true;
+ signal = value;
+ started.resolve();
+ await finish.promise;
+ starting = false;
+ },
+ fence() {},
+ relinquish: async () => {
+ assert.equal(starting, false);
+ stops++;
+ },
+ drained: async () => {},
+ onError: (error) => errors.push(error),
+ pollMs: 10,
+ validityMs: 100,
+ startupTimeoutMs: 500,
+ });
+ controller.start();
+ await started.promise;
+ const flush = () => new Promise<void>((resolve) => setImmediate(resolve));
+ for (let tick = 0; tick < 30; tick++) {
+ t.mock.timers.tick(10);
+ await flush();
+ assert.equal(controller.canClaim(), true);
+ assert.equal(signal?.aborted, false);
+ assert.equal((await store.get()).members[0]?.ready, false);
+ }
+ assert.equal(starts, 1);
+ assert.equal(stops, 0);
+ if (outcome === "transition") {
+ await store.transition({
+ expectedGeneration: 0,
+ requestId: "switch",
+ desiredDeploymentId: null,
+ bootstrapTaskArns: ["task:a"],
+ });
+ }
+ if (outcome === "read-failure") readMode = "fail";
+ if (outcome === "read-stall" || outcome === "stop") readMode = "stall";
+ t.mock.timers.tick(10);
+ await flush();
+ const stopping = outcome === "stop" ? controller.stop() : undefined;
+ if (outcome === "startup-timeout") {
+ for (let tick = 0; tick < 30; tick++) {
+ t.mock.timers.tick(10);
+ await flush();
+ }
+ assert.equal(starts, 1);
+ assert.equal(stops, 0);
+ assert.equal((await store.get()).members[0]?.ready, false);
+ }
+ if (outcome === "read-stall" || outcome === "stop") {
+ t.mock.timers.tick(110);
+ await flush();
+ assert.equal(signal?.aborted, true);
+ readMode = "normal";
+ read.resolve();
+ await flush();
+ }
+ assert.equal(controller.canClaim(), outcome === "complete");
+ assert.equal(signal?.aborted, outcome !== "complete");
+ const reconciled = controller.reconcile();
+ finish.resolve();
+ await reconciled;
+ await stopping;
+ assert.equal((await store.get()).members[0]?.ready, outcome === "complete");
+ assert.equal(starts, 1);
+ assert.equal(maxReads, 1);
+ assert.equal(errors.length, outcome === "read-failure" || outcome === "startup-timeout" ? 1 : 0);
+ await controller.stop();
+ await controller.drained();
+ assert.equal(stops, 1);
+ });
+}