Multiplayer agent harness for work.
Drain legacy worker claims during ownership enrollment (#1295)
Regan Bell2372971abc65
diff --git a/docs/background-ownership.md b/docs/background-ownership.mdindex f3f4edde8..89007bdc8 100644--- a/docs/background-ownership.md+++ b/docs/background-ownership.md@@ -90,6 +90,26 @@ prove success; the caller must receive and verify the complete final object.Request IDs are durably consumed across replicas and cannot be replayed. Eachprocess permits one check at a time, including its cleanup. Disconnecting thecaller does not cancel the check or release that guard before cleanup finishes.-The deployment lease still serializes operator requests across the cohort.+Singleflight is per process, and durable request IDs prevent replay. Requests with distinct IDs can run concurrently on different replicas. The release workflow serializes its own requests.An uncertain result must fail the release; do not retry automatically or fallback to a second canary execution.++## Enrolling an active legacy deployment++Before bootstrap, a controlled process with its legacy boot flag enabled publishes+an enrollment-specific legacy build heartbeat only after its generation-zero+membership is admitted and ready. Older legacy workers then stop claiming new+runs even when both processes use the same image. Controlled processes ignore+legacy supersession results and continue using durable ownership admission.+Inactive, unready, or fenced processes do not publish this compatibility heartbeat;+publishing also stops once durable ownership is enabled.++This bridge preserves the legacy worker drain behavior and its running-turn task+protection. It does not add missing lifecycle controls to older binaries: their+Slack ingress, cron callbacks, and inline HTTP execution can remain active.+Never treat the heartbeat as a deployment relinquishment acknowledgment or as+proof that a legacy task is safe to terminate. Bootstrap still requires explicit+infrastructure proof that every legacy task has retired, including pending tasks.+Do not clear task protection or terminate live turns to finish enrollment.+If enrollment is abandoned, the legacy heartbeat expiry permits older workers+to resume their existing claim loop.diff --git a/src/runs/instance-registry.ts b/src/runs/instance-registry.tsindex cf96237a6..b076f728b 100644--- a/src/runs/instance-registry.ts+++ b/src/runs/instance-registry.ts@@ -57,3 +57,15 @@ export function createPostgresInstanceRegistry(},};}++export function createLegacyEnrollmentBridge(+ registry: InstanceRegistry,+ eligible: () => Promise<boolean>,+): InstanceRegistry {+ return {+ async beat() {+ if (await eligible()) await registry.beat();+ return false;+ },+ };+}diff --git a/src/wiring.ts b/src/wiring.tsindex 0e7cb7f47..55e09a7ac 100644--- a/src/wiring.ts+++ b/src/wiring.ts@@ -303,6 +303,7 @@ import { isTerminal, type Run, type RunStore } from "./runs/run-store.ts";import { createWorker, type Worker } from "./runs/worker.ts";import {createNoopInstanceRegistry,+ createLegacyEnrollmentBridge,createPostgresInstanceRegistry,type InstanceRegistry,} from "./runs/instance-registry.ts";@@ -2205,14 +2206,30 @@ export function buildApp(deploymentId: config.backgroundDeploymentId,}: undefined;- const instanceRegistry: InstanceRegistry =- config.buildSha && pgArtifactMap && !config.backgroundDeploymentId+ const legacyRegistry =+ pgArtifactMap && (config.buildSha || backgroundOwnership)? createPostgresInstanceRegistry(pgArtifactMap.pool, {instanceId: randomUUID(),- buildSha: config.buildSha,+ buildSha: backgroundOwnership ? `enrollment:${backgroundOwnership.deploymentId}` : config.buildSha!,startedAt: Date.now(),}): createNoopInstanceRegistry();+ const instanceRegistry: InstanceRegistry = backgroundOwnership+ ? createLegacyEnrollmentBridge(legacyRegistry, async () => {+ if (!config.backgroundWorkEnabled || !backgroundAdmission()) return false;+ const state = await backgroundOwnership.store.get();+ const member = state.members.find((entry) => entry.instanceId === backgroundOwnership.instanceId);+ return (+ !state.enabled &&+ state.generation === 0 &&+ member?.generation === 0 &&+ !member.retired &&+ member.state === "admitted" &&+ member.ready &&+ backgroundAdmission()+ );+ })+ : legacyRegistry;const taskProtection: TaskProtection | null =config.ecsTaskProtection && config.ecsAgentUri ? createEcsTaskProtection(config.ecsAgentUri) : null;const drain: DrainController = createDrainController({diff --git a/test/legacy-enrollment-bridge.test.ts b/test/legacy-enrollment-bridge.test.tsnew file mode 100644index 000000000..b388db37e--- /dev/null+++ b/test/legacy-enrollment-bridge.test.ts@@ -0,0 +1,42 @@+import assert from "node:assert/strict";+import test from "node:test";+import { createLegacyEnrollmentBridge } from "../src/runs/instance-registry.ts";++test("legacy enrollment publishes only when admitted and ignores legacy supersession", async () => {+ let eligible = false;+ let beats = 0;+ const bridge = createLegacyEnrollmentBridge(+ {+ beat: async () => {+ beats++;+ return true;+ },+ },+ async () => eligible,+ );+ assert.equal(await bridge.beat(), false);+ assert.equal(beats, 0);+ eligible = true;+ assert.equal(await bridge.beat(), false);+ assert.equal(beats, 1);+ eligible = false;+ assert.equal(await bridge.beat(), false);+ assert.equal(beats, 1);+});++test("legacy enrollment propagates failed eligibility reads without publishing", async () => {+ let beats = 0;+ const bridge = createLegacyEnrollmentBridge(+ {+ beat: async () => {+ beats++;+ return false;+ },+ },+ async () => {+ throw new Error("database unavailable");+ },+ );+ await assert.rejects(bridge.beat(), /database unavailable/);+ assert.equal(beats, 0);+});diff --git a/test/postgres-instance-registry.test.ts b/test/postgres-instance-registry.test.tsindex 16c3aa74c..dcaeba4a1 100644--- a/test/postgres-instance-registry.test.ts+++ b/test/postgres-instance-registry.test.ts@@ -1,7 +1,7 @@import { test, before } from "node:test";import assert from "node:assert/strict";import { createPostgresMapFactory } from "../src/persistence/durable-map.ts";-import { createPostgresInstanceRegistry } from "../src/runs/instance-registry.ts";+import { createPostgresInstanceRegistry, createLegacyEnrollmentBridge } from "../src/runs/instance-registry.ts";const URL = process.env.DATABASE_URL;const skip = URL ? false : "set DATABASE_URL (a Postgres) to run the instance-registry tests";@@ -51,3 +51,44 @@ test(assert.equal(await old.beat(), false, "the newer build's beats went stale (failed deploy): claiming resumes");},);++test("controlled enrollment drains same-image legacy workers without self-supersession", { skip }, async () => {+ const factory = createPostgresMapFactory(URL!);+ const old = createPostgresInstanceRegistry(factory.pool, {+ instanceId: "migration-old",+ buildSha: "same-image",+ startedAt: 10_000,+ livenessMs: 200,+ });+ let eligible = false;+ const bridge = createLegacyEnrollmentBridge(+ createPostgresInstanceRegistry(factory.pool, {+ instanceId: "migration-new",+ buildSha: "enrollment:cohort-a",+ startedAt: 20_000,+ livenessMs: 200,+ }),+ async () => eligible,+ );+ try {+ assert.equal(await bridge.beat(), false);+ assert.equal(await old.beat(), false);+ eligible = true;+ assert.equal(await bridge.beat(), false);+ assert.equal(await old.beat(), true);+ const newer = createPostgresInstanceRegistry(factory.pool, {+ instanceId: "migration-peer",+ buildSha: "another-image",+ startedAt: 30_000,+ livenessMs: 200,+ });+ await newer.beat();+ assert.equal(await bridge.beat(), false);+ eligible = false;+ await new Promise((resolve) => setTimeout(resolve, 250));+ assert.equal(await bridge.beat(), false);+ assert.equal(await old.beat(), false);+ } finally {+ await factory.pool.close();+ }+});