Multiplayer agent harness for work.
Fence Slack reconciliation while ingress is stopping (#1288)
Regan Bell3d48d4d3d267
diff --git a/src/slack/index.ts b/src/slack/index.tsindex 712b45b2b..7035cae2a 100644--- a/src/slack/index.ts+++ b/src/slack/index.ts@@ -1,3 +1,4 @@+import { SlackPluginStartCleanupError } from "../surfaces/slack-runtime.ts";import { createSlackRateLimitNotice } from "./rate-limit-notice.ts";import { createSlackHistoryReader } from "./history.ts";import { errMessage, swallow, swallowAs } from "../util/errors.ts";@@ -397,7 +398,13 @@ export async function startSlackPlugin(} catch (err) {stopped = true;await devIntrospection?.close().catch(swallowAs("slack: dev-introspection close on failed start", undefined));- await app.stop().catch(swallowAs("slack: app.stop on failed start", undefined));+ try {+ await app.stop();+ } catch (cleanupError) {+ throw new SlackPluginStartCleanupError(err, cleanupError, async () => {+ await app.stop();+ });+ }throw err;}devIntrospection?.ready({ connectedAs: auth.user ?? "", botUserId: ids.botUserId, teamId: ids.ownTeamId });diff --git a/src/surfaces/slack-runtime.ts b/src/surfaces/slack-runtime.tsindex 55b26c3c4..12058b7be 100644--- a/src/surfaces/slack-runtime.ts+++ b/src/surfaces/slack-runtime.ts@@ -1,3 +1,12 @@+export class SlackPluginStartCleanupError extends AggregateError {+ readonly cleanup: () => Promise<void>;++ constructor(startError: unknown, cleanupError: unknown, cleanup: () => Promise<void>) {+ super([startError, cleanupError], "Slack startup failed and socket cleanup remains incomplete");+ this.cleanup = cleanup;+ }+}+export interface ReloadableSlackConfig<Config> {version: string;config: Config;@@ -12,9 +21,17 @@ export function createSlackRuntimeReconciler<Config>(opts: {let active: { plugin: { stop(): Promise<void> }; version: string; config: Config } | null = null;let timer: NodeJS.Timeout | null = null;let inFlight: Promise<void> | null = null;+ let stopping: Promise<void> | null = null;+ let stopped = false;+ let pendingCleanup: (() => Promise<void>) | null = null;const reconcile = async (): Promise<void> => {+ if (pendingCleanup) {+ await pendingCleanup();+ pendingCleanup = null;+ }const desired = await opts.load();+ if (stopped) return;if (!desired) {if (active) {await active.plugin.stop();@@ -28,15 +45,21 @@ export function createSlackRuntimeReconciler<Config>(opts: {await previous.plugin.stop();active = null;}+ if (stopped) return;try {const plugin = await opts.startPlugin(desired.config);active = { plugin, version: desired.version, config: desired.config };} catch (error) {- if (previous) {+ if (error instanceof SlackPluginStartCleanupError) {+ pendingCleanup = error.cleanup;+ throw error;+ }+ if (previous && !stopped) {try {const plugin = await opts.startPlugin(previous.config);active = { plugin, version: previous.version, config: previous.config };} catch (rollbackError) {+ if (rollbackError instanceof SlackPluginStartCleanupError) pendingCleanup = rollbackError.cleanup;throw new AggregateError([error, rollbackError], "Slack reload and rollback both failed", {cause: rollbackError,});@@ -47,6 +70,7 @@ export function createSlackRuntimeReconciler<Config>(opts: {};const run = (): Promise<void> => {+ if (stopped) return Promise.resolve();if (inFlight) return inFlight;inFlight = reconcile().finally(() => {inFlight = null;@@ -59,19 +83,32 @@ export function createSlackRuntimeReconciler<Config>(opts: {return {start() {+ if (timer || stopping || (stopped && (active || pendingCleanup))) return;+ stopped = false;tick();timer = setInterval(tick, opts.intervalMs ?? 5_000);timer.unref();},reconcile: run,- async stop() {+ stop(): Promise<void> {+ if (stopping) return stopping;+ stopped = true;if (timer) clearInterval(timer);timer = null;- await inFlight;- if (active) {- await active.plugin.stop();- active = null;- }+ stopping = (async () => {+ await inFlight?.catch(() => {});+ if (pendingCleanup) {+ await pendingCleanup();+ pendingCleanup = null;+ }+ if (active) {+ await active.plugin.stop();+ active = null;+ }+ })().finally(() => {+ stopping = null;+ });+ return stopping;},};}diff --git a/test/slack-runtime.test.ts b/test/slack-runtime.test.tsindex 2c0a2e663..079e7e788 100644--- a/test/slack-runtime.test.ts+++ b/test/slack-runtime.test.ts@@ -1,6 +1,6 @@import assert from "node:assert/strict";import test from "node:test";-import { createSlackRuntimeReconciler } from "../src/surfaces/slack-runtime.ts";+import { createSlackRuntimeReconciler, SlackPluginStartCleanupError } from "../src/surfaces/slack-runtime.ts";test("Slack runtime activates, reloads, and removes durable admin configuration", async () => {let desired: { version: string; config: { botToken: string } } | null = null;@@ -80,3 +80,215 @@ test("Slack runtime keeps retrying a failed stop before starting replacement creassert.deepEqual(events, ["start:first", "stop:first", "stop:first", "start:second"]);await runtime.stop();});++test("Slack stop finishes cleanup even when the pending configuration load fails", async () => {+ const loading = Promise.withResolvers<void>();+ let loads = 0;+ let stops = 0;+ const runtime = createSlackRuntimeReconciler({+ load: async () => {+ if (++loads > 1) {+ await loading.promise;+ throw new Error("database disconnected");+ }+ return { version: "1", config: "first" };+ },+ startPlugin: async () => ({+ stop: async () => {+ stops++;+ },+ }),+ });+ await runtime.reconcile();+ const pending = assert.rejects(runtime.reconcile(), /database disconnected/);+ const stopping = runtime.stop();+ loading.resolve();+ await Promise.all([pending, stopping]);+ assert.equal(stops, 1);+ await runtime.reconcile();+ assert.equal(loads, 2);+});++test("Slack stop fences a pending load and repeated starts leave no polling timer behind", async () => {+ const loading = Promise.withResolvers<void>();+ let loads = 0;+ let starts = 0;+ const runtime = createSlackRuntimeReconciler({+ load: async () => {+ loads++;+ await loading.promise;+ return { version: "1", config: "first" };+ },+ startPlugin: async () => {+ starts++;+ return { stop: async () => {} };+ },+ intervalMs: 5,+ });+ runtime.start();+ runtime.start();+ const stopping = runtime.stop();+ loading.resolve();+ await stopping;+ await new Promise((resolve) => setTimeout(resolve, 20));+ assert.equal(loads, 1);+ assert.equal(starts, 0);+ runtime.start();+ await runtime.reconcile();+ await runtime.stop();+ assert.equal(starts, 1);+});++test("Slack stop waits for an opening socket, collapses repeated stops, and permits a clean resume", async () => {+ const opening = Promise.withResolvers<void>();+ const opened = Promise.withResolvers<void>();+ let starts = 0;+ let stops = 0;+ const runtime = createSlackRuntimeReconciler({+ load: async () => ({ version: "1", config: "first" }),+ startPlugin: async () => {+ starts++;+ opened.resolve();+ await opening.promise;+ return {+ stop: async () => {+ stops++;+ },+ };+ },+ });+ runtime.start();+ await opened.promise;+ const stopping = runtime.stop();+ assert.equal(runtime.stop(), stopping);+ runtime.start();+ opening.resolve();+ await stopping;+ assert.equal(starts, 1);+ assert.equal(stops, 1);+ runtime.start();+ await runtime.reconcile();+ await runtime.stop();+ assert.equal(starts, 2);+ assert.equal(stops, 2);+});++test("Slack stop during a failed replacement does not resurrect rollback credentials", async () => {+ const failing = Promise.withResolvers<void>();+ const opening = Promise.withResolvers<void>();+ let desired = { version: "1", config: "first" };+ const events: string[] = [];+ const runtime = createSlackRuntimeReconciler({+ load: async () => desired,+ startPlugin: async (config) => {+ events.push(`start:${config}`);+ if (config === "second") {+ opening.resolve();+ await failing.promise;+ throw new Error("failed activation");+ }+ return {+ stop: async () => {+ events.push(`stop:${config}`);+ },+ };+ },+ });+ await runtime.reconcile();+ desired = { version: "2", config: "second" };+ const pending = assert.rejects(runtime.reconcile(), /failed activation/);+ await opening.promise;+ const stopping = runtime.stop();+ failing.resolve();+ await Promise.all([pending, stopping]);+ assert.deepEqual(events, ["start:first", "stop:first", "start:second"]);+});++test("Slack cannot resume after failed socket closure until stop succeeds", async () => {+ let starts = 0;+ let stops = 0;+ const runtime = createSlackRuntimeReconciler({+ load: async () => ({ version: "1", config: "first" }),+ startPlugin: async () => {+ starts++;+ return {+ stop: async () => {+ if (++stops === 1) throw new Error("socket close failed");+ },+ };+ },+ });+ await runtime.reconcile();+ await assert.rejects(runtime.stop(), /socket close failed/);+ runtime.start();+ await runtime.reconcile();+ assert.equal(starts, 1);+ await runtime.stop();+ runtime.start();+ await runtime.reconcile();+ await runtime.stop();+ assert.equal(starts, 2);+ assert.equal(stops, 3);+});++test("uncertain failed startup cleanup blocks repeated stop acknowledgments and rollback until it closes", async () => {+ let desired = { version: "1", config: "first" };+ let cleanupAllowed = false;+ const events: string[] = [];+ const runtime = createSlackRuntimeReconciler({+ load: async () => desired,+ startPlugin: async (config) => {+ events.push(`start:${config}`);+ if (config === "second")+ throw new SlackPluginStartCleanupError(new Error("start failed"), new Error("close failed"), async () => {+ events.push("cleanup:second");+ if (!cleanupAllowed) throw new Error("still open");+ });+ return {+ stop: async () => {+ events.push(`stop:${config}`);+ },+ };+ },+ });+ await runtime.reconcile();+ desired = { version: "2", config: "second" };+ await assert.rejects(runtime.reconcile(), SlackPluginStartCleanupError);+ assert.deepEqual(events, ["start:first", "stop:first", "start:second"]);+ await assert.rejects(runtime.stop(), /still open/);+ await assert.rejects(runtime.stop(), /still open/);+ runtime.start();+ await runtime.reconcile();+ assert.equal(events.filter((event) => event.startsWith("start:")).length, 2);+ cleanupAllowed = true;+ await runtime.stop();+ desired = { version: "3", config: "third" };+ runtime.start();+ await runtime.reconcile();+ await runtime.stop();+ assert.equal(events.at(-2), "start:third");+ assert.equal(events.at(-1), "stop:third");+});++test("a rollback startup with incomplete cleanup also blocks relinquishment", async () => {+ let desired = { version: "1", config: "first" };+ let firstStarts = 0;+ let cleanupAllowed = false;+ const runtime = createSlackRuntimeReconciler({+ load: async () => desired,+ startPlugin: async (config) => {+ if (config === "second") throw new Error("replacement failed");+ if (++firstStarts > 1)+ throw new SlackPluginStartCleanupError(new Error("rollback failed"), new Error("close failed"), async () => {+ if (!cleanupAllowed) throw new Error("rollback still open");+ });+ return { stop: async () => {} };+ },+ });+ await runtime.reconcile();+ desired = { version: "2", config: "second" };+ await assert.rejects(runtime.reconcile(), /reload and rollback both failed/);+ await assert.rejects(runtime.stop(), /rollback still open/);+ cleanupAllowed = true;+ await runtime.stop();+});