Skip to content

armgit / qm

Public Imported from GitHub
0

Multiplayer agent harness for work.

Fence Slack reconciliation while ingress is stopping (#1288)

Regan Bell3d48d4d3d267
diff --git a/src/slack/index.ts b/src/slack/index.ts
index 712b45b2b..7035cae2a 100644
--- a/src/slack/index.ts
+++ b/src/slack/index.ts
@@ -1,3 +1,4 @@
+import { SlackPluginStartCleanupError } from "../surfaces/slack-runtime.ts";
import { createSlackRateLimitNotice } from "./rate-limit-notice.ts";
import { createSlackHistoryReader } from "./history.ts";
import { errMessage, swallow, swallowAs } from "../util/errors.ts";
@@ -397,7 +398,13 @@ export async function startSlackPlugin(
} catch (err) {
stopped = true;
await devIntrospection?.close().catch(swallowAs("slack: dev-introspection close on failed start", undefined));
- await app.stop().catch(swallowAs("slack: app.stop on failed start", undefined));
+ try {
+ await app.stop();
+ } catch (cleanupError) {
+ throw new SlackPluginStartCleanupError(err, cleanupError, async () => {
+ await app.stop();
+ });
+ }
throw err;
}
devIntrospection?.ready({ connectedAs: auth.user ?? "", botUserId: ids.botUserId, teamId: ids.ownTeamId });
diff --git a/src/surfaces/slack-runtime.ts b/src/surfaces/slack-runtime.ts
index 55b26c3c4..12058b7be 100644
--- a/src/surfaces/slack-runtime.ts
+++ b/src/surfaces/slack-runtime.ts
@@ -1,3 +1,12 @@
+export class SlackPluginStartCleanupError extends AggregateError {
+ readonly cleanup: () => Promise<void>;
+
+ constructor(startError: unknown, cleanupError: unknown, cleanup: () => Promise<void>) {
+ super([startError, cleanupError], "Slack startup failed and socket cleanup remains incomplete");
+ this.cleanup = cleanup;
+ }
+}
+
export interface ReloadableSlackConfig<Config> {
version: string;
config: Config;
@@ -12,9 +21,17 @@ export function createSlackRuntimeReconciler<Config>(opts: {
let active: { plugin: { stop(): Promise<void> }; version: string; config: Config } | null = null;
let timer: NodeJS.Timeout | null = null;
let inFlight: Promise<void> | null = null;
+ let stopping: Promise<void> | null = null;
+ let stopped = false;
+ let pendingCleanup: (() => Promise<void>) | null = null;
const reconcile = async (): Promise<void> => {
+ if (pendingCleanup) {
+ await pendingCleanup();
+ pendingCleanup = null;
+ }
const desired = await opts.load();
+ if (stopped) return;
if (!desired) {
if (active) {
await active.plugin.stop();
@@ -28,15 +45,21 @@ export function createSlackRuntimeReconciler<Config>(opts: {
await previous.plugin.stop();
active = null;
}
+ if (stopped) return;
try {
const plugin = await opts.startPlugin(desired.config);
active = { plugin, version: desired.version, config: desired.config };
} catch (error) {
- if (previous) {
+ if (error instanceof SlackPluginStartCleanupError) {
+ pendingCleanup = error.cleanup;
+ throw error;
+ }
+ if (previous && !stopped) {
try {
const plugin = await opts.startPlugin(previous.config);
active = { plugin, version: previous.version, config: previous.config };
} catch (rollbackError) {
+ if (rollbackError instanceof SlackPluginStartCleanupError) pendingCleanup = rollbackError.cleanup;
throw new AggregateError([error, rollbackError], "Slack reload and rollback both failed", {
cause: rollbackError,
});
@@ -47,6 +70,7 @@ export function createSlackRuntimeReconciler<Config>(opts: {
};
const run = (): Promise<void> => {
+ if (stopped) return Promise.resolve();
if (inFlight) return inFlight;
inFlight = reconcile().finally(() => {
inFlight = null;
@@ -59,19 +83,32 @@ export function createSlackRuntimeReconciler<Config>(opts: {
return {
start() {
+ if (timer || stopping || (stopped && (active || pendingCleanup))) return;
+ stopped = false;
tick();
timer = setInterval(tick, opts.intervalMs ?? 5_000);
timer.unref();
},
reconcile: run,
- async stop() {
+ stop(): Promise<void> {
+ if (stopping) return stopping;
+ stopped = true;
if (timer) clearInterval(timer);
timer = null;
- await inFlight;
- if (active) {
- await active.plugin.stop();
- active = null;
- }
+ stopping = (async () => {
+ await inFlight?.catch(() => {});
+ if (pendingCleanup) {
+ await pendingCleanup();
+ pendingCleanup = null;
+ }
+ if (active) {
+ await active.plugin.stop();
+ active = null;
+ }
+ })().finally(() => {
+ stopping = null;
+ });
+ return stopping;
},
};
}
diff --git a/test/slack-runtime.test.ts b/test/slack-runtime.test.ts
index 2c0a2e663..079e7e788 100644
--- a/test/slack-runtime.test.ts
+++ b/test/slack-runtime.test.ts
@@ -1,6 +1,6 @@
import assert from "node:assert/strict";
import test from "node:test";
-import { createSlackRuntimeReconciler } from "../src/surfaces/slack-runtime.ts";
+import { createSlackRuntimeReconciler, SlackPluginStartCleanupError } from "../src/surfaces/slack-runtime.ts";
test("Slack runtime activates, reloads, and removes durable admin configuration", async () => {
let desired: { version: string; config: { botToken: string } } | null = null;
@@ -80,3 +80,215 @@ test("Slack runtime keeps retrying a failed stop before starting replacement cre
assert.deepEqual(events, ["start:first", "stop:first", "stop:first", "start:second"]);
await runtime.stop();
});
+
+test("Slack stop finishes cleanup even when the pending configuration load fails", async () => {
+ const loading = Promise.withResolvers<void>();
+ let loads = 0;
+ let stops = 0;
+ const runtime = createSlackRuntimeReconciler({
+ load: async () => {
+ if (++loads > 1) {
+ await loading.promise;
+ throw new Error("database disconnected");
+ }
+ return { version: "1", config: "first" };
+ },
+ startPlugin: async () => ({
+ stop: async () => {
+ stops++;
+ },
+ }),
+ });
+ await runtime.reconcile();
+ const pending = assert.rejects(runtime.reconcile(), /database disconnected/);
+ const stopping = runtime.stop();
+ loading.resolve();
+ await Promise.all([pending, stopping]);
+ assert.equal(stops, 1);
+ await runtime.reconcile();
+ assert.equal(loads, 2);
+});
+
+test("Slack stop fences a pending load and repeated starts leave no polling timer behind", async () => {
+ const loading = Promise.withResolvers<void>();
+ let loads = 0;
+ let starts = 0;
+ const runtime = createSlackRuntimeReconciler({
+ load: async () => {
+ loads++;
+ await loading.promise;
+ return { version: "1", config: "first" };
+ },
+ startPlugin: async () => {
+ starts++;
+ return { stop: async () => {} };
+ },
+ intervalMs: 5,
+ });
+ runtime.start();
+ runtime.start();
+ const stopping = runtime.stop();
+ loading.resolve();
+ await stopping;
+ await new Promise((resolve) => setTimeout(resolve, 20));
+ assert.equal(loads, 1);
+ assert.equal(starts, 0);
+ runtime.start();
+ await runtime.reconcile();
+ await runtime.stop();
+ assert.equal(starts, 1);
+});
+
+test("Slack stop waits for an opening socket, collapses repeated stops, and permits a clean resume", async () => {
+ const opening = Promise.withResolvers<void>();
+ const opened = Promise.withResolvers<void>();
+ let starts = 0;
+ let stops = 0;
+ const runtime = createSlackRuntimeReconciler({
+ load: async () => ({ version: "1", config: "first" }),
+ startPlugin: async () => {
+ starts++;
+ opened.resolve();
+ await opening.promise;
+ return {
+ stop: async () => {
+ stops++;
+ },
+ };
+ },
+ });
+ runtime.start();
+ await opened.promise;
+ const stopping = runtime.stop();
+ assert.equal(runtime.stop(), stopping);
+ runtime.start();
+ opening.resolve();
+ await stopping;
+ assert.equal(starts, 1);
+ assert.equal(stops, 1);
+ runtime.start();
+ await runtime.reconcile();
+ await runtime.stop();
+ assert.equal(starts, 2);
+ assert.equal(stops, 2);
+});
+
+test("Slack stop during a failed replacement does not resurrect rollback credentials", async () => {
+ const failing = Promise.withResolvers<void>();
+ const opening = Promise.withResolvers<void>();
+ let desired = { version: "1", config: "first" };
+ const events: string[] = [];
+ const runtime = createSlackRuntimeReconciler({
+ load: async () => desired,
+ startPlugin: async (config) => {
+ events.push(`start:${config}`);
+ if (config === "second") {
+ opening.resolve();
+ await failing.promise;
+ throw new Error("failed activation");
+ }
+ return {
+ stop: async () => {
+ events.push(`stop:${config}`);
+ },
+ };
+ },
+ });
+ await runtime.reconcile();
+ desired = { version: "2", config: "second" };
+ const pending = assert.rejects(runtime.reconcile(), /failed activation/);
+ await opening.promise;
+ const stopping = runtime.stop();
+ failing.resolve();
+ await Promise.all([pending, stopping]);
+ assert.deepEqual(events, ["start:first", "stop:first", "start:second"]);
+});
+
+test("Slack cannot resume after failed socket closure until stop succeeds", async () => {
+ let starts = 0;
+ let stops = 0;
+ const runtime = createSlackRuntimeReconciler({
+ load: async () => ({ version: "1", config: "first" }),
+ startPlugin: async () => {
+ starts++;
+ return {
+ stop: async () => {
+ if (++stops === 1) throw new Error("socket close failed");
+ },
+ };
+ },
+ });
+ await runtime.reconcile();
+ await assert.rejects(runtime.stop(), /socket close failed/);
+ runtime.start();
+ await runtime.reconcile();
+ assert.equal(starts, 1);
+ await runtime.stop();
+ runtime.start();
+ await runtime.reconcile();
+ await runtime.stop();
+ assert.equal(starts, 2);
+ assert.equal(stops, 3);
+});
+
+test("uncertain failed startup cleanup blocks repeated stop acknowledgments and rollback until it closes", async () => {
+ let desired = { version: "1", config: "first" };
+ let cleanupAllowed = false;
+ const events: string[] = [];
+ const runtime = createSlackRuntimeReconciler({
+ load: async () => desired,
+ startPlugin: async (config) => {
+ events.push(`start:${config}`);
+ if (config === "second")
+ throw new SlackPluginStartCleanupError(new Error("start failed"), new Error("close failed"), async () => {
+ events.push("cleanup:second");
+ if (!cleanupAllowed) throw new Error("still open");
+ });
+ return {
+ stop: async () => {
+ events.push(`stop:${config}`);
+ },
+ };
+ },
+ });
+ await runtime.reconcile();
+ desired = { version: "2", config: "second" };
+ await assert.rejects(runtime.reconcile(), SlackPluginStartCleanupError);
+ assert.deepEqual(events, ["start:first", "stop:first", "start:second"]);
+ await assert.rejects(runtime.stop(), /still open/);
+ await assert.rejects(runtime.stop(), /still open/);
+ runtime.start();
+ await runtime.reconcile();
+ assert.equal(events.filter((event) => event.startsWith("start:")).length, 2);
+ cleanupAllowed = true;
+ await runtime.stop();
+ desired = { version: "3", config: "third" };
+ runtime.start();
+ await runtime.reconcile();
+ await runtime.stop();
+ assert.equal(events.at(-2), "start:third");
+ assert.equal(events.at(-1), "stop:third");
+});
+
+test("a rollback startup with incomplete cleanup also blocks relinquishment", async () => {
+ let desired = { version: "1", config: "first" };
+ let firstStarts = 0;
+ let cleanupAllowed = false;
+ const runtime = createSlackRuntimeReconciler({
+ load: async () => desired,
+ startPlugin: async (config) => {
+ if (config === "second") throw new Error("replacement failed");
+ if (++firstStarts > 1)
+ throw new SlackPluginStartCleanupError(new Error("rollback failed"), new Error("close failed"), async () => {
+ if (!cleanupAllowed) throw new Error("rollback still open");
+ });
+ return { stop: async () => {} };
+ },
+ });
+ await runtime.reconcile();
+ desired = { version: "2", config: "second" };
+ await assert.rejects(runtime.reconcile(), /reload and rollback both failed/);
+ await assert.rejects(runtime.stop(), /rollback still open/);
+ cleanupAllowed = true;
+ await runtime.stop();
+});