Skip to content

Free, Open Source, Self-Hosted WhatsApp API Gateway

fix(session): take a failed delete's stop back after overlaps

Yudhi Armyndharisae92309de279
diff --git a/src/modules/session/session-delete-claim.spec.ts b/src/modules/session/session-delete-claim.spec.ts
index 968d85d58..68c9fcfbd 100644
--- a/src/modules/session/session-delete-claim.spec.ts
+++ b/src/modules/session/session-delete-claim.spec.ts
@@ -269,6 +269,15 @@ describe('A session deleted on one node while a peer shares the database', () =>
};
}
+ /** What a settled delete or stop must not leave behind: its share of a stop record, a stop write counted. */
+ function bookkeeping(id: string): { deletes: boolean; stopWrites: boolean } {
+ const service = a.service as unknown as {
+ deleteStops: Map<string, unknown>;
+ stopWritesInFlight: Map<string, number>;
+ };
+ return { deletes: service.deleteStops.has(id), stopWrites: service.stopWritesInFlight.has(id) };
+ }
+
const gone = { sessions: 0, messages: 0, outbox: 0, deliveryFailures: 0 };
it.each([
@@ -522,6 +531,280 @@ describe('A session deleted on one node while a peer shares the database', () =>
expect((await sessions.findOneByOrFail({ id })).desiredState).toBe('stopped');
});
+ it('keeps a stop whose record has landed but not yet returned when its delete fails', async () => {
+ const id = await seed({ status: SessionStatus.DISCONNECTED });
+ const sessions = a.db.getRepository(Session);
+ const update = sessions.update.bind(sessions);
+ const reading = deferred();
+ const read = deferred();
+ jest.spyOn(sessions, 'findOne').mockImplementationOnce(async () => {
+ reading.resolve();
+ await read.promise;
+ throw new Error('connection reset');
+ });
+ const deleting = a.service.delete(id);
+ await reading.promise;
+ const landed = deferred();
+ const answer = deferred();
+ // The stop's record is on the row; its answer is held until the delete has failed.
+ jest.spyOn(sessions, 'update').mockImplementationOnce(async (...args: Parameters<typeof update>) => {
+ const result = await update(...args);
+ landed.resolve();
+ await answer.promise;
+ return result;
+ });
+ const stopping = a.service.stop(id);
+ await landed.promise;
+ read.resolve();
+
+ await expect(deleting).rejects.toThrow('connection reset');
+ answer.resolve();
+ await stopping;
+
+ expect((await sessions.findOneByOrFail({ id })).desiredState).toBe('stopped');
+ });
+
+ it('takes its stop back when another delete of it is refused at the first fence too', async () => {
+ const id = await seed({ status: SessionStatus.DISCONNECTED });
+ await a.service.start(id, { explicit: true });
+ jest
+ .spyOn(SessionLifecycleFences.prototype, 'awaitPendingTeardown')
+ .mockRejectedValueOnce(new ConflictException('a logout teardown is running'))
+ .mockRejectedValueOnce(new ConflictException('a logout teardown is running'));
+
+ const outcomes = await Promise.allSettled([a.service.delete(id), a.service.delete(id)]);
+
+ expect(outcomes.map(outcome => outcome.status)).toEqual(['rejected', 'rejected']);
+ expect(a.service.getEngine(id)).toBe(a.engines[0]);
+ expect((await a.db.getRepository(Session).findOneByOrFail({ id })).desiredState).toBeNull();
+ expect(bookkeeping(id)).toEqual({ deletes: false, stopWrites: false });
+ });
+
+ // Each of these begins before the delete and ends, having recorded no stop, before the delete fails.
+ it.each([
+ ['a force-kill with no engine to kill', 'kill-refused', 'BadRequestException'],
+ ['a force-kill that cannot record its stop', 'kill-write', 'ok'],
+ ['a stop whose row read fails', 'stop-read', 'Error'],
+ ['a stop that cannot record itself', 'stop-write', 'Error'],
+ ])('takes its stop back when %s overlaps it', async (_case, other, answer) => {
+ const id = await seed({ status: SessionStatus.DISCONNECTED });
+ if (other === 'kill-write') await a.service.start(id, { explicit: true });
+ const sessions = a.db.getRepository(Session);
+ const findOne = sessions.findOne.bind(sessions);
+ const reached = deferred();
+ const go = deferred();
+ // The other request is held at its row read or at its stop record until the delete has begun.
+ const hold = async (): Promise<void> => {
+ reached.resolve();
+ await go.promise;
+ };
+ if (other === 'kill-refused') {
+ jest.spyOn(sessions, 'findOne').mockImplementationOnce(async (...args: Parameters<typeof findOne>) => {
+ await hold();
+ return findOne(...args);
+ });
+ } else if (other === 'stop-read') {
+ jest.spyOn(sessions, 'findOne').mockImplementationOnce(async () => {
+ await hold();
+ throw new Error('connection reset');
+ });
+ } else {
+ jest.spyOn(sessions, 'update').mockImplementationOnce(async () => {
+ await hold();
+ throw new Error('connection reset');
+ });
+ }
+ // The delete waits at its first fence until the other request has answered, and is refused there.
+ const fenced = deferred();
+ const refuse = deferred();
+ jest.spyOn(SessionLifecycleFences.prototype, 'awaitPendingTeardown').mockImplementationOnce(async () => {
+ fenced.resolve();
+ await refuse.promise;
+ throw new ConflictException('a logout teardown is running');
+ });
+ const request = other.startsWith('kill') ? a.service.forceKill(id) : a.service.stop(id);
+ const answered = request.then(
+ () => 'ok',
+ (error: unknown) => (error as Error).constructor.name,
+ );
+ await reached.promise;
+ const deleting = a.service.delete(id);
+ await fenced.promise;
+ go.resolve();
+ expect(await answered).toBe(answer);
+ refuse.resolve();
+
+ await expect(deleting).rejects.toBeInstanceOf(ConflictException);
+
+ expect((await sessions.findOneByOrFail({ id })).desiredState).toBeNull();
+ expect(bookkeeping(id)).toEqual({ deletes: false, stopWrites: false });
+ });
+
+ it.each([
+ ['records it', 'stopped'],
+ ['fails to read the session', null],
+ ])('leaves the session to a stop still reading it when its delete fails, which then %s', async (_case, after) => {
+ const id = await seed({ status: SessionStatus.DISCONNECTED });
+ const sessions = a.db.getRepository(Session);
+ const findOne = sessions.findOne.bind(sessions);
+ const reading = deferred();
+ const read = deferred();
+ jest.spyOn(sessions, 'findOne').mockImplementationOnce(async (...args: Parameters<typeof findOne>) => {
+ reading.resolve();
+ await read.promise;
+ if (after === null) throw new Error('connection reset');
+ return findOne(...args);
+ });
+ jest
+ .spyOn(SessionLifecycleFences.prototype, 'awaitPendingTeardown')
+ .mockRejectedValueOnce(new ConflictException('a logout teardown is running'));
+ const stopping = a.service.stop(id).catch((error: unknown) => error);
+ await reading.promise;
+
+ await expect(a.service.delete(id)).rejects.toBeInstanceOf(ConflictException);
+ read.resolve();
+ await stopping;
+
+ expect((await sessions.findOneByOrFail({ id })).desiredState).toBe(after);
+ });
+
+ // Two deletes of one session share the stop the first of them recorded; the last one out decides.
+ it('keeps the session down for a delete still running when another delete of it is refused', async () => {
+ const id = await seed({ status: SessionStatus.DISCONNECTED });
+ const sessions = a.db.getRepository(Session);
+ const findOne = sessions.findOne.bind(sessions);
+ const reading = deferred();
+ const read = deferred();
+ // The first delete's row read, which comes after its stop record, is held.
+ jest.spyOn(sessions, 'findOne').mockImplementationOnce(async (...args: Parameters<typeof findOne>) => {
+ reading.resolve();
+ await read.promise;
+ return findOne(...args);
+ });
+ const reached = deferred();
+ const release = deferred();
+ a.hooks.execute.mockImplementation(async (event: string, data: unknown) => {
+ if (event === 'session:deleted') {
+ reached.resolve();
+ await release.promise;
+ }
+ return { continue: true, data };
+ });
+ finish = release.resolve;
+ deletion = a.service.delete(id);
+ await reading.promise;
+ jest
+ .spyOn(SessionLifecycleFences.prototype, 'awaitPendingTeardown')
+ .mockRejectedValueOnce(new ConflictException('a logout teardown is running'));
+
+ await expect(a.service.delete(id)).rejects.toBeInstanceOf(ConflictException);
+ expect((await sessions.findOneByOrFail({ id })).desiredState).toBe('stopped');
+
+ read.resolve();
+ await reached.promise;
+ // The rows are being removed: a process that died here would leave the session down.
+ expect((await sessions.findOneByOrFail({ id })).desiredState).toBe('stopped');
+ release.resolve();
+ await deletion;
+
+ expect(await rowsOf(id)).toEqual(gone);
+ });
+
+ /**
+ * A failed delete's take-back is held while `next` runs: until `next` has written a stop record or 200 ms
+ * have passed, then let go.
+ */
+ async function duringTakeBack(
+ next: (id: string) => Promise<unknown>,
+ { takeBackFails = false, refusals = 1 } = {},
+ ): Promise<string> {
+ const id = await seed({ status: SessionStatus.DISCONNECTED });
+ const sessions = a.db.getRepository(Session);
+ const update = sessions.update.bind(sessions);
+ const clearing = deferred();
+ const cleared = deferred();
+ let held = false;
+ let stopsWritten = 0;
+ // Only the first take-back is held, and fails with `takeBackFails`; any later one runs at once.
+ jest.spyOn(sessions, 'update').mockImplementation(async (...args: Parameters<typeof update>) => {
+ if (args[1].desiredState === null && !held) {
+ held = true;
+ clearing.resolve();
+ await cleared.promise;
+ if (takeBackFails) throw new Error('connection reset');
+ }
+ const result = await update(...args);
+ if (args[1].desiredState === 'stopped') stopsWritten++;
+ return result;
+ });
+ // The failed delete is refused at its first fence, and so are the next `refusals - 1` deletes.
+ const fence = jest.spyOn(SessionLifecycleFences.prototype, 'awaitPendingTeardown');
+ for (let refused = 0; refused < refusals; refused++) {
+ fence.mockRejectedValueOnce(new ConflictException('a logout teardown is running'));
+ }
+ const failing = a.service.delete(id).catch((error: unknown) => error);
+ await clearing.promise;
+ const written = stopsWritten;
+ const running = next(id);
+ for (let waited = 0; waited < 200 && stopsWritten === written; waited += 5) {
+ await new Promise(resolve => setTimeout(resolve, 5));
+ }
+ cleared.resolve();
+ expect(await failing).toBeInstanceOf(ConflictException);
+ await running;
+ return id;
+ }
+
+ it('records its own stop when it begins while a failed delete takes its stop back', async () => {
+ const reached = deferred();
+ const release = deferred();
+ a.hooks.execute.mockImplementation(async (event: string, data: unknown) => {
+ if (event === 'session:deleted') {
+ reached.resolve();
+ await release.promise;
+ }
+ return { continue: true, data };
+ });
+ finish = release.resolve;
+ const id = await duringTakeBack(async next => {
+ deletion = a.service.delete(next);
+ await reached.promise;
+ });
+
+ // The rows are being removed: a process that died here would leave the session down.
+ expect((await a.db.getRepository(Session).findOneByOrFail({ id })).desiredState).toBe('stopped');
+ release.resolve();
+ await deletion;
+ expect(await rowsOf(id)).toEqual(gone);
+ });
+
+ it('keeps a stop sent while a failed delete takes its stop back', async () => {
+ const id = await duringTakeBack(next => a.service.stop(next));
+
+ expect((await a.db.getRepository(Session).findOneByOrFail({ id })).desiredState).toBe('stopped');
+ expect(bookkeeping(id)).toEqual({ deletes: false, stopWrites: false });
+ });
+
+ it('leaves the take-back to the failed delete when one begun during it ends before recording its own', async () => {
+ const id = await duringTakeBack(async next => {
+ jest.spyOn(a.ownership, 'claim').mockRejectedValueOnce(new Error('Connection terminated unexpectedly'));
+ await expect(a.service.delete(next)).rejects.toBeInstanceOf(ServiceUnavailableException);
+ await a.service.stop(next);
+ });
+
+ expect((await a.db.getRepository(Session).findOneByOrFail({ id })).desiredState).toBe('stopped');
+ });
+
+ it('takes the stop back after a failed take-back when a delete begun during it fails too', async () => {
+ const id = await duringTakeBack(next => expect(a.service.delete(next)).rejects.toBeInstanceOf(ConflictException), {
+ takeBackFails: true,
+ refusals: 2,
+ });
+
+ expect((await a.db.getRepository(Session).findOneByOrFail({ id })).desiredState).toBeNull();
+ expect(bookkeeping(id)).toEqual({ deletes: false, stopWrites: false });
+ });
+
it('refuses the delete and removes nothing when the claim cannot be written', async () => {
const id = await seed({ status: SessionStatus.DISCONNECTED });
await a.service.start(id, { explicit: true });
diff --git a/src/modules/session/session-engine-controls.ts b/src/modules/session/session-engine-controls.ts
index 1296bfce2..8e32e2281 100644
--- a/src/modules/session/session-engine-controls.ts
+++ b/src/modules/session/session-engine-controls.ts
@@ -149,6 +149,8 @@ export interface StopHooks {
onReadFailed?: () => void;
/** forceKill() only: runs synchronously once there is an engine to kill; records the stop request. */
onEngineFound?: () => void;
+ /** forceKill() only: runs the write of the kill's stop record, which it is handed; without it, the write runs alone. */
+ recordStop?: (write: () => Promise<unknown>) => Promise<void>;
/** delete() only: runs once the delete starts removing the session's data, the session:deleted hook first. */
onRemoving?: () => void;
}
@@ -747,15 +749,14 @@ export class SessionEngineControls {
// explicit start. Recorded only once there is an engine to kill, and before the teardown, so a
// POST /start that lands after the eviction clears it and stands. A failed write must not keep a
// wedged engine alive, so it is logged and the kill goes on.
- await this.sessionRepository
- .update(id, { desiredState: 'stopped' })
- .catch((error: unknown) =>
- this.logger.error(
- 'Failed to record the force-kill as a stop',
- error instanceof Error ? error.message : String(error),
- { sessionId: id, action: 'force_kill_keep_down_failed' },
- ),
- );
+ const write = (): Promise<unknown> => this.sessionRepository.update(id, { desiredState: 'stopped' });
+ await (hooks.recordStop ? hooks.recordStop(write) : write()).catch((error: unknown) =>
+ this.logger.error(
+ 'Failed to record the force-kill as a stop',
+ error instanceof Error ? error.message : String(error),
+ { sessionId: id, action: 'force_kill_keep_down_failed' },
+ ),
+ );
// Announced by the write below rather than from the engine callback, as in stop().
this.operatorTeardowns.set(id, engine);
diff --git a/src/modules/session/session.service.ts b/src/modules/session/session.service.ts
index db960f5ad..40210bb3f 100644
--- a/src/modules/session/session.service.ts
+++ b/src/modules/session/session.service.ts
@@ -103,6 +103,19 @@ export interface SessionListOptions extends ListOptions {
name?: string;
}
+/** The deletes of one session running on this node, and the stop record they share; see SessionService.delete. */
+interface DeleteStops {
+ running: number;
+ /** One of them wrote the record on a row that had none. */
+ recorded: boolean;
+ /** One of them began removing the session's data. */
+ removing: boolean;
+ /** An operator stop or a kill recorded a stop while they ran. */
+ overtaken: boolean;
+ /** The last one out taking the record back. */
+ givingBack?: Promise<void>;
+}
+
/**
* Statuses that assert an engine is running somewhere. The boot reset clears them for every row this
* node may claim; markLapsedDisconnected clears them for a row whose holder never came back, a
@@ -145,8 +158,10 @@ export class SessionService implements OnModuleDestroy, OnModuleInit, OnApplicat
* after it began from a mark left over by an earlier stop, which start() clears by design.
*/
private readonly stopRequests = new Map<string, number>();
- /** stop() and forceKill() calls in flight per session, whose stop record may still be written; see delete(). */
- private readonly stopsInFlight = new Map<string, number>();
+ /** Stop records of stop() and forceKill() being written per session, which may land before a take-back; see delete(). */
+ private readonly stopWritesInFlight = new Map<string, number>();
+ /** Per session, the deletes running here; see delete(). */
+ private readonly deleteStops = new Map<string, DeleteStops>();
/**
* Starts past start()'s cap check, held until they return. That check counts them with the
* engine's slot holders, so two starts at the last free slot cannot both pass it while their claims
@@ -619,20 +634,14 @@ export class SessionService implements OnModuleDestroy, OnModuleInit, OnApplicat
// post-INITIALIZING check by the time that write settles; awaiting anything first — the fence's
// COUNT, or delete()'s own requireSession — would let the mark land after that window. A mark
// left behind when the fence refuses (409) is harmless and is cleared by the next start().
- // A stop or kill already running may record its stop after this delete records its own.
- const stopInFlight = this.stopsInFlight.has(id);
this.markStopping(id);
// Running from here, before the claim: a claim with nothing running behind it would read as idle to the
// heartbeat, and a stop or a failed start here would hand it back while the rows are removed. It ends
// before the claim is handed back.
const endDelete = this.engineLifecycle.holdDelete(id);
- // This delete's own stop count, and every other stop counted before it: a stop counted while it runs is
- // the operator's, and outlives a failed delete.
- let counted = 1;
- const otherStopsBefore = (this.stopRequests.get(id) ?? 0) - counted;
+ // From here an operator stop or kill that records a stop is seen, also one whose record is already on its way.
+ const stops = this.joinDeleteStops(id);
let claimed = false;
- let keptDown = false;
- let removing = false;
try {
try {
if (this.ownership) {
@@ -641,19 +650,16 @@ export class SessionService implements OnModuleDestroy, OnModuleInit, OnApplicat
claimed = true;
}
// A failed write took nothing down, so the mark and count are undone as for a failed claim.
- keptDown = await this.keepDownForDelete(id).catch((error: unknown) => {
+ await this.keepDownForDelete(id, stops).catch((error: unknown) => {
this.engineLifecycle.clearStopping(id);
this.uncountStopRequest(id);
throw error;
});
await this.engineLifecycle.delete(id, {
// A failed row read deleted nothing: the lifecycle drops the mark, and the count goes with it.
- onReadFailed: () => {
- counted = 0;
- this.uncountStopRequest(id);
- },
+ onReadFailed: () => this.uncountStopRequest(id),
onRemoving: () => {
- removing = true;
+ stops.removing = true;
},
});
} finally {
@@ -664,14 +670,50 @@ export class SessionService implements OnModuleDestroy, OnModuleInit, OnApplicat
} catch (error) {
// A delete that removed nothing leaves the session as it found it. One that began to remove it leaves
// it stopped, as a process that dies part-way does.
- const stoppedMeanwhile = stopInFlight || (this.stopRequests.get(id) ?? 0) - counted !== otherStopsBefore;
- if (keptDown && !removing && !stoppedMeanwhile) await this.clearDeleteStop(id);
+ await this.leaveDelete(id, stops);
// The row survived. A claim left on it lapses with nothing to renew it, and a peer would then
// adopt and restart the session the operator was deleting.
if (claimed) await this.releaseAfterTeardown(id);
this.discardStopMarkForMissingSession(id, error);
throw error;
}
+ await this.leaveDelete(id, stops);
+ }
+
+ private joinDeleteStops(id: string): DeleteStops {
+ let stops = this.deleteStops.get(id);
+ if (!stops) {
+ stops = { running: 0, recorded: false, removing: false, overtaken: false };
+ this.deleteStops.set(id, stops);
+ }
+ stops.running += 1;
+ return stops;
+ }
+
+ /**
+ * End a delete's share of the stop record. Another delete of the session still running here relies on it, so
+ * only the last one out takes it back, and only when they wrote it, none began removing the session, and no
+ * operator stop or kill recorded one meanwhile or is still writing one (it may land before the take-back).
+ */
+ private async leaveDelete(id: string, stops: DeleteStops): Promise<void> {
+ stops.running -= 1;
+ if (
+ stops.running === 0 &&
+ stops.recorded &&
+ !stops.removing &&
+ !stops.overtaken &&
+ !this.stopWritesInFlight.has(id)
+ ) {
+ // Not theirs while it is cleared, so a delete that begins and ends meanwhile leaves it alone.
+ stops.recorded = false;
+ stops.givingBack = this.clearDeleteStop(id).then(cleared => {
+ // Still on the row: a delete that began meanwhile shares it.
+ if (!cleared) stops.recorded = true;
+ stops.givingBack = undefined;
+ });
+ await stops.givingBack;
+ }
+ if (stops.running === 0 && !stops.givingBack) this.deleteStops.delete(id);
}
/**
@@ -705,26 +747,33 @@ export class SessionService implements OnModuleDestroy, OnModuleInit, OnApplicat
/**
* Record the delete as a stop on a row that has none, so a session whose delete ends part-way, with the
- * process that ran it, stays down: no peer adopts it and no boot starts it. Answers whether it wrote one.
+ * process that ran it, stays down: no peer adopts it and no boot starts it. Written after a take-back in
+ * progress, which would otherwise clear it.
*/
- private async keepDownForDelete(id: string): Promise<boolean> {
+ private async keepDownForDelete(id: string, stops: DeleteStops): Promise<void> {
+ if (stops.givingBack) await stops.givingBack;
const { affected } = await this.sessionRepository.update(
{ id, desiredState: IsNull() },
{ desiredState: 'stopped' },
);
- return affected === 1;
+ if (affected === 1) stops.recorded = true;
}
- /** Undo keepDownForDelete. Best effort: the delete's own error is the answer, and a stop left on is safe. */
- private async clearDeleteStop(id: string): Promise<void> {
+ /**
+ * Undo keepDownForDelete; answers whether the record is off the row. Best effort: the delete's own error is
+ * the answer, and a stop left on is safe.
+ */
+ private async clearDeleteStop(id: string): Promise<boolean> {
try {
await this.sessionRepository.update({ id, desiredState: 'stopped' }, { desiredState: null });
+ return true;
} catch (error) {
this.logger.warn(`Could not clear the stop a failed delete recorded for session ${id}`, {
sessionId: id,
action: 'session_delete_stop_clear_failed',
error: error instanceof Error ? error.message : String(error),
});
+ return false;
}
}
@@ -933,7 +982,6 @@ export class SessionService implements OnModuleDestroy, OnModuleInit, OnApplicat
async stop(id: string): Promise<Session> {
// Synchronous stop-mark before the awaited fence — see delete() for why.
this.markStopping(id);
- const endStop = this.countStopInFlight(id);
let session: Session;
try {
if (this.ownership) await this.assertNotHeldElsewhere(id);
@@ -959,8 +1007,6 @@ export class SessionService implements OnModuleDestroy, OnModuleInit, OnApplicat
if (error instanceof BadGatewayException) await this.releaseAfterTeardown(id);
this.discardStopMarkForMissingSession(id, error);
throw error;
- } finally {
- endStop();
}
// Handed back on the way out so a peer can pick it up immediately rather than waiting for the
// lease to lapse. Stop is the deliberate end of this process's ownership — but a start() that
@@ -1004,10 +1050,13 @@ export class SessionService implements OnModuleDestroy, OnModuleInit, OnApplicat
this.uncountStopRequest(id);
};
if (this.engines.get(id)) count();
- const endStop = this.countStopInFlight(id);
try {
// The engine kill records the stop itself, once it has an engine to kill.
- const session = await this.engineLifecycle.forceKill(id, { onReadFailed: uncount, onEngineFound: count });
+ const session = await this.engineLifecycle.forceKill(id, {
+ onReadFailed: uncount,
+ onEngineFound: count,
+ recordStop: write => this.recordStop(id, write),
+ });
await this.releaseAfterTeardown(id);
return session;
} catch (error) {
@@ -1016,23 +1065,32 @@ export class SessionService implements OnModuleDestroy, OnModuleInit, OnApplicat
if (error instanceof BadRequestException) uncount();
if (!(error instanceof BadRequestException)) await this.releaseAfterTeardown(id);
throw error;
- } finally {
- endStop();
}
}
- private countStopInFlight(id: string): () => void {
- this.stopsInFlight.set(id, (this.stopsInFlight.get(id) ?? 0) + 1);
- return () => {
- const left = (this.stopsInFlight.get(id) ?? 1) - 1;
- if (left > 0) this.stopsInFlight.set(id, left);
- else this.stopsInFlight.delete(id);
- };
- }
-
/** Persist the operator's stop so boot auto-start and the takeover sweep leave the session down. */
private async keepDown(id: string): Promise<void> {
- await this.sessionRepository.update(id, { desiredState: 'stopped' });
+ await this.recordStop(id, () => this.sessionRepository.update(id, { desiredState: 'stopped' }));
+ }
+
+ /**
+ * Write an operator stop's or a kill's record of a stop. The deletes of the session running here keep their
+ * own record once it lands, and leave it alone while it is being written; it is written after a take-back
+ * in progress, which would otherwise clear it.
+ */
+ private async recordStop(id: string, write: () => Promise<unknown>): Promise<void> {
+ this.stopWritesInFlight.set(id, (this.stopWritesInFlight.get(id) ?? 0) + 1);
+ try {
+ const givingBack = this.deleteStops.get(id)?.givingBack;
+ if (givingBack) await givingBack;
+ await write();
+ const stops = this.deleteStops.get(id);
+ if (stops) stops.overtaken = true;
+ } finally {
+ const left = (this.stopWritesInFlight.get(id) ?? 1) - 1;
+ if (left > 0) this.stopWritesInFlight.set(id, left);
+ else this.stopWritesInFlight.delete(id);
+ }
}
/**