Skip to content

Free, Open Source, Self-Hosted WhatsApp API Gateway

69.4 KB
Raw Blame
import {
  BadGatewayException,
  BadRequestException,
  ConflictException,
  HttpStatus,
  NotFoundException,
  ServiceUnavailableException,
} from '@nestjs/common';
import { ConfigService } from '@nestjs/config';
import { EntityTarget, EntityManager, In, IsNull, Raw, Repository, DataSource } from 'typeorm';
import { Session, SessionStatus } from './entities/session.entity';
import { Message } from '../message/entities/message.entity';
import { MessageBatch } from '../message/entities/message-batch.entity';
import { failUnfinishedSessionBatches } from '../message/batch-media.util';
import { SendIdempotencyKey } from '../message/entities/send-idempotency-key.entity';
import { Webhook } from '../webhook/entities/webhook.entity';
import { WebhookOutboxEvent } from '../webhook/entities/webhook-outbox-event.entity';
import { WebhookDeliveryFailure } from '../webhook/entities/webhook-delivery-failure.entity';
import { IntegrationDeliveryFailure } from '../integration/entities/integration-delivery-failure.entity';
import { Template } from '../template/entities/template.entity';
import { BaileysStoredMessage } from '../../engine';
import { ChatState } from '../../engine/adapters/baileys-chat-state.entity';
import { StatusUpdate } from '../status-store/entities/status-update.entity';
import { closeSessionWrites } from '../../common/utils/session-write-scope';
import { awaitNoOpenTransaction } from '../../common/utils/shared-connection';
import { EngineFactory } from '../../engine/engine.factory';
import { EngineRegistry } from '../../engine/engine-registry.service';
import { IWhatsAppEngine } from '../../engine/interfaces/whatsapp-engine.interface';
import { SessionErrorStore } from './session-error-store.service';
import { SessionRestrictionStore } from './session-restriction-store.service';
import { PresenceStore } from './presence-store.service';
import { type createLogger } from '../../common/services/logger.service';
import { HookManager } from '../../core/hooks';
import { SessionLifecycleFences } from './session-lifecycle-fences';
import { SessionStatusBroadcaster } from './session-status-broadcaster';
// The two pure config resolvers stay exported from the lifecycle module (its export surface is
// unchanged — session.service re-exports them from there), so this unit imports them back. The
// resulting import cycle is deferred-usage only — the bindings are read inside method bodies,
// never at module-eval time — which the CommonJS loader resolves safely.
import {
  resolveMaxConcurrentSessions,
  resolveReconnectConfig,
  startSlotHolders,
  type ReconnectState,
} from './session-engine-lifecycle.service';

/**
 * Rows of the tables that grow with traffic (messages, webhook outbox events, webhook delivery failures)
 * removed per statement when a session is deleted. On SQLite this is the most a batch may take: far below the
 * bind-parameter limit, and the loop sizes each batch to the time the last one took (see
 * SESSION_DELETE_BATCH_TARGET_MS_SQLITE), because one writer serves every session and each message row also
 * fires the full-text index trigger. A larger batch costs a little less per row but stalls longer: a
 * 300k-message session deletes 1.2 to 1.7 times faster at 2000 rows than at 500 (it depends on the load of the
 * host), with a longer worst statement. Most of the gain over the first batched delete comes from leaving the
 * session out of the delete's condition, see deleteTrafficRowsInBatches.
 */
export const SESSION_DELETE_BATCH_SIZE_SQLITE = 2000;

/**
 * How long a SQLite batch should take. The driver runs a statement on the event loop, so this is the longest
 * the process is held for one batch. The cost per row rises with the size of the database (about 75 microseconds
 * per row at 300k messages, 170 at 1.2M), which a fixed row count turned into a stall that grew with the history.
 * A batch costs less per row the larger it is, so a shorter target bounds the stall more tightly and lengthens
 * the whole delete: at 150 ms it takes about 1.4 times as long as at 2000 rows a batch, at 250 ms 1.0 to 1.2 times.
 */
export const SESSION_DELETE_BATCH_TARGET_MS_SQLITE = 150;

/** The first SQLite batch and the least one may shrink to, so a slow database is never started on a large batch. */
export const SESSION_DELETE_BATCH_MIN_SQLITE = 250;

/**
 * The pause after a SQLite batch. Yielding for one turn of the event loop lets a request take one step, but a
 * request takes several turns and each was followed by the next batch, so it waited out several batches. A
 * short pause lets it finish before the next one starts.
 */
export const SESSION_DELETE_PAUSE_MS_SQLITE = 5;

/**
 * The same on PostgreSQL, where a statement does not hold the process and a larger batch buys little (a million
 * small rows on an analyzed table delete in about the same few seconds at 500 or 2000 rows). What bounds a
 * statement there is its time under the statement timeout, and that follows the bytes of the rows it deletes: a
 * message that keeps its media inline stores megabytes per row. So it keeps the smaller batch and its headroom.
 */
export const SESSION_DELETE_BATCH_SIZE_POSTGRES = 500;

/**
 * How many batches of a webhook outbox or delivery failure pass one read feeds on PostgreSQL. These two tables
 * have no index that orders a session's rows, so each read goes back over the session's whole index range,
 * including the entries of rows already deleted that a vacuum has not yet removed; a read after every batch made
 * the delete take longer than linear, by seconds at 300,000 rows and far more beyond.
 */
export const SESSION_DELETE_READ_AHEAD_POSTGRES = 20;

/**
 * The size of the next SQLite batch: the last one scaled to the target time, growing by at most a factor of two
 * so a fast first batch cannot overshoot into a slow one, and kept between the floor and the ceiling.
 */
export function nextSqliteDeleteBatchSize(rows: number, tookMs: number): number {
  const scaled = Math.round((rows * SESSION_DELETE_BATCH_TARGET_MS_SQLITE) / Math.max(tookMs, 1));
  return Math.max(SESSION_DELETE_BATCH_MIN_SQLITE, Math.min(SESSION_DELETE_BATCH_SIZE_SQLITE, rows * 2, scaled));
}

/** One table's pass of the batched delete: the repository to delete from and the reader of the next page of ids. */
type TrafficPass = { rows: Repository<{ id: string }>; next: (take: number) => Promise<string[]> };

/** A start refused because an operator stopped the session. Same 409 on the wire as any conflict. */
export class SessionStoppedException extends ConflictException {}

/** A start refused because the gateway is shutting down. Retryable once it is back. */
export class ServerShuttingDownException extends ServiceUnavailableException {
  constructor() {
    super('Server is shutting down');
  }
}

/**
 * The stop marks. Each add() stamps the id afresh, even one already marked, so start() can tell a mark
 * set while it waited from one that was already there when it began.
 */
export class StopMarks extends Set<string> {
  private seq = 0;
  private readonly stamps = new Map<string, number>();

  override add(id: string): this {
    this.stamps.set(id, ++this.seq);
    return super.add(id);
  }

  override delete(id: string): boolean {
    this.stamps.delete(id);
    return super.delete(id);
  }

  /** The stamp of the id's current mark, undefined when it has none. */
  stamp(id: string): number | undefined {
    return this.stamps.get(id);
  }
}

/** Caller callbacks for stop(), delete(), forceKill() and logout(), run around the session read that opens each verb. */
export interface StopHooks {
  /** delete() only: the ownership captured by its caller still covers this operation. */
  ownsClaim?: () => boolean;
  /** stop() only: runs once the session read succeeds and before any teardown; records the stop. */
  afterRead?: () => Promise<void>;
  /**
   * Runs when the first session read fails (for stop() and delete(), after the stop mark is dropped):
   * withdraws the caller's stop request.
   */
  onReadFailed?: () => void;
  /** forceKill()/logout(): runs synchronously once there is an engine to retire. */
  onEngineFound?: () => void;
  /** forceKill() only: persist the stop with the caller's current ownership predicate. */
  recordStop?: () => Promise<void>;
  /** delete() only: runs once the delete starts removing the session's data, the session:deleted hook first. */
  onRemoving?: () => void;
}

/**
 * The deps + core call-ins SessionEngineControls needs from the lifecycle. Built ONCE in the
 * lifecycle's constructor. The dependency VALUES (sessionRepository … broadcaster) are captured at
 * construction — the same reads the pre-extraction code made of the lifecycle's readonly
 * constructor fields — while the shared state containers (stoppingSessions, reconnectStates,
 * stuckAuthRecoveryUsed, initializingSessions) are handed over BY REFERENCE, so this unit, the
 * lifecycle core, and every spec poking them through the lifecycle always observe the same
 * instances. The core call-ins (cancelReconnect … updateStatus) are deliberately NON-async
 * passthrough closures onto the lifecycle's live methods (the Task-1 delegate rule: an `async`
 * wrapper would adopt the inner promise and add settlement hops the retirement-race specs assert
 * against). `dataSource` is a LIVE closure — never a captured value — because specs replace the
 * lifecycle's dataSource on the instance after construction (logout-teardown-race.spec) and
 * delete()'s transaction must observe the current one at call time.
 */
export interface SessionEngineControlsHost {
  sessionRepository: Repository<Session>;
  engineFactory: EngineFactory;
  engines: EngineRegistry;
  sessionErrors: SessionErrorStore;
  sessionRestrictions: SessionRestrictionStore;
  presence: PresenceStore;
  hookManager: HookManager;
  configService?: ConfigService;
  nodeId?: string;
  logger: ReturnType<typeof createLogger>;
  /** LIVE closure — read at call time, never captured (specs replace lifecycle.dataSource at runtime). */
  dataSource(): DataSource;
  fences: SessionLifecycleFences;
  broadcaster: SessionStatusBroadcaster;
  cancelReconnect(id: string): void;
  initializeEngine(id: string, session: Session): Promise<void>;
  isSessionRetired(id: string): Promise<boolean>;
  prepareStart(id: string, desiredState: Session['desiredState']): Promise<boolean>;
  purgeAuthDirsIfDeleted(id: string): Promise<void>;
  updateStatus(id: string, status: SessionStatus): Promise<void>;
  /** Ownership gate, same contract as SessionEngineWiringHost.ownsSession. */
  ownsSession(id: string): boolean;
  /** True once shutdown() has begun; the lifecycle never reopens. */
  isClosed(): boolean;
  stoppingSessions: StopMarks;
  /** The engine each operator-initiated teardown is retiring; see the lifecycle field of the same name. */
  operatorTeardowns: Map<string, IWhatsAppEngine>;
  reconnectStates: Map<string, ReconnectState>;
  stuckAuthRecoveryUsed: Set<string>;
  initializingSessions: Set<string>;
}

/**
 * The 7 public control verbs extracted from SessionEngineLifecycle: start/stop/logout/forceKill/
 * delete/shutdown/stopOrphanEngines, plus the private findOne-or-404 requireSession. Plain class
 * (NOT a NestJS provider — the lifecycle's constructor signature is frozen by specs), built inside
 * the lifecycle's constructor; the lifecycle keeps all 7 verbs as one-line NON-async delegates, so
 * SessionService's public API path and every spec calling the verbs on the lifecycle stay
 * byte-identical. The dependency values and shared state containers are assigned to same-named
 * fields (by reference), so the method bodies below read like the inline originals; every former
 * core call-in (`this.cancelReconnect`, `this.initializeEngine`, `this.isSessionRetired`,
 * `this.purgeAuthDirsIfDeleted`, `this.updateStatus`) goes through `host`, the fences through the
 * `fences` unit, and delete()'s transaction reads `this.host.dataSource()` — the LIVE closure — at
 * call time. Method bodies and their comments/docstrings moved verbatim.
 */
export class SessionEngineControls {
  private readonly sessionRepository: Repository<Session>;
  private readonly engineFactory: EngineFactory;
  private readonly engines: EngineRegistry;
  private readonly sessionErrors: SessionErrorStore;
  private readonly sessionRestrictions: SessionRestrictionStore;
  private readonly presence: PresenceStore;
  private readonly hookManager: HookManager;
  private readonly configService?: ConfigService;
  private readonly logger: ReturnType<typeof createLogger>;
  private readonly fences: SessionLifecycleFences;
  private readonly broadcaster: SessionStatusBroadcaster;
  private readonly stoppingSessions: StopMarks;
  private readonly operatorTeardowns: Map<string, IWhatsAppEngine>;
  private readonly reconnectStates: Map<string, ReconnectState>;
  private readonly stuckAuthRecoveryUsed: Set<string>;
  private readonly initializingSessions: Set<string>;
  /**
   * Deletes in flight per id, counted so that two overlapping deletes of one id do not end each other's hold.
   * The stop mark of a delete cannot refuse a start for as long as the delete runs: start() takes a mark that
   * was already set when it began for a stale one and clears it.
   */
  private readonly deleting = new Map<string, number>();

  constructor(private readonly host: SessionEngineControlsHost) {
    this.sessionRepository = host.sessionRepository;
    this.engineFactory = host.engineFactory;
    this.engines = host.engines;
    this.sessionErrors = host.sessionErrors;
    this.sessionRestrictions = host.sessionRestrictions;
    this.presence = host.presence;
    this.hookManager = host.hookManager;
    this.configService = host.configService;
    this.logger = host.logger;
    this.fences = host.fences;
    this.broadcaster = host.broadcaster;
    this.stoppingSessions = host.stoppingSessions;
    this.operatorTeardowns = host.operatorTeardowns;
    this.reconnectStates = host.reconnectStates;
    this.stuckAuthRecoveryUsed = host.stuckAuthRecoveryUsed;
    this.initializingSessions = host.initializingSessions;
  }

  /**
   * True while a delete of the session runs: from before SessionService.delete claims the session until it
   * returns or throws, credential purge included.
   */
  isDeleting(id: string): boolean {
    return this.deleting.has(id);
  }

  /** Count a delete of the session as running until the returned call; see isDeleting. */
  holdDelete(id: string): () => void {
    this.deleting.set(id, (this.deleting.get(id) ?? 0) + 1);
    return () => {
      const running = (this.deleting.get(id) ?? 1) - 1;
      if (running > 0) this.deleting.set(id, running);
      else this.deleting.delete(id);
    };
  }

  /** findOne-or-404 with the runtime-state projection, mirroring SessionService.findOne. */
  private async requireSession(id: string): Promise<Session> {
    const session = await this.sessionRepository.findOne({ where: { id } });
    if (!session) {
      throw new NotFoundException(`Session with id '${id}' not found`);
    }
    return this.sessionRestrictions.attachTo(this.sessionErrors.attachTo(session));
  }

  /**
   * requireSession for stop()/delete(), whose caller sets the stop mark before this read. A read
   * that fails has retired nothing, so the mark must not outlive it: left on a running session, its
   * next disconnect would never reconnect and start() would keep refusing it as already started.
   */
  private async requireSessionOrDropStopMark(id: string, onReadFailed?: () => void): Promise<Session> {
    try {
      return await this.requireSession(id);
    } catch (error) {
      this.stoppingSessions.delete(id);
      onReadFailed?.();
      throw error;
    }
  }

  /**
   * `explicit` marks an operator's POST /start, the only start that may clear an operator's stop
   * (desiredState), and only a stop that was already on the row when it began. It clears it only
   * once past every refusal below, so a refused start leaves the stop exactly as it found it.
   * `stoppedSince` answers whether a stop or delete of the session issued after the caller's start
   * began still stands; the start yields to one, also when its mark is already gone.
   */
  async start(
    id: string,
    { explicit = false, stoppedSince }: { explicit?: boolean; stoppedSince?: () => boolean } = {},
  ): Promise<Session> {
    // Reserve the slot SYNCHRONOUSLY at entry — before even the requireSession await. Two
    // near-simultaneous start() calls must not both pass the check and orphan an engine (the has()
    // -> engines.set() window spans the awaited hook below), and the infra import pre-flight
    // (getActiveSessionIds) must see an in-flight start during the findOne round-trip too: registered
    // only after the row read, a start would be invisible in that window and a stopOrphans import
    // could DELETE the session row while an engine for it is being created. The finally clears the
    // reservation on success AND failure so a failed start never wedges at "already starting".
    if (this.initializingSessions.has(id)) {
      throw new BadRequestException('Session is already starting');
    }
    this.initializingSessions.add(id);
    // A stop mark already set when this start began is stale (a start is how one is cleared). One set
    // while it waits below comes from a retirement that saw this start in flight, stopOrphanEngines
    // above all, which writes nothing else this start could read and counts on it aborting. That
    // includes a mark set again on top of a stale one, which only its stamp tells apart.
    const markAtEntry = this.stoppingSessions.stamp(id);

    try {
      const session = await this.requireSession(id);

      // An operator's stop outranks any start but POST /start, so this refuses boot auto-start and
      // the takeover sweep, which pick their rows once and launch them seconds apart, when a stop
      // landed after that read. A delete's record ('deleting') counts as one.
      if (session.desiredState && !explicit) {
        throw new SessionStoppedException(`Session ${id} was stopped`);
      }

      if (this.engines.has(id)) {
        throw new BadRequestException('Session is already started');
      }
      const maxConcurrentSessions = resolveMaxConcurrentSessions(this.configService);
      if (maxConcurrentSessions !== null) {
        // Count each OTHER session once. A session mid-initialization is transiently in BOTH
        // `engines` (set at the start of initializeEngine) and `initializingSessions` (until
        // start()'s finally), so summing the two sizes would double-count it; and `id` itself is
        // already reserved in `initializingSessions` (added at entry), so it must not count
        // against the cap it is being checked against. A session waiting out a failed relaunch holds
        // no engine (the attempt evicted it) but re-registers one when its timer fires, without a cap
        // check of its own, so it keeps its slot here.
        const activeIds = startSlotHolders(this.engines, this.reconnectStates);
        activeIds.delete(id);
        if (activeIds.size >= maxConcurrentSessions) {
          throw new BadRequestException(`Maximum concurrent sessions reached (${maxConcurrentSessions})`);
        }
      }

      // Credential-teardown fence — runs IMMEDIATELY after the read-only session read + duplicate/cap
      // checks and BEFORE any lifecycle mutation (stop-mark clear, hook, reconnect-state, engine
      // creation, recovery-budget reset) or auth-dir access. A logout teardown that lost its deadline
      // race is still running and ends in an fs.rm of this session's on-disk profile — the same path
      // initializeEngine is about to populate. The fence is keyed by session NAME (see
      // awaitPendingTeardown) and FAIL CLOSED: a still-wedged teardown could still rm the fresh
      // profile, so refuse with a retryable 409 instead of proceeding. On a 409, no lifecycle state is touched
      // (the stop mark, reconnect timer, engine, last status/error, and recovery budget are left as
      // they were) — start() simply did not happen. The one transient exception is the
      // initializingSessions reservation added synchronously at start() entry: its finally removes it
      // again, so a refused start does not leave a false "already starting" mark behind (briefly held).
      await this.fences.awaitPendingTeardown(session.name);

      // A delete still running outranks every start, however long it has run: the mark it set is not a
      // stale one, and clearing it below would let an engine start under the delete that is removing the
      // session (the engine would outlive the row). Checked here, with no await before the clear.
      if (this.deleting.has(id)) {
        throw new SessionStoppedException(`Session ${id} is being deleted`);
      }

      // A fresh start intentionally (re-)creates the engine — clear any stale stop/delete mark, but
      // yield to one set after this start began, leaving it in place.
      if (this.stoppingSessions.has(id) && this.stoppingSessions.stamp(id) !== markAtEntry) {
        throw new SessionStoppedException(`Session ${id} was stopped`);
      }
      this.stoppingSessions.delete(id);

      // Cancel any reconnect timer a prior failed executeReconnect left pending, BEFORE the awaited
      // session:starting hook and engine init — otherwise the stale timer can fire during that I/O
      // and destroy/replace the engine this start() is about to create (or orphan the Chromium
      // process). Idempotent: a no-op when no reconnect state exists (the common fresh-start case).
      this.host.cancelReconnect(id);

      // The operator's stop, read again past every refusal and after the mark is cleared. A stop that
      // finished while this start waited above had no engine to take down and just lost its mark,
      // but its record is on the row: only a POST /start sent to an already stopped session clears
      // it, and any other start yields to it with the mark put back. A stop from here on sets the
      // mark again and retires this start after its init. A launch that fails after the clear still
      // counts as the start the operator asked for. A delete issued since this start began that failed once it
      // began removing has dropped its mark and left a record this start may have read too: it still stands.
      const { desiredState } = await this.requireSession(id);
      if ((desiredState && (!explicit || desiredState !== session.desiredState)) || stoppedSince?.()) {
        this.stoppingSessions.add(id);
        throw new SessionStoppedException(`Session ${id} was stopped`);
      }
      // Shutdown may have begun during the reads above, past SessionService's own checks. A start the
      // exiting process will not keep must not clear the stop or run the hook. A start that yielded to
      // a stop answers as the stop, never as a retryable 503: one already on the row is refused above,
      // and the mark (cleared above, so set by a stop or delete that began since) covers one whose row
      // write has not landed yet.
      const closedRefusal = (): Error =>
        this.stoppingSessions.has(id)
          ? new SessionStoppedException(`Session ${id} was stopped`)
          : new ServerShuttingDownException();
      if (this.host.isClosed()) {
        throw closedRefusal();
      }
      if (!(await this.host.prepareStart(id, desiredState))) {
        await this.requireSession(id);
        this.stoppingSessions.add(id);
        throw new SessionStoppedException(`Session ${id} was stopped`);
      }
      // The clear stands, but a shutdown during the write must not let the hook run.
      if (this.host.isClosed()) {
        throw closedRefusal();
      }

      // Execute hook before starting
      await this.hookManager.execute(
        'session:starting',
        { sessionId: id },
        {
          sessionId: id,
          source: 'SessionService',
        },
      );

      // A local teardown already retired this launch; preserve its normal inactive-session response.
      if (this.stoppingSessions.has(id)) return this.requireSession(id);

      // The hook may outlive the lease. Renew this claim only if no stop or owner has replaced it,
      // then register synchronously: a peer stop either changes the row first, or sees the live renewal.
      const prepared = await this.host.prepareStart(id, null);
      // A local stop can arrive during admission too. It owns the inactive response even when its
      // write made admission fail; a delete may already have dropped its mark but retain its request.
      if (this.stoppingSessions.has(id) || stoppedSince?.()) return this.requireSession(id);
      if (!prepared) {
        await this.requireSession(id);
        this.stoppingSessions.add(id);
        throw new SessionStoppedException(`Session ${id} was stopped`);
      }
      if (this.host.isClosed()) throw closedRefusal();

      // Initialize reconnect state from the (untrusted) opaque session.config — coerced + clamped
      // so a poisoned value can't drive a NaN/immediate-relaunch storm or an unbounded loop.
      const { maxAttempts, baseDelay } = resolveReconnectConfig(session.config);
      const reconnectState: ReconnectState = { attempts: 0, timer: null, maxAttempts, baseDelay };
      this.reconnectStates.set(id, reconnectState);

      // An accepted top-level start() re-arms the stuck-auth recovery budget: every fence above
      // (duplicate-start, cap, credential-teardown) passed, so this is a deliberate, operator-initiated
      // (re)start — not an automatic reconnect. The budget is hoisted to the session so an automatic
      // reconnect can't reset it per generation; only a fresh top-level episode may spend it again.
      // Boot auto-start reaches here through this same method, so it re-arms too.
      this.stuckAuthRecoveryUsed.delete(id);

      // initializeEngine registers its engine synchronously, before its first await, so the entry
      // read right after the call is this start's own engine (undefined if creation threw first).
      // The catch must use this capture, not a later lookup: a disconnect of this engine mid-init can
      // schedule a reconnect that registers a replacement before initialize() rejects.
      const before = this.engines.get(id);
      const init = this.host.initializeEngine(id, session);
      const registered = this.engines.get(id);
      const mine = registered !== before ? registered : undefined;
      let failedWritten = false;
      try {
        await init;
      } catch (err) {
        // engine.initialize() failed AFTER the engine was registered. Evict + tear it down so the
        // session doesn't wedge at "already started" with a leaked Chromium/socket permanently holding
        // a concurrency slot. Identity-checked: a replacement registered by a reconnect is left alone,
        // and so is its status (whoever evicted this start's engine owns the teardown).
        //
        // Use forceDestroy(), not destroy(): initialize() failing usually means the underlying
        // browser/CDP connection is already broken (e.g. a "Target closed" crash mid-injection), so
        // a graceful destroy() has nothing live to talk to — it can only time out via
        // teardownEngineSafely's race, after which the orphaned Chromium process is never actually
        // killed. forceDestroy() SIGKILLs the OS process directly, the same recovery force-kill uses
        // for a wedged engine, which is exactly the state this catch block is handling.
        if (mine && this.engines.deleteIfLive(id, mine)) {
          this.sessionErrors.set(id, err instanceof Error ? err.message : String(err));
          await this.fences.teardownEngineSafely(id, mine, e => e.forceDestroy(), 'force-destroy');
          // Fenced on ownership like the engine callbacks: initializeEngine can await a slow
          // Chromium launch for minutes, and this node's lease can lapse and be taken over inside
          // that window. FAILED is excluded from the boot reset AND from the takeover sweep, so
          // writing it onto a row a peer now owns strands the session on every node. Skipped during
          // shutdown too: an init that rejects then usually does so because shutdown destroyed its
          // engine, and the INITIALIZING row is reset by the next boot, where FAILED would not be.
          if (this.host.ownsSession(id) && !this.host.isClosed()) {
            failedWritten = true;
            await this.host.updateStatus(id, SessionStatus.FAILED).catch(() => undefined);
          }
        }
        // Drop the reconnect state this start() armed up front when no engine is registered: nothing
        // will ever fire it, and leaving it behind is dead state a later liveness check would have to
        // reason about. A retry builds its own. A reconnect that already registered a replacement
        // keeps its state.
        if (!this.engines.has(id) && this.reconnectStates.get(id) === reconnectState && !reconnectState.started) {
          this.host.cancelReconnect(id);
        }
        // Shutdown's destroy is the usual cause of an init rejecting now; the adapter's error would
        // read as a fault (or, if transient, buy a retry shutdown refuses anyway). A start a stop
        // retired keeps the adapter's error, as it would without shutdown. So does one whose FAILED
        // went out before shutdown began: a transient error then reaches the retry, whose refusal
        // hands the row back as DISCONNECTED for the next boot.
        throw this.host.isClosed() && !failedWritten && !this.stoppingSessions.has(id)
          ? new ServerShuttingDownException()
          : err;
      }

      // A stop()/delete() may have landed while we awaited engine.initialize() — if so, tear down the
      // engine we just registered so the session isn't resurrected to READY (mirrors the post-init
      // guard in executeReconnect; initialize()'s callbacks can also fire async after this returns).
      // delete() clears its teardown mark before this slow init resolves, so re-check the session row
      // exists, not just the mark; the requireSession below then surfaces a deleted session as NotFound.
      // Guarded like the same read in executeReconnect: a failed read assumes not-retired, so a
      // transient database error cannot fail a start whose engine is already registered and running.
      let retired: boolean;
      try {
        retired = await this.host.isSessionRetired(id);
      } catch {
        retired = false;
      }
      if (retired) {
        await this.reapRetiredEngine(id, mine, reconnectState);
      } else if (this.host.isClosed()) {
        // Shutdown destroys every engine it finds, this one included, and skips initialize() for an
        // engine it caught mid-start. Either way the session is not kept, so this is no success. A
        // start retired above answers as the stop or delete that retired it instead.
        throw new ServerShuttingDownException();
      }
      try {
        return await this.requireSession(id);
      } catch (error) {
        // The engine is registered and running. A failed read (a missing row stays NotFound) must not
        // report a start that worked as failed: the retry a transient database error buys would answer
        // "already started". The row read on entry stands in for the one that failed.
        // A row found gone only now is the retirement the failed read above could not report: the engine
        // goes with it.
        if (!retired && error instanceof NotFoundException) await this.reapRetiredEngine(id, mine, reconnectState);
        if (retired || error instanceof NotFoundException) throw error;
        return session;
      }
    } finally {
      this.initializingSessions.delete(id);
    }
  }

  /**
   * Tears down the engine a start registered for a session that was stopped or deleted meanwhile, and drops
   * the reconnect state it armed for it.
   */
  private async reapRetiredEngine(
    id: string,
    resurrected: IWhatsAppEngine | undefined,
    reconnectState?: ReconnectState,
  ): Promise<void> {
    if (resurrected && this.engines.isLive(id, resurrected)) {
      await this.fences.destroyWithEscalation(id, resurrected);
      this.engines.deleteIfLive(id, resurrected);
    }
    // The reconnect state this start armed up front serves no engine now, and the stop or delete that retired it
    // may have run before the state was armed, so nothing else would drop it. A replacement engine that
    // registered meanwhile keeps its own.
    if (!this.engines.has(id) && this.reconnectStates.get(id) === reconnectState && !reconnectState?.started) {
      this.host.cancelReconnect(id);
    }
    // A delete() that raced this start purged the on-disk auth dirs BEFORE this init re-created them, so
    // purge again: the window then leaves no credential residue behind (no-op for a stop()).
    await this.host.purgeAuthDirsIfDeleted(id);
  }

  async stop(id: string, hooks: StopHooks = {}): Promise<Session> {
    const session = await this.requireSessionOrDropStopMark(id, hooks.onReadFailed);
    // Recorded only once the row is known to exist, and before anything is torn down.
    await hooks.afterRead?.();

    // Mark as tearing down BEFORE cleanup so an in-flight reconnect can't resurrect it.
    this.stoppingSessions.add(id);
    // Cancel any reconnection attempts
    this.host.cancelReconnect(id);

    // Disconnect the engine — time-bounded + isolated so a stuck socket can't wedge the stop; the
    // Map is reconciled regardless. (The stop mark is intentionally left set, matching the prior
    // behaviour: a later start() clears it; it guards against a late reconnect resurrecting the id.)
    const engine = this.engines.get(id);
    if (engine) {
      // This teardown's own DISCONNECTED is not announced from the engine callback (see
      // operatorTeardowns): the adapter reports it before the eviction, so consumers would learn the
      // session is down while it still reads as engine-loaded. The write below announces it instead.
      this.operatorTeardowns.set(id, engine);
      try {
        // Await THIS engine's in-flight INITIALIZING write before teardown / the final DISCONNECTED
        // write so a delayed pre-initialize status update can never settle after the retirement and
        // become the last persisted status. Identity-checked: only the captured engine's promise.
        await this.fences.awaitInitialStatus(id, engine);
        let tornDown = await this.fences.teardownEngineSafely(id, engine, e => e.disconnect(), 'disconnect');
        if (!tornDown) {
          // The graceful disconnect threw or timed out, so the engine may be half-attached (a leaked
          // Chromium process or a live socket). Escalate to the hard kill, the same forceDestroy()
          // that forceKill() uses for a wedged engine, before reporting the stop.
          this.logger.warn(`Graceful disconnect failed for session ${session.name}; escalating to force-destroy`, {
            sessionId: id,
            action: 'stop_escalate_force_destroy',
          });
          tornDown = await this.fences.teardownEngineSafely(id, engine, e => e.forceDestroy(), 'force-destroy');
        }
        // Reconciled regardless of the outcome: a wedged engine must not keep holding a concurrency
        // slot or read as "already started" to a later start().
        this.engines.deleteIfLive(id, engine);
        if (!tornDown) {
          // The hard kill failed too, so the engine's process may still be alive. Local state is
          // settled (Map reconciled, status DISCONNECTED — mirroring logout()'s incomplete path), but
          // the stop is reported as incomplete instead of claimed clean: a retryable 502 with a stable
          // code, and no success log (the controller audits SESSION_STOPPED only after this resolves).
          await this.host.updateStatus(id, SessionStatus.DISCONNECTED);
          throw new BadGatewayException({
            statusCode: HttpStatus.BAD_GATEWAY,
            message:
              'Session was stopped locally, but the engine teardown did not complete: the engine ' +
              'process may still be running. Retry the stop; restart the node to reap a leaked process.',
            error: 'Bad Gateway',
            code: 'SESSION_STOP_INCOMPLETE',
          });
        }
      } finally {
        // Identity-checked, so a verb that captured a different instance keeps its own mark.
        if (this.operatorTeardowns.get(id) === engine) {
          this.operatorTeardowns.delete(id);
        }
      }
    }

    this.logger.log(`Session stopped: ${session.name}`, {
      sessionId: id,
      action: 'stop',
    });
    await this.host.updateStatus(id, SessionStatus.DISCONNECTED);
    return this.requireSession(id);
  }

  /**
   * Log out of WhatsApp — attempts an engine-native unlink of this device, then tears the session
   * down locally regardless of the unlink outcome.
   *
   * Differs from stop() in the one way that matters to a user: logout() asks WhatsApp to remove the
   * companion device, so a completed unlink eventually makes the entry disappear from the account
   * holder's Linked Devices list. stop() and delete() only release things locally (delete also
   * purges the on-disk auth dirs).
   *
   * Completion (HTTP 200) means the engine-native unlink operation completed AND the required local
   * credential cleanup completed — for Baileys, a valid companion identity, an acknowledged
   * `remove-companion-device` IQ response, and removal of the on-disk auth dir; for whatsapp-web.js,
   * `Client.logout()` including `LocalAuth.logout()` settled. 200 is NOT an independent observation
   * that the handset UI no longer shows the linked device — only the linked-device canary observes
   * that, and the dashboard must not claim otherwise.
   *
   * Must run while the engine is still live — logout is a network round-trip to WhatsApp, so it
   * cannot be performed after destroy()/forceDestroy(). Mirrors stop()'s lifecycle otherwise
   * (stop-mark + cancel-reconnect + bounded, isolated teardown + Map reconciliation).
   *
   * Requires a started session: with no engine loaded there is nothing to send the unlink through,
   * so the request is rejected with 400 rather than reporting an unlink that never happened. A 400
   * does NOT change the row. To just release a stopped session locally, use stop()/delete().
   *
   * Throws a retryable BadGatewayException (502) carrying a stable `code: 'SESSION_LOGOUT_INCOMPLETE'`
   * when an accepted engine-backed attempt stopped locally but the unlink operation did NOT complete
   * — no identity/no send, no IQ acknowledgement, timeout/transport error, OR local credential
   * cleanup failure. After EVERY engine-backed attempt (200 OR 502) the session is torn down locally
   * (map reconciled, status DISCONNECTED) and `phone` is cleared so the boot auto-start does not
   * resurrect the session into an uncertain/invalid credential state. No success audit is written on
   * the 502 path (the controller audits SESSION_LOGGED_OUT only after the service resolves). The
   * operator can start the session and retry the logout.
   */
  async logout(id: string, hooks: StopHooks = {}): Promise<Session> {
    const session = await this.requireSession(id).catch((error: unknown) => {
      hooks.onReadFailed?.();
      throw error;
    });
    const engine = this.engines.get(id);

    if (!engine) {
      throw new BadRequestException('Session is not started. Call POST /sessions/:sessionId/start first.');
    }
    hooks.onEngineFound?.();

    // Mark as tearing down BEFORE cleanup so an in-flight reconnect can't resurrect it.
    this.stoppingSessions.add(id);
    // Cancel any reconnection attempts
    this.host.cancelReconnect(id);

    // Announced by the write below rather than from the engine callback, as in stop().
    this.operatorTeardowns.set(id, engine);
    let unlinked: boolean;
    try {
      // Await THIS engine's in-flight INITIALIZING write before teardown / the final DISCONNECTED
      // write so a delayed pre-initialize status update can never settle after the retirement.
      await this.fences.awaitInitialStatus(id, engine);
      // The credential fence is keyed by session NAME (see awaitPendingTeardown). Captured immutably
      // here so a raw logout that outlives its deadline race is tracked under the right name even if
      // the row is later deleted/recreated.
      unlinked = await this.fences.teardownEngineSafely(id, engine, e => e.logout(), 'logout', session.name);
      this.engines.deleteIfLive(id, engine);
      // Clear `phone` AFTER the attempt, whatever its outcome, to take the session out of the boot
      // auto-start query (phone IS NOT NULL). A completed engine-backed unlink wipes the stored
      // credentials, so the session can never reach READY again without a fresh QR/pairing; an
      // incomplete one tore the local credentials down, so re-entering auto-start would only wedge it.
      // Before the status write, so a failed status write cannot leave the flag behind and resurrect
      // the session into a QR it can never pass on every restart. onReady rewrites it on the next link.
      await this.sessionRepository.update(id, { phone: null });
      await this.host.updateStatus(id, SessionStatus.DISCONNECTED);
    } finally {
      if (this.operatorTeardowns.get(id) === engine) {
        this.operatorTeardowns.delete(id);
      }
    }

    if (!unlinked) {
      this.logger.warn(`Session stopped locally but the logout operation did not complete: ${session.name}`, {
        sessionId: id,
        action: 'logout_incomplete',
      });
      // The local teardown already ran (map reconciled, `phone` cleared, status DISCONNECTED), but the
      // unlink operation did not complete. The retryable 502 carries a stable machine code so the
      // dashboard can branch on origin without guessing from the status/message; no success audit is
      // written on this path.
      throw new BadGatewayException({
        statusCode: HttpStatus.BAD_GATEWAY,
        message:
          'Session was stopped locally, but the logout operation is incomplete — the device may ' +
          'still be linked. Start the session and retry the logout.',
        error: 'Bad Gateway',
        code: 'SESSION_LOGOUT_INCOMPLETE',
      });
    }

    this.logger.log(`Session logged out: ${session.name}`, {
      sessionId: id,
      action: 'logout',
    });
    return this.requireSession(id);
  }

  /**
   * Force-recover a stuck session: SIGKILL its engine's own resources (a wedged Chromium for the
   * whatsapp-web.js engine) and tear it down, even when a normal stop()/delete() can't because the
   * engine is hung. Mirrors stop()'s lifecycle (stop-mark + cancel-reconnect + bounded, isolated
   * teardown + Map reconciliation) but uses the engine's forceDestroy().
   */
  async forceKill(id: string, hooks: StopHooks = {}): Promise<Session> {
    // No stop mark is set yet, so a failed read leaves a concurrent stop's mark alone.
    const session = await this.requireSession(id).catch((error: unknown) => {
      hooks.onReadFailed?.();
      throw error;
    });
    const engine = this.engines.get(id);

    // No live engine means there is nothing to SIGKILL. Resolving would let the controller write a
    // SESSION_FORCE_KILLED audit row for a kill that never happened — mirror logout()'s not-started
    // refusal. A wedged engine torn down earlier is reaped by the next start()'s orphan sweep (and
    // by process exit), not by force-kill.
    if (!engine) {
      throw new BadRequestException('Session is not started. Call POST /sessions/:sessionId/start first.');
    }
    // In the same synchronous step as the engine lookup: a refused or failed kill never gets here.
    hooks.onEngineFound?.();

    // Mark as tearing down BEFORE cleanup so an in-flight reconnect can't resurrect it.
    this.stoppingSessions.add(id);
    this.host.cancelReconnect(id);

    // A force-kill is a stop too: the session stays down across restarts and takeover until an
    // explicit start. Recorded only once there is an engine to kill, and before the teardown, so a
    // POST /start that lands after the eviction clears it and stands. A failed write must not keep a
    // wedged engine alive, so it is logged and the kill goes on.
    const write = (): Promise<unknown> =>
      hooks.recordStop ? hooks.recordStop() : this.sessionRepository.update(id, { desiredState: 'stopped' });
    await write().catch((error: unknown) =>
      this.logger.error(
        'Failed to record the force-kill as a stop',
        error instanceof Error ? error.message : String(error),
        { sessionId: id, action: 'force_kill_keep_down_failed' },
      ),
    );

    // Announced by the write below rather than from the engine callback, as in stop().
    this.operatorTeardowns.set(id, engine);
    try {
      // Await THIS engine's in-flight INITIALIZING write before teardown / the final DISCONNECTED
      // write so a delayed pre-initialize status update can never settle after the retirement.
      await this.fences.awaitInitialStatus(id, engine);
      const tornDown = await this.fences.teardownEngineSafely(id, engine, e => e.forceDestroy(), 'force-destroy');
      this.engines.deleteIfLive(id, engine);
      if (!tornDown) {
        // As in stop(): local state is settled, but a kill that may have left the process alive is
        // reported as incomplete, not claimed clean (the controller audits only after a resolve).
        await this.host.updateStatus(id, SessionStatus.DISCONNECTED);
        throw new BadGatewayException({
          statusCode: HttpStatus.BAD_GATEWAY,
          message:
            'Session was stopped locally, but the engine force-kill did not complete: the engine ' +
            'process may still be running. Restart the node to reap a leaked process.',
          error: 'Bad Gateway',
          code: 'SESSION_FORCE_KILL_INCOMPLETE',
        });
      }

      this.logger.warn(`Session force-killed: ${session.name}`, {
        sessionId: id,
        action: 'force_kill',
      });
      await this.host.updateStatus(id, SessionStatus.DISCONNECTED);
    } finally {
      if (this.operatorTeardowns.get(id) === engine) {
        this.operatorTeardowns.delete(id);
      }
    }
    return this.requireSession(id);
  }

  async delete(id: string, hooks: StopHooks = {}): Promise<void> {
    const session = await this.requireSessionOrDropStopMark(id, hooks.onReadFailed);

    // FENCE #1 — fail-fast on an ALREADY-PENDING credential teardown for this session NAME, BEFORE
    // any lifecycle mutation. A logout teardown that lost its deadline race is still running and ends
    // in an fs.rm of this session's on-disk auth dir. Running the purge below while that rm is live
    // would leave the two removals racing over the same tree, and a stale rm must not outlive the
    // row. The fence is keyed by session NAME and fails CLOSED (409). On a 409 NOTHING else runs:
    // no stop mark, no reconnect cancel, no engine teardown, no state cleanup — delete() simply did
    // not happen, and the entry stays reserved.
    await this.fences.awaitPendingTeardown(session.name);

    // Mark as tearing down BEFORE cleanup so an in-flight reconnect can't resurrect it.
    this.stoppingSessions.add(id);
    // Cancel any reconnection attempts
    this.host.cancelReconnect(id);

    // Set only after the transaction actually removes the parent row. lastDispatchedStatus /
    // sessionErrors (and the recovery budget, Task 8) are cleared ONLY on a committed delete; a
    // rejected 409 from fence #2 leaves them intact (the session still exists).
    let parentDeleted = false;
    const writes = closeSessionWrites(this.host.dataSource(), id);
    const endDelete = this.holdDelete(id);

    try {
      // Stop engine if running — time-bounded + isolated so a stuck Chromium can't wedge the delete;
      // the Map is reconciled and the DB removal proceeds regardless of the outcome. Use forceDestroy()
      // (SIGKILL) rather than a graceful destroy(): the session is being removed permanently, so there is
      // no session state worth saving, and a wedged Chromium must be reaped, not left to time out.
      const engine = this.engines.get(id);
      if (engine) {
        // Await THIS engine's in-flight INITIALIZING write before teardown and the parent-row
        // deletion below, so a delayed pre-initialize status update can never settle after the row
        // is gone (delete cannot be followed by a late status write). Identity-checked.
        await this.fences.awaitInitialStatus(id, engine);
        await this.fences.teardownEngineSafely(id, engine, e => e.forceDestroy(), 'force-destroy');
        this.engines.deleteIfLive(id, engine);
      }

      // FENCE #2 — immediately after the current engine is evicted, BEFORE the session:deleted hook
      // and the DB transaction. A logout that started concurrently AFTER fence #1 but captured this
      // engine before eviction registers its destructive promise synchronously (via the engine's
      // onCredentialTeardownStarted callback), so this fence sees it and refuses — the row/name stay
      // reserved and the transaction does not run. After eviction a NEW logout can't create a
      // destructive promise (no live engine); one that already captured the engine is observed here.
      //
      // Unlike fence #1, a refusal HERE leaves the row behind with its engine already destroyed, so
      // the persisted status would keep reading whatever it was (READY, authenticating, …) for a
      // session that can no longer answer anything. Reconcile it to DISCONNECTED before propagating
      // the 409 — the retry the message asks for should not have to look past a status that lies.
      // Fence #1 needs none of this: nothing has run there yet, so its status is still accurate.
      // Best-effort: a failed status write must not mask the 409 the caller has to act on.
      try {
        await this.fences.awaitPendingTeardown(session.name);
      } catch (fenceError) {
        if (engine) {
          await this.host.updateStatus(id, SessionStatus.DISCONNECTED).catch(() => undefined);
        }
        throw fenceError;
      }

      await writes.drain();
      hooks.onRemoving?.();
      // Execute hook BEFORE delete so plugins can access session data
      await this.hookManager.execute(
        'session:deleted',
        {
          id: session.id,
          name: session.name,
          phone: session.phone,
          pushName: session.pushName,
        },
        {
          sessionId: id,
          source: 'SessionService',
        },
      );

      // DB removal is NOT best-effort: a genuine failure must surface (500) rather than be swallowed.
      // Delete every child row explicitly, children before the parent: the three tables that grow with traffic
      // go first, in batches (a failure part-way keeps the session with fewer rows), then the rest and the
      // parent in one transaction. For
      // messages/message_batches this is load-bearing: they carry a plain sessionId with no FK, so
      // nothing else would ever remove them. webhooks/templates/baileys_stored_messages (and
      // automation_rules) DO declare an ON DELETE CASCADE FK that fires on BOTH engines —
      // better-sqlite3 defaults `foreign_keys` ON and TypeORM's driver re-asserts it at connection
      // creation — so their explicit deletes are belt-and-braces rather than required; they stay
      // because depending on a pragma neither this file nor a test pins is a thinner guarantee than
      // an explicit delete, and the ordering mirrors the restore path's explicit-clear. chat_states and
      // status_updates are in the no-FK group too, and so are the session's webhook outbox rows,
      // webhook delivery-failure records, integration dead-letter rows (which keep full payloads) and
      // send idempotency claims (which keep the stored response of a send for 24 h, so a session
      // re-created under the same id could not replay one). The webhook and dead-letter records are
      // point-in-time: a delivery still in flight can write one after the commit, and their retention
      // windows cover it.
      //
      // Deliberately kept: conversation_mappings (a re-paired session is rebound to its provider
      // conversations, see ConversationMappingService.rebindSession), ingress_events (the
      // instance-keyed inbound dedup record, payload already dropped after dispatch, always pruned
      // after INGRESS_DEDUP_RETENTION_DAYS), lid_mappings (a global cache by design) and audit_logs (the audit trail).
      await this.deleteTrafficRowsInBatches(session, hooks.ownsClaim);
      const dataSource = this.host.dataSource();
      await this.withDeleteClaim(session, hooks.ownsClaim, async manager => {
        await manager.delete(Message, { sessionId: id });
        await manager.delete(MessageBatch, { sessionId: id });
        await manager.delete(ChatState, { sessionId: id });
        await manager.delete(StatusUpdate, { sessionId: id });
        await manager.delete(Webhook, { sessionId: id });
        await manager.delete(Template, { sessionId: id });
        await manager.delete(BaileysStoredMessage, { sessionId: id });
        await manager.delete(WebhookOutboxEvent, { sessionId: id });
        await manager.delete(WebhookDeliveryFailure, { sessionId: id });
        await manager.delete(IntegrationDeliveryFailure, { sessionId: id });
        await manager.delete(SendIdempotencyKey, { sessionId: id });
        await manager.remove(session);
      });
      parentDeleted = true;
      this.logger.log(`Session deleted: ${session.name}`, {
        sessionId: id,
        action: 'delete',
      });

      // Purge the persistent on-disk auth/store dirs — BOTH engine shapes (see EngineFactory), since
      // an engine switch may have left a live link for the other engine behind. They're keyed by
      // session ID and live independently of the (now torn-down, and on delete often never-loaded)
      // engine instance, so the teardown above doesn't touch them. Without this, the deleted
      // session's WhatsApp credentials stay on the volume. Best-effort inside the factory — never
      // fails an otherwise-successful delete. By this point both fences passed, so no old remover is
      // live against this session's directories. The name goes too: it is the key the directories
      // carried before 0.23.5, and the boot migration keeps a legacy one it could not rename. Unless
      // it is another session's id, whatever its shape (an import accepts any safe key): the dirs it
      // names are then that session's live login, the same exact-id guard the migration keeps. A
      // failed lookup withholds the name rather than fail a delete that has already committed. The id
      // column is varchar on PostgreSQL too (an import keeps any safe key), so a name of any shape can be
      // a session's id there; it is compared as text, so only a name that really is an id withholds itself.
      // Keyed by `id`, not `session.id`: TypeORM clears a removed entity's primary key, so session.id
      // is undefined here and the purge would refuse it as an unsafe key and remove nothing.
      const legacyId =
        dataSource.options.type === 'postgres'
          ? Raw(alias => `CAST(${alias} AS text) = :legacyId`, { legacyId: session.name })
          : session.name;
      const nameIsAnId = await this.sessionRepository.exists({ where: { id: legacyId } }).catch(() => true);
      await this.engineFactory.purgeSessionData(id, nameIsAnId ? undefined : session.name);
    } catch (error) {
      // The session survives, but the write scopes closed above stay retired for good: a bulk batch
      // running for it stops at its next item without recording anything, and one not yet picked up
      // never starts. Nothing else moves a PENDING or PROCESSING row on while this process lives.
      if (!parentDeleted) await this.failUnfinishedBatches(session, hooks.ownsClaim);
      throw error;
    } finally {
      writes.release();
      endDelete();
      // Always clear the teardown mark so a later recreate/start with this id isn't suppressed. This
      // stop mark was set after fence #1, so clearing it on a rejected 409 only undoes what THIS
      // delete added — it does NOT touch reconnect timer / engine / last status / error / recovery
      // state (those are gated on parentDeleted below).
      this.stoppingSessions.delete(id);
      if (parentDeleted) {
        this.broadcaster.clear(id);
        // Drop the FAILED-reason entry too: it's keyed by a now-deleted UUID that can never be read
        // again, so leaving it would grow the map without bound across create/fail/delete churn.
        this.sessionErrors.clear(id);
        // Same unbounded-growth argument for the restriction entry, and the gauge it feeds must not
        // keep counting an account whose session no longer exists.
        this.sessionRestrictions.clear(id);
        // Presence is per-connection state that the deleted session can never receive again, and it
        // is keyed by an id that will never be read.
        this.presence.clear(id);
        // The stuck-auth recovery budget is keyed by id; a committed delete frees it (and a recreated
        // session under the same name gets a fresh UUID + fresh budget). Left only on a committed
        // delete so a failed/409 delete — the session still exists — keeps the budget intact.
        this.stuckAuthRecoveryUsed.delete(id);
      }
      // NOTE: pendingTeardowns is intentionally NOT cleared here. It is keyed by session NAME and
      // its entries self-remove on settlement (identity-checked). A delete that refused at either
      // fence MUST leave the entry in place — the name is still reserved against the live remover —
      // and a delete that succeeded already saw both fences pass, so no live remover exists.
    }
  }

  /**
   * Fail the session's unfinished bulk batches after a delete that closed its write scopes but did not
   * remove it. Written against the table rather than through the batch service, which this module cannot
   * reach without a cycle. Best effort: the database that just failed the delete may fail this too, and the
   * boot reaper still gets the row on the next start.
   */
  private async failUnfinishedBatches(session: Session, ownsClaim?: () => boolean): Promise<void> {
    const sessionId = session.id;
    try {
      await this.withDeleteClaim(session, ownsClaim, manager => failUnfinishedSessionBatches(manager, sessionId));
    } catch (error) {
      this.logger.warn(`Could not fail the unfinished batches of session ${sessionId} after a failed delete`, {
        sessionId,
        action: 'session_delete_batch_fail_failed',
        error: error instanceof Error ? error.message : String(error),
      });
    }
  }

  /**
   * Remove a session's rows of the tables that grow with traffic (its messages, its webhook outbox records
   * and its webhook delivery failures) in batches, ahead of the transaction that removes everything else.
   * One DELETE over a session that has stored millions of rows ran past PostgreSQL's statement timeout,
   * which failed the whole delete (every retry the same) and, on SQLite, blocked the one event loop for the
   * length of the statement while each message row also fired the full-text index trigger.
   *
   * The batches are not one atomic unit: a failure part-way leaves the session with fewer rows, and
   * repeating the delete finishes the job. Nothing new can land meanwhile, because the engine is gone and
   * the closing write scope refuses outbound writes. SQLite resolves each statement within the same tick, so
   * the loop pauses between batches to let the event loop run, and sizes each batch to the time the last one took.
   *
   * A batch is removed by primary key alone. Its ids were read for this session a statement earlier, so a
   * sessionId condition beside the id list adds no safety, and it costs: SQLite plans the delete as a walk
   * over the session's remaining rows that filters on the list, once per batch (PostgreSQL does too while its
   * statistics are missing or stale), which made a long history take longer than linear.
   */
  private async deleteTrafficRowsInBatches(session: Session, ownsClaim?: () => boolean): Promise<void> {
    const sessionId = session.id;
    const dataSource = this.host.dataSource();
    const postgres = dataSource.options.type === 'postgres';
    let size = postgres ? SESSION_DELETE_BATCH_SIZE_POSTGRES : SESSION_DELETE_BATCH_MIN_SQLITE;
    const readAhead = postgres ? SESSION_DELETE_READ_AHEAD_POSTGRES : 1;
    const head = (entity: EntityTarget<{ id: string; sessionId: string }>): TrafficPass => {
      const repository = dataSource.getRepository(entity);
      let ahead: string[] = [];
      let more = true;
      return {
        rows: repository,
        next: async take => {
          // What is left of the last read is still in the table and comes first in the next, so it is replaced.
          if (ahead.length < take && more) {
            const rows = take * readAhead;
            ahead = (await repository.find({ select: { id: true }, where: { sessionId }, take: rows })).map(
              row => row.id,
            );
            more = ahead.length === rows;
          }
          return ahead.splice(0, take);
        },
      };
    };
    const messages = dataSource.getRepository(Message);
    const passes: TrafficPass[] = [
      { rows: messages, next: this.stampedMessageIds(sessionId) },
      { rows: messages, next: this.unstampedMessageIds(sessionId) },
      head(WebhookOutboxEvent),
      head(WebhookDeliveryFailure),
    ];
    for (const { rows, next } of passes) {
      for (;;) {
        const started = performance.now();
        const ids = await next(size);
        if (ids.length === 0) break;
        await this.withDeleteClaim(session, ownsClaim, async manager => {
          await manager.getRepository(rows.target).delete({ id: In(ids) });
        });
        if (ids.length < size) break;
        if (!postgres) size = nextSqliteDeleteBatchSize(ids.length, performance.now() - started);
        await new Promise<void>(resolve =>
          postgres ? setImmediate(resolve) : setTimeout(resolve, SESSION_DELETE_PAUSE_MS_SQLITE),
        );
      }
    }
  }

  /** Lock the owner row with each destructive unit, never across a hook or between batches. */
  private async withDeleteClaim(
    session: Session,
    ownsClaim: (() => boolean) | undefined,
    remove: (manager: EntityManager) => Promise<void>,
  ): Promise<void> {
    const dataSource = this.host.dataSource();
    // A savepoint on SQLite's shared connection could be rolled back by an unrelated caller.
    if (!(await awaitNoOpenTransaction(dataSource))) {
      throw new ServiceUnavailableException(
        `Session ${session.id} was not deleted: another database transaction stayed open. Retry the delete.`,
      );
    }
    await dataSource.transaction(async manager => {
      if (this.host.nodeId !== undefined) {
        const { affected } = await manager
          .createQueryBuilder()
          .update(Session)
          .set({ nodeId: this.host.nodeId, updatedAt: () => '"updatedAt"' })
          .where({ id: session.id, nodeId: this.host.nodeId, claimedAt: session.claimedAt ?? IsNull() })
          .execute();
        if (affected !== 1) throw new ConflictException(`Session ${session.id} is no longer owned by this node`);
      }
      if (ownsClaim?.() === false)
        throw new ConflictException(`Session ${session.id} is no longer owned by this delete`);
      await remove(manager);
    });
  }

  /**
   * The messages that carry a timestamp, a page at a time in the order of IDX_messages_session_timestamp_id,
   * each page starting after the last row of the one before. A select that restarted from the head of the
   * session's range every time stepped over every entry the earlier batches had deleted: PostgreSQL keeps
   * those in the index until a vacuum removes them, so the cost of the loop grew with the square of the history.
   */
  private stampedMessageIds(sessionId: string): (take: number) => Promise<string[]> {
    const messages = this.host.dataSource().getRepository(Message);
    let last: { timestamp: number; id: string } | undefined;
    return async take => {
      const query = messages
        .createQueryBuilder('message')
        .select('message.id', 'id')
        .addSelect('message.timestamp', 'timestamp')
        .where('message.sessionId = :sessionId AND message.timestamp IS NOT NULL', { sessionId });
      if (last) query.andWhere('(message.timestamp, message.id) > (:timestamp, :id)', last);
      const page = await query
        .orderBy('message.timestamp', 'ASC')
        .addOrderBy('message.id', 'ASC')
        .limit(take)
        .getRawMany<{ id: string; timestamp: string | number }>();
      const end = page[page.length - 1];
      if (end) last = { timestamp: Number(end.timestamp), id: end.id };
      return page.map(row => row.id);
    };
  }

  /**
   * The messages without a timestamp (a send that was never acknowledged), after the stamped ones, paged by
   * id. Ordering by the null timestamp as well is what lets PostgreSQL read the index in order instead of
   * sorting every remaining row for each page.
   */
  private unstampedMessageIds(sessionId: string): (take: number) => Promise<string[]> {
    const messages = this.host.dataSource().getRepository(Message);
    let lastId: string | undefined;
    return async take => {
      const query = messages
        .createQueryBuilder('message')
        .select('message.id', 'id')
        .where('message.sessionId = :sessionId AND message.timestamp IS NULL', { sessionId });
      if (lastId !== undefined) query.andWhere('message.id > :lastId', { lastId });
      const page = await query
        .orderBy('message.timestamp', 'ASC')
        .addOrderBy('message.id', 'ASC')
        .limit(take)
        .getRawMany<{ id: string }>();
      if (page.length > 0) lastId = page[page.length - 1].id;
      return page.map(row => row.id);
    };
  }

  /**
   * The engine half of SessionService.onModuleDestroy: stop reconnect timers FIRST so nothing
   * reschedules mid-teardown (and so this always runs even if an engine.destroy() below hangs or
   * throws), then destroy engines in parallel, each isolated + time-bounded, so one stuck Chromium
   * can neither stall the shutdown nor abort teardown of the other sessions.
   */
  async shutdown(): Promise<void> {
    for (const [, state] of this.reconnectStates) {
      if (state.timer) {
        clearTimeout(state.timer);
      }
    }
    this.reconnectStates.clear();

    await Promise.allSettled(
      [...this.engines].map(([sessionId, engine]) => this.fences.destroyEngineSafely(sessionId, engine)),
    );
    this.engines.clear();
  }

  /**
   * Stop the engines for the given session ids WITHOUT touching the sessions DB row (the caller,
   * the infra import path, is about to DELETE that row as part of a full replace). This is the one
   * stop path that bypasses the session-row-keyed controls — every other path keys through the row,
   * so an engine orphaned by a restore was previously unstoppable until process restart.
   *
   * Each id is handled in isolation and time-bounded: a stuck Chromium/socket on one orphan can
   * neither stall nor abort the others, and the whole call is bounded by two of teardownEngineSafely's
   * 10s deadlines per engine (destroy, then the forceDestroy escalation). The mark + reconnect-cancel happen first so an in-flight reconnect
   * cannot resurrect the id while teardown runs. Engines that are mid-initialization (no entry in
   * `engines` yet) are marked but cannot be torn down here — their start() will see the stop mark
   * via its existing guard and self-abort; the caller learns about them in `notRunning`. A session
   * waiting to relaunch after a failed reconnect has no engine either: the reconnect cancel above is
   * what stops it, and it is reported in `notRunning` too.
   *
   * Always resolves. Best-effort: a `failed` entry means destroy and the forceDestroy escalation both
   * threw or timed out, and the engine is removed from the Map regardless so it stops holding a
   * concurrency slot.
   */
  async stopOrphanEngines(
    sessionIds: string[],
  ): Promise<{ stopped: string[]; notRunning: string[]; failed: string[] }> {
    const stopped: string[] = [];
    const notRunning: string[] = [];
    const failed: string[] = [];

    if (sessionIds.length === 0) return { stopped, notRunning, failed };

    await Promise.allSettled(
      sessionIds.map(async id => {
        // Mark + cancel BEFORE teardown so a late reconnect cannot resurrect the id mid-teardown.
        this.stoppingSessions.add(id);
        this.host.cancelReconnect(id);

        const engine = this.engines.get(id);
        if (!engine) {
          // Either never started, or still inside initializeEngine (no Map entry yet). The stop mark
          // above is what aborts the initializing case via start()'s existing guard.
          notRunning.push(id);
          return;
        }
        try {
          const tornDown = await this.fences.destroyWithEscalation(id, engine);
          // The engine leaves the Map regardless of teardown outcome so it stops holding a
          // concurrency slot — but only a completed teardown counts as `stopped`. A throw/timeout
          // means the Chromium/socket may still be alive and writing, so the id lands in `failed`
          // and the caller (the infra import) can flag restartRequired instead of reporting a
          // clean stop for a wedged engine.
          this.engines.deleteIfLive(id, engine);
          if (tornDown) {
            stopped.push(id);
          } else {
            failed.push(id);
          }
        } catch (err) {
          // destroyWithEscalation never throws today (it isolates via teardownEngineSafely), but defend
          // against a future change so a single orphan cannot abort the batch.
          this.logger.error(`Failed to stop orphan engine for session ${id}`, String(err), {
            sessionId: id,
            action: 'stop_orphan_failed',
          });
          this.engines.deleteIfLive(id, engine);
          failed.push(id);
        }
      }),
    );

    return { stopped, notRunning, failed };
  }
}