Skip to content

OpenPanel is an open-source web and product analytics platform, an open-source alternative to Mixpanel with optional self-hosting.

4.1 KB
Raw Blame
// The graceful-shutdown sequence, kept out of `main.ts` so its ORDER can be asserted by a test.
//
// A SIGTERM restart once permanently lost accepted events: the event buffer's in-process `pendingEvents`
// was never flushed to Redis and the offsets for those events were already committed, so nothing was
// redelivered. A crash commits nothing, so Kafka replays the window.
//
// kafkajs commits a batch's offsets as soon as `eachBatch` returns, so events buffered inside a batch are made
// durable in the batch handler (`flushBufferedEvents`). What is left here is everything buffered OUTSIDE a
// Kafka batch (a session-end job's event, an import): one rpush of at most `microBatchMaxSize` events,
// well inside `SHUTDOWN_FORCE_EXIT_MS`. It is not a Redis -> ClickHouse drain; the next process drains the list.

/** Everything succeeded; the offsets are committed and the events are durable. */
export const SHUTDOWN_EXIT_OK = 0;
/**
 * Something failed. Exiting non-zero without committing is deliberately the
 * crash shape — the safe one: Kafka redelivers and a duplicate is
 * recoverable where a loss is not.
 */
export const SHUTDOWN_EXIT_FAILED = 1;

export interface ShutdownLogger {
  info: (obj: object, msg?: string) => void;
  error: (obj: object, msg?: string) => void;
  fatal: (obj: object, msg?: string) => void;
}

/**
 * Each step is a no-op in the roles that do not have the thing it tears down,
 * so the sequence itself is unconditional and reads top to bottom.
 */
export interface ShutdownSteps {
  /** Stop serving HTTP. Readiness already answers 503 by this point. */
  stopHttpServer: () => Promise<void>;
  /** Let the cron queue finish what it started, if this process owns it. */
  drainCron: () => Promise<void>;
  /** BullMQ stops taking jobs and its in-flight handlers finish. */
  closeWorkers: () => Promise<void>;
  /** The Kafka consumer stops fetching and its in-flight batch finishes. */
  stopConsuming: () => Promise<void>;
  /** The one step that must not be concurrent with anything: rpush or throw. */
  flushEventBuffer: () => Promise<void>;
  /** The consumer leaves the group and disconnects. */
  stopConsumer: () => Promise<void>;
  closeProducers: () => Promise<void>;
  disconnectKafka: () => Promise<void>;
}

/**
 * @returns the process exit code. The caller owns `process.exit` and the
 * force-exit timer, so this function stays callable from a test.
 */
export async function runShutdownSequence(
  steps: ShutdownSteps,
  logger: ShutdownLogger
): Promise<number> {
  try {
    await steps.stopHttpServer();
    // Only if THIS process consumes cron: waiting on a queue another replica
    // owns would stall the whole shutdown budget on someone else's jobs.
    await steps.drainCron();
    // After this line nothing new can enter the event buffer: BullMQ stopped taking jobs and the Kafka consumer
    // stopped fetching. The two teardowns are unrelated, so they run concurrently.
    await Promise.all([steps.closeWorkers(), steps.stopConsuming()]);
  } catch (error) {
    logger.error({ err: error }, 'Error during graceful shutdown');
    return SHUTDOWN_EXIT_FAILED;
  }

  // Must COMPLETE before the consumer is torn down — never alongside it — and
  // must be able to stop the rest of the sequence: Redis being unreachable is
  // exactly when the rpush cannot land, and finishing a tidy shutdown then
  // would drop those events for good.
  try {
    await steps.flushEventBuffer();
  } catch (error) {
    logger.fatal(
      { err: error },
      'Event buffer flush to Redis failed — refusing to finish the shutdown so Kafka redelivers'
    );
    return SHUTDOWN_EXIT_FAILED;
  }

  try {
    await steps.stopConsumer();
    await steps.closeProducers();
    // Core opens the Kafka producer lazily on the first /track and the
    // consumer at boot; `deps.producers` is the BullMQ handle, not this one.
    // Without this an idempotent producer with in-flight batches is dropped.
    await steps.disconnectKafka();
  } catch (error) {
    logger.error({ err: error }, 'Error during graceful shutdown');
    return SHUTDOWN_EXIT_FAILED;
  }

  logger.info({}, 'Graceful shutdown completed');
  return SHUTDOWN_EXIT_OK;
}