Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

15.4 KB
Raw Blame
import amqp from "amqplib";
import { randomUUID } from "node:crypto";
import { config } from "../config";
import { logger as _logger } from "./logger";
import {
  recordSpanException,
  setSpanAttributes,
  withSpan,
} from "./otel-tracer";

const DELAY_MS = 24 * 60 * 60 * 1000;
const EXCHANGE = "zdr.cleanup";
const DELAY_QUEUE = "zdr.cleanup.delayed.24h";
const READY_QUEUE = "zdr.cleanup.ready";
const RETRY_QUEUE = "zdr.cleanup.retry";
const DLQ = "zdr.cleanup.dlq";
const READY_ROUTING_KEY = "ready";
const DLQ_ROUTING_KEY = "dead";
const RECONNECT_BASE_DELAY_MS = 1_000;
const RECONNECT_MAX_DELAY_MS = 30_000;
// 5 attempts of at most 5s each, with 1+2+4+8s between them: at most 40s,
// well inside the API pod's shutdown grace.
const PUBLISH_MAX_ATTEMPTS = 5;
const PUBLISH_ATTEMPT_TIMEOUT_MS = 5_000;
const RETRY_DELAY_MS = 5 * 60 * 1000;
const MAX_ATTEMPTS = 24;
const ATTEMPT_HEADER = "zdr-attempt";

const logger = _logger.child({ module: "zdr-queue" });

// A request-level job resolves its blobs from ClickHouse; a blob job names one blob.
export type ZdrCleanupJob = { requestId: string } | { blobId: string };

type CleanupHandler = (job: ZdrCleanupJob) => Promise<void>;

let connection: amqp.ChannelModel | null = null;
let connectionPromise: Promise<amqp.ChannelModel> | null = null;
let publishChannel: amqp.ConfirmChannel | null = null;
let publishChannelPromise: Promise<amqp.ConfirmChannel> | null = null;
let consumeChannel: amqp.Channel | null = null;
let consumeChannelPromise: Promise<amqp.Channel> | null = null;
let registeredHandler: CleanupHandler | null = null;
let subscribed = false;
let reconnectTimer: NodeJS.Timeout | null = null;
let reconnectDelayMs = RECONNECT_BASE_DELAY_MS;
let closed = false;
const pendingPublishes = new WeakMap<
  amqp.ConfirmChannel,
  Map<string, { returned: boolean }>
>();

async function assertTopology(ch: amqp.Channel): Promise<void> {
  await ch.assertExchange(EXCHANGE, "direct", { durable: true });

  await ch.assertQueue(DLQ, {
    durable: true,
    arguments: {
      "x-queue-type": "quorum",
      "x-overflow": "reject-publish",
    },
  });
  await ch.bindQueue(DLQ, EXCHANGE, DLQ_ROUTING_KEY);

  await ch.assertQueue(READY_QUEUE, {
    durable: true,
    arguments: {
      "x-queue-type": "quorum",
      "x-overflow": "reject-publish",
      "x-delivery-limit": -1,
      "x-dead-letter-strategy": "at-least-once",
      "x-dead-letter-exchange": EXCHANGE,
      "x-dead-letter-routing-key": DLQ_ROUTING_KEY,
    },
  });
  await ch.bindQueue(READY_QUEUE, EXCHANGE, READY_ROUTING_KEY);

  await ch.assertQueue(DELAY_QUEUE, {
    durable: true,
    arguments: {
      "x-queue-type": "quorum",
      "x-overflow": "reject-publish",
      "x-message-ttl": DELAY_MS,
      "x-dead-letter-strategy": "at-least-once",
      "x-dead-letter-exchange": EXCHANGE,
      "x-dead-letter-routing-key": READY_ROUTING_KEY,
    },
  });

  await ch.assertQueue(RETRY_QUEUE, {
    durable: true,
    arguments: {
      "x-queue-type": "quorum",
      "x-overflow": "reject-publish",
      "x-message-ttl": RETRY_DELAY_MS,
      "x-dead-letter-strategy": "at-least-once",
      "x-dead-letter-exchange": EXCHANGE,
      "x-dead-letter-routing-key": READY_ROUTING_KEY,
    },
  });
}

function resetCachedState(): void {
  connection = null;
  connectionPromise = null;
  publishChannel = null;
  publishChannelPromise = null;
  consumeChannel = null;
  consumeChannelPromise = null;
  subscribed = false;
}

function handleConnectionDrop(reason: string, error?: unknown): void {
  if (connection || publishChannel || consumeChannel) {
    logger.warn("ZDR queue connection dropped", { reason, error });
  }
  resetCachedState();
  if (registeredHandler && !closed) scheduleReconnect();
}

function scheduleReconnect(): void {
  if (reconnectTimer || closed) return;

  const delayMs = reconnectDelayMs;
  reconnectTimer = setTimeout(async () => {
    reconnectTimer = null;
    try {
      if (registeredHandler) {
        const ch = await getConsumeChannel();
        if (closed) {
          await ch.close().catch(() => {});
          await connection?.close().catch(() => {});
          resetCachedState();
          return;
        }
        await subscribe(ch, registeredHandler);
      }
      reconnectDelayMs = RECONNECT_BASE_DELAY_MS;
      logger.info("ZDR queue reconnected; consumer re-subscribed");
    } catch (error) {
      logger.error("ZDR queue reconnect failed", { error, delayMs });
      resetCachedState();
      reconnectDelayMs = Math.min(delayMs * 2, RECONNECT_MAX_DELAY_MS);
      scheduleReconnect();
    }
  }, delayMs);
  reconnectTimer.unref?.();
}

async function getConnection(): Promise<amqp.ChannelModel> {
  if (connection) return connection;
  if (!connectionPromise) {
    connectionPromise = amqp
      .connect(config.NUQ_RABBITMQ_URL!, {
        clientProperties: { connection_name: "zdr-cleanup" },
      })
      .then(conn => {
        if (closed) {
          void conn.close().catch(() => {});
          throw new Error("ZDR queue closed while connecting");
        }
        connection = conn;
        conn.on("close", () => handleConnectionDrop("connection closed"));
        conn.on("error", error =>
          logger.error("ZDR queue connection error", { error }),
        );
        return conn;
      })
      .catch(error => {
        connectionPromise = null;
        throw error;
      });
  }
  return connectionPromise;
}

async function createPublishChannel(): Promise<amqp.ConfirmChannel> {
  const conn = await getConnection();
  const ch = await conn.createConfirmChannel();
  await assertTopology(ch);
  ch.on("close", () => {
    if (publishChannel === ch) {
      publishChannel = null;
      publishChannelPromise = null;
    }
  });
  ch.on("error", error => logger.error("ZDR publish channel error", { error }));
  const publishes = new Map<string, { returned: boolean }>();
  pendingPublishes.set(ch, publishes);
  ch.on("return", msg => {
    const correlationId = msg.properties.correlationId;
    if (correlationId) {
      const publish = publishes.get(correlationId);
      if (publish) publish.returned = true;
    }
  });
  return ch;
}

async function createConsumeChannel(): Promise<amqp.Channel> {
  const conn = await getConnection();
  const ch = await conn.createChannel();
  await assertTopology(ch);
  ch.on("close", () => {
    if (consumeChannel === ch) {
      consumeChannel = null;
      consumeChannelPromise = null;
      subscribed = false;
      if (!closed) scheduleReconnect();
    }
  });
  ch.on("error", error => logger.error("ZDR consume channel error", { error }));
  return ch;
}

async function getPublishChannel(): Promise<amqp.ConfirmChannel> {
  if (publishChannel) return publishChannel;
  if (!publishChannelPromise) {
    publishChannelPromise = createPublishChannel()
      .then(ch => {
        publishChannel = ch;
        return ch;
      })
      .catch(error => {
        publishChannelPromise = null;
        throw error;
      });
  }
  return publishChannelPromise;
}

async function getConsumeChannel(): Promise<amqp.Channel> {
  if (consumeChannel) return consumeChannel;
  if (!consumeChannelPromise) {
    consumeChannelPromise = createConsumeChannel()
      .then(ch => {
        consumeChannel = ch;
        return ch;
      })
      .catch(error => {
        consumeChannelPromise = null;
        throw error;
      });
  }
  return consumeChannelPromise;
}

function jobId(job: ZdrCleanupJob): string {
  return "blobId" in job ? job.blobId : job.requestId;
}

function publishConfirmed(
  ch: amqp.ConfirmChannel,
  queue: string,
  job: ZdrCleanupJob,
  attempt: number,
): Promise<void> {
  const correlationId = randomUUID();
  const publish = { returned: false };
  const publishes = pendingPublishes.get(ch);
  publishes?.set(correlationId, publish);

  return new Promise((resolve, reject) => {
    ch.sendToQueue(
      queue,
      Buffer.from(JSON.stringify(job)),
      {
        persistent: true,
        contentType: "application/json",
        messageId: jobId(job),
        correlationId,
        timestamp: Math.floor(Date.now() / 1000),
        mandatory: true,
        headers: { [ATTEMPT_HEADER]: attempt },
      },
      error => {
        setImmediate(() => {
          publishes?.delete(correlationId);
          if (error) reject(error);
          else if (publish.returned) {
            reject(new Error("RabbitMQ returned unroutable ZDR cleanup job"));
          } else resolve();
        });
      },
    );
  });
}

async function publishUntilConfirmed(
  queue: string,
  job: ZdrCleanupJob,
  attempt: number,
): Promise<void> {
  let delayMs = RECONNECT_BASE_DELAY_MS;
  for (let publishAttempt = 1; !closed; publishAttempt++) {
    let attemptTimer: NodeJS.Timeout | undefined;
    try {
      await withSpan("zdr.rabbitmq.publish", async span => {
        setSpanAttributes(span, {
          "messaging.system": "rabbitmq",
          "messaging.destination.name": queue,
          "messaging.operation.name": "publish",
          "messaging.message.id": jobId(job),
          "zdr.rabbitmq.retry_delay_ms": delayMs,
        });
        await Promise.race([
          getPublishChannel().then(ch =>
            publishConfirmed(ch, queue, job, attempt),
          ),
          new Promise<never>((_, reject) => {
            attemptTimer = setTimeout(
              () => reject(new Error("ZDR publish attempt timed out")),
              PUBLISH_ATTEMPT_TIMEOUT_MS,
            );
          }),
        ]);
        setSpanAttributes(span, {
          "zdr.rabbitmq.outcome": "confirmed",
        });
      });
      return;
    } catch (error) {
      if (publishAttempt >= PUBLISH_MAX_ATTEMPTS) {
        logger.error("Failed to enqueue ZDR cleanup job; giving up", {
          error,
          queue,
          messageId: jobId(job),
          attempts: publishAttempt,
        });
        throw new Error(
          `ZDR cleanup job not confirmed after ${publishAttempt} attempts`,
        );
      }
      logger.error("Failed to durably enqueue ZDR cleanup job; retrying", {
        error,
        queue,
        messageId: jobId(job),
        delayMs,
      });
      await publishChannel?.close().catch(() => {});
      publishChannel = null;
      publishChannelPromise = null;
      await new Promise(resolve => setTimeout(resolve, delayMs));
      delayMs = Math.min(delayMs * 2, RECONNECT_MAX_DELAY_MS);
    } finally {
      clearTimeout(attemptTimer);
    }
  }

  throw new Error("ZDR queue closed before the cleanup job was confirmed");
}

/** Schedules a cleanup job 24 hours out; throws if the broker never confirms it. */
export async function enqueueZdrCleanupJob(job: ZdrCleanupJob): Promise<void> {
  if (!config.NUQ_RABBITMQ_URL) {
    throw new Error("NUQ_RABBITMQ_URL is not configured");
  }
  await publishUntilConfirmed(DELAY_QUEUE, job, 1);
}

function isZdrCleanupJob(value: unknown): value is ZdrCleanupJob {
  if (typeof value !== "object" || value === null) return false;
  const id =
    "blobId" in value
      ? value.blobId
      : "requestId" in value
        ? value.requestId
        : undefined;
  return typeof id === "string" && id.length > 0;
}

async function subscribe(
  ch: amqp.Channel,
  handler: CleanupHandler,
): Promise<void> {
  if (subscribed) return;
  await ch.prefetch(10);
  await ch.consume(
    READY_QUEUE,
    async msg => {
      if (!msg) {
        subscribed = false;
        scheduleReconnect();
        return;
      }
      const messageId = msg.properties.messageId;
      try {
        await withSpan("zdr.rabbitmq.consume", async span => {
          setSpanAttributes(span, {
            "messaging.system": "rabbitmq",
            "messaging.destination.name": READY_QUEUE,
            "messaging.operation.name": "process",
            "messaging.message.id": messageId,
            "messaging.rabbitmq.delivery_tag": msg.fields.deliveryTag,
            "messaging.rabbitmq.redelivered": msg.fields.redelivered,
          });

          let job: unknown;
          try {
            job = JSON.parse(msg.content.toString());
          } catch (error) {
            recordSpanException(span, error);
            setSpanAttributes(span, {
              "zdr.rabbitmq.outcome": "dead_lettered",
            });
            logger.error("Dead-lettering malformed ZDR cleanup job", { error });
            ch.nack(msg, false, false);
            return;
          }

          if (!isZdrCleanupJob(job)) {
            const error = new Error("Invalid ZDR cleanup job");
            recordSpanException(span, error);
            setSpanAttributes(span, {
              "zdr.rabbitmq.outcome": "dead_lettered",
            });
            logger.error("Dead-lettering invalid ZDR cleanup job", { job });
            ch.nack(msg, false, false);
            return;
          }

          const attempt = Number(msg.properties.headers?.[ATTEMPT_HEADER]) || 1;
          setSpanAttributes(span, {
            ...("blobId" in job
              ? { "zdr.blob_id": job.blobId }
              : { "zdr.request_id": job.requestId }),
            "zdr.attempt": attempt,
          });

          try {
            // An unsettled earlier delivery (worker died, channel closed) counts as a failed attempt.
            if (msg.fields.redelivered) {
              throw new Error("ZDR cleanup job was redelivered unsettled");
            }
            await handler(job);
          } catch (error) {
            recordSpanException(span, error);
            if (attempt >= MAX_ATTEMPTS) {
              logger.error(
                "ZDR cleanup job failed its last attempt; dead-lettering",
                {
                  error,
                  messageId,
                  attempt,
                },
              );
              ch.nack(msg, false, false);
              setSpanAttributes(span, {
                "zdr.rabbitmq.outcome": "dead_lettered",
              });
              return;
            }
            logger.warn("ZDR cleanup job failed; scheduling a retry", {
              error,
              messageId,
              attempt,
              retryDelayMs: RETRY_DELAY_MS,
            });
            await publishUntilConfirmed(RETRY_QUEUE, job, attempt + 1);
            ch.ack(msg);
            setSpanAttributes(span, {
              "zdr.rabbitmq.outcome": "retry_scheduled",
            });
            return;
          }

          ch.ack(msg);
          setSpanAttributes(span, {
            "zdr.rabbitmq.outcome": "acknowledged",
          });
        });
      } catch (error) {
        // Left unacked, the broker redelivers it once this channel closes.
        logger.error("Could not settle ZDR cleanup job", { error, messageId });
      }
    },
    { noAck: false },
  );
  subscribed = true;
  logger.info("Started consuming ZDR cleanup jobs");
}

export async function consumeZdrCleanupJobs(
  handler: CleanupHandler,
): Promise<void> {
  registeredHandler = handler;
  closed = false;
  try {
    const ch = await getConsumeChannel();
    if (closed) {
      await ch.close().catch(() => {});
      await connection?.close().catch(() => {});
      resetCachedState();
      return;
    }
    await subscribe(ch, handler);
  } catch (error) {
    if (closed) return;
    logger.error("Failed to start ZDR queue consumer; retrying", { error });
    resetCachedState();
    scheduleReconnect();
  }
}

export async function shutdownZdrQueue(): Promise<void> {
  closed = true;
  registeredHandler = null;
  if (reconnectTimer) clearTimeout(reconnectTimer);
  reconnectTimer = null;
  await publishChannel?.close().catch(() => {});
  await consumeChannel?.close().catch(() => {});
  await connection?.close().catch(() => {});
  resetCachedState();
}