Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

3.8 KB
Raw Blame
import "dotenv/config";
import { clickhouseClient } from "./clickhouse-client";
import { removeJobFromGCS } from "./gcs-jobs";
import { logger as _logger } from "./logger";
import { config } from "../config";
import { setSpanAttributes, withSpan } from "./otel-tracer";
import type { ZdrCleanupJob } from "./zdr-queue";

export async function sendZdrHeartbeat() {
  if (config.ZDRCLEANER_HEARTBEAT_URL) {
    fetch(config.ZDRCLEANER_HEARTBEAT_URL).catch(() => {});
  }
}

// A request can resolve to a very large number of blobs; issuing every delete
// at once exhausts the worker's memory.
const BLOB_DELETE_CONCURRENCY = 50;

async function removeBlobs(blobIds: string[]): Promise<unknown[]> {
  const logger = _logger.child({
    module: "zdrcleaner",
    method: "removeBlobs",
    zeroDataRetention: true,
  });
  const errors: unknown[] = [];
  let next = 0;
  const worker = async () => {
    while (next < blobIds.length) {
      const blobId = blobIds[next++];
      try {
        await removeJobFromGCS(blobId, logger);
      } catch (error) {
        errors.push(error);
      }
    }
  };
  await Promise.all(
    Array.from(
      { length: Math.min(BLOB_DELETE_CONCURRENCY, blobIds.length) },
      worker,
    ),
  );
  return errors;
}

/**
 * Every result blob a request produced, from the `request_children` table in
 * the analytics ClickHouse service, which materialized views fill from the
 * Pub/Sub-ingested scrapes, searches, extracts, maps, llmstxts and
 * deep_researches rows. The table is created by hand (statements in the PR
 * that introduced this reader), not by this codebase. Cleanup runs 24 hours after the request, far
 * beyond ClickPipes ingest lag, so the index is complete by the time it is
 * read.
 */
async function getRequestBlobIds(requestId: string): Promise<string[]> {
  return withSpan("zdr.clickhouse.read_blobs", async span => {
    setSpanAttributes(span, {
      "db.system": "clickhouse",
      "db.operation.name": "select",
      "db.collection.name": "request_children",
      "zdr.request_id": requestId,
    });
    if (clickhouseClient === null) {
      throw new Error(
        "ClickHouse is not configured; cannot resolve ZDR request blobs",
      );
    }
    const result = await clickhouseClient.query({
      query:
        "SELECT DISTINCT id FROM request_children WHERE request_id = {requestId: UUID}",
      query_params: { requestId },
      format: "JSONEachRow",
    });
    const rows = await result.json<{ id: string }>();
    const blobIds = rows.map(row => row.id);
    setSpanAttributes(span, {
      "db.response.returned_rows": blobIds.length,
    });
    return blobIds;
  });
}

export async function cleanZdrRequest(requestId: string): Promise<void> {
  await withSpan("zdr.cleanup.request", async span => {
    setSpanAttributes(span, { "zdr.request_id": requestId });
    const blobIds = await getRequestBlobIds(requestId);
    setSpanAttributes(span, { "zdr.blob_count": blobIds.length });
    // A request that failed before producing a job row has no children; the
    // span attribute above is the signal for that, so this stays at debug.
    if (blobIds.length === 0) {
      _logger.debug("ZDR request has no indexed result blobs", {
        module: "zdrcleaner",
        method: "cleanZdrRequest",
        zeroDataRetention: true,
        requestId,
      });
    }
    const errors = await removeBlobs(blobIds);
    if (errors.length > 0) {
      throw new AggregateError(
        errors,
        `Failed to remove ${errors.length} blobs for ZDR request ${requestId}`,
      );
    }
    setSpanAttributes(span, { "zdr.cleanup.outcome": "completed" });
  });
}

export async function cleanZdrJob(job: ZdrCleanupJob): Promise<void> {
  if ("blobId" in job) {
    await removeJobFromGCS(job.blobId);
    return;
  }
  await cleanZdrRequest(job.requestId);
}