Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

feat(api): resolve ZDR cleanup blobs from ClickHouse (#4681)

Gergő Móriczb52afbe6acbd
diff --git a/apps/api/src/db/rpc.ts b/apps/api/src/db/rpc.ts
index 86d7510e0..eae478508 100644
--- a/apps/api/src/db/rpc.ts
+++ b/apps/api/src/db/rpc.ts
@@ -113,15 +113,6 @@ export function diffGetLastScrape(
);
}
-export function getZdrCleanupBatch(
- p_limit: number,
-): Promise<{ request_id: string; ids: string[] }[]> {
- return execRows(
- db,
- sql`select * from get_zdr_cleanup_batch_2(p_limit => ${p_limit})`,
- );
-}
-
export function monitoringClaimDueMonitors<T = Record<string, any>>(params: {
workerId: string;
limit: number;
diff --git a/apps/api/src/lib/zdrcleaner.test.ts b/apps/api/src/lib/zdrcleaner.test.ts
index 396bae881..bb0156f54 100644
--- a/apps/api/src/lib/zdrcleaner.test.ts
+++ b/apps/api/src/lib/zdrcleaner.test.ts
@@ -1,74 +1,35 @@
import { vi } from "vitest";
-const {
- tables,
- select,
- update,
- set,
- where,
- getZdrCleanupBatch,
- removeJobFromGCS,
- spans,
- withSpan,
- setSpanAttributes,
-} = vi.hoisted(() => {
- const makeTable = (name: string) => ({
- name,
- id: `${name}.id`,
- request_id: `${name}.request_id`,
- });
- const tables = {
- requests: makeTable("requests"),
- scrapes: makeTable("scrapes"),
- searches: makeTable("searches"),
- extracts: makeTable("extracts"),
- maps: makeTable("maps"),
- llmstxts: makeTable("llmstxts"),
- deep_researches: makeTable("deep_researches"),
- };
- const where = vi.fn(async () => {});
- const set = vi.fn(() => ({ where }));
- const update = vi.fn(() => ({ set }));
- const select = vi.fn(() => ({
- from: (table: { name: string }) => ({
- where: vi.fn(async () => [{ id: `${table.name}-blob` }]),
- }),
- }));
- const spans: Array<{ name: string; attributes: Record<string, unknown> }> =
- [];
- const withSpan = vi.fn(async (name: string, fn: (span: any) => any) => {
- const span = { attributes: {} };
- spans.push({ name, attributes: span.attributes });
- return fn(span);
+const { query, spans, withSpan, setSpanAttributes, removeJobFromGCS, client } =
+ vi.hoisted(() => {
+ const query = vi.fn();
+ const spans: Array<{ name: string; attributes: Record<string, unknown> }> =
+ [];
+ const withSpan = vi.fn(async (name: string, fn: (span: any) => any) => {
+ const span = { attributes: {} };
+ spans.push({ name, attributes: span.attributes });
+ return fn(span);
+ });
+ const setSpanAttributes = vi.fn(
+ (span: { attributes: Record<string, unknown> }, attributes: object) => {
+ Object.assign(span.attributes, attributes);
+ },
+ );
+ return {
+ query,
+ spans,
+ withSpan,
+ setSpanAttributes,
+ removeJobFromGCS: vi.fn<(id: string) => Promise<void>>(async () => {}),
+ client: { current: { query } as { query: typeof query } | null },
+ };
});
- const setSpanAttributes = vi.fn(
- (span: { attributes: Record<string, unknown> }, attributes: object) => {
- Object.assign(span.attributes, attributes);
- },
- );
- return {
- tables,
- select,
- update,
- set,
- where,
- getZdrCleanupBatch: vi.fn<
- (limit: number) => Promise<Array<{ request_id: string; ids: string[] }>>
- >(async () => []),
- removeJobFromGCS: vi.fn<(id: string) => Promise<void>>(async () => {}),
- spans,
- withSpan,
- setSpanAttributes,
- };
-});
-vi.mock("drizzle-orm", () => ({
- eq: vi.fn(() => "eq"),
- inArray: vi.fn(() => "inArray"),
+vi.mock("./clickhouse-client", () => ({
+ get clickhouseClient() {
+ return client.current;
+ },
}));
-vi.mock("../db/connection", () => ({ db: { select, update } }));
-vi.mock("../db/schema", () => tables);
-vi.mock("../db/rpc", () => ({ getZdrCleanupBatch }));
vi.mock("./gcs-jobs", () => ({ removeJobFromGCS }));
vi.mock("../config", () => ({ config: {} }));
vi.mock("./logger", () => {
@@ -82,27 +43,39 @@ vi.mock("./logger", () => {
});
vi.mock("./otel-tracer", () => ({ withSpan, setSpanAttributes }));
-import { cleanZdrRequest, zdrcleaner } from "./zdrcleaner";
+import { cleanZdrRequest } from "./zdrcleaner";
+import { logger } from "./logger";
+
+function mockChildren(ids: string[]) {
+ query.mockResolvedValueOnce({
+ json: vi.fn(async () => ids.map(id => ({ id }))),
+ });
+}
describe("ZDR cleaner", () => {
beforeEach(() => {
vi.clearAllMocks();
- getZdrCleanupBatch.mockResolvedValue([]);
+ client.current = { query };
removeJobFromGCS.mockResolvedValue(undefined);
spans.length = 0;
});
- it("loads and removes every blob type for a queued request", async () => {
+ it("removes every indexed blob for a queued request", async () => {
+ mockChildren(["scrape-blob", "search-blob", "map-blob"]);
+
await cleanZdrRequest("request-1");
- expect(select).toHaveBeenCalledTimes(6);
+ expect(query).toHaveBeenCalledOnce();
+ expect(query.mock.calls[0][0]).toMatchObject({
+ query:
+ "SELECT DISTINCT id FROM request_children WHERE request_id = {requestId: UUID}",
+ query_params: { requestId: "request-1" },
+ format: "JSONEachRow",
+ });
expect(removeJobFromGCS.mock.calls.map(([id]) => id)).toEqual([
- "scrapes-blob",
- "searches-blob",
- "extracts-blob",
- "maps-blob",
- "llmstxts-blob",
- "deep_researches-blob",
+ "scrape-blob",
+ "search-blob",
+ "map-blob",
]);
expect(spans).toEqual(
expect.arrayContaining([
@@ -110,42 +83,62 @@ describe("ZDR cleaner", () => {
name: "zdr.cleanup.request",
attributes: expect.objectContaining({
"zdr.request_id": "request-1",
- "zdr.blob_count": 6,
+ "zdr.blob_count": 3,
"zdr.cleanup.outcome": "completed",
}),
},
{
- name: "zdr.postgres.read_blobs",
+ name: "zdr.clickhouse.read_blobs",
attributes: expect.objectContaining({
- "db.system": "postgresql",
- "db.operation.name": "select",
- "db.response.returned_rows": 6,
+ "db.system": "clickhouse",
+ "db.collection.name": "request_children",
+ "db.response.returned_rows": 3,
}),
},
]),
);
+ expect(logger.warn).not.toHaveBeenCalled();
});
- it("fails a queued request if any blob could not be removed", async () => {
- removeJobFromGCS.mockRejectedValueOnce(new Error("GCS unavailable"));
+ it("completes quietly when a request has no indexed blobs", async () => {
+ mockChildren([]);
- await expect(cleanZdrRequest("request-2")).rejects.toThrow(
- "Failed to remove 1 blobs for ZDR request request-2",
+ await cleanZdrRequest("request-empty");
+
+ expect(removeJobFromGCS).not.toHaveBeenCalled();
+ expect(logger.warn).not.toHaveBeenCalled();
+ expect(logger.debug).toHaveBeenCalledWith(
+ "ZDR request has no indexed result blobs",
+ expect.objectContaining({ requestId: "request-empty" }),
);
});
- it("keeps draining and clearing legacy PostgreSQL schedules", async () => {
- getZdrCleanupBatch.mockResolvedValueOnce([
- { request_id: "legacy-1", ids: ["blob-1"] },
- { request_id: "legacy-2", ids: ["blob-2"] },
- ]);
+ it("fails a queued request if any blob could not be removed", async () => {
+ mockChildren(["blob-1", "blob-2"]);
removeJobFromGCS.mockImplementation(async id => {
if (id === "blob-2") throw new Error("GCS unavailable");
});
- await zdrcleaner();
+ await expect(cleanZdrRequest("request-2")).rejects.toThrow(
+ "Failed to remove 1 blobs for ZDR request request-2",
+ );
+ });
- expect(set).toHaveBeenCalledWith({ dr_clean_by: null });
- expect(where).toHaveBeenCalledOnce();
+ it("fails a queued request when ClickHouse is not configured", async () => {
+ client.current = null;
+
+ await expect(cleanZdrRequest("request-3")).rejects.toThrow(
+ "ClickHouse is not configured",
+ );
+ expect(removeJobFromGCS).not.toHaveBeenCalled();
+ });
+
+ it("propagates ClickHouse read failures so the job is retried", async () => {
+ query.mockRejectedValueOnce(new Error("ClickHouse unavailable"));
+
+ await expect(cleanZdrRequest("request-4")).rejects.toThrow(
+ "ClickHouse unavailable",
+ );
+ expect(removeJobFromGCS).not.toHaveBeenCalled();
});
});
diff --git a/apps/api/src/lib/zdrcleaner.ts b/apps/api/src/lib/zdrcleaner.ts
index 029ac9377..2b3f1b55d 100644
--- a/apps/api/src/lib/zdrcleaner.ts
+++ b/apps/api/src/lib/zdrcleaner.ts
@@ -1,14 +1,11 @@
import "dotenv/config";
-import { eq, inArray } from "drizzle-orm";
-import { db } from "../db/connection";
-import * as schema from "../db/schema";
-import { getZdrCleanupBatch } from "../db/rpc";
+import { clickhouseClient } from "./clickhouse-client";
import { removeJobFromGCS } from "./gcs-jobs";
import { logger as _logger } from "./logger";
import { config } from "../config";
import { setSpanAttributes, withSpan } from "./otel-tracer";
-async function sendHeartbeat() {
+export async function sendZdrHeartbeat() {
if (config.ZDRCLEANER_HEARTBEAT_URL) {
fetch(config.ZDRCLEANER_HEARTBEAT_URL).catch(() => {});
}
@@ -28,49 +25,36 @@ async function removeBlobs(blobIds: string[]): Promise<unknown[]> {
);
}
+/**
+ * Every result blob a request produced, from the `request_children` table in
+ * the analytics ClickHouse service, which materialized views fill from the
+ * Pub/Sub-ingested scrapes, searches, extracts, maps, llmstxts and
+ * deep_researches rows. The table is created by hand (statements in the PR
+ * that introduced this reader), not by this codebase. Cleanup runs 24 hours after the request, far
+ * beyond ClickPipes ingest lag, so the index is complete by the time it is
+ * read.
+ */
async function getRequestBlobIds(requestId: string): Promise<string[]> {
- return withSpan("zdr.postgres.read_blobs", async span => {
+ return withSpan("zdr.clickhouse.read_blobs", async span => {
setSpanAttributes(span, {
- "db.system": "postgresql",
+ "db.system": "clickhouse",
"db.operation.name": "select",
+ "db.collection.name": "request_children",
"zdr.request_id": requestId,
});
- const [scrapes, searches, extracts, maps, llmstxts, deepResearches] =
- await Promise.all([
- db
- .select({ id: schema.scrapes.id })
- .from(schema.scrapes)
- .where(eq(schema.scrapes.request_id, requestId)),
- db
- .select({ id: schema.searches.id })
- .from(schema.searches)
- .where(eq(schema.searches.request_id, requestId)),
- db
- .select({ id: schema.extracts.id })
- .from(schema.extracts)
- .where(eq(schema.extracts.request_id, requestId)),
- db
- .select({ id: schema.maps.id })
- .from(schema.maps)
- .where(eq(schema.maps.request_id, requestId)),
- db
- .select({ id: schema.llmstxts.id })
- .from(schema.llmstxts)
- .where(eq(schema.llmstxts.request_id, requestId)),
- db
- .select({ id: schema.deep_researches.id })
- .from(schema.deep_researches)
- .where(eq(schema.deep_researches.request_id, requestId)),
- ]);
-
- const blobIds = [
- ...scrapes,
- ...searches,
- ...extracts,
- ...maps,
- ...llmstxts,
- ...deepResearches,
- ].map(row => row.id);
+ if (clickhouseClient === null) {
+ throw new Error(
+ "ClickHouse is not configured; cannot resolve ZDR request blobs",
+ );
+ }
+ const result = await clickhouseClient.query({
+ query:
+ "SELECT DISTINCT id FROM request_children WHERE request_id = {requestId: UUID}",
+ query_params: { requestId },
+ format: "JSONEachRow",
+ });
+ const rows = await result.json<{ id: string }>();
+ const blobIds = rows.map(row => row.id);
setSpanAttributes(span, {
"db.response.returned_rows": blobIds.length,
});
@@ -83,6 +67,16 @@ export async function cleanZdrRequest(requestId: string): Promise<void> {
setSpanAttributes(span, { "zdr.request_id": requestId });
const blobIds = await getRequestBlobIds(requestId);
setSpanAttributes(span, { "zdr.blob_count": blobIds.length });
+ // A request that failed before producing a job row has no children; the
+ // span attribute above is the signal for that, so this stays at debug.
+ if (blobIds.length === 0) {
+ _logger.debug("ZDR request has no indexed result blobs", {
+ module: "zdrcleaner",
+ method: "cleanZdrRequest",
+ zeroDataRetention: true,
+ requestId,
+ });
+ }
const errors = await removeBlobs(blobIds);
if (errors.length > 0) {
throw new AggregateError(
@@ -93,81 +87,3 @@ export async function cleanZdrRequest(requestId: string): Promise<void> {
setSpanAttributes(span, { "zdr.cleanup.outcome": "completed" });
});
}
-
-export async function zdrcleaner() {
- const logger = _logger.child({
- module: "zdrcleaner",
- method: "zdrcleaner",
- });
-
- const start = Date.now();
- try {
- // Call the RPC to get all blobs (scrapes, searches, extracts, maps, llmstxts, deep_researches)
- // associated with requests that need cleanup.
- // The RPC handles the dr_clean_by filtering logic (team-specific vs scheduled)
- const rows: { request_id: string; ids: string[] }[] =
- await getZdrCleanupBatch(1000);
-
- if (!rows || rows.length === 0) {
- logger.debug("zdrcleaner batch completed with no rows to process", {
- canonicalLog: "zdrcleaner",
- success: true,
- timeMs: Date.now() - start,
- });
- await sendHeartbeat();
- await new Promise(resolve => setTimeout(resolve, 1000));
- return;
- }
-
- const deleteErrors: any[] = [];
- const fullyCleanedRequests: string[] = [];
- for (let j = 0; j < Math.ceil(rows.length / 50); j++) {
- const batch = rows.slice(j * 50, (j + 1) * 50);
- await Promise.all(
- batch.map(async row => {
- const errors = await removeBlobs(row.ids);
- if (errors.length === 0) fullyCleanedRequests.push(row.request_id);
- else deleteErrors.push(...errors);
- }),
- );
- }
-
- const updateErrors: any[] = [];
-
- if (fullyCleanedRequests.length > 0) {
- try {
- await db
- .update(schema.requests)
- .set({ dr_clean_by: null })
- .where(inArray(schema.requests.id, fullyCleanedRequests));
- } catch (error) {
- updateErrors.push(error);
- }
- }
-
- await sendHeartbeat();
- if (deleteErrors.length > 0 || updateErrors.length > 0) {
- logger.warn("zdrcleaner batch completed with errors", {
- canonicalLog: "zdrcleaner",
- success: true,
- deleteErrors,
- updateErrors,
- timeMs: Date.now() - start,
- });
- } else {
- logger.debug("zdrcleaner batch completed", {
- canonicalLog: "zdrcleaner",
- success: true,
- deleteErrors,
- timeMs: Date.now() - start,
- });
- }
- } catch (error) {
- logger.error(`Error looping through cleanup batch`, {
- canonicalLog: "zdrcleaner",
- success: false,
- timeMs: Date.now() - start,
- error,
- });
- }
-}
diff --git a/apps/api/src/services/zdr-worker.ts b/apps/api/src/services/zdr-worker.ts
index 7702f9f7d..05cdc9e38 100644
--- a/apps/api/src/services/zdr-worker.ts
+++ b/apps/api/src/services/zdr-worker.ts
@@ -1,7 +1,7 @@
import "dotenv/config";
import { shutdownTracing } from "../otel";
import { logger } from "../lib/logger";
-import { cleanZdrRequest, zdrcleaner } from "../lib/zdrcleaner";
+import { cleanZdrRequest, sendZdrHeartbeat } from "../lib/zdrcleaner";
import { consumeZdrCleanupJobs, shutdownZdrQueue } from "../lib/zdr-queue";
let isShuttingDown = false;
@@ -19,8 +19,11 @@ process.on("SIGTERM", () => {
(async () => {
await consumeZdrCleanupJobs(job => cleanZdrRequest(job.requestId));
+ // Cleanup is driven entirely by the delayed RabbitMQ queue; this loop only
+ // keeps the liveness heartbeat going while the consumer runs.
while (!isShuttingDown) {
- await zdrcleaner();
+ await sendZdrHeartbeat();
+ await new Promise(resolve => setTimeout(resolve, 1000));
}
await shutdownZdrQueue();