Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(api): resolve ZDR cleanup blobs from ClickHouse (#4681)
Gergő Móriczb52afbe6acbd
diff --git a/apps/api/src/db/rpc.ts b/apps/api/src/db/rpc.tsindex 86d7510e0..eae478508 100644--- a/apps/api/src/db/rpc.ts+++ b/apps/api/src/db/rpc.ts@@ -113,15 +113,6 @@ export function diffGetLastScrape();}-export function getZdrCleanupBatch(- p_limit: number,-): Promise<{ request_id: string; ids: string[] }[]> {- return execRows(- db,- sql`select * from get_zdr_cleanup_batch_2(p_limit => ${p_limit})`,- );-}-export function monitoringClaimDueMonitors<T = Record<string, any>>(params: {workerId: string;limit: number;diff --git a/apps/api/src/lib/zdrcleaner.test.ts b/apps/api/src/lib/zdrcleaner.test.tsindex 396bae881..bb0156f54 100644--- a/apps/api/src/lib/zdrcleaner.test.ts+++ b/apps/api/src/lib/zdrcleaner.test.ts@@ -1,74 +1,35 @@import { vi } from "vitest";-const {- tables,- select,- update,- set,- where,- getZdrCleanupBatch,- removeJobFromGCS,- spans,- withSpan,- setSpanAttributes,-} = vi.hoisted(() => {- const makeTable = (name: string) => ({- name,- id: `${name}.id`,- request_id: `${name}.request_id`,- });- const tables = {- requests: makeTable("requests"),- scrapes: makeTable("scrapes"),- searches: makeTable("searches"),- extracts: makeTable("extracts"),- maps: makeTable("maps"),- llmstxts: makeTable("llmstxts"),- deep_researches: makeTable("deep_researches"),- };- const where = vi.fn(async () => {});- const set = vi.fn(() => ({ where }));- const update = vi.fn(() => ({ set }));- const select = vi.fn(() => ({- from: (table: { name: string }) => ({- where: vi.fn(async () => [{ id: `${table.name}-blob` }]),- }),- }));- const spans: Array<{ name: string; attributes: Record<string, unknown> }> =- [];- const withSpan = vi.fn(async (name: string, fn: (span: any) => any) => {- const span = { attributes: {} };- spans.push({ name, attributes: span.attributes });- return fn(span);+const { query, spans, withSpan, setSpanAttributes, removeJobFromGCS, client } =+ vi.hoisted(() => {+ const query = vi.fn();+ const spans: Array<{ name: string; attributes: Record<string, unknown> }> =+ [];+ const withSpan = vi.fn(async (name: string, fn: (span: any) => any) => {+ const span = { attributes: {} };+ spans.push({ name, attributes: span.attributes });+ return fn(span);+ });+ const setSpanAttributes = vi.fn(+ (span: { attributes: Record<string, unknown> }, attributes: object) => {+ Object.assign(span.attributes, attributes);+ },+ );+ return {+ query,+ spans,+ withSpan,+ setSpanAttributes,+ removeJobFromGCS: vi.fn<(id: string) => Promise<void>>(async () => {}),+ client: { current: { query } as { query: typeof query } | null },+ };});- const setSpanAttributes = vi.fn(- (span: { attributes: Record<string, unknown> }, attributes: object) => {- Object.assign(span.attributes, attributes);- },- );- return {- tables,- select,- update,- set,- where,- getZdrCleanupBatch: vi.fn<- (limit: number) => Promise<Array<{ request_id: string; ids: string[] }>>- >(async () => []),- removeJobFromGCS: vi.fn<(id: string) => Promise<void>>(async () => {}),- spans,- withSpan,- setSpanAttributes,- };-});-vi.mock("drizzle-orm", () => ({- eq: vi.fn(() => "eq"),- inArray: vi.fn(() => "inArray"),+vi.mock("./clickhouse-client", () => ({+ get clickhouseClient() {+ return client.current;+ },}));-vi.mock("../db/connection", () => ({ db: { select, update } }));-vi.mock("../db/schema", () => tables);-vi.mock("../db/rpc", () => ({ getZdrCleanupBatch }));vi.mock("./gcs-jobs", () => ({ removeJobFromGCS }));vi.mock("../config", () => ({ config: {} }));vi.mock("./logger", () => {@@ -82,27 +43,39 @@ vi.mock("./logger", () => {});vi.mock("./otel-tracer", () => ({ withSpan, setSpanAttributes }));-import { cleanZdrRequest, zdrcleaner } from "./zdrcleaner";+import { cleanZdrRequest } from "./zdrcleaner";+import { logger } from "./logger";++function mockChildren(ids: string[]) {+ query.mockResolvedValueOnce({+ json: vi.fn(async () => ids.map(id => ({ id }))),+ });+}describe("ZDR cleaner", () => {beforeEach(() => {vi.clearAllMocks();- getZdrCleanupBatch.mockResolvedValue([]);+ client.current = { query };removeJobFromGCS.mockResolvedValue(undefined);spans.length = 0;});- it("loads and removes every blob type for a queued request", async () => {+ it("removes every indexed blob for a queued request", async () => {+ mockChildren(["scrape-blob", "search-blob", "map-blob"]);+await cleanZdrRequest("request-1");- expect(select).toHaveBeenCalledTimes(6);+ expect(query).toHaveBeenCalledOnce();+ expect(query.mock.calls[0][0]).toMatchObject({+ query:+ "SELECT DISTINCT id FROM request_children WHERE request_id = {requestId: UUID}",+ query_params: { requestId: "request-1" },+ format: "JSONEachRow",+ });expect(removeJobFromGCS.mock.calls.map(([id]) => id)).toEqual([- "scrapes-blob",- "searches-blob",- "extracts-blob",- "maps-blob",- "llmstxts-blob",- "deep_researches-blob",+ "scrape-blob",+ "search-blob",+ "map-blob",]);expect(spans).toEqual(expect.arrayContaining([@@ -110,42 +83,62 @@ describe("ZDR cleaner", () => {name: "zdr.cleanup.request",attributes: expect.objectContaining({"zdr.request_id": "request-1",- "zdr.blob_count": 6,+ "zdr.blob_count": 3,"zdr.cleanup.outcome": "completed",}),},{- name: "zdr.postgres.read_blobs",+ name: "zdr.clickhouse.read_blobs",attributes: expect.objectContaining({- "db.system": "postgresql",- "db.operation.name": "select",- "db.response.returned_rows": 6,+ "db.system": "clickhouse",+ "db.collection.name": "request_children",+ "db.response.returned_rows": 3,}),},]),);+ expect(logger.warn).not.toHaveBeenCalled();});- it("fails a queued request if any blob could not be removed", async () => {- removeJobFromGCS.mockRejectedValueOnce(new Error("GCS unavailable"));+ it("completes quietly when a request has no indexed blobs", async () => {+ mockChildren([]);- await expect(cleanZdrRequest("request-2")).rejects.toThrow(- "Failed to remove 1 blobs for ZDR request request-2",+ await cleanZdrRequest("request-empty");++ expect(removeJobFromGCS).not.toHaveBeenCalled();+ expect(logger.warn).not.toHaveBeenCalled();+ expect(logger.debug).toHaveBeenCalledWith(+ "ZDR request has no indexed result blobs",+ expect.objectContaining({ requestId: "request-empty" }),);});- it("keeps draining and clearing legacy PostgreSQL schedules", async () => {- getZdrCleanupBatch.mockResolvedValueOnce([- { request_id: "legacy-1", ids: ["blob-1"] },- { request_id: "legacy-2", ids: ["blob-2"] },- ]);+ it("fails a queued request if any blob could not be removed", async () => {+ mockChildren(["blob-1", "blob-2"]);removeJobFromGCS.mockImplementation(async id => {if (id === "blob-2") throw new Error("GCS unavailable");});- await zdrcleaner();+ await expect(cleanZdrRequest("request-2")).rejects.toThrow(+ "Failed to remove 1 blobs for ZDR request request-2",+ );+ });- expect(set).toHaveBeenCalledWith({ dr_clean_by: null });- expect(where).toHaveBeenCalledOnce();+ it("fails a queued request when ClickHouse is not configured", async () => {+ client.current = null;++ await expect(cleanZdrRequest("request-3")).rejects.toThrow(+ "ClickHouse is not configured",+ );+ expect(removeJobFromGCS).not.toHaveBeenCalled();+ });++ it("propagates ClickHouse read failures so the job is retried", async () => {+ query.mockRejectedValueOnce(new Error("ClickHouse unavailable"));++ await expect(cleanZdrRequest("request-4")).rejects.toThrow(+ "ClickHouse unavailable",+ );+ expect(removeJobFromGCS).not.toHaveBeenCalled();});});diff --git a/apps/api/src/lib/zdrcleaner.ts b/apps/api/src/lib/zdrcleaner.tsindex 029ac9377..2b3f1b55d 100644--- a/apps/api/src/lib/zdrcleaner.ts+++ b/apps/api/src/lib/zdrcleaner.ts@@ -1,14 +1,11 @@import "dotenv/config";-import { eq, inArray } from "drizzle-orm";-import { db } from "../db/connection";-import * as schema from "../db/schema";-import { getZdrCleanupBatch } from "../db/rpc";+import { clickhouseClient } from "./clickhouse-client";import { removeJobFromGCS } from "./gcs-jobs";import { logger as _logger } from "./logger";import { config } from "../config";import { setSpanAttributes, withSpan } from "./otel-tracer";-async function sendHeartbeat() {+export async function sendZdrHeartbeat() {if (config.ZDRCLEANER_HEARTBEAT_URL) {fetch(config.ZDRCLEANER_HEARTBEAT_URL).catch(() => {});}@@ -28,49 +25,36 @@ async function removeBlobs(blobIds: string[]): Promise<unknown[]> {);}+/**+ * Every result blob a request produced, from the `request_children` table in+ * the analytics ClickHouse service, which materialized views fill from the+ * Pub/Sub-ingested scrapes, searches, extracts, maps, llmstxts and+ * deep_researches rows. The table is created by hand (statements in the PR+ * that introduced this reader), not by this codebase. Cleanup runs 24 hours after the request, far+ * beyond ClickPipes ingest lag, so the index is complete by the time it is+ * read.+ */async function getRequestBlobIds(requestId: string): Promise<string[]> {- return withSpan("zdr.postgres.read_blobs", async span => {+ return withSpan("zdr.clickhouse.read_blobs", async span => {setSpanAttributes(span, {- "db.system": "postgresql",+ "db.system": "clickhouse","db.operation.name": "select",+ "db.collection.name": "request_children","zdr.request_id": requestId,});- const [scrapes, searches, extracts, maps, llmstxts, deepResearches] =- await Promise.all([- db- .select({ id: schema.scrapes.id })- .from(schema.scrapes)- .where(eq(schema.scrapes.request_id, requestId)),- db- .select({ id: schema.searches.id })- .from(schema.searches)- .where(eq(schema.searches.request_id, requestId)),- db- .select({ id: schema.extracts.id })- .from(schema.extracts)- .where(eq(schema.extracts.request_id, requestId)),- db- .select({ id: schema.maps.id })- .from(schema.maps)- .where(eq(schema.maps.request_id, requestId)),- db- .select({ id: schema.llmstxts.id })- .from(schema.llmstxts)- .where(eq(schema.llmstxts.request_id, requestId)),- db- .select({ id: schema.deep_researches.id })- .from(schema.deep_researches)- .where(eq(schema.deep_researches.request_id, requestId)),- ]);-- const blobIds = [- ...scrapes,- ...searches,- ...extracts,- ...maps,- ...llmstxts,- ...deepResearches,- ].map(row => row.id);+ if (clickhouseClient === null) {+ throw new Error(+ "ClickHouse is not configured; cannot resolve ZDR request blobs",+ );+ }+ const result = await clickhouseClient.query({+ query:+ "SELECT DISTINCT id FROM request_children WHERE request_id = {requestId: UUID}",+ query_params: { requestId },+ format: "JSONEachRow",+ });+ const rows = await result.json<{ id: string }>();+ const blobIds = rows.map(row => row.id);setSpanAttributes(span, {"db.response.returned_rows": blobIds.length,});@@ -83,6 +67,16 @@ export async function cleanZdrRequest(requestId: string): Promise<void> {setSpanAttributes(span, { "zdr.request_id": requestId });const blobIds = await getRequestBlobIds(requestId);setSpanAttributes(span, { "zdr.blob_count": blobIds.length });+ // A request that failed before producing a job row has no children; the+ // span attribute above is the signal for that, so this stays at debug.+ if (blobIds.length === 0) {+ _logger.debug("ZDR request has no indexed result blobs", {+ module: "zdrcleaner",+ method: "cleanZdrRequest",+ zeroDataRetention: true,+ requestId,+ });+ }const errors = await removeBlobs(blobIds);if (errors.length > 0) {throw new AggregateError(@@ -93,81 +87,3 @@ export async function cleanZdrRequest(requestId: string): Promise<void> {setSpanAttributes(span, { "zdr.cleanup.outcome": "completed" });});}--export async function zdrcleaner() {- const logger = _logger.child({- module: "zdrcleaner",- method: "zdrcleaner",- });-- const start = Date.now();- try {- // Call the RPC to get all blobs (scrapes, searches, extracts, maps, llmstxts, deep_researches)- // associated with requests that need cleanup.- // The RPC handles the dr_clean_by filtering logic (team-specific vs scheduled)- const rows: { request_id: string; ids: string[] }[] =- await getZdrCleanupBatch(1000);-- if (!rows || rows.length === 0) {- logger.debug("zdrcleaner batch completed with no rows to process", {- canonicalLog: "zdrcleaner",- success: true,- timeMs: Date.now() - start,- });- await sendHeartbeat();- await new Promise(resolve => setTimeout(resolve, 1000));- return;- }-- const deleteErrors: any[] = [];- const fullyCleanedRequests: string[] = [];- for (let j = 0; j < Math.ceil(rows.length / 50); j++) {- const batch = rows.slice(j * 50, (j + 1) * 50);- await Promise.all(- batch.map(async row => {- const errors = await removeBlobs(row.ids);- if (errors.length === 0) fullyCleanedRequests.push(row.request_id);- else deleteErrors.push(...errors);- }),- );- }-- const updateErrors: any[] = [];-- if (fullyCleanedRequests.length > 0) {- try {- await db- .update(schema.requests)- .set({ dr_clean_by: null })- .where(inArray(schema.requests.id, fullyCleanedRequests));- } catch (error) {- updateErrors.push(error);- }- }-- await sendHeartbeat();- if (deleteErrors.length > 0 || updateErrors.length > 0) {- logger.warn("zdrcleaner batch completed with errors", {- canonicalLog: "zdrcleaner",- success: true,- deleteErrors,- updateErrors,- timeMs: Date.now() - start,- });- } else {- logger.debug("zdrcleaner batch completed", {- canonicalLog: "zdrcleaner",- success: true,- deleteErrors,- timeMs: Date.now() - start,- });- }- } catch (error) {- logger.error(`Error looping through cleanup batch`, {- canonicalLog: "zdrcleaner",- success: false,- timeMs: Date.now() - start,- error,- });- }-}diff --git a/apps/api/src/services/zdr-worker.ts b/apps/api/src/services/zdr-worker.tsindex 7702f9f7d..05cdc9e38 100644--- a/apps/api/src/services/zdr-worker.ts+++ b/apps/api/src/services/zdr-worker.ts@@ -1,7 +1,7 @@import "dotenv/config";import { shutdownTracing } from "../otel";import { logger } from "../lib/logger";-import { cleanZdrRequest, zdrcleaner } from "../lib/zdrcleaner";+import { cleanZdrRequest, sendZdrHeartbeat } from "../lib/zdrcleaner";import { consumeZdrCleanupJobs, shutdownZdrQueue } from "../lib/zdr-queue";let isShuttingDown = false;@@ -19,8 +19,11 @@ process.on("SIGTERM", () => {(async () => {await consumeZdrCleanupJobs(job => cleanZdrRequest(job.requestId));+ // Cleanup is driven entirely by the delayed RabbitMQ queue; this loop only+ // keeps the liveness heartbeat going while the consumer runs.while (!isShuttingDown) {- await zdrcleaner();+ await sendZdrHeartbeat();+ await new Promise(resolve => setTimeout(resolve, 1000));}await shutdownZdrQueue();