Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
import "dotenv/config";
import { clickhouseClient } from "./clickhouse-client";
import { removeJobFromGCS } from "./gcs-jobs";
import { logger as _logger } from "./logger";
import { config } from "../config";
import { setSpanAttributes, withSpan } from "./otel-tracer";
import type { ZdrCleanupJob } from "./zdr-queue";
export async function sendZdrHeartbeat() {
if (config.ZDRCLEANER_HEARTBEAT_URL) {
fetch(config.ZDRCLEANER_HEARTBEAT_URL).catch(() => {});
}
}
// A request can resolve to a very large number of blobs; issuing every delete
// at once exhausts the worker's memory.
const BLOB_DELETE_CONCURRENCY = 50;
async function removeBlobs(blobIds: string[]): Promise<unknown[]> {
const logger = _logger.child({
module: "zdrcleaner",
method: "removeBlobs",
zeroDataRetention: true,
});
const errors: unknown[] = [];
let next = 0;
const worker = async () => {
while (next < blobIds.length) {
const blobId = blobIds[next++];
try {
await removeJobFromGCS(blobId, logger);
} catch (error) {
errors.push(error);
}
}
};
await Promise.all(
Array.from(
{ length: Math.min(BLOB_DELETE_CONCURRENCY, blobIds.length) },
worker,
),
);
return errors;
}
/**
* Every result blob a request produced, from the `request_children` table in
* the analytics ClickHouse service, which materialized views fill from the
* Pub/Sub-ingested scrapes, searches, extracts, maps, llmstxts and
* deep_researches rows. The table is created by hand (statements in the PR
* that introduced this reader), not by this codebase. Cleanup runs 24 hours after the request, far
* beyond ClickPipes ingest lag, so the index is complete by the time it is
* read.
*/
async function getRequestBlobIds(requestId: string): Promise<string[]> {
return withSpan("zdr.clickhouse.read_blobs", async span => {
setSpanAttributes(span, {
"db.system": "clickhouse",
"db.operation.name": "select",
"db.collection.name": "request_children",
"zdr.request_id": requestId,
});
if (clickhouseClient === null) {
throw new Error(
"ClickHouse is not configured; cannot resolve ZDR request blobs",
);
}
const result = await clickhouseClient.query({
query:
"SELECT DISTINCT id FROM request_children WHERE request_id = {requestId: UUID}",
query_params: { requestId },
format: "JSONEachRow",
});
const rows = await result.json<{ id: string }>();
const blobIds = rows.map(row => row.id);
setSpanAttributes(span, {
"db.response.returned_rows": blobIds.length,
});
return blobIds;
});
}
export async function cleanZdrRequest(requestId: string): Promise<void> {
await withSpan("zdr.cleanup.request", async span => {
setSpanAttributes(span, { "zdr.request_id": requestId });
const blobIds = await getRequestBlobIds(requestId);
setSpanAttributes(span, { "zdr.blob_count": blobIds.length });
// A request that failed before producing a job row has no children; the
// span attribute above is the signal for that, so this stays at debug.
if (blobIds.length === 0) {
_logger.debug("ZDR request has no indexed result blobs", {
module: "zdrcleaner",
method: "cleanZdrRequest",
zeroDataRetention: true,
requestId,
});
}
const errors = await removeBlobs(blobIds);
if (errors.length > 0) {
throw new AggregateError(
errors,
`Failed to remove ${errors.length} blobs for ZDR request ${requestId}`,
);
}
setSpanAttributes(span, { "zdr.cleanup.outcome": "completed" });
});
}
export async function cleanZdrJob(job: ZdrCleanupJob): Promise<void> {
if ("blobId" in job) {
await removeJobFromGCS(job.blobId);
return;
}
await cleanZdrRequest(job.requestId);
}