Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

fix(api): keep no error text on zero data retention scrape logs (#5432)

Gergő Móricz52e655a4ba41
diff --git a/apps/api/src/__tests__/snips/v2/zdr.test.ts b/apps/api/src/__tests__/snips/v2/zdr.test.ts
index 81a2749eb..565437a84 100644
--- a/apps/api/src/__tests__/snips/v2/zdr.test.ts
+++ b/apps/api/src/__tests__/snips/v2/zdr.test.ts
@@ -1,6 +1,7 @@
import { getJobFromGCS } from "../../../lib/gcs-jobs";
import {
scrape,
+ scrapeRaw,
crawl,
batchScrape,
scrapeStatusRaw,
@@ -26,7 +27,7 @@ import {
expectScrapesAreFullyCleanedAfterZDRCleaner,
expectBatchScrapeIsCleanedUp,
} from "../zdr-helpers";
-import { HAS_JOB_LOG } from "../job-log";
+import { HAS_JOB_LOG, waitForJobLogRows } from "../job-log";
describeIf(TEST_PRODUCTION)("Zero Data Retention", () => {
const scopes = ["Team-scoped", "Request-scoped"] as const;
@@ -64,6 +65,43 @@ describeIf(TEST_PRODUCTION)("Zero Data Retention", () => {
}
}, 60000);
+ it(
+ "should keep no error text for a failed scrape",
+ async () => {
+ const identity = await idmux({
+ name: `zdr/${scope}/failed-scrape`,
+ credits: 10000,
+ flags: {
+ scrapeZDR: scope === "Team-scoped" ? "forced" : "allowed",
+ },
+ });
+
+ // An unresolvable host fails the scrape with an error naming the host.
+ const hostname = `zdr-error-${crypto.randomUUID()}.invalid`;
+ const response = await scrapeRaw(
+ {
+ url: `https://${hostname}/`,
+ zeroDataRetention: scope === "Request-scoped" ? true : undefined,
+ },
+ identity,
+ );
+ expect(response.body.success).toBe(false);
+
+ const rows = await waitForJobLogRows(
+ "scrapes",
+ "team_id = {teamId: UUID}",
+ { teamId: identity.teamId! },
+ );
+ expect(rows).toHaveLength(1);
+ expect(rows[0].is_successful).toBe(false);
+ expect(rows[0].error).toBeTruthy();
+ expect(String(rows[0].error)).not.toContain(hostname);
+
+ await expectScrapeIsCleanedUp(String(rows[0].id));
+ },
+ scrapeTimeout,
+ );
+
it(
"should clean up a crawl",
async () => {
diff --git a/apps/api/src/__tests__/snips/zdr-helpers.ts b/apps/api/src/__tests__/snips/zdr-helpers.ts
index 419cbf948..4e51c38b6 100644
--- a/apps/api/src/__tests__/snips/zdr-helpers.ts
+++ b/apps/api/src/__tests__/snips/zdr-helpers.ts
@@ -87,10 +87,15 @@ export async function getLogs() {
type JobRow = Record<string, unknown>;
+// A failed ZDR scrape keeps at most an error code, never the error's text.
+const ZDR_STORED_ERROR =
+ /^(?:[A-Za-z_]*|<redacted due to zero data retention>)$/;
+
// Scrape and crawl options are stored as a JSON string; ZDR rows carry none.
function expectNoStoredContent(row: JobRow) {
expect(String(row.url)).not.toContain("://"); // no url stored
expect(row.options ?? null).toBeNull();
+ expect(String(row.error ?? "")).toMatch(ZDR_STORED_ERROR);
}
export async function expectScrapeIsCleanedUp(scrapeId: string) {
diff --git a/apps/api/src/services/logging/log_job.ts b/apps/api/src/services/logging/log_job.ts
index 805f9c86b..1c7eaea87 100644
--- a/apps/api/src/services/logging/log_job.ts
+++ b/apps/api/src/services/logging/log_job.ts
@@ -21,6 +21,7 @@ import type {
SearchResultCategoriesBySource,
} from "../../lib/entities";
import type { CostTracking } from "../../lib/cost-tracking";
+import type { ErrorCodes } from "../../lib/error";
import type { Logger } from "winston";
import { saveExtractResult } from "../../lib/extract/extract-redis";
import { trackFirstSurfaceUse } from "../posthog";
@@ -588,6 +589,8 @@ export type LoggedScrape = {
url: string;
is_successful: boolean;
error?: string;
+ /** Code of a failed scrape's error; the only error detail kept under zero data retention. */
+ error_code?: ErrorCodes;
doc?: Document;
time_taken: number;
team_id: string;
@@ -716,7 +719,11 @@ async function logScrapeInternal(scrape: LoggedScrape, hooks?: LogScrapeHooks) {
? "<redacted due to zero data retention>"
: scrape.url,
is_successful: scrape.is_successful,
- error: scrape.error ?? null,
+ error: scrape.zeroDataRetention
+ ? scrape.is_successful
+ ? null
+ : (scrape.error_code ?? "<redacted due to zero data retention>")
+ : (scrape.error ?? null),
time_taken: scrape.time_taken,
team_id: storedTeamId,
options: scrape.zeroDataRetention ? null : scrape.options,
diff --git a/apps/api/src/services/worker/scrape-worker.ts b/apps/api/src/services/worker/scrape-worker.ts
index e5ad3f7f9..e5ebe8949 100644
--- a/apps/api/src/services/worker/scrape-worker.ts
+++ b/apps/api/src/services/worker/scrape-worker.ts
@@ -1241,6 +1241,7 @@ async function processJob(job: NuQJob<ScrapeJobSingleUrls>) {
? error
: (error.message ??
"Something went wrong... Contact [email protected]"),
+ error_code: error instanceof TransportableError ? error.code : undefined,
time_taken: timeTakenInSeconds,
team_id: job.data.team_id,
options: job.data.scrapeOptions,