Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(api): keep no error text on zero data retention scrape logs (#5432)
Gergő Móricz52e655a4ba41
diff --git a/apps/api/src/__tests__/snips/v2/zdr.test.ts b/apps/api/src/__tests__/snips/v2/zdr.test.tsindex 81a2749eb..565437a84 100644--- a/apps/api/src/__tests__/snips/v2/zdr.test.ts+++ b/apps/api/src/__tests__/snips/v2/zdr.test.ts@@ -1,6 +1,7 @@import { getJobFromGCS } from "../../../lib/gcs-jobs";import {scrape,+ scrapeRaw,crawl,batchScrape,scrapeStatusRaw,@@ -26,7 +27,7 @@ import {expectScrapesAreFullyCleanedAfterZDRCleaner,expectBatchScrapeIsCleanedUp,} from "../zdr-helpers";-import { HAS_JOB_LOG } from "../job-log";+import { HAS_JOB_LOG, waitForJobLogRows } from "../job-log";describeIf(TEST_PRODUCTION)("Zero Data Retention", () => {const scopes = ["Team-scoped", "Request-scoped"] as const;@@ -64,6 +65,43 @@ describeIf(TEST_PRODUCTION)("Zero Data Retention", () => {}}, 60000);+ it(+ "should keep no error text for a failed scrape",+ async () => {+ const identity = await idmux({+ name: `zdr/${scope}/failed-scrape`,+ credits: 10000,+ flags: {+ scrapeZDR: scope === "Team-scoped" ? "forced" : "allowed",+ },+ });++ // An unresolvable host fails the scrape with an error naming the host.+ const hostname = `zdr-error-${crypto.randomUUID()}.invalid`;+ const response = await scrapeRaw(+ {+ url: `https://${hostname}/`,+ zeroDataRetention: scope === "Request-scoped" ? true : undefined,+ },+ identity,+ );+ expect(response.body.success).toBe(false);++ const rows = await waitForJobLogRows(+ "scrapes",+ "team_id = {teamId: UUID}",+ { teamId: identity.teamId! },+ );+ expect(rows).toHaveLength(1);+ expect(rows[0].is_successful).toBe(false);+ expect(rows[0].error).toBeTruthy();+ expect(String(rows[0].error)).not.toContain(hostname);++ await expectScrapeIsCleanedUp(String(rows[0].id));+ },+ scrapeTimeout,+ );+it("should clean up a crawl",async () => {diff --git a/apps/api/src/__tests__/snips/zdr-helpers.ts b/apps/api/src/__tests__/snips/zdr-helpers.tsindex 419cbf948..4e51c38b6 100644--- a/apps/api/src/__tests__/snips/zdr-helpers.ts+++ b/apps/api/src/__tests__/snips/zdr-helpers.ts@@ -87,10 +87,15 @@ export async function getLogs() {type JobRow = Record<string, unknown>;+// A failed ZDR scrape keeps at most an error code, never the error's text.+const ZDR_STORED_ERROR =+ /^(?:[A-Za-z_]*|<redacted due to zero data retention>)$/;+// Scrape and crawl options are stored as a JSON string; ZDR rows carry none.function expectNoStoredContent(row: JobRow) {expect(String(row.url)).not.toContain("://"); // no url storedexpect(row.options ?? null).toBeNull();+ expect(String(row.error ?? "")).toMatch(ZDR_STORED_ERROR);}export async function expectScrapeIsCleanedUp(scrapeId: string) {diff --git a/apps/api/src/services/logging/log_job.ts b/apps/api/src/services/logging/log_job.tsindex 805f9c86b..1c7eaea87 100644--- a/apps/api/src/services/logging/log_job.ts+++ b/apps/api/src/services/logging/log_job.ts@@ -21,6 +21,7 @@ import type {SearchResultCategoriesBySource,} from "../../lib/entities";import type { CostTracking } from "../../lib/cost-tracking";+import type { ErrorCodes } from "../../lib/error";import type { Logger } from "winston";import { saveExtractResult } from "../../lib/extract/extract-redis";import { trackFirstSurfaceUse } from "../posthog";@@ -588,6 +589,8 @@ export type LoggedScrape = {url: string;is_successful: boolean;error?: string;+ /** Code of a failed scrape's error; the only error detail kept under zero data retention. */+ error_code?: ErrorCodes;doc?: Document;time_taken: number;team_id: string;@@ -716,7 +719,11 @@ async function logScrapeInternal(scrape: LoggedScrape, hooks?: LogScrapeHooks) {? "<redacted due to zero data retention>": scrape.url,is_successful: scrape.is_successful,- error: scrape.error ?? null,+ error: scrape.zeroDataRetention+ ? scrape.is_successful+ ? null+ : (scrape.error_code ?? "<redacted due to zero data retention>")+ : (scrape.error ?? null),time_taken: scrape.time_taken,team_id: storedTeamId,options: scrape.zeroDataRetention ? null : scrape.options,diff --git a/apps/api/src/services/worker/scrape-worker.ts b/apps/api/src/services/worker/scrape-worker.tsindex e5ad3f7f9..e5ebe8949 100644--- a/apps/api/src/services/worker/scrape-worker.ts+++ b/apps/api/src/services/worker/scrape-worker.ts@@ -1241,6 +1241,7 @@ async function processJob(job: NuQJob<ScrapeJobSingleUrls>) {? error: (error.message ??"Something went wrong... Contact [email protected]"),+ error_code: error instanceof TransportableError ? error.code : undefined,time_taken: timeTakenInSeconds,team_id: job.data.team_id,options: job.data.scrapeOptions,