Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

Update Sentry for ZDR compliance (#2529)

Abimael Martell109f5148a8ff
diff --git a/apps/api/src/controllers/v1/search.ts b/apps/api/src/controllers/v1/search.ts
index 899387e62..1ee1b3c27 100644
--- a/apps/api/src/controllers/v1/search.ts
+++ b/apps/api/src/controllers/v1/search.ts
@@ -15,7 +15,6 @@ import { addScrapeJob, waitForJob } from "../../services/queue-jobs";
import { logSearch, logRequest } from "../../services/logging/log_job";
import { search } from "../../search";
import { isUrlBlocked } from "../../scraper/WebScraper/utils/blocklist";
-import * as Sentry from "@sentry/node";
import { BLOCKLISTED_URL_MESSAGE } from "../../lib/strings";
import { logger as _logger } from "../../lib/logger";
import type { Logger } from "winston";
@@ -26,6 +25,10 @@ import { supabase_service } from "../../services/supabase";
import { fromV1ScrapeOptions } from "../v2/types";
import { ScrapeJobTimeoutError } from "../../lib/error";
import { scrapeQueue } from "../../services/worker/nuq";
+import {
+ applyZdrScope,
+ captureExceptionWithZdrCheck,
+} from "../../services/sentry";
interface DocumentWithCostTracking {
document: Document;
@@ -93,7 +96,8 @@ async function scrapeSearchResult(
const costTracking = new CostTracking();
- const zeroDataRetention = flags?.forceZDR || false;
+ const zeroDataRetention = flags?.forceZDR ?? false;
+ applyZdrScope(zeroDataRetention);
try {
if (isUrlBlocked(searchResult.url, flags)) {
@@ -499,7 +503,9 @@ export async function searchController(
});
}
- Sentry.captureException(error);
+ captureExceptionWithZdrCheck(error, {
+ extra: { zeroDataRetention: false },
+ });
logger.error("Unhandled error occurred in search", {
version: "v1",
error,
diff --git a/apps/api/src/controllers/v1/x402-search.ts b/apps/api/src/controllers/v1/x402-search.ts
index 131d0e27f..42fd81976 100644
--- a/apps/api/src/controllers/v1/x402-search.ts
+++ b/apps/api/src/controllers/v1/x402-search.ts
@@ -14,7 +14,6 @@ import { addScrapeJob, waitForJob } from "../../services/queue-jobs";
import { logSearch, logRequest } from "../../services/logging/log_job";
import { search } from "../../search";
import { isUrlBlocked } from "../../scraper/WebScraper/utils/blocklist";
-import * as Sentry from "@sentry/node";
import { BLOCKLISTED_URL_MESSAGE } from "../../lib/strings";
import { logger as _logger } from "../../lib/logger";
import type { Logger } from "winston";
@@ -23,6 +22,10 @@ import { supabase_service } from "../../services/supabase";
import { fromV1ScrapeOptions } from "../v2/types";
import { ScrapeJobTimeoutError } from "../../lib/error";
import { scrapeQueue } from "../../services/worker/nuq";
+import {
+ applyZdrScope,
+ captureExceptionWithZdrCheck,
+} from "../../services/sentry";
import { getJobPriority } from "../../lib/job-priority";
interface DocumentWithCostTracking {
@@ -48,7 +51,8 @@ async function scrapeX402SearchResult(
const costTracking = new CostTracking();
- const zeroDataRetention = flags?.forceZDR || false;
+ const zeroDataRetention = flags?.forceZDR ?? false;
+ applyZdrScope(zeroDataRetention);
try {
if (isUrlBlocked(searchResult.url, flags)) {
@@ -351,7 +355,9 @@ export async function x402SearchController(
});
}
- Sentry.captureException(error);
+ captureExceptionWithZdrCheck(error, {
+ extra: { zeroDataRetention: false },
+ });
logger.error("Unhandled error occurred in search [x402]", { error });
return res.status(500).json({
success: false,
diff --git a/apps/api/src/controllers/v2/search.ts b/apps/api/src/controllers/v2/search.ts
index 5dc6d07dc..fb67b2feb 100644
--- a/apps/api/src/controllers/v2/search.ts
+++ b/apps/api/src/controllers/v2/search.ts
@@ -15,7 +15,6 @@ import { addScrapeJob, waitForJob } from "../../services/queue-jobs";
import { logSearch, logRequest } from "../../services/logging/log_job";
import { search } from "../../search/v2";
import { isUrlBlocked } from "../../scraper/WebScraper/utils/blocklist";
-import * as Sentry from "@sentry/node";
import { logger as _logger } from "../../lib/logger";
import type { Logger } from "winston";
import { getJobPriority } from "../../lib/job-priority";
@@ -31,6 +30,10 @@ import {
getCategoryFromUrl,
CategoryOption,
} from "../../lib/search-query-builder";
+import {
+ applyZdrScope,
+ captureExceptionWithZdrCheck,
+} from "../../services/sentry";
interface DocumentWithCostTracking {
document: Document;
@@ -242,6 +245,8 @@ export async function searchController(
let credits_billed = 0;
+ let zeroDataRetention = false;
+
try {
req.body = searchRequestSchema.parse(req.body);
@@ -254,6 +259,8 @@ export async function searchController(
const isZDR = req.body.enterprise?.includes("zdr");
const isAnon = req.body.enterprise?.includes("anon");
const isZDROrAnon = isZDR || isAnon;
+ zeroDataRetention = isZDROrAnon ?? false;
+ applyZdrScope(isZDROrAnon ?? false);
await logRequest({
id: jobId,
@@ -756,7 +763,9 @@ export async function searchController(
});
}
- Sentry.captureException(error);
+ captureExceptionWithZdrCheck(error, {
+ extra: { zeroDataRetention },
+ });
logger.error("Unhandled error occurred in search", {
version: "v2",
error,
diff --git a/apps/api/src/controllers/v2/x402-search.ts b/apps/api/src/controllers/v2/x402-search.ts
index 89f5265b1..72d48b2cf 100644
--- a/apps/api/src/controllers/v2/x402-search.ts
+++ b/apps/api/src/controllers/v2/x402-search.ts
@@ -14,7 +14,6 @@ import { addScrapeJob, waitForJob } from "../../services/queue-jobs";
import { logSearch, logRequest } from "../../services/logging/log_job";
import { search } from "../../search/v2";
import { isUrlBlocked } from "../../scraper/WebScraper/utils/blocklist";
-import * as Sentry from "@sentry/node";
import { logger as _logger } from "../../lib/logger";
import type { Logger } from "winston";
import { getJobPriority } from "../../lib/job-priority";
@@ -29,6 +28,10 @@ import {
getCategoryFromUrl,
CategoryOption,
} from "../../lib/search-query-builder";
+import {
+ applyZdrScope,
+ captureExceptionWithZdrCheck,
+} from "../../services/sentry";
interface DocumentWithCostTracking {
document: Document;
@@ -58,7 +61,8 @@ async function startX420ScrapeJob(
): Promise<string> {
const jobId = uuidv7();
- const zeroDataRetention = flags?.forceZDR || false;
+ const zeroDataRetention = flags?.forceZDR ?? false;
+ applyZdrScope(zeroDataRetention);
logger.info("Adding scrape job [x402]", {
scrapeId: jobId,
@@ -639,7 +643,9 @@ export async function x402SearchController(
});
}
- Sentry.captureException(error);
+ captureExceptionWithZdrCheck(error, {
+ extra: { zeroDataRetention: false },
+ });
logger.error("Unhandled error occurred in search [x402]", { error });
return res.status(500).json({
success: false,
diff --git a/apps/api/src/scraper/scrapeURL/index.ts b/apps/api/src/scraper/scrapeURL/index.ts
index c0b858af3..98db64ec4 100644
--- a/apps/api/src/scraper/scrapeURL/index.ts
+++ b/apps/api/src/scraper/scrapeURL/index.ts
@@ -2,6 +2,7 @@ import { Logger } from "winston";
import { config } from "../../config";
import * as Sentry from "@sentry/node";
import { withSpan, setSpanAttributes } from "../../lib/otel-tracer";
+import { captureExceptionWithZdrCheck } from "../../services/sentry";
import {
type Document,
@@ -1213,7 +1214,11 @@ export async function scrapeURL(
errorType = "AbortManagerThrownError";
throw error.inner;
} else {
- Sentry.captureException(error);
+ captureExceptionWithZdrCheck(error, {
+ extra: {
+ zeroDataRetention: internalOptions.zeroDataRetention ?? false,
+ },
+ });
meta.logger.error("scrapeURL: Unexpected error happened", { error });
// TODO: results?
}
diff --git a/apps/api/src/services/sentry.ts b/apps/api/src/services/sentry.ts
index d7e51838c..1aa1d1d6a 100644
--- a/apps/api/src/services/sentry.ts
+++ b/apps/api/src/services/sentry.ts
@@ -2,6 +2,22 @@ import * as Sentry from "@sentry/node";
import { logger } from "../lib/logger";
import { config } from "../config";
+type CaptureContext = {
+ tags?: Record<string, string>;
+ extra?: Record<string, any>;
+ level?: Sentry.SeverityLevel;
+ fingerprint?: string[];
+ contexts?: Record<string, any>;
+ user?: {
+ id?: string;
+ username?: string;
+ email?: string;
+ ip_address?: string;
+ [key: string]: any;
+ };
+ [key: string]: any;
+};
+
if (config.SENTRY_DSN) {
logger.info("Setting up Sentry...");
@@ -24,6 +40,16 @@ if (config.SENTRY_DSN) {
serverName: config.NUQ_POD_NAME,
environment: config.SENTRY_ENVIRONMENT,
beforeSend(event, hint) {
+ const zeroDataRetention =
+ event.tags?.zeroDataRetention === "true" ||
+ event.tags?.zero_data_retention === "true" ||
+ event.extra?.zeroDataRetention === true ||
+ event.extra?.zero_data_retention === true;
+
+ if (zeroDataRetention) {
+ return null;
+ }
+
const error = hint?.originalException;
if (error && typeof error === "object") {
@@ -61,6 +87,34 @@ if (config.SENTRY_DSN) {
});
}
+export function captureExceptionWithZdrCheck(
+ error: any,
+ context?: (CaptureContext & { zeroDataRetention?: boolean }) | null,
+) {
+ const zeroDataRetention =
+ context?.zeroDataRetention ??
+ (context?.extra as any)?.zeroDataRetention ??
+ (context?.data as any)?.zeroDataRetention;
+
+ if (zeroDataRetention) {
+ return;
+ }
+
+ const { zeroDataRetention: _zdr, ...sentryContext } = context || {};
+
+ return Sentry.captureException(error, sentryContext);
+}
+
+export function applyZdrScope(zeroDataRetention?: boolean) {
+ if (!zeroDataRetention) {
+ return;
+ }
+
+ const scope = Sentry.getCurrentScope();
+ scope.setTag("zeroDataRetention", "true");
+ scope.setExtra("zeroDataRetention", true);
+}
+
/**
* Set the service type tag for this Sentry instance
* This helps distinguish between API server and worker errors in Sentry
diff --git a/apps/api/src/services/worker/scrape-worker.ts b/apps/api/src/services/worker/scrape-worker.ts
index 0d665da87..78f628b4a 100644
--- a/apps/api/src/services/worker/scrape-worker.ts
+++ b/apps/api/src/services/worker/scrape-worker.ts
@@ -1,6 +1,7 @@
import { configDotenv } from "dotenv";
import { config } from "../../config";
import * as Sentry from "@sentry/node";
+import { applyZdrScope, captureExceptionWithZdrCheck } from "../sentry";
import http from "http";
import https from "https";
@@ -145,7 +146,9 @@ async function billScrapeJob(
`Failed to add billing job to queue for team ${job.data.team_id} for ${creditsToBeBilled} credits`,
{ error },
);
- Sentry.captureException(error);
+ captureExceptionWithZdrCheck(error, {
+ extra: { zeroDataRetention: job.data.zeroDataRetention ?? false },
+ });
return creditsToBeBilled;
}
}
@@ -164,6 +167,7 @@ async function processJob(job: NuQJob<ScrapeJobSingleUrls>) {
teamId: job.data?.team_id ?? undefined,
zeroDataRetention: job.data?.zeroDataRetention ?? false,
});
+ applyZdrScope(job.data?.zeroDataRetention);
logger.info(`🐂 Worker taking job ${job.id}`, { url: job.data.url });
const start = job.data.startTime ?? Date.now();
const remainingTime = job.data.scrapeOptions.timeout
@@ -599,10 +603,11 @@ async function processJob(job: NuQJob<ScrapeJobSingleUrls>) {
// Filter out TransportableErrors (flow control)
if (!(error instanceof TransportableError)) {
- Sentry.captureException(error, {
+ captureExceptionWithZdrCheck(error, {
data: {
job: job.id,
},
+ extra: { zeroDataRetention: job.data.zeroDataRetention ?? false },
});
}
@@ -1192,7 +1197,9 @@ async function processJobWithTracing(job: NuQJob<ScrapeJobData>, logger: any) {
// Filter out TransportableErrors (flow control)
if (!(error instanceof TransportableError)) {
- Sentry.captureException(error);
+ captureExceptionWithZdrCheck(error, {
+ extra: { zeroDataRetention: job.data.zeroDataRetention ?? false },
+ });
}
if (job.data.skipNuq) {