Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
Update Sentry for ZDR compliance (#2529)
Abimael Martell109f5148a8ff
diff --git a/apps/api/src/controllers/v1/search.ts b/apps/api/src/controllers/v1/search.tsindex 899387e62..1ee1b3c27 100644--- a/apps/api/src/controllers/v1/search.ts+++ b/apps/api/src/controllers/v1/search.ts@@ -15,7 +15,6 @@ import { addScrapeJob, waitForJob } from "../../services/queue-jobs";import { logSearch, logRequest } from "../../services/logging/log_job";import { search } from "../../search";import { isUrlBlocked } from "../../scraper/WebScraper/utils/blocklist";-import * as Sentry from "@sentry/node";import { BLOCKLISTED_URL_MESSAGE } from "../../lib/strings";import { logger as _logger } from "../../lib/logger";import type { Logger } from "winston";@@ -26,6 +25,10 @@ import { supabase_service } from "../../services/supabase";import { fromV1ScrapeOptions } from "../v2/types";import { ScrapeJobTimeoutError } from "../../lib/error";import { scrapeQueue } from "../../services/worker/nuq";+import {+ applyZdrScope,+ captureExceptionWithZdrCheck,+} from "../../services/sentry";interface DocumentWithCostTracking {document: Document;@@ -93,7 +96,8 @@ async function scrapeSearchResult(const costTracking = new CostTracking();- const zeroDataRetention = flags?.forceZDR || false;+ const zeroDataRetention = flags?.forceZDR ?? false;+ applyZdrScope(zeroDataRetention);try {if (isUrlBlocked(searchResult.url, flags)) {@@ -499,7 +503,9 @@ export async function searchController(});}- Sentry.captureException(error);+ captureExceptionWithZdrCheck(error, {+ extra: { zeroDataRetention: false },+ });logger.error("Unhandled error occurred in search", {version: "v1",error,diff --git a/apps/api/src/controllers/v1/x402-search.ts b/apps/api/src/controllers/v1/x402-search.tsindex 131d0e27f..42fd81976 100644--- a/apps/api/src/controllers/v1/x402-search.ts+++ b/apps/api/src/controllers/v1/x402-search.ts@@ -14,7 +14,6 @@ import { addScrapeJob, waitForJob } from "../../services/queue-jobs";import { logSearch, logRequest } from "../../services/logging/log_job";import { search } from "../../search";import { isUrlBlocked } from "../../scraper/WebScraper/utils/blocklist";-import * as Sentry from "@sentry/node";import { BLOCKLISTED_URL_MESSAGE } from "../../lib/strings";import { logger as _logger } from "../../lib/logger";import type { Logger } from "winston";@@ -23,6 +22,10 @@ import { supabase_service } from "../../services/supabase";import { fromV1ScrapeOptions } from "../v2/types";import { ScrapeJobTimeoutError } from "../../lib/error";import { scrapeQueue } from "../../services/worker/nuq";+import {+ applyZdrScope,+ captureExceptionWithZdrCheck,+} from "../../services/sentry";import { getJobPriority } from "../../lib/job-priority";interface DocumentWithCostTracking {@@ -48,7 +51,8 @@ async function scrapeX402SearchResult(const costTracking = new CostTracking();- const zeroDataRetention = flags?.forceZDR || false;+ const zeroDataRetention = flags?.forceZDR ?? false;+ applyZdrScope(zeroDataRetention);try {if (isUrlBlocked(searchResult.url, flags)) {@@ -351,7 +355,9 @@ export async function x402SearchController(});}- Sentry.captureException(error);+ captureExceptionWithZdrCheck(error, {+ extra: { zeroDataRetention: false },+ });logger.error("Unhandled error occurred in search [x402]", { error });return res.status(500).json({success: false,diff --git a/apps/api/src/controllers/v2/search.ts b/apps/api/src/controllers/v2/search.tsindex 5dc6d07dc..fb67b2feb 100644--- a/apps/api/src/controllers/v2/search.ts+++ b/apps/api/src/controllers/v2/search.ts@@ -15,7 +15,6 @@ import { addScrapeJob, waitForJob } from "../../services/queue-jobs";import { logSearch, logRequest } from "../../services/logging/log_job";import { search } from "../../search/v2";import { isUrlBlocked } from "../../scraper/WebScraper/utils/blocklist";-import * as Sentry from "@sentry/node";import { logger as _logger } from "../../lib/logger";import type { Logger } from "winston";import { getJobPriority } from "../../lib/job-priority";@@ -31,6 +30,10 @@ import {getCategoryFromUrl,CategoryOption,} from "../../lib/search-query-builder";+import {+ applyZdrScope,+ captureExceptionWithZdrCheck,+} from "../../services/sentry";interface DocumentWithCostTracking {document: Document;@@ -242,6 +245,8 @@ export async function searchController(let credits_billed = 0;+ let zeroDataRetention = false;+try {req.body = searchRequestSchema.parse(req.body);@@ -254,6 +259,8 @@ export async function searchController(const isZDR = req.body.enterprise?.includes("zdr");const isAnon = req.body.enterprise?.includes("anon");const isZDROrAnon = isZDR || isAnon;+ zeroDataRetention = isZDROrAnon ?? false;+ applyZdrScope(isZDROrAnon ?? false);await logRequest({id: jobId,@@ -756,7 +763,9 @@ export async function searchController(});}- Sentry.captureException(error);+ captureExceptionWithZdrCheck(error, {+ extra: { zeroDataRetention },+ });logger.error("Unhandled error occurred in search", {version: "v2",error,diff --git a/apps/api/src/controllers/v2/x402-search.ts b/apps/api/src/controllers/v2/x402-search.tsindex 89f5265b1..72d48b2cf 100644--- a/apps/api/src/controllers/v2/x402-search.ts+++ b/apps/api/src/controllers/v2/x402-search.ts@@ -14,7 +14,6 @@ import { addScrapeJob, waitForJob } from "../../services/queue-jobs";import { logSearch, logRequest } from "../../services/logging/log_job";import { search } from "../../search/v2";import { isUrlBlocked } from "../../scraper/WebScraper/utils/blocklist";-import * as Sentry from "@sentry/node";import { logger as _logger } from "../../lib/logger";import type { Logger } from "winston";import { getJobPriority } from "../../lib/job-priority";@@ -29,6 +28,10 @@ import {getCategoryFromUrl,CategoryOption,} from "../../lib/search-query-builder";+import {+ applyZdrScope,+ captureExceptionWithZdrCheck,+} from "../../services/sentry";interface DocumentWithCostTracking {document: Document;@@ -58,7 +61,8 @@ async function startX420ScrapeJob(): Promise<string> {const jobId = uuidv7();- const zeroDataRetention = flags?.forceZDR || false;+ const zeroDataRetention = flags?.forceZDR ?? false;+ applyZdrScope(zeroDataRetention);logger.info("Adding scrape job [x402]", {scrapeId: jobId,@@ -639,7 +643,9 @@ export async function x402SearchController(});}- Sentry.captureException(error);+ captureExceptionWithZdrCheck(error, {+ extra: { zeroDataRetention: false },+ });logger.error("Unhandled error occurred in search [x402]", { error });return res.status(500).json({success: false,diff --git a/apps/api/src/scraper/scrapeURL/index.ts b/apps/api/src/scraper/scrapeURL/index.tsindex c0b858af3..98db64ec4 100644--- a/apps/api/src/scraper/scrapeURL/index.ts+++ b/apps/api/src/scraper/scrapeURL/index.ts@@ -2,6 +2,7 @@ import { Logger } from "winston";import { config } from "../../config";import * as Sentry from "@sentry/node";import { withSpan, setSpanAttributes } from "../../lib/otel-tracer";+import { captureExceptionWithZdrCheck } from "../../services/sentry";import {type Document,@@ -1213,7 +1214,11 @@ export async function scrapeURL(errorType = "AbortManagerThrownError";throw error.inner;} else {- Sentry.captureException(error);+ captureExceptionWithZdrCheck(error, {+ extra: {+ zeroDataRetention: internalOptions.zeroDataRetention ?? false,+ },+ });meta.logger.error("scrapeURL: Unexpected error happened", { error });// TODO: results?}diff --git a/apps/api/src/services/sentry.ts b/apps/api/src/services/sentry.tsindex d7e51838c..1aa1d1d6a 100644--- a/apps/api/src/services/sentry.ts+++ b/apps/api/src/services/sentry.ts@@ -2,6 +2,22 @@ import * as Sentry from "@sentry/node";import { logger } from "../lib/logger";import { config } from "../config";+type CaptureContext = {+ tags?: Record<string, string>;+ extra?: Record<string, any>;+ level?: Sentry.SeverityLevel;+ fingerprint?: string[];+ contexts?: Record<string, any>;+ user?: {+ id?: string;+ username?: string;+ email?: string;+ ip_address?: string;+ [key: string]: any;+ };+ [key: string]: any;+};+if (config.SENTRY_DSN) {logger.info("Setting up Sentry...");@@ -24,6 +40,16 @@ if (config.SENTRY_DSN) {serverName: config.NUQ_POD_NAME,environment: config.SENTRY_ENVIRONMENT,beforeSend(event, hint) {+ const zeroDataRetention =+ event.tags?.zeroDataRetention === "true" ||+ event.tags?.zero_data_retention === "true" ||+ event.extra?.zeroDataRetention === true ||+ event.extra?.zero_data_retention === true;++ if (zeroDataRetention) {+ return null;+ }+const error = hint?.originalException;if (error && typeof error === "object") {@@ -61,6 +87,34 @@ if (config.SENTRY_DSN) {});}+export function captureExceptionWithZdrCheck(+ error: any,+ context?: (CaptureContext & { zeroDataRetention?: boolean }) | null,+) {+ const zeroDataRetention =+ context?.zeroDataRetention ??+ (context?.extra as any)?.zeroDataRetention ??+ (context?.data as any)?.zeroDataRetention;++ if (zeroDataRetention) {+ return;+ }++ const { zeroDataRetention: _zdr, ...sentryContext } = context || {};++ return Sentry.captureException(error, sentryContext);+}++export function applyZdrScope(zeroDataRetention?: boolean) {+ if (!zeroDataRetention) {+ return;+ }++ const scope = Sentry.getCurrentScope();+ scope.setTag("zeroDataRetention", "true");+ scope.setExtra("zeroDataRetention", true);+}+/*** Set the service type tag for this Sentry instance* This helps distinguish between API server and worker errors in Sentrydiff --git a/apps/api/src/services/worker/scrape-worker.ts b/apps/api/src/services/worker/scrape-worker.tsindex 0d665da87..78f628b4a 100644--- a/apps/api/src/services/worker/scrape-worker.ts+++ b/apps/api/src/services/worker/scrape-worker.ts@@ -1,6 +1,7 @@import { configDotenv } from "dotenv";import { config } from "../../config";import * as Sentry from "@sentry/node";+import { applyZdrScope, captureExceptionWithZdrCheck } from "../sentry";import http from "http";import https from "https";@@ -145,7 +146,9 @@ async function billScrapeJob(`Failed to add billing job to queue for team ${job.data.team_id} for ${creditsToBeBilled} credits`,{ error },);- Sentry.captureException(error);+ captureExceptionWithZdrCheck(error, {+ extra: { zeroDataRetention: job.data.zeroDataRetention ?? false },+ });return creditsToBeBilled;}}@@ -164,6 +167,7 @@ async function processJob(job: NuQJob<ScrapeJobSingleUrls>) {teamId: job.data?.team_id ?? undefined,zeroDataRetention: job.data?.zeroDataRetention ?? false,});+ applyZdrScope(job.data?.zeroDataRetention);logger.info(`🐂 Worker taking job ${job.id}`, { url: job.data.url });const start = job.data.startTime ?? Date.now();const remainingTime = job.data.scrapeOptions.timeout@@ -599,10 +603,11 @@ async function processJob(job: NuQJob<ScrapeJobSingleUrls>) {// Filter out TransportableErrors (flow control)if (!(error instanceof TransportableError)) {- Sentry.captureException(error, {+ captureExceptionWithZdrCheck(error, {data: {job: job.id,},+ extra: { zeroDataRetention: job.data.zeroDataRetention ?? false },});}@@ -1192,7 +1197,9 @@ async function processJobWithTracing(job: NuQJob<ScrapeJobData>, logger: any) {// Filter out TransportableErrors (flow control)if (!(error instanceof TransportableError)) {- Sentry.captureException(error);+ captureExceptionWithZdrCheck(error, {+ extra: { zeroDataRetention: job.data.zeroDataRetention ?? false },+ });}if (job.data.skipNuq) {