Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. πŸ”₯

Filter Sentry Exceptions (#2453)

Abimael Martell8cf9f582a92c
diff --git a/apps/api/src/lib/error-serde.ts b/apps/api/src/lib/error-serde.ts
index 004909c99..680c2137b 100644
--- a/apps/api/src/lib/error-serde.ts
+++ b/apps/api/src/lib/error-serde.ts
@@ -1,8 +1,10 @@
import {
+ CrawlDenialError,
ErrorCodes,
MapTimeoutError,
RacedRedirectError,
ScrapeJobTimeoutError,
+ SitemapError,
TransportableError,
UnknownError,
} from "./error";
@@ -41,6 +43,8 @@ const errorMap: Record<ErrorCodes, any> = {
SCRAPE_UNSUPPORTED_FILE_ERROR: UnsupportedFileError,
SCRAPE_ACTION_ERROR: ActionError,
SCRAPE_RACED_REDIRECT_ERROR: RacedRedirectError,
+ SCRAPE_SITEMAP_ERROR: SitemapError,
+ CRAWL_DENIAL: CrawlDenialError,
// Zod errors
BAD_REQUEST: null,
diff --git a/apps/api/src/lib/error.ts b/apps/api/src/lib/error.ts
index 05378a775..9bf3c90cd 100644
--- a/apps/api/src/lib/error.ts
+++ b/apps/api/src/lib/error.ts
@@ -16,6 +16,8 @@ export type ErrorCodes =
| "SCRAPE_UNSUPPORTED_FILE_ERROR"
| "SCRAPE_ACTION_ERROR"
| "SCRAPE_RACED_REDIRECT_ERROR"
+ | "SCRAPE_SITEMAP_ERROR"
+ | "CRAWL_DENIAL"
| "BAD_REQUEST_INVALID_JSON"
| "BAD_REQUEST";
@@ -128,3 +130,44 @@ export class RacedRedirectError extends TransportableError {
return x;
}
}
+
+export class SitemapError extends TransportableError {
+ constructor(message: string, cause?: unknown) {
+ super("SCRAPE_SITEMAP_ERROR", message, { cause });
+ }
+
+ serialize() {
+ return super.serialize();
+ }
+
+ static deserialize(
+ _: ErrorCodes,
+ data: ReturnType<typeof this.prototype.serialize>,
+ ) {
+ const x = new SitemapError(data.message, data.cause);
+ x.stack = data.stack;
+ return x;
+ }
+}
+
+export class CrawlDenialError extends TransportableError {
+ constructor(public reason: string) {
+ super("CRAWL_DENIAL", reason);
+ }
+
+ serialize() {
+ return {
+ ...super.serialize(),
+ reason: this.reason,
+ };
+ }
+
+ static deserialize(
+ _: ErrorCodes,
+ data: ReturnType<typeof this.prototype.serialize> & { reason: string },
+ ) {
+ const x = new CrawlDenialError(data.reason);
+ x.stack = data.stack;
+ return x;
+ }
+}
diff --git a/apps/api/src/scraper/crawler/sitemap.ts b/apps/api/src/scraper/crawler/sitemap.ts
index d4c69d76e..9a458bc48 100644
--- a/apps/api/src/scraper/crawler/sitemap.ts
+++ b/apps/api/src/scraper/crawler/sitemap.ts
@@ -8,6 +8,7 @@ import { processSitemap } from "@mendable/firecrawl-rs";
import { fetchFileToBuffer } from "../scrapeURL/engines/utils/downloadFile";
import { gunzip } from "node:zlib";
import { promisify } from "node:util";
+import { SitemapError } from "../../lib/error";
const useFireEngine =
process.env.FIRE_ENGINE_BETA_URL !== "" &&
@@ -90,11 +91,12 @@ async function getSitemapXML(options: SitemapScrapeOptions): Promise<string> {
) {
return response.document.rawHtml!;
} else if (!response.success) {
- throw new Error("Failed to scrape sitemap", { cause: response.error });
+ throw new SitemapError("Failed to scrape sitemap", response.error);
} else {
- throw new Error("Failed to scrape sitemap", {
- cause: response.document.metadata.statusCode,
- });
+ throw new SitemapError(
+ "Failed to scrape sitemap",
+ response.document.metadata.statusCode,
+ );
}
}
@@ -118,7 +120,20 @@ export async function scrapeSitemap(
logger.info("Processing sitemap");
- const instructions = await processSitemap(xml);
+ let instructions;
+ try {
+ instructions = await processSitemap(xml);
+ } catch (error) {
+ // Wrap XML parsing errors (user's broken sitemap) in SitemapError
+ const errorMessage = error instanceof Error ? error.message : String(error);
+ if (
+ errorMessage.includes("XML parsing error") ||
+ errorMessage.includes("Parse sitemap error")
+ ) {
+ throw new SitemapError(errorMessage, error);
+ }
+ throw error;
+ }
const sitemapData: SitemapData = {
urls: [],
diff --git a/apps/api/src/scraper/scrapeURL/index.ts b/apps/api/src/scraper/scrapeURL/index.ts
index ad05a836f..0ba49a4b2 100644
--- a/apps/api/src/scraper/scrapeURL/index.ts
+++ b/apps/api/src/scraper/scrapeURL/index.ts
@@ -69,7 +69,7 @@ import {
AbortManager,
AbortManagerThrownError,
} from "./lib/abortManager";
-import { ScrapeJobTimeoutError } from "../../lib/error";
+import { ScrapeJobTimeoutError, CrawlDenialError } from "../../lib/error";
import { htmlTransform } from "./lib/removeUnwantedElements";
import { postprocessors } from "./postprocessors";
@@ -923,10 +923,10 @@ export async function scrapeURL(
setSpanAttributes(span, {
"scrape.blocked_by_robots": true,
});
- throw new Error("URL blocked by robots.txt");
+ throw new CrawlDenialError("URL blocked by robots.txt");
}
} catch (error) {
- if (error.message === "URL blocked by robots.txt") {
+ if (error instanceof CrawlDenialError) {
throw error;
}
meta.logger.debug("Failed to fetch robots.txt, allowing scrape", {
diff --git a/apps/api/src/services/extract-worker.ts b/apps/api/src/services/extract-worker.ts
index ba39204eb..d82b05dd6 100644
--- a/apps/api/src/services/extract-worker.ts
+++ b/apps/api/src/services/extract-worker.ts
@@ -20,6 +20,7 @@ import Express from "express";
import { robustFetch } from "../scraper/scrapeURL/lib/fetch";
import { BullMQOtel } from "bullmq-otel";
import { getErrorContactMessage } from "../lib/deployment";
+import { TransportableError } from "../lib/error";
import { initializeBlocklist } from "../scraper/WebScraper/utils/blocklist";
import { initializeEngineForcing } from "../scraper/WebScraper/utils/engine-forcing";
@@ -131,11 +132,14 @@ const processExtractJobInternal = async (
} catch (error) {
logger.error(`🚫 Job errored ${job.id} - ${error}`, { error });
- Sentry.captureException(error, {
- data: {
- job: job.id,
- },
- });
+ // Filter out TransportableErrors (flow control)
+ if (!(error instanceof TransportableError)) {
+ Sentry.captureException(error, {
+ data: {
+ job: job.id,
+ },
+ });
+ }
try {
// Move job to failed state in Redis
diff --git a/apps/api/src/services/queue-worker.ts b/apps/api/src/services/queue-worker.ts
index 78089667c..566043c3e 100644
--- a/apps/api/src/services/queue-worker.ts
+++ b/apps/api/src/services/queue-worker.ts
@@ -25,6 +25,7 @@ import { initializeEngineForcing } from "../scraper/WebScraper/utils/engine-forc
import { crawlFinishedQueue, NuQJob, scrapeQueue } from "./worker/nuq";
import { finishCrawlSuper } from "./worker/crawl-logic";
import { getCrawl } from "../lib/crawl-redis";
+import { TransportableError } from "../lib/error";
configDotenv();
@@ -98,11 +99,14 @@ const processDeepResearchJobInternal = async (
} catch (error) {
logger.error(`🚫 Job errored ${job.id} - ${error}`, { error });
- Sentry.captureException(error, {
- data: {
- job: job.id,
- },
- });
+ // Filter out TransportableErrors (flow control)
+ if (!(error instanceof TransportableError)) {
+ Sentry.captureException(error, {
+ data: {
+ job: job.id,
+ },
+ });
+ }
try {
// Move job to failed state in Redis
@@ -173,11 +177,14 @@ const processGenerateLlmsTxtJobInternal = async (
} catch (error) {
logger.error(`🚫 Job errored ${job.id} - ${error}`, { error });
- Sentry.captureException(error, {
- data: {
- job: job.id,
- },
- });
+ // Filter out TransportableErrors (flow control)
+ if (!(error instanceof TransportableError)) {
+ Sentry.captureException(error, {
+ data: {
+ job: job.id,
+ },
+ });
+ }
try {
await job.moveToFailed(error, token, false);
diff --git a/apps/api/src/services/sentry.ts b/apps/api/src/services/sentry.ts
index 61ff3b9db..d17056efb 100644
--- a/apps/api/src/services/sentry.ts
+++ b/apps/api/src/services/sentry.ts
@@ -10,9 +10,45 @@ if (process.env.SENTRY_DSN) {
dsn: process.env.SENTRY_DSN,
integrations: integrations => [...integrations, nodeProfilingIntegration()],
tracesSampleRate: 0,
+ sampleRate: 0.05,
serverName: process.env.NUQ_POD_NAME,
environment: process.env.SENTRY_ENVIRONMENT ?? "production",
skipOpenTelemetrySetup: true,
+ beforeSend(event, hint) {
+ const error = hint?.originalException;
+
+ if (error && typeof error === "object") {
+ const errorCode = "code" in error ? String(error.code) : "";
+
+ const transportableErrorCodes = [
+ "SCRAPE_ALL_ENGINES_FAILED",
+ "SCRAPE_DNS_RESOLUTION_ERROR",
+ "SCRAPE_SITE_ERROR",
+ "SCRAPE_SSL_ERROR",
+ "SCRAPE_PROXY_SELECTION_ERROR",
+ "SCRAPE_ZDR_VIOLATION_ERROR",
+ "SCRAPE_UNSUPPORTED_FILE_ERROR",
+ "SCRAPE_PDF_ANTIBOT_ERROR",
+ "SCRAPE_ACTION_ERROR",
+ "SCRAPE_PDF_INSUFFICIENT_TIME_ERROR",
+ "SCRAPE_PDF_PREFETCH_FAILED",
+ "SCRAPE_DOCUMENT_ANTIBOT_ERROR",
+ "SCRAPE_DOCUMENT_PREFETCH_FAILED",
+ "SCRAPE_TIMEOUT",
+ "MAP_TIMEOUT",
+ "SCRAPE_UNKNOWN_ERROR",
+ "SCRAPE_RACED_REDIRECT_ERROR",
+ "SCRAPE_SITEMAP_ERROR",
+ "CRAWL_DENIAL",
+ ];
+
+ if (transportableErrorCodes.includes(errorCode)) {
+ return null;
+ }
+ }
+
+ return event;
+ },
});
}
diff --git a/apps/api/src/services/worker/scrape-worker.ts b/apps/api/src/services/worker/scrape-worker.ts
index 35bc26ddb..5c3f01be2 100644
--- a/apps/api/src/services/worker/scrape-worker.ts
+++ b/apps/api/src/services/worker/scrape-worker.ts
@@ -51,6 +51,7 @@ import { calculateCreditsToBeBilled } from "../../lib/scrape-billing";
import { getBillingQueue } from "../queue-service";
import type { Logger } from "winston";
import {
+ CrawlDenialError,
RacedRedirectError,
ScrapeJobTimeoutError,
TransportableError,
@@ -567,8 +568,8 @@ async function processJob(job: NuQJob<ScrapeJobSingleUrls>) {
if (
job.data.crawl_id &&
job.data.crawlerOptions !== null &&
- error instanceof Error &&
- error.message === "URL blocked by robots.txt"
+ error instanceof CrawlDenialError &&
+ error.reason === "URL blocked by robots.txt"
) {
await recordRobotsBlocked(job.data.crawl_id, job.data.url);
}
@@ -606,11 +607,14 @@ async function processJob(job: NuQJob<ScrapeJobSingleUrls>) {
} else {
logger.error(`πŸ‚ Job errored ${job.id} - ${error}`, { error });
- Sentry.captureException(error, {
- data: {
- job: job.id,
- },
- });
+ // Filter out TransportableErrors (flow control)
+ if (!(error instanceof TransportableError)) {
+ Sentry.captureException(error, {
+ data: {
+ job: job.id,
+ },
+ });
+ }
if (error instanceof CustomError) {
// Here we handle the error, then save the failed job
@@ -1197,7 +1201,11 @@ async function processJobWithTracing(job: NuQJob<ScrapeJobData>, logger: any) {
}
} catch (error) {
logger.warn("Job failed", { error });
- Sentry.captureException(error);
+
+ // Filter out TransportableErrors (flow control)
+ if (!(error instanceof TransportableError)) {
+ Sentry.captureException(error);
+ }
if (job.data.skipNuq) {
throw error;