Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. π₯
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
Filter Sentry Exceptions (#2453)
Abimael Martell8cf9f582a92c
diff --git a/apps/api/src/lib/error-serde.ts b/apps/api/src/lib/error-serde.tsindex 004909c99..680c2137b 100644--- a/apps/api/src/lib/error-serde.ts+++ b/apps/api/src/lib/error-serde.ts@@ -1,8 +1,10 @@import {+ CrawlDenialError,ErrorCodes,MapTimeoutError,RacedRedirectError,ScrapeJobTimeoutError,+ SitemapError,TransportableError,UnknownError,} from "./error";@@ -41,6 +43,8 @@ const errorMap: Record<ErrorCodes, any> = {SCRAPE_UNSUPPORTED_FILE_ERROR: UnsupportedFileError,SCRAPE_ACTION_ERROR: ActionError,SCRAPE_RACED_REDIRECT_ERROR: RacedRedirectError,+ SCRAPE_SITEMAP_ERROR: SitemapError,+ CRAWL_DENIAL: CrawlDenialError,// Zod errorsBAD_REQUEST: null,diff --git a/apps/api/src/lib/error.ts b/apps/api/src/lib/error.tsindex 05378a775..9bf3c90cd 100644--- a/apps/api/src/lib/error.ts+++ b/apps/api/src/lib/error.ts@@ -16,6 +16,8 @@ export type ErrorCodes =| "SCRAPE_UNSUPPORTED_FILE_ERROR"| "SCRAPE_ACTION_ERROR"| "SCRAPE_RACED_REDIRECT_ERROR"+ | "SCRAPE_SITEMAP_ERROR"+ | "CRAWL_DENIAL"| "BAD_REQUEST_INVALID_JSON"| "BAD_REQUEST";@@ -128,3 +130,44 @@ export class RacedRedirectError extends TransportableError {return x;}}++export class SitemapError extends TransportableError {+ constructor(message: string, cause?: unknown) {+ super("SCRAPE_SITEMAP_ERROR", message, { cause });+ }++ serialize() {+ return super.serialize();+ }++ static deserialize(+ _: ErrorCodes,+ data: ReturnType<typeof this.prototype.serialize>,+ ) {+ const x = new SitemapError(data.message, data.cause);+ x.stack = data.stack;+ return x;+ }+}++export class CrawlDenialError extends TransportableError {+ constructor(public reason: string) {+ super("CRAWL_DENIAL", reason);+ }++ serialize() {+ return {+ ...super.serialize(),+ reason: this.reason,+ };+ }++ static deserialize(+ _: ErrorCodes,+ data: ReturnType<typeof this.prototype.serialize> & { reason: string },+ ) {+ const x = new CrawlDenialError(data.reason);+ x.stack = data.stack;+ return x;+ }+}diff --git a/apps/api/src/scraper/crawler/sitemap.ts b/apps/api/src/scraper/crawler/sitemap.tsindex d4c69d76e..9a458bc48 100644--- a/apps/api/src/scraper/crawler/sitemap.ts+++ b/apps/api/src/scraper/crawler/sitemap.ts@@ -8,6 +8,7 @@ import { processSitemap } from "@mendable/firecrawl-rs";import { fetchFileToBuffer } from "../scrapeURL/engines/utils/downloadFile";import { gunzip } from "node:zlib";import { promisify } from "node:util";+import { SitemapError } from "../../lib/error";const useFireEngine =process.env.FIRE_ENGINE_BETA_URL !== "" &&@@ -90,11 +91,12 @@ async function getSitemapXML(options: SitemapScrapeOptions): Promise<string> {) {return response.document.rawHtml!;} else if (!response.success) {- throw new Error("Failed to scrape sitemap", { cause: response.error });+ throw new SitemapError("Failed to scrape sitemap", response.error);} else {- throw new Error("Failed to scrape sitemap", {- cause: response.document.metadata.statusCode,- });+ throw new SitemapError(+ "Failed to scrape sitemap",+ response.document.metadata.statusCode,+ );}}@@ -118,7 +120,20 @@ export async function scrapeSitemap(logger.info("Processing sitemap");- const instructions = await processSitemap(xml);+ let instructions;+ try {+ instructions = await processSitemap(xml);+ } catch (error) {+ // Wrap XML parsing errors (user's broken sitemap) in SitemapError+ const errorMessage = error instanceof Error ? error.message : String(error);+ if (+ errorMessage.includes("XML parsing error") ||+ errorMessage.includes("Parse sitemap error")+ ) {+ throw new SitemapError(errorMessage, error);+ }+ throw error;+ }const sitemapData: SitemapData = {urls: [],diff --git a/apps/api/src/scraper/scrapeURL/index.ts b/apps/api/src/scraper/scrapeURL/index.tsindex ad05a836f..0ba49a4b2 100644--- a/apps/api/src/scraper/scrapeURL/index.ts+++ b/apps/api/src/scraper/scrapeURL/index.ts@@ -69,7 +69,7 @@ import {AbortManager,AbortManagerThrownError,} from "./lib/abortManager";-import { ScrapeJobTimeoutError } from "../../lib/error";+import { ScrapeJobTimeoutError, CrawlDenialError } from "../../lib/error";import { htmlTransform } from "./lib/removeUnwantedElements";import { postprocessors } from "./postprocessors";@@ -923,10 +923,10 @@ export async function scrapeURL(setSpanAttributes(span, {"scrape.blocked_by_robots": true,});- throw new Error("URL blocked by robots.txt");+ throw new CrawlDenialError("URL blocked by robots.txt");}} catch (error) {- if (error.message === "URL blocked by robots.txt") {+ if (error instanceof CrawlDenialError) {throw error;}meta.logger.debug("Failed to fetch robots.txt, allowing scrape", {diff --git a/apps/api/src/services/extract-worker.ts b/apps/api/src/services/extract-worker.tsindex ba39204eb..d82b05dd6 100644--- a/apps/api/src/services/extract-worker.ts+++ b/apps/api/src/services/extract-worker.ts@@ -20,6 +20,7 @@ import Express from "express";import { robustFetch } from "../scraper/scrapeURL/lib/fetch";import { BullMQOtel } from "bullmq-otel";import { getErrorContactMessage } from "../lib/deployment";+import { TransportableError } from "../lib/error";import { initializeBlocklist } from "../scraper/WebScraper/utils/blocklist";import { initializeEngineForcing } from "../scraper/WebScraper/utils/engine-forcing";@@ -131,11 +132,14 @@ const processExtractJobInternal = async (} catch (error) {logger.error(`π« Job errored ${job.id} - ${error}`, { error });- Sentry.captureException(error, {- data: {- job: job.id,- },- });+ // Filter out TransportableErrors (flow control)+ if (!(error instanceof TransportableError)) {+ Sentry.captureException(error, {+ data: {+ job: job.id,+ },+ });+ }try {// Move job to failed state in Redisdiff --git a/apps/api/src/services/queue-worker.ts b/apps/api/src/services/queue-worker.tsindex 78089667c..566043c3e 100644--- a/apps/api/src/services/queue-worker.ts+++ b/apps/api/src/services/queue-worker.ts@@ -25,6 +25,7 @@ import { initializeEngineForcing } from "../scraper/WebScraper/utils/engine-forcimport { crawlFinishedQueue, NuQJob, scrapeQueue } from "./worker/nuq";import { finishCrawlSuper } from "./worker/crawl-logic";import { getCrawl } from "../lib/crawl-redis";+import { TransportableError } from "../lib/error";configDotenv();@@ -98,11 +99,14 @@ const processDeepResearchJobInternal = async (} catch (error) {logger.error(`π« Job errored ${job.id} - ${error}`, { error });- Sentry.captureException(error, {- data: {- job: job.id,- },- });+ // Filter out TransportableErrors (flow control)+ if (!(error instanceof TransportableError)) {+ Sentry.captureException(error, {+ data: {+ job: job.id,+ },+ });+ }try {// Move job to failed state in Redis@@ -173,11 +177,14 @@ const processGenerateLlmsTxtJobInternal = async (} catch (error) {logger.error(`π« Job errored ${job.id} - ${error}`, { error });- Sentry.captureException(error, {- data: {- job: job.id,- },- });+ // Filter out TransportableErrors (flow control)+ if (!(error instanceof TransportableError)) {+ Sentry.captureException(error, {+ data: {+ job: job.id,+ },+ });+ }try {await job.moveToFailed(error, token, false);diff --git a/apps/api/src/services/sentry.ts b/apps/api/src/services/sentry.tsindex 61ff3b9db..d17056efb 100644--- a/apps/api/src/services/sentry.ts+++ b/apps/api/src/services/sentry.ts@@ -10,9 +10,45 @@ if (process.env.SENTRY_DSN) {dsn: process.env.SENTRY_DSN,integrations: integrations => [...integrations, nodeProfilingIntegration()],tracesSampleRate: 0,+ sampleRate: 0.05,serverName: process.env.NUQ_POD_NAME,environment: process.env.SENTRY_ENVIRONMENT ?? "production",skipOpenTelemetrySetup: true,+ beforeSend(event, hint) {+ const error = hint?.originalException;++ if (error && typeof error === "object") {+ const errorCode = "code" in error ? String(error.code) : "";++ const transportableErrorCodes = [+ "SCRAPE_ALL_ENGINES_FAILED",+ "SCRAPE_DNS_RESOLUTION_ERROR",+ "SCRAPE_SITE_ERROR",+ "SCRAPE_SSL_ERROR",+ "SCRAPE_PROXY_SELECTION_ERROR",+ "SCRAPE_ZDR_VIOLATION_ERROR",+ "SCRAPE_UNSUPPORTED_FILE_ERROR",+ "SCRAPE_PDF_ANTIBOT_ERROR",+ "SCRAPE_ACTION_ERROR",+ "SCRAPE_PDF_INSUFFICIENT_TIME_ERROR",+ "SCRAPE_PDF_PREFETCH_FAILED",+ "SCRAPE_DOCUMENT_ANTIBOT_ERROR",+ "SCRAPE_DOCUMENT_PREFETCH_FAILED",+ "SCRAPE_TIMEOUT",+ "MAP_TIMEOUT",+ "SCRAPE_UNKNOWN_ERROR",+ "SCRAPE_RACED_REDIRECT_ERROR",+ "SCRAPE_SITEMAP_ERROR",+ "CRAWL_DENIAL",+ ];++ if (transportableErrorCodes.includes(errorCode)) {+ return null;+ }+ }++ return event;+ },});}diff --git a/apps/api/src/services/worker/scrape-worker.ts b/apps/api/src/services/worker/scrape-worker.tsindex 35bc26ddb..5c3f01be2 100644--- a/apps/api/src/services/worker/scrape-worker.ts+++ b/apps/api/src/services/worker/scrape-worker.ts@@ -51,6 +51,7 @@ import { calculateCreditsToBeBilled } from "../../lib/scrape-billing";import { getBillingQueue } from "../queue-service";import type { Logger } from "winston";import {+ CrawlDenialError,RacedRedirectError,ScrapeJobTimeoutError,TransportableError,@@ -567,8 +568,8 @@ async function processJob(job: NuQJob<ScrapeJobSingleUrls>) {if (job.data.crawl_id &&job.data.crawlerOptions !== null &&- error instanceof Error &&- error.message === "URL blocked by robots.txt"+ error instanceof CrawlDenialError &&+ error.reason === "URL blocked by robots.txt") {await recordRobotsBlocked(job.data.crawl_id, job.data.url);}@@ -606,11 +607,14 @@ async function processJob(job: NuQJob<ScrapeJobSingleUrls>) {} else {logger.error(`π Job errored ${job.id} - ${error}`, { error });- Sentry.captureException(error, {- data: {- job: job.id,- },- });+ // Filter out TransportableErrors (flow control)+ if (!(error instanceof TransportableError)) {+ Sentry.captureException(error, {+ data: {+ job: job.id,+ },+ });+ }if (error instanceof CustomError) {// Here we handle the error, then save the failed job@@ -1197,7 +1201,11 @@ async function processJobWithTracing(job: NuQJob<ScrapeJobData>, logger: any) {}} catch (error) {logger.warn("Job failed", { error });- Sentry.captureException(error);++ // Filter out TransportableErrors (flow control)+ if (!(error instanceof TransportableError)) {+ Sentry.captureException(error);+ }if (job.data.skipNuq) {throw error;