Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(api): ignore queue full errors in Sentry (#3635)
Abimael Martell188eb2e2f690
diff --git a/apps/api/src/index.ts b/apps/api/src/index.tsindex e2ac0b823..8038bef59 100644--- a/apps/api/src/index.ts+++ b/apps/api/src/index.ts@@ -26,7 +26,7 @@ import {ResponseWithSentry,} from "./controllers/v1/types";import { ZodError } from "zod";-import { QueueFullError } from "./lib/concurrency-limit";+import { QueueFullError } from "./lib/queue-full-error";import { v7 as uuidv7 } from "uuid";import { attachWsProxy } from "./services/agentLivecastWS";import { cacheableLookup } from "./scraper/scrapeURL/lib/cacheableLookup";diff --git a/apps/api/src/lib/concurrency-limit.ts b/apps/api/src/lib/concurrency-limit.tsindex 029a67145..34922fadf 100644--- a/apps/api/src/lib/concurrency-limit.ts+++ b/apps/api/src/lib/concurrency-limit.ts@@ -5,16 +5,7 @@ import { getCrawl, StoredCrawl } from "./crawl-redis";import { logger } from "./logger";import { abTestJob } from "../services/ab-test";import { scrapeQueue, type NuQJob } from "../services/worker/nuq";--export class QueueFullError extends Error {- statusCode = 429;- constructor(queueSize: number, queueLimit: number) {- super(- `Queue limit reached: your team has ${queueSize} jobs queued (limit: ${queueLimit}). Please wait for existing jobs to complete before adding more, or upgrade your plan for a higher limit. For more info, see https://docs.firecrawl.dev/rate-limits#concurrent-browser-limits`,- );- this.name = "QueueFullError";- }-}+export { QueueFullError } from "./queue-full-error";// min 50k, max 2M, 2000 per concurrent browserexport function getTeamQueueLimit(concurrencyLimit: number): number {diff --git a/apps/api/src/lib/queue-full-error.ts b/apps/api/src/lib/queue-full-error.tsnew file mode 100644index 000000000..5e9e44245--- /dev/null+++ b/apps/api/src/lib/queue-full-error.ts@@ -0,0 +1,51 @@+const QUEUE_FULL_ERROR_NAME = "QueueFullError";+const QUEUE_FULL_ERROR_MESSAGE_PREFIX = "Queue limit reached:";++export class QueueFullError extends Error {+ statusCode = 429;++ constructor(queueSize: number, queueLimit: number) {+ super(+ `Queue limit reached: your team has ${queueSize} jobs queued (limit: ${queueLimit}). Please wait for existing jobs to complete before adding more, or upgrade your plan for a higher limit. For more info, see https://docs.firecrawl.dev/rate-limits#concurrent-browser-limits`,+ );+ this.name = QUEUE_FULL_ERROR_NAME;+ }+}++function getStringField(+ error: Record<string, unknown>,+ fields: string[],+): string {+ for (const field of fields) {+ if (field in error && error[field] !== undefined && error[field] !== null) {+ return String(error[field]);+ }+ }++ return "";+}++export function isQueueFullError(error: unknown): boolean {+ const objectError =+ error && typeof error === "object"+ ? (error as Record<string, unknown>)+ : null;++ const errorName = objectError+ ? getStringField(objectError, ["name", "type"])+ : "";+ if (errorName === QUEUE_FULL_ERROR_NAME) {+ return true;+ }++ const errorMessage =+ error instanceof Error+ ? error.message+ : typeof error === "string"+ ? error+ : objectError+ ? getStringField(objectError, ["message", "value"])+ : "";++ return errorMessage.startsWith(QUEUE_FULL_ERROR_MESSAGE_PREFIX);+}diff --git a/apps/api/src/services/sentry.test.ts b/apps/api/src/services/sentry.test.tsnew file mode 100644index 000000000..71f1c80bc--- /dev/null+++ b/apps/api/src/services/sentry.test.ts@@ -0,0 +1,122 @@+import { jest } from "@jest/globals";++const captureException = jest.fn();+const init = jest.fn();+const setTag = jest.fn();+const scopeSetExtra = jest.fn();+const scopeSetTag = jest.fn();++jest.mock("@sentry/node", () => ({+ captureException,+ getCurrentScope: () => ({+ setExtra: scopeSetExtra,+ setTag: scopeSetTag,+ }),+ init,+ setTag,+ vercelAIIntegration: jest.fn(() => ({})),+}));++jest.mock("../config", () => ({+ config: {+ NUQ_POD_NAME: "test-pod",+ SENTRY_DSN: undefined,+ SENTRY_ENVIRONMENT: "test",+ SENTRY_ERROR_SAMPLE_RATE: 1,+ SENTRY_TRACE_SAMPLE_RATE: 0,+ },+}));++jest.mock("../lib/logger", () => ({+ logger: {+ info: jest.fn(),+ },+}));++class AddFeatureError extends Error {}+class RemoveFeatureError extends Error {}+class EngineError extends Error {}+class AbortManagerThrownError extends Error {}+class JobCancelledError extends Error {}++jest.mock("../scraper/scrapeURL/error", () => ({+ AddFeatureError,+ RemoveFeatureError,+ EngineError,+}));++jest.mock("../scraper/scrapeURL/lib/abortManager", () => ({+ AbortManagerThrownError,+}));++jest.mock("../lib/error", () => ({+ JobCancelledError,+}));++import { QueueFullError } from "../lib/queue-full-error";+import {+ captureExceptionWithZdrCheck,+ shouldIgnoreSentryException,+} from "./sentry";++describe("sentry filtering", () => {+ beforeEach(() => {+ jest.clearAllMocks();+ });++ it("does not capture queue full errors", () => {+ captureExceptionWithZdrCheck(new QueueFullError(50000, 50000));++ expect(captureException).not.toHaveBeenCalled();+ });++ it("recognizes Sentry exception payloads for queue full errors", () => {+ expect(+ shouldIgnoreSentryException({+ type: "QueueFullError",+ value:+ "Queue limit reached: your team has 50000 jobs queued (limit: 50000).",+ }),+ ).toBe(true);+ });++ it("recognizes serialized transportable errors in Sentry exception values", () => {+ expect(+ shouldIgnoreSentryException({+ type: "Error",+ value: 'SCRAPE_TIMEOUT|{"message":"timeout"}',+ }),+ ).toBe(true);+ });++ it("recognizes cancellation errors in Sentry exception values", () => {+ expect(+ shouldIgnoreSentryException({+ type: "Error",+ value: "Parent crawl/batch scrape was cancelled",+ }),+ ).toBe(true);+ });++ it("does not ignore unexpected Sentry exception values", () => {+ expect(+ shouldIgnoreSentryException({+ type: "Error",+ value: "boom",+ }),+ ).toBe(false);+ });++ it("still captures unexpected errors", () => {+ const error = new Error("boom");++ captureExceptionWithZdrCheck(error, {+ tags: { module: "test" },+ zeroDataRetention: false,+ });++ expect(captureException).toHaveBeenCalledWith(error, {+ tags: { module: "test" },+ });+ });+});diff --git a/apps/api/src/services/sentry.ts b/apps/api/src/services/sentry.tsindex 681f95e17..a84550a47 100644--- a/apps/api/src/services/sentry.ts+++ b/apps/api/src/services/sentry.ts@@ -8,6 +8,7 @@ import {} from "../scraper/scrapeURL/error";import { AbortManagerThrownError } from "../scraper/scrapeURL/lib/abortManager";import { JobCancelledError } from "../lib/error";+import { isQueueFullError } from "../lib/queue-full-error";type CaptureContext = {tags?: Record<string, string>;@@ -25,6 +26,101 @@ type CaptureContext = {[key: string]: any;};+const transportableErrorCodes = [+ "SCRAPE_ALL_ENGINES_FAILED",+ "SCRAPE_DNS_RESOLUTION_ERROR",+ "SCRAPE_SITE_ERROR",+ "SCRAPE_SSL_ERROR",+ "SCRAPE_PROXY_SELECTION_ERROR",+ "SCRAPE_ZDR_VIOLATION_ERROR",+ "SCRAPE_UNSUPPORTED_FILE_ERROR",+ "SCRAPE_PDF_ANTIBOT_ERROR",+ "SCRAPE_ACTION_ERROR",+ "SCRAPE_PDF_INSUFFICIENT_TIME_ERROR",+ "SCRAPE_PDF_PREFETCH_FAILED",+ "SCRAPE_DOCUMENT_ANTIBOT_ERROR",+ "SCRAPE_DOCUMENT_PREFETCH_FAILED",+ "SCRAPE_TIMEOUT",+ "MAP_TIMEOUT",+ "SCRAPE_UNKNOWN_ERROR",+ "SCRAPE_RACED_REDIRECT_ERROR",+ "SCRAPE_SITEMAP_ERROR",+ "CRAWL_DENIAL",+];++function getStringField(+ error: Record<string, unknown>,+ fields: string[],+): string {+ for (const field of fields) {+ if (error[field] !== undefined && error[field] !== null) {+ return String(error[field]);+ }+ }++ return "";+}++function getErrorMessage(error: unknown): string {+ if (error instanceof Error) {+ return String(error.message || "");+ }++ if (typeof error === "string") {+ return error;+ }++ if (error && typeof error === "object") {+ return getStringField(error as Record<string, unknown>, [+ "message",+ "value",+ ]);+ }++ return "";+}++export function shouldIgnoreSentryException(error: unknown): boolean {+ if (isQueueFullError(error)) {+ return true;+ }++ if (!error || typeof error !== "object") {+ return false;+ }++ if (+ error instanceof AddFeatureError ||+ error instanceof RemoveFeatureError ||+ error instanceof AbortManagerThrownError ||+ error instanceof EngineError ||+ error instanceof JobCancelledError+ ) {+ return true;+ }++ // We sometimes rethrow TransportableErrors across process boundaries as a+ // plain Error with message like `${code}|${json}` (see error-serde.ts).+ // In that case, `error.code` is missing, so extract it from the message.+ const errorCodeFromField = "code" in error ? String(error.code) : "";+ const errorMessage = getErrorMessage(error);++ // Ignore cancellation errors (fallback for serialized errors)+ if (+ errorMessage === "Parent crawl/batch scrape was cancelled" ||+ errorMessage.includes("Parent crawl/batch scrape was cancelled")+ ) {+ return true;+ }++ const errorCodeFromMessage =+ errorCodeFromField ||+ (errorMessage.includes("|") ? errorMessage.split("|", 1)[0] : "");+ const errorCode = errorCodeFromMessage;++ return transportableErrorCodes.includes(errorCode);+}+if (config.SENTRY_DSN) {logger.info("Setting up Sentry...");@@ -58,63 +154,13 @@ if (config.SENTRY_DSN) {}const error = hint?.originalException;+ const sentryException = event.exception?.values?.[0];- if (error && typeof error === "object") {- if (- error instanceof AddFeatureError ||- error instanceof RemoveFeatureError ||- error instanceof AbortManagerThrownError ||- error instanceof EngineError ||- error instanceof JobCancelledError- ) {- return null;- }-- // We sometimes rethrow TransportableErrors across process boundaries as a- // plain Error with message like `${code}|${json}` (see error-serde.ts).- // In that case, `error.code` is missing, so extract it from the message.- const errorCodeFromField = "code" in error ? String(error.code) : "";- const errorMessage =- error instanceof Error ? String(error.message || "") : "";-- // Ignore cancellation errors (fallback for serialized errors)- if (- errorMessage === "Parent crawl/batch scrape was cancelled" ||- errorMessage.includes("Parent crawl/batch scrape was cancelled")- ) {- return null;- }-- const errorCodeFromMessage =- errorCodeFromField ||- (errorMessage.includes("|") ? errorMessage.split("|", 1)[0] : "");- const errorCode = errorCodeFromMessage;-- const transportableErrorCodes = [- "SCRAPE_ALL_ENGINES_FAILED",- "SCRAPE_DNS_RESOLUTION_ERROR",- "SCRAPE_SITE_ERROR",- "SCRAPE_SSL_ERROR",- "SCRAPE_PROXY_SELECTION_ERROR",- "SCRAPE_ZDR_VIOLATION_ERROR",- "SCRAPE_UNSUPPORTED_FILE_ERROR",- "SCRAPE_PDF_ANTIBOT_ERROR",- "SCRAPE_ACTION_ERROR",- "SCRAPE_PDF_INSUFFICIENT_TIME_ERROR",- "SCRAPE_PDF_PREFETCH_FAILED",- "SCRAPE_DOCUMENT_ANTIBOT_ERROR",- "SCRAPE_DOCUMENT_PREFETCH_FAILED",- "SCRAPE_TIMEOUT",- "MAP_TIMEOUT",- "SCRAPE_UNKNOWN_ERROR",- "SCRAPE_RACED_REDIRECT_ERROR",- "SCRAPE_SITEMAP_ERROR",- "CRAWL_DENIAL",- ];-- if (transportableErrorCodes.includes(errorCode)) {- return null;- }+ if (+ shouldIgnoreSentryException(error) ||+ shouldIgnoreSentryException(sentryException)+ ) {+ return null;}return event;@@ -135,6 +181,10 @@ export function captureExceptionWithZdrCheck(return;}+ if (shouldIgnoreSentryException(error)) {+ return;+ }+const { zeroDataRetention: _zdr, ...sentryContext } = context || {};return Sentry.captureException(error, sentryContext);