Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
Update error messages for self-hosted instances (#2119)
devin-ai-integration[bot]96b2e8ef7752
diff --git a/apps/api/src/controllers/v0/crawl.ts b/apps/api/src/controllers/v0/crawl.tsindex 287cd5a3e..d2442d811 100644--- a/apps/api/src/controllers/v0/crawl.ts+++ b/apps/api/src/controllers/v0/crawl.ts@@ -31,6 +31,7 @@ import { url as urlSchema } from "../v1/types";import { ZodError } from "zod";import { BLOCKLISTED_URL_MESSAGE } from "../../lib/strings";import { fromV0ScrapeOptions } from "../v2/types";+import { isSelfHosted } from "../../lib/deployment";export async function crawlController(req: Request, res: Response) {try {@@ -114,8 +115,9 @@ export async function crawlController(req: Request, res: Response) {if (!creditsCheckSuccess) {return res.status(402).json({- error:- "Insufficient credits. You may be requesting with a higher limit than the amount of credits you have left. If not, upgrade your plan at https://firecrawl.dev/pricing or contact us at [email protected]",+ error: isSelfHosted()+ ? "Insufficient credits. You may be requesting with a higher limit than the amount of credits you have left. Please check your server configuration."+ : "Insufficient credits. You may be requesting with a higher limit than the amount of credits you have left. If not, upgrade your plan at https://firecrawl.dev/pricing or contact us at [email protected]",});}diff --git a/apps/api/src/controllers/v0/scrape.ts b/apps/api/src/controllers/v0/scrape.tsindex 3d48d832b..94dd1ba30 100644--- a/apps/api/src/controllers/v0/scrape.ts+++ b/apps/api/src/controllers/v0/scrape.ts@@ -23,6 +23,7 @@ import { BLOCKLISTED_URL_MESSAGE } from "../../lib/strings";import { fromV0Combo } from "../v2/types";import { ScrapeJobTimeoutError } from "../../lib/error";import { scrapeQueue } from "../../services/worker/nuq";+import { getErrorContactMessage } from "../../lib/deployment";async function scrapeHelper(jobId: string,@@ -230,8 +231,7 @@ export async function scrapeController(req: Request, res: Response) {logger.error(error);earlyReturn = true;return res.status(500).json({- error:- "Error checking team credits. Please contact [email protected] for help.",+ error: getErrorContactMessage(),});}diff --git a/apps/api/src/controllers/v1/crawl-status-ws.ts b/apps/api/src/controllers/v1/crawl-status-ws.tsindex f1ec30606..6c4c6155b 100644--- a/apps/api/src/controllers/v1/crawl-status-ws.ts+++ b/apps/api/src/controllers/v1/crawl-status-ws.ts@@ -20,6 +20,7 @@ import { getJobs, PseudoJob } from "./crawl-status";import * as Sentry from "@sentry/node";import { getConcurrencyLimitedJobs } from "../../lib/concurrency-limit";import { scrapeQueue, NuQJobStatus } from "../../services/worker/nuq";+import { getErrorContactMessage } from "../../lib/deployment";type ErrorMessage = {type: "error";@@ -211,9 +212,7 @@ export async function crawlStatusWSController();return close(ws, 1011, {type: "error",- error:- "An unexpected error occurred. Please contact [email protected] for help. Your exception ID is " +- id,+ error: getErrorContactMessage(id),});}}diff --git a/apps/api/src/controllers/v2/crawl-status-ws.ts b/apps/api/src/controllers/v2/crawl-status-ws.tsindex f1ec30606..6c4c6155b 100644--- a/apps/api/src/controllers/v2/crawl-status-ws.ts+++ b/apps/api/src/controllers/v2/crawl-status-ws.ts@@ -20,6 +20,7 @@ import { getJobs, PseudoJob } from "./crawl-status";import * as Sentry from "@sentry/node";import { getConcurrencyLimitedJobs } from "../../lib/concurrency-limit";import { scrapeQueue, NuQJobStatus } from "../../services/worker/nuq";+import { getErrorContactMessage } from "../../lib/deployment";type ErrorMessage = {type: "error";@@ -211,9 +212,7 @@ export async function crawlStatusWSController();return close(ws, 1011, {type: "error",- error:- "An unexpected error occurred. Please contact [email protected] for help. Your exception ID is " +- id,+ error: getErrorContactMessage(id),});}}diff --git a/apps/api/src/index.ts b/apps/api/src/index.tsindex c8ca7fd54..1c95201af 100644--- a/apps/api/src/index.ts+++ b/apps/api/src/index.ts@@ -38,6 +38,7 @@ import { resourceFromAttributes } from "@opentelemetry/resources";import { BatchSpanProcessor } from "@opentelemetry/sdk-trace-node";import { OTLPTraceExporter } from "@opentelemetry/exporter-trace-otlp-grpc";import { nuqShutdown } from "./services/worker/nuq";+import { getErrorContactMessage } from "./lib/deployment";const { createBullBoard } = require("@bull-board/api");const { BullMQAdapter } = require("@bull-board/api/bullMQAdapter");@@ -245,9 +246,7 @@ app.use(res.status(500).json({success: false,code: "UNKNOWN_ERROR",- error:- "An unexpected error occurred. Please contact [email protected] for help. Your exception ID is " +- id,+ error: getErrorContactMessage(id),});},);diff --git a/apps/api/src/lib/deployment.ts b/apps/api/src/lib/deployment.tsnew file mode 100644index 000000000..e83e4063e--- /dev/null+++ b/apps/api/src/lib/deployment.ts@@ -0,0 +1,15 @@+export function isSelfHosted(): boolean {+ return process.env.USE_DB_AUTHENTICATION !== "true";+}++export function getErrorContactMessage(errorId?: string): string {+ if (isSelfHosted()) {+ return errorId+ ? `An error occurred. Please check your logs for more details. Error ID: ${errorId}`+ : "An error occurred. Please check your logs for more details.";+ } else {+ return errorId+ ? `An unexpected error occurred. Please contact [email protected] for help. Your exception ID is ${errorId}`+ : "An unexpected error occurred. Please contact [email protected] for help.";+ }+}diff --git a/apps/api/src/lib/extract/extraction-service.ts b/apps/api/src/lib/extract/extraction-service.tsindex fca7153d6..487605293 100644--- a/apps/api/src/lib/extract/extraction-service.ts+++ b/apps/api/src/lib/extract/extraction-service.ts@@ -20,6 +20,7 @@ import { transformArrayToObject } from "./helpers/transform-array-to-obj";import { mixSchemaObjects } from "./helpers/mix-schema-objs";import Ajv from "ajv";const ajv = new Ajv();+import { getErrorContactMessage } from "../deployment";import { ExtractStep, updateExtract } from "./extract-redis";import { deduplicateObjectsArray } from "./helpers/deduplicate-objs-array";@@ -717,8 +718,7 @@ export async function performExtraction(});return {success: false,- error:- "An unexpected error occurred. Please contact [email protected] for help.",+ error: getErrorContactMessage(),extractId,urlTrace: urlTraces,totalUrlsScraped,diff --git a/apps/api/src/lib/extract/fire-0/extraction-service-f0.ts b/apps/api/src/lib/extract/fire-0/extraction-service-f0.tsindex c6389546d..c6a2d92be 100644--- a/apps/api/src/lib/extract/fire-0/extraction-service-f0.ts+++ b/apps/api/src/lib/extract/fire-0/extraction-service-f0.ts@@ -11,6 +11,7 @@ import { logJob } from "../../../services/logging/log_job";import { spreadSchemas_F0 } from "./helpers/spread-schemas-f0";import Ajv from "ajv";const ajv = new Ajv();+import { getErrorContactMessage } from "../../deployment";import { ExtractStep, updateExtract } from "../extract-redis";import { CUSTOM_U_TEAMS } from "../config";@@ -628,8 +629,7 @@ export async function performExtraction_F0(});return {success: false,- error:- "An unexpected error occurred. Please contact [email protected] for help.",+ error: getErrorContactMessage(),extractId,urlTrace: urlTraces,totalUrlsScraped,diff --git a/apps/api/src/lib/strings.ts b/apps/api/src/lib/strings.tsindex 4e278d2b3..bf6156d28 100644--- a/apps/api/src/lib/strings.ts+++ b/apps/api/src/lib/strings.ts@@ -1,2 +1,5 @@-export const BLOCKLISTED_URL_MESSAGE =- "This website is no longer supported, please reach out to [email protected] for more info on how to activate it on your account.";+import { isSelfHosted } from "./deployment";++export const BLOCKLISTED_URL_MESSAGE = isSelfHosted()+ ? "This website is no longer supported. Please check your server configuration and logs for more details."+ : "This website is no longer supported, please reach out to [email protected] for more info on how to activate it on your account.";diff --git a/apps/api/src/routes/shared.ts b/apps/api/src/routes/shared.tsindex caf0e6137..080df803c 100644--- a/apps/api/src/routes/shared.ts+++ b/apps/api/src/routes/shared.ts@@ -16,6 +16,7 @@ import { logger } from "../lib/logger";import { BLOCKLISTED_URL_MESSAGE } from "../lib/strings";import { addDomainFrequencyJob } from "../services";import * as geoip from "geoip-country";+import { isSelfHosted } from "../lib/deployment";export function checkCreditsMiddleware(_minimum?: number,@@ -229,8 +230,9 @@ export function countryCheck(});return res.status(403).json({success: false,- error:- "Use of headers, actions, and the FIRE-1 agent is not allowed by default in your country. Please contact us at [email protected]",+ error: isSelfHosted()+ ? "Use of headers, actions, and the FIRE-1 agent is not allowed by default in your country. Please check your server configuration."+ : "Use of headers, actions, and the FIRE-1 agent is not allowed by default in your country. Please contact us at [email protected]",});}diff --git a/apps/api/src/scraper/scrapeURL/error.ts b/apps/api/src/scraper/scrapeURL/error.tsindex faae68f1d..40e3da160 100644--- a/apps/api/src/scraper/scrapeURL/error.ts+++ b/apps/api/src/scraper/scrapeURL/error.ts@@ -1,6 +1,7 @@import { ErrorCodes, TransportableError } from "../../lib/error";import { Meta } from ".";import { Engine, FeatureFlag } from "./engines";+import { isSelfHosted } from "../../lib/deployment";export class EngineError extends Error {constructor(message?: string, options?: ErrorOptions) {@@ -12,10 +13,11 @@ export class NoEnginesLeftError extends TransportableError {public fallbackList: Engine[];constructor(fallbackList: Engine[]) {- super(- "SCRAPE_ALL_ENGINES_FAILED",- "All scraping engines failed! -- Double check the URL to make sure it's not broken. If the issue persists, contact us at [email protected].",- );+ const message = isSelfHosted()+ ? "All scraping engines failed! -- Double check the URL to make sure it's not broken. Check your server logs for more details."+ : "All scraping engines failed! -- Double check the URL to make sure it's not broken. If the issue persists, contact us at [email protected].";++ super("SCRAPE_ALL_ENGINES_FAILED", message);this.fallbackList = fallbackList;}@@ -293,10 +295,11 @@ export class ZDRViolationError extends TransportableError {export class PDFPrefetchFailed extends TransportableError {constructor() {- super(- "SCRAPE_PDF_PREFETCH_FAILED",- "Failed to prefetch PDF that is protected by anti-bot. Please contact [email protected]",- );+ const message = isSelfHosted()+ ? "Failed to prefetch PDF that is protected by anti-bot. Please check your logs for more details."+ : "Failed to prefetch PDF that is protected by anti-bot. Please contact [email protected]";++ super("SCRAPE_PDF_PREFETCH_FAILED", message);}serialize() {diff --git a/apps/api/src/services/queue-worker.ts b/apps/api/src/services/queue-worker.tsindex 83494420f..bf517105d 100644--- a/apps/api/src/services/queue-worker.ts+++ b/apps/api/src/services/queue-worker.ts@@ -37,6 +37,7 @@ import { resourceFromAttributes } from "@opentelemetry/resources";import { OTLPTraceExporter } from "@opentelemetry/exporter-trace-otlp-grpc";import { BullMQOtel } from "bullmq-otel";import { pathToFileURL } from "url";+import { getErrorContactMessage } from "../lib/deployment";configDotenv();@@ -164,20 +165,13 @@ const processExtractJobInternal = async (await job.moveToCompleted(result, token, false);await updateExtract(job.data.extractId, {- status: "failed",- error:- result?.error ??- "Unknown error, please contact [email protected]. Extract id: " +- job.data.extractId,+ error: result?.error ?? getErrorContactMessage(job.data.extractId),});if (sender) {sender.send(WebhookEvent.EXTRACT_FAILED, {success: false,- error:- result?.error ??- "Unknown error, please contact [email protected]. Extract id: " +- job.data.extractId,+ error: result?.error ?? getErrorContactMessage(job.data.extractId),});}@@ -201,30 +195,20 @@ const processExtractJobInternal = async (await updateExtract(job.data.extractId, {status: "failed",- error:- error.error ??- error ??- "Unknown error, please contact [email protected]. Extract id: " +- job.data.extractId,+ error: error.error ?? error ?? getErrorContactMessage(job.data.extractId),});if (sender) {sender.send(WebhookEvent.EXTRACT_FAILED, {success: false,error:- (error as any)?.message ??- "Unknown error, please contact [email protected]. Extract id: " +- job.data.extractId,+ (error as any)?.message ?? getErrorContactMessage(job.data.extractId),});}return {success: false,- error:- error.error ??- error ??- "Unknown error, please contact [email protected]. Extract id: " +- job.data.extractId,+ error: error.error ?? error ?? getErrorContactMessage(job.data.extractId),};// throw error;} finally {