Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
Merge
Gergő Móriczb7d7b8e5d8ce
diff --cc apps/api/src/services/indexing/index-worker.tsindex f65ea6bbf,758169c25..806349478--- a/apps/api/src/services/indexing/index-worker.ts+++ b/apps/api/src/services/indexing/index-worker.ts@@@ -14,9 -14,10 +14,10 @@@ import systemMonitor from "../system-moimport { v4 as uuidv4 } from "uuid";import { index_supabase_service, processIndexInsertJobs, processIndexRFInsertJobs, processOMCEJobs, processDomainFrequencyJobs } from "..";import { processWebhookInsertJobs } from "../webhook";-import { scrapeOptions as scrapeOptionsSchema, crawlRequestSchema, toLegacyCrawlerOptions } from "../../controllers/v1/types";+import { scrapeOptions as scrapeOptionsSchema, crawlRequestSchema, toV0CrawlerOptions } from "../../controllers/v2/types";import { StoredCrawl, crawlToCrawler, saveCrawl } from "../../lib/crawl-redis";import { _addScrapeJobToBullMQ } from "../queue-jobs";+ import { BullMQOtel } from "bullmq-otel";const workerLockDuration = Number(process.env.WORKER_LOCK_DURATION) || 60000;const workerStalledCheckInterval =diff --cc apps/api/src/services/queue-worker.tsindex b8c11e85f,03c8998b1..13cfa3529--- a/apps/api/src/services/queue-worker.ts+++ b/apps/api/src/services/queue-worker.ts@@@ -67,16 -73,22 +71,22 @@@ const runningJobs: Set<string> = new SecacheableLookup.install(http.globalAgent);cacheableLookup.install(https.globalAgent);- const langfuseOtel = process.env.LANGFUSE_PUBLIC_KEY ? new NodeSDK({- traceExporter: new LangfuseExporter(),- instrumentations: [getNodeAutoInstrumentations({- '@opentelemetry/instrumentation-undici': { enabled: false },- '@opentelemetry/instrumentation-http': { enabled: false },- })],+ const shouldOtel = process.env.LANGFUSE_PUBLIC_KEY || process.env.OTEL_EXPORTER_OTLP_ENDPOINT;+ const otelSdk = shouldOtel ? new NodeSDK({+ resource: resourceFromAttributes({+ [ATTR_SERVICE_NAME]: "firecrawl-worker",+ }),+ spanProcessors: [+ ...(process.env.LANGFUSE_PUBLIC_KEY ? [new BatchSpanProcessor(new LangfuseExporter())] : []),+ ...(process.env.OTEL_EXPORTER_OTLP_ENDPOINT ? [new BatchSpanProcessor(new OTLPTraceExporter({+ url: process.env.OTEL_EXPORTER_OTLP_ENDPOINT,+ }))] : []),+ ],+ instrumentations: [getNodeAutoInstrumentations()],}) : null;--if (otelSdk) {+- if (langfuseOtel) {- langfuseOtel.start();++if (otelSdk) {+ otelSdk.start();}const processExtractJobInternal = async (@@@ -353,7 -361,7 +363,7 @@@ const separateWorkerFun =const maxOldSpaceSize = process.env.SCRAPE_WORKER_MAX_OLD_SPACE_SIZE || process.execArgv.find(arg => arg.startsWith('--max-old-space-size='))?.split('=')[1];--++// Filter out the invalid flag for worker threadsconst filteredExecArgv = process.execArgv.filter(arg => !arg.startsWith('--max-old-space-size'));@@@ -467,7 -477,7 +479,7 @@@ app.get("/liveness", (req, res) =>const host = process.env.FIRECRAWL_APP_HOST || "firecrawl-app-service";const port = process.env.FIRECRAWL_APP_PORT || "3002";const scheme = process.env.FIRECRAWL_APP_SCHEME || "http";--++robustFetch({url: `${scheme}://${host}:${port}`,method: "GET",@@@ -590,8 -575,8 +600,8 @@@ app.listen(workerPort, () =>await scrapeQueueEvents.close();console.log("All jobs finished. Worker out!");- if (langfuseOtel) {- await langfuseOtel.shutdown();+ if (otelSdk) {+ await otelSdk.shutdown();}process.exit(0);-})();+})();diff --cc apps/api/src/services/worker/scrape-worker.tsindex a761ba6b2,8cd7541d0..57e3f25ea--- a/apps/api/src/services/worker/scrape-worker.ts+++ b/apps/api/src/services/worker/scrape-worker.ts@@@ -49,8 -48,10 +49,12 @@@ import { finishCrawlIfNeeded } from "./import { LangfuseExporter } from "langfuse-vercel";import { getNodeAutoInstrumentations } from "@opentelemetry/auto-instrumentations-node";import { NodeSDK } from "@opentelemetry/sdk-node";+import { ScrapeJobTimeoutError, TransportableError, UnknownError } from "../../lib/error";+import { serializeTransportableError } from "../../lib/error-serde";+ import { BatchSpanProcessor } from "@opentelemetry/sdk-trace-node";+ import { OTLPTraceExporter } from "@opentelemetry/exporter-trace-otlp-grpc";+ import { resourceFromAttributes } from "@opentelemetry/resources";+ import { ATTR_SERVICE_NAME } from "@opentelemetry/semantic-conventions";class RacedRedirectError extends Error {constructor() {@@@ -796,7 -797,7 +800,7 @@@ export const processJobInternal = asyncrawlId: job.data?.crawl_id ?? undefined,zeroDataRetention: job.data?.zeroDataRetention ?? false,});--++try {try {let extendLockInterval: NodeJS.Timeout | null = null;@@@ -856,13 -853,22 +860,22 @@@}};- const langfuseOtel = process.env.LANGFUSE_PUBLIC_KEY ? new NodeSDK({- traceExporter: new LangfuseExporter(),+ const shouldOtel = process.env.LANGFUSE_PUBLIC_KEY || process.env.OTEL_EXPORTER_OTLP_ENDPOINT;+ const otelSdk = shouldOtel ? new NodeSDK({+ resource: resourceFromAttributes({+ [ATTR_SERVICE_NAME]: "firecrawl-worker-scrape",+ }),+ spanProcessors: [+ ...(process.env.LANGFUSE_PUBLIC_KEY ? [new BatchSpanProcessor(new LangfuseExporter())] : []),+ ...(process.env.OTEL_EXPORTER_OTLP_ENDPOINT ? [new BatchSpanProcessor(new OTLPTraceExporter({+ url: process.env.OTEL_EXPORTER_OTLP_ENDPOINT,+ }))] : []),+ ],instrumentations: [getNodeAutoInstrumentations()],}) : null;--- if (langfuseOtel) {- langfuseOtel.start();-if (otelSdk) {++++if (otelSdk) {+ otelSdk.start();}module.exports = processJobInternal;@@@ -878,4 -884,4 +891,4 @@@ const exitHandler = () =>process.on("SIGINT", exitHandler);process.on("SIGTERM", exitHandler);--process.on("exit", exitHandler);++process.on("exit", exitHandler);