Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

Merge

Gergő Móriczb7d7b8e5d8ce
diff --cc apps/api/src/services/indexing/index-worker.ts
index f65ea6bbf,758169c25..806349478
--- a/apps/api/src/services/indexing/index-worker.ts
+++ b/apps/api/src/services/indexing/index-worker.ts
@@@ -14,9 -14,10 +14,10 @@@ import systemMonitor from "../system-mo
import { v4 as uuidv4 } from "uuid";
import { index_supabase_service, processIndexInsertJobs, processIndexRFInsertJobs, processOMCEJobs, processDomainFrequencyJobs } from "..";
import { processWebhookInsertJobs } from "../webhook";
-import { scrapeOptions as scrapeOptionsSchema, crawlRequestSchema, toLegacyCrawlerOptions } from "../../controllers/v1/types";
+import { scrapeOptions as scrapeOptionsSchema, crawlRequestSchema, toV0CrawlerOptions } from "../../controllers/v2/types";
import { StoredCrawl, crawlToCrawler, saveCrawl } from "../../lib/crawl-redis";
import { _addScrapeJobToBullMQ } from "../queue-jobs";
+ import { BullMQOtel } from "bullmq-otel";
const workerLockDuration = Number(process.env.WORKER_LOCK_DURATION) || 60000;
const workerStalledCheckInterval =
diff --cc apps/api/src/services/queue-worker.ts
index b8c11e85f,03c8998b1..13cfa3529
--- a/apps/api/src/services/queue-worker.ts
+++ b/apps/api/src/services/queue-worker.ts
@@@ -67,16 -73,22 +71,22 @@@ const runningJobs: Set<string> = new Se
cacheableLookup.install(http.globalAgent);
cacheableLookup.install(https.globalAgent);
- const langfuseOtel = process.env.LANGFUSE_PUBLIC_KEY ? new NodeSDK({
- traceExporter: new LangfuseExporter(),
- instrumentations: [getNodeAutoInstrumentations({
- '@opentelemetry/instrumentation-undici': { enabled: false },
- '@opentelemetry/instrumentation-http': { enabled: false },
- })],
+ const shouldOtel = process.env.LANGFUSE_PUBLIC_KEY || process.env.OTEL_EXPORTER_OTLP_ENDPOINT;
+ const otelSdk = shouldOtel ? new NodeSDK({
+ resource: resourceFromAttributes({
+ [ATTR_SERVICE_NAME]: "firecrawl-worker",
+ }),
+ spanProcessors: [
+ ...(process.env.LANGFUSE_PUBLIC_KEY ? [new BatchSpanProcessor(new LangfuseExporter())] : []),
+ ...(process.env.OTEL_EXPORTER_OTLP_ENDPOINT ? [new BatchSpanProcessor(new OTLPTraceExporter({
+ url: process.env.OTEL_EXPORTER_OTLP_ENDPOINT,
+ }))] : []),
+ ],
+ instrumentations: [getNodeAutoInstrumentations()],
}) : null;
-
-if (otelSdk) {
+
- if (langfuseOtel) {
- langfuseOtel.start();
++if (otelSdk) {
+ otelSdk.start();
}
const processExtractJobInternal = async (
@@@ -353,7 -361,7 +363,7 @@@ const separateWorkerFun =
const maxOldSpaceSize = process.env.SCRAPE_WORKER_MAX_OLD_SPACE_SIZE || process.execArgv
.find(arg => arg.startsWith('--max-old-space-size='))
?.split('=')[1];
--
++
// Filter out the invalid flag for worker threads
const filteredExecArgv = process.execArgv
.filter(arg => !arg.startsWith('--max-old-space-size'));
@@@ -467,7 -477,7 +479,7 @@@ app.get("/liveness", (req, res) =>
const host = process.env.FIRECRAWL_APP_HOST || "firecrawl-app-service";
const port = process.env.FIRECRAWL_APP_PORT || "3002";
const scheme = process.env.FIRECRAWL_APP_SCHEME || "http";
--
++
robustFetch({
url: `${scheme}://${host}:${port}`,
method: "GET",
@@@ -590,8 -575,8 +600,8 @@@ app.listen(workerPort, () =>
await scrapeQueueEvents.close();
console.log("All jobs finished. Worker out!");
- if (langfuseOtel) {
- await langfuseOtel.shutdown();
+ if (otelSdk) {
+ await otelSdk.shutdown();
}
process.exit(0);
-})();
+})();
diff --cc apps/api/src/services/worker/scrape-worker.ts
index a761ba6b2,8cd7541d0..57e3f25ea
--- a/apps/api/src/services/worker/scrape-worker.ts
+++ b/apps/api/src/services/worker/scrape-worker.ts
@@@ -49,8 -48,10 +49,12 @@@ import { finishCrawlIfNeeded } from "./
import { LangfuseExporter } from "langfuse-vercel";
import { getNodeAutoInstrumentations } from "@opentelemetry/auto-instrumentations-node";
import { NodeSDK } from "@opentelemetry/sdk-node";
+import { ScrapeJobTimeoutError, TransportableError, UnknownError } from "../../lib/error";
+import { serializeTransportableError } from "../../lib/error-serde";
+ import { BatchSpanProcessor } from "@opentelemetry/sdk-trace-node";
+ import { OTLPTraceExporter } from "@opentelemetry/exporter-trace-otlp-grpc";
+ import { resourceFromAttributes } from "@opentelemetry/resources";
+ import { ATTR_SERVICE_NAME } from "@opentelemetry/semantic-conventions";
class RacedRedirectError extends Error {
constructor() {
@@@ -796,7 -797,7 +800,7 @@@ export const processJobInternal = asyn
crawlId: job.data?.crawl_id ?? undefined,
zeroDataRetention: job.data?.zeroDataRetention ?? false,
});
--
++
try {
try {
let extendLockInterval: NodeJS.Timeout | null = null;
@@@ -856,13 -853,22 +860,22 @@@
}
};
- const langfuseOtel = process.env.LANGFUSE_PUBLIC_KEY ? new NodeSDK({
- traceExporter: new LangfuseExporter(),
+ const shouldOtel = process.env.LANGFUSE_PUBLIC_KEY || process.env.OTEL_EXPORTER_OTLP_ENDPOINT;
+ const otelSdk = shouldOtel ? new NodeSDK({
+ resource: resourceFromAttributes({
+ [ATTR_SERVICE_NAME]: "firecrawl-worker-scrape",
+ }),
+ spanProcessors: [
+ ...(process.env.LANGFUSE_PUBLIC_KEY ? [new BatchSpanProcessor(new LangfuseExporter())] : []),
+ ...(process.env.OTEL_EXPORTER_OTLP_ENDPOINT ? [new BatchSpanProcessor(new OTLPTraceExporter({
+ url: process.env.OTEL_EXPORTER_OTLP_ENDPOINT,
+ }))] : []),
+ ],
instrumentations: [getNodeAutoInstrumentations()],
}) : null;
--
- if (langfuseOtel) {
- langfuseOtel.start();
-if (otelSdk) {
++
++if (otelSdk) {
+ otelSdk.start();
}
module.exports = processJobInternal;
@@@ -878,4 -884,4 +891,4 @@@ const exitHandler = () =>
process.on("SIGINT", exitHandler);
process.on("SIGTERM", exitHandler);
--process.on("exit", exitHandler);
++process.on("exit", exitHandler);