Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

fix(otel):

Gergő Móriczc1b78d2cbd8e
diff --git a/apps/api/package.json b/apps/api/package.json
index 8d88296a1..c29baf861 100644
--- a/apps/api/package.json
+++ b/apps/api/package.json
@@ -79,12 +79,8 @@
"@openrouter/ai-sdk-provider": "^0.4.5",
"@opentelemetry/auto-instrumentations-node": "^0.62.0",
"@opentelemetry/core": "^2.0.1",
- "@opentelemetry/exporter-trace-otlp-grpc": "^0.203.0",
"@opentelemetry/exporter-trace-otlp-http": "^0.205.0",
- "@opentelemetry/resources": "^2.0.1",
"@opentelemetry/sdk-node": "^0.203.0",
- "@opentelemetry/sdk-trace-node": "^2.0.1",
- "@opentelemetry/semantic-conventions": "^1.36.0",
"@sentry/cli": "^2.33.1",
"@sentry/node": "^9.40.0",
"@sentry/profiling-node": "^9.40.0",
@@ -112,8 +108,6 @@
"joplin-turndown-plugin-gfm": "^1.0.12",
"jsdom": "^26.0.0",
"koffi": "^2.9.0",
- "langfuse": "^3.38.4",
- "langfuse-vercel": "^3.38.4",
"lodash": "^4.17.21",
"marked": "^14.1.2",
"ollama-ai-provider": "^1.2.0",
diff --git a/apps/api/pnpm-lock.yaml b/apps/api/pnpm-lock.yaml
index c4c67b301..6e7dc8b3d 100644
--- a/apps/api/pnpm-lock.yaml
+++ b/apps/api/pnpm-lock.yaml
@@ -66,24 +66,12 @@ importers:
'@opentelemetry/core':
specifier: ^2.0.1
version: 2.0.1(@opentelemetry/[email protected])
- '@opentelemetry/exporter-trace-otlp-grpc':
- specifier: ^0.203.0
- version: 0.203.0(@opentelemetry/[email protected])
'@opentelemetry/exporter-trace-otlp-http':
specifier: ^0.205.0
version: 0.205.0(@opentelemetry/[email protected])
- '@opentelemetry/resources':
- specifier: ^2.0.1
- version: 2.0.1(@opentelemetry/[email protected])
'@opentelemetry/sdk-node':
specifier: ^0.203.0
version: 0.203.0(@opentelemetry/[email protected])
- '@opentelemetry/sdk-trace-node':
- specifier: ^2.0.1
- version: 2.0.1(@opentelemetry/[email protected])
- '@opentelemetry/semantic-conventions':
- specifier: ^1.36.0
- version: 1.36.0
'@sentry/cli':
specifier: ^2.33.1
version: 2.33.1([email protected])
@@ -165,12 +153,6 @@ importers:
koffi:
specifier: ^2.9.0
version: 2.9.0
- langfuse:
- specifier: ^3.38.4
- version: 3.38.4
- langfuse-vercel:
- specifier: ^3.38.4
lodash:
specifier: ^4.17.21
version: 4.17.21
@@ -5222,20 +5204,6 @@ packages:
resolution: {integrity: sha512-Xq9nH7KlWZmXAtodXDDRE7vs6DU1gTU8zYDHDiWLSip45Egwq3plLHzPn27NgvzL2r1LMPC1vdqh98sQxtqj4A==}
- resolution: {integrity: sha512-onTAqcEGhoXuBgqDFXe2t+bt9Vi+5YChRgdz3voM49JKoHwtVZQiUdqTfjSivGR75eSbYoiaIL8IRoio+jaqwg==}
- engines: {node: '>=18'}
-
- resolution: {integrity: sha512-gyhQuim2b1qpI7yiGnww9qzqsdpD59XagD+QB3eGHIrX+1ao7vbcrwL14H0GlsLIUM10Nud9wtEbxiFZP1PB7g==}
- engines: {node: '>=18'}
- peerDependencies:
- ai: '>=3.2.44'
-
- resolution: {integrity: sha512-2UqMeHLl3DGNX1Nh/cO4jGhk7TzDJ6gjQLlyS9rwFCKVO81xot6b58yeTsTB5YrWupWsOxQtMNoQYIQGOUlH9Q==}
- engines: {node: '>=18'}
-
resolution: {integrity: sha512-y+SqErxb8h7nE/fiEX07jsbuhrpO9lL8eca7/Y1nuWV2moNlXhyd59iDGcRf6moVyDMbmTNzL40SUyrFU/yDpg==}
@@ -5483,10 +5451,6 @@ packages:
resolution: {integrity: sha512-HoMUjhH9T8DDBNT+6xzkrd9ga/XiBI4xLr58LJACwK6G3HTOPeMz4nB4KJs33L2BelrIJa7P0VuNaVF3hMYfjg==}
- resolution: {integrity: sha512-71ippSywq5Yb7/tVYyGbkBggbU8H3u5Rz56fH60jGFgr8uHwxs+aSKeqmluIVzM0m0kB7xQjKS6qPfd0b2ZoqQ==}
- hasBin: true
-
resolution: {integrity: sha512-WWdIxpyjEn+FhQJQQv9aQAYlHoNVdzIzUySNV1gHUPDSdZJ3yZn7pAAbQcV7B56Mvu881q9FZV+0Vx2xC44VWA==}
engines: {node: ^18.17.0 || >=20.5.0}
@@ -13742,20 +13706,6 @@ snapshots:
- dependencies:
- mustache: 4.2.0
-
- dependencies:
- langfuse: 3.38.4
- langfuse-core: 3.38.4
-
- dependencies:
- langfuse-core: 3.38.4
-
@@ -13989,8 +13939,6 @@ snapshots:
-
diff --git a/apps/api/src/index.ts b/apps/api/src/index.ts
index 3ccbbcc2c..598565536 100644
--- a/apps/api/src/index.ts
+++ b/apps/api/src/index.ts
@@ -31,13 +31,8 @@ import { cacheableLookup } from "./scraper/scrapeURL/lib/cacheableLookup";
import { v2Router } from "./routes/v2";
import domainFrequencyRouter from "./routes/domain-frequency";
import { NodeSDK } from "@opentelemetry/sdk-node";
-import { LangfuseExporter } from "langfuse-vercel";
import { getNodeAutoInstrumentations } from "@opentelemetry/auto-instrumentations-node";
-import { ATTR_SERVICE_NAME } from "@opentelemetry/semantic-conventions";
-import { resourceFromAttributes } from "@opentelemetry/resources";
-import { BatchSpanProcessor } from "@opentelemetry/sdk-trace-node";
-import { OTLPTraceExporter } from "@opentelemetry/exporter-trace-otlp-grpc";
-import { OTLPTraceExporter as OTLPHTTPTraceExporter } from "@opentelemetry/exporter-trace-otlp-http";
+import { OTLPTraceExporter } from "@opentelemetry/exporter-trace-otlp-http";
import { nuqShutdown } from "./services/worker/nuq";
import { getErrorContactMessage } from "./lib/deployment";
import { initializeBlocklist } from "./scraper/WebScraper/utils/blocklist";
@@ -58,48 +53,12 @@ logger.info("Network info dump", {
cacheableLookup.install(http.globalAgent);
cacheableLookup.install(https.globalAgent);
-const shouldOtel =
- process.env.LANGFUSE_PUBLIC_KEY ||
- process.env.OTEL_EXPORTER_OTLP_ENDPOINT ||
- process.env.HONEYCOMB_TEAM_ID;
-const otelSdk = shouldOtel
- ? new NodeSDK({
- resource: resourceFromAttributes({
- [ATTR_SERVICE_NAME]: "firecrawl-app",
- }),
- spanProcessors: [
- ...(process.env.LANGFUSE_PUBLIC_KEY
- ? [new BatchSpanProcessor(new LangfuseExporter())]
- : []),
- ...(process.env.OTEL_EXPORTER_OTLP_ENDPOINT
- ? [
- new BatchSpanProcessor(
- new OTLPTraceExporter({
- url: process.env.OTEL_EXPORTER_OTLP_ENDPOINT,
- }),
- ),
- ]
- : []),
- ...(process.env.HONEYCOMB_TEAM_ID
- ? [
- new BatchSpanProcessor(
- new OTLPHTTPTraceExporter({
- url: "https://api.honeycomb.io",
- headers: {
- "x-honeycomb-team": process.env.HONEYCOMB_TEAM_ID,
- },
- }),
- ),
- ]
- : []),
- ],
- instrumentations: [getNodeAutoInstrumentations()],
- })
- : null;
-
-if (otelSdk) {
- otelSdk.start();
-}
+const otelSdk = new NodeSDK({
+ traceExporter: new OTLPTraceExporter(),
+ instrumentations: [getNodeAutoInstrumentations()],
+});
+
+otelSdk.start();
// Initialize Express with WebSocket support
const expressApp = express();
@@ -183,14 +142,10 @@ async function startServer(port = DEFAULT_PORT) {
logger.info("Server closed.");
nuqShutdown().finally(() => {
logger.info("NUQ shutdown complete");
- if (otelSdk) {
- otelSdk.shutdown().finally(() => {
- logger.info("OTEL shutdown");
- process.exit(0);
- });
- } else {
+ otelSdk.shutdown().finally(() => {
+ logger.info("OTEL shutdown");
process.exit(0);
- }
+ });
});
});
};
diff --git a/apps/api/src/lib/extract/extraction-service.ts b/apps/api/src/lib/extract/extraction-service.ts
index 4abb8cf9b..4e70fc516 100644
--- a/apps/api/src/lib/extract/extraction-service.ts
+++ b/apps/api/src/lib/extract/extraction-service.ts
@@ -40,7 +40,6 @@ import { normalizeUrl } from "../canonical-url";
import { search } from "../../search";
import { buildRephraseToSerpPrompt } from "./build-prompts";
import { getACUCTeam } from "../../controllers/auth";
-import { langfuse } from "../../services/langfuse";
import { CostLimitExceededError, CostTracking } from "../cost-tracking";
interface ExtractServiceOptions {
@@ -103,16 +102,6 @@ export async function performExtraction(
teamId,
});
- langfuse.trace({
- id: "extract:" + extractId,
- name: "performExtraction",
- metadata: {
- teamId,
- extractId,
- model: "fire-1",
- },
- });
-
try {
// If no URLs are provided, generate URLs from the prompt
if ((!request.urls || request.urls.length === 0) && request.prompt) {
diff --git a/apps/api/src/lib/extract/fire-0/extraction-service-f0.ts b/apps/api/src/lib/extract/fire-0/extraction-service-f0.ts
index 8de1cb842..951c0a536 100644
--- a/apps/api/src/lib/extract/fire-0/extraction-service-f0.ts
+++ b/apps/api/src/lib/extract/fire-0/extraction-service-f0.ts
@@ -39,7 +39,6 @@ import {
} from "./usage/llm-cost-f0";
import { SourceTracker_F0 } from "./helpers/source-tracker-f0";
import { getACUCTeam } from "../../../controllers/auth";
-import { langfuse } from "../../../services/langfuse";
interface ExtractServiceOptions {
request: ExtractRequest;
@@ -94,16 +93,6 @@ export async function performExtraction_F0(
teamId,
});
- langfuse.trace({
- id: "extract:" + extractId,
- name: "performExtraction",
- metadata: {
- teamId,
- extractId,
- model: "fire-0",
- },
- });
-
// If no URLs are provided, generate URLs from the prompt
if ((!request.urls || request.urls.length === 0) && request.prompt) {
logger.debug("Generating URLs from prompt...", {
diff --git a/apps/api/src/services/langfuse.ts b/apps/api/src/services/langfuse.ts
deleted file mode 100644
index 90aa50155..000000000
--- a/apps/api/src/services/langfuse.ts
+++ /dev/null
@@ -1,3 +0,0 @@
-import { Langfuse } from "langfuse";
-
-export const langfuse = new Langfuse();
diff --git a/apps/api/src/services/queue-worker.ts b/apps/api/src/services/queue-worker.ts
index bbae4a02b..e128d7ed1 100644
--- a/apps/api/src/services/queue-worker.ts
+++ b/apps/api/src/services/queue-worker.ts
@@ -30,13 +30,8 @@ import { cacheableLookup } from "../scraper/scrapeURL/lib/cacheableLookup";
import { robustFetch } from "../scraper/scrapeURL/lib/fetch";
import { NodeSDK } from "@opentelemetry/sdk-node";
import { getNodeAutoInstrumentations } from "@opentelemetry/auto-instrumentations-node";
-import { LangfuseExporter } from "langfuse-vercel";
-import { BatchSpanProcessor } from "@opentelemetry/sdk-trace-node";
-import { ATTR_SERVICE_NAME } from "@opentelemetry/semantic-conventions";
-import { resourceFromAttributes } from "@opentelemetry/resources";
-import { OTLPTraceExporter } from "@opentelemetry/exporter-trace-otlp-grpc";
+import { OTLPTraceExporter } from "@opentelemetry/exporter-trace-otlp-http";
import { BullMQOtel } from "bullmq-otel";
-import { pathToFileURL } from "url";
import { getErrorContactMessage } from "../lib/deployment";
import { initializeBlocklist } from "../scraper/WebScraper/utils/blocklist";
@@ -61,48 +56,12 @@ const runningJobs: Set<string> = new Set();
cacheableLookup.install(http.globalAgent);
cacheableLookup.install(https.globalAgent);
-const shouldOtel =
- process.env.LANGFUSE_PUBLIC_KEY ||
- process.env.OTEL_EXPORTER_OTLP_ENDPOINT ||
- process.env.HONEYCOMB_TEAM_ID;
-const otelSdk = shouldOtel
- ? new NodeSDK({
- resource: resourceFromAttributes({
- [ATTR_SERVICE_NAME]: "firecrawl-worker",
- }),
- spanProcessors: [
- ...(process.env.LANGFUSE_PUBLIC_KEY
- ? [new BatchSpanProcessor(new LangfuseExporter())]
- : []),
- ...(process.env.OTEL_EXPORTER_OTLP_ENDPOINT
- ? [
- new BatchSpanProcessor(
- new OTLPTraceExporter({
- url: process.env.OTEL_EXPORTER_OTLP_ENDPOINT,
- }),
- ),
- ]
- : []),
- ...(process.env.HONEYCOMB_TEAM_ID
- ? [
- new BatchSpanProcessor(
- new OTLPTraceExporter({
- url: "https://api.honeycomb.io",
- headers: {
- "x-honeycomb-team": process.env.HONEYCOMB_TEAM_ID,
- },
- }),
- ),
- ]
- : []),
- ],
- instrumentations: [getNodeAutoInstrumentations()],
- })
- : null;
+const otelSdk = new NodeSDK({
+ traceExporter: new OTLPTraceExporter(),
+ instrumentations: [getNodeAutoInstrumentations()],
+});
-if (otelSdk) {
- otelSdk.start();
-}
+otelSdk.start();
const processExtractJobInternal = async (
token: string,
@@ -529,8 +488,6 @@ app.listen(workerPort, () => {
}
console.log("All jobs finished. Worker out!");
- if (otelSdk) {
- await otelSdk.shutdown();
- }
+ await otelSdk.shutdown();
process.exit(0);
})();
diff --git a/apps/api/src/services/worker/scrape-worker.ts b/apps/api/src/services/worker/scrape-worker.ts
index 6ebd829d2..536a760c1 100644
--- a/apps/api/src/services/worker/scrape-worker.ts
+++ b/apps/api/src/services/worker/scrape-worker.ts
@@ -50,7 +50,6 @@ import { calculateCreditsToBeBilled } from "../../lib/scrape-billing";
import { getBillingQueue } from "../queue-service";
import type { Logger } from "winston";
import { finishCrawlIfNeeded } from "./crawl-logic";
-import { LangfuseExporter } from "langfuse-vercel";
import { getNodeAutoInstrumentations } from "@opentelemetry/auto-instrumentations-node";
import { NodeSDK } from "@opentelemetry/sdk-node";
import {
@@ -60,10 +59,7 @@ import {
UnknownError,
} from "../../lib/error";
import { serializeTransportableError } from "../../lib/error-serde";
-import { BatchSpanProcessor } from "@opentelemetry/sdk-trace-node";
-import { OTLPTraceExporter } from "@opentelemetry/exporter-trace-otlp-grpc";
-import { resourceFromAttributes } from "@opentelemetry/resources";
-import { ATTR_SERVICE_NAME } from "@opentelemetry/semantic-conventions";
+import { OTLPTraceExporter } from "@opentelemetry/exporter-trace-otlp-http";
import type { NuQJob } from "./nuq";
import {
ScrapeJobData,
@@ -1144,56 +1140,18 @@ export const processJobInternal = async (job: NuQJob<ScrapeJobData>) => {
}
};
-const shouldOtel =
- process.env.LANGFUSE_PUBLIC_KEY ||
- process.env.OTEL_EXPORTER_OTLP_ENDPOINT ||
- process.env.HONEYCOMB_TEAM_ID;
-const otelSdk = shouldOtel
- ? new NodeSDK({
- resource: resourceFromAttributes({
- [ATTR_SERVICE_NAME]: "firecrawl-worker-scrape",
- }),
- spanProcessors: [
- ...(process.env.LANGFUSE_PUBLIC_KEY
- ? [new BatchSpanProcessor(new LangfuseExporter())]
- : []),
- ...(process.env.OTEL_EXPORTER_OTLP_ENDPOINT
- ? [
- new BatchSpanProcessor(
- new OTLPTraceExporter({
- url: process.env.OTEL_EXPORTER_OTLP_ENDPOINT,
- }),
- ),
- ]
- : []),
- ...(process.env.HONEYCOMB_TEAM_ID
- ? [
- new BatchSpanProcessor(
- new OTLPTraceExporter({
- url: "https://api.honeycomb.io",
- headers: {
- "x-honeycomb-team": process.env.HONEYCOMB_TEAM_ID,
- },
- }),
- ),
- ]
- : []),
- ],
- instrumentations: [getNodeAutoInstrumentations()],
- })
- : null;
-
-if (otelSdk) {
- otelSdk.start();
-}
+const otelSdk = new NodeSDK({
+ traceExporter: new OTLPTraceExporter(),
+ instrumentations: [getNodeAutoInstrumentations()],
+});
+
+otelSdk.start();
const exitHandler = () => {
- if (otelSdk) {
- otelSdk.shutdown().finally(() => {
- _logger.debug("OTEL shutdown");
- process.exit(0);
- });
- }
+ otelSdk.shutdown().finally(() => {
+ _logger.debug("OTEL shutdown");
+ process.exit(0);
+ });
};
process.on("SIGINT", exitHandler);