Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(otel):
Gergő Móriczc1b78d2cbd8e
diff --git a/apps/api/package.json b/apps/api/package.jsonindex 8d88296a1..c29baf861 100644--- a/apps/api/package.json+++ b/apps/api/package.json@@ -79,12 +79,8 @@"@openrouter/ai-sdk-provider": "^0.4.5","@opentelemetry/auto-instrumentations-node": "^0.62.0","@opentelemetry/core": "^2.0.1",- "@opentelemetry/exporter-trace-otlp-grpc": "^0.203.0","@opentelemetry/exporter-trace-otlp-http": "^0.205.0",- "@opentelemetry/resources": "^2.0.1","@opentelemetry/sdk-node": "^0.203.0",- "@opentelemetry/sdk-trace-node": "^2.0.1",- "@opentelemetry/semantic-conventions": "^1.36.0","@sentry/cli": "^2.33.1","@sentry/node": "^9.40.0","@sentry/profiling-node": "^9.40.0",@@ -112,8 +108,6 @@"joplin-turndown-plugin-gfm": "^1.0.12","jsdom": "^26.0.0","koffi": "^2.9.0",- "langfuse": "^3.38.4",- "langfuse-vercel": "^3.38.4","lodash": "^4.17.21","marked": "^14.1.2","ollama-ai-provider": "^1.2.0",diff --git a/apps/api/pnpm-lock.yaml b/apps/api/pnpm-lock.yamlindex c4c67b301..6e7dc8b3d 100644--- a/apps/api/pnpm-lock.yaml+++ b/apps/api/pnpm-lock.yaml@@ -66,24 +66,12 @@ importers:'@opentelemetry/core':specifier: ^2.0.1version: 2.0.1(@opentelemetry/[email protected])- '@opentelemetry/exporter-trace-otlp-grpc':- specifier: ^0.203.0- version: 0.203.0(@opentelemetry/[email protected])'@opentelemetry/exporter-trace-otlp-http':specifier: ^0.205.0version: 0.205.0(@opentelemetry/[email protected])- '@opentelemetry/resources':- specifier: ^2.0.1- version: 2.0.1(@opentelemetry/[email protected])'@opentelemetry/sdk-node':specifier: ^0.203.0version: 0.203.0(@opentelemetry/[email protected])- '@opentelemetry/sdk-trace-node':- specifier: ^2.0.1- version: 2.0.1(@opentelemetry/[email protected])- '@opentelemetry/semantic-conventions':- specifier: ^1.36.0- version: 1.36.0'@sentry/cli':specifier: ^2.33.1version: 2.33.1([email protected])@@ -165,12 +153,6 @@ importers:koffi:specifier: ^2.9.0version: 2.9.0- langfuse:- specifier: ^3.38.4- version: 3.38.4- langfuse-vercel:- specifier: ^3.38.4lodash:specifier: ^4.17.21version: 4.17.21@@ -5222,20 +5204,6 @@ packages:resolution: {integrity: sha512-Xq9nH7KlWZmXAtodXDDRE7vs6DU1gTU8zYDHDiWLSip45Egwq3plLHzPn27NgvzL2r1LMPC1vdqh98sQxtqj4A==}- resolution: {integrity: sha512-onTAqcEGhoXuBgqDFXe2t+bt9Vi+5YChRgdz3voM49JKoHwtVZQiUdqTfjSivGR75eSbYoiaIL8IRoio+jaqwg==}- engines: {node: '>=18'}-- resolution: {integrity: sha512-gyhQuim2b1qpI7yiGnww9qzqsdpD59XagD+QB3eGHIrX+1ao7vbcrwL14H0GlsLIUM10Nud9wtEbxiFZP1PB7g==}- engines: {node: '>=18'}- peerDependencies:- ai: '>=3.2.44'-- resolution: {integrity: sha512-2UqMeHLl3DGNX1Nh/cO4jGhk7TzDJ6gjQLlyS9rwFCKVO81xot6b58yeTsTB5YrWupWsOxQtMNoQYIQGOUlH9Q==}- engines: {node: '>=18'}-resolution: {integrity: sha512-y+SqErxb8h7nE/fiEX07jsbuhrpO9lL8eca7/Y1nuWV2moNlXhyd59iDGcRf6moVyDMbmTNzL40SUyrFU/yDpg==}@@ -5483,10 +5451,6 @@ packages:resolution: {integrity: sha512-HoMUjhH9T8DDBNT+6xzkrd9ga/XiBI4xLr58LJACwK6G3HTOPeMz4nB4KJs33L2BelrIJa7P0VuNaVF3hMYfjg==}- resolution: {integrity: sha512-71ippSywq5Yb7/tVYyGbkBggbU8H3u5Rz56fH60jGFgr8uHwxs+aSKeqmluIVzM0m0kB7xQjKS6qPfd0b2ZoqQ==}- hasBin: true-resolution: {integrity: sha512-WWdIxpyjEn+FhQJQQv9aQAYlHoNVdzIzUySNV1gHUPDSdZJ3yZn7pAAbQcV7B56Mvu881q9FZV+0Vx2xC44VWA==}engines: {node: ^18.17.0 || >=20.5.0}@@ -13742,20 +13706,6 @@ snapshots:- dependencies:- mustache: 4.2.0-- dependencies:- ai: 4.3.5([email protected])([email protected])- langfuse: 3.38.4- langfuse-core: 3.38.4-- dependencies:- langfuse-core: 3.38.4-@@ -13989,8 +13939,6 @@ snapshots:- [email protected]: {}-diff --git a/apps/api/src/index.ts b/apps/api/src/index.tsindex 3ccbbcc2c..598565536 100644--- a/apps/api/src/index.ts+++ b/apps/api/src/index.ts@@ -31,13 +31,8 @@ import { cacheableLookup } from "./scraper/scrapeURL/lib/cacheableLookup";import { v2Router } from "./routes/v2";import domainFrequencyRouter from "./routes/domain-frequency";import { NodeSDK } from "@opentelemetry/sdk-node";-import { LangfuseExporter } from "langfuse-vercel";import { getNodeAutoInstrumentations } from "@opentelemetry/auto-instrumentations-node";-import { ATTR_SERVICE_NAME } from "@opentelemetry/semantic-conventions";-import { resourceFromAttributes } from "@opentelemetry/resources";-import { BatchSpanProcessor } from "@opentelemetry/sdk-trace-node";-import { OTLPTraceExporter } from "@opentelemetry/exporter-trace-otlp-grpc";-import { OTLPTraceExporter as OTLPHTTPTraceExporter } from "@opentelemetry/exporter-trace-otlp-http";+import { OTLPTraceExporter } from "@opentelemetry/exporter-trace-otlp-http";import { nuqShutdown } from "./services/worker/nuq";import { getErrorContactMessage } from "./lib/deployment";import { initializeBlocklist } from "./scraper/WebScraper/utils/blocklist";@@ -58,48 +53,12 @@ logger.info("Network info dump", {cacheableLookup.install(http.globalAgent);cacheableLookup.install(https.globalAgent);-const shouldOtel =- process.env.LANGFUSE_PUBLIC_KEY ||- process.env.OTEL_EXPORTER_OTLP_ENDPOINT ||- process.env.HONEYCOMB_TEAM_ID;-const otelSdk = shouldOtel- ? new NodeSDK({- resource: resourceFromAttributes({- [ATTR_SERVICE_NAME]: "firecrawl-app",- }),- spanProcessors: [- ...(process.env.LANGFUSE_PUBLIC_KEY- ? [new BatchSpanProcessor(new LangfuseExporter())]- : []),- ...(process.env.OTEL_EXPORTER_OTLP_ENDPOINT- ? [- new BatchSpanProcessor(- new OTLPTraceExporter({- url: process.env.OTEL_EXPORTER_OTLP_ENDPOINT,- }),- ),- ]- : []),- ...(process.env.HONEYCOMB_TEAM_ID- ? [- new BatchSpanProcessor(- new OTLPHTTPTraceExporter({- url: "https://api.honeycomb.io",- headers: {- "x-honeycomb-team": process.env.HONEYCOMB_TEAM_ID,- },- }),- ),- ]- : []),- ],- instrumentations: [getNodeAutoInstrumentations()],- })- : null;--if (otelSdk) {- otelSdk.start();-}+const otelSdk = new NodeSDK({+ traceExporter: new OTLPTraceExporter(),+ instrumentations: [getNodeAutoInstrumentations()],+});++otelSdk.start();// Initialize Express with WebSocket supportconst expressApp = express();@@ -183,14 +142,10 @@ async function startServer(port = DEFAULT_PORT) {logger.info("Server closed.");nuqShutdown().finally(() => {logger.info("NUQ shutdown complete");- if (otelSdk) {- otelSdk.shutdown().finally(() => {- logger.info("OTEL shutdown");- process.exit(0);- });- } else {+ otelSdk.shutdown().finally(() => {+ logger.info("OTEL shutdown");process.exit(0);- }+ });});});};diff --git a/apps/api/src/lib/extract/extraction-service.ts b/apps/api/src/lib/extract/extraction-service.tsindex 4abb8cf9b..4e70fc516 100644--- a/apps/api/src/lib/extract/extraction-service.ts+++ b/apps/api/src/lib/extract/extraction-service.ts@@ -40,7 +40,6 @@ import { normalizeUrl } from "../canonical-url";import { search } from "../../search";import { buildRephraseToSerpPrompt } from "./build-prompts";import { getACUCTeam } from "../../controllers/auth";-import { langfuse } from "../../services/langfuse";import { CostLimitExceededError, CostTracking } from "../cost-tracking";interface ExtractServiceOptions {@@ -103,16 +102,6 @@ export async function performExtraction(teamId,});- langfuse.trace({- id: "extract:" + extractId,- name: "performExtraction",- metadata: {- teamId,- extractId,- model: "fire-1",- },- });-try {// If no URLs are provided, generate URLs from the promptif ((!request.urls || request.urls.length === 0) && request.prompt) {diff --git a/apps/api/src/lib/extract/fire-0/extraction-service-f0.ts b/apps/api/src/lib/extract/fire-0/extraction-service-f0.tsindex 8de1cb842..951c0a536 100644--- a/apps/api/src/lib/extract/fire-0/extraction-service-f0.ts+++ b/apps/api/src/lib/extract/fire-0/extraction-service-f0.ts@@ -39,7 +39,6 @@ import {} from "./usage/llm-cost-f0";import { SourceTracker_F0 } from "./helpers/source-tracker-f0";import { getACUCTeam } from "../../../controllers/auth";-import { langfuse } from "../../../services/langfuse";interface ExtractServiceOptions {request: ExtractRequest;@@ -94,16 +93,6 @@ export async function performExtraction_F0(teamId,});- langfuse.trace({- id: "extract:" + extractId,- name: "performExtraction",- metadata: {- teamId,- extractId,- model: "fire-0",- },- });-// If no URLs are provided, generate URLs from the promptif ((!request.urls || request.urls.length === 0) && request.prompt) {logger.debug("Generating URLs from prompt...", {diff --git a/apps/api/src/services/langfuse.ts b/apps/api/src/services/langfuse.tsdeleted file mode 100644index 90aa50155..000000000--- a/apps/api/src/services/langfuse.ts+++ /dev/null@@ -1,3 +0,0 @@-import { Langfuse } from "langfuse";--export const langfuse = new Langfuse();diff --git a/apps/api/src/services/queue-worker.ts b/apps/api/src/services/queue-worker.tsindex bbae4a02b..e128d7ed1 100644--- a/apps/api/src/services/queue-worker.ts+++ b/apps/api/src/services/queue-worker.ts@@ -30,13 +30,8 @@ import { cacheableLookup } from "../scraper/scrapeURL/lib/cacheableLookup";import { robustFetch } from "../scraper/scrapeURL/lib/fetch";import { NodeSDK } from "@opentelemetry/sdk-node";import { getNodeAutoInstrumentations } from "@opentelemetry/auto-instrumentations-node";-import { LangfuseExporter } from "langfuse-vercel";-import { BatchSpanProcessor } from "@opentelemetry/sdk-trace-node";-import { ATTR_SERVICE_NAME } from "@opentelemetry/semantic-conventions";-import { resourceFromAttributes } from "@opentelemetry/resources";-import { OTLPTraceExporter } from "@opentelemetry/exporter-trace-otlp-grpc";+import { OTLPTraceExporter } from "@opentelemetry/exporter-trace-otlp-http";import { BullMQOtel } from "bullmq-otel";-import { pathToFileURL } from "url";import { getErrorContactMessage } from "../lib/deployment";import { initializeBlocklist } from "../scraper/WebScraper/utils/blocklist";@@ -61,48 +56,12 @@ const runningJobs: Set<string> = new Set();cacheableLookup.install(http.globalAgent);cacheableLookup.install(https.globalAgent);-const shouldOtel =- process.env.LANGFUSE_PUBLIC_KEY ||- process.env.OTEL_EXPORTER_OTLP_ENDPOINT ||- process.env.HONEYCOMB_TEAM_ID;-const otelSdk = shouldOtel- ? new NodeSDK({- resource: resourceFromAttributes({- [ATTR_SERVICE_NAME]: "firecrawl-worker",- }),- spanProcessors: [- ...(process.env.LANGFUSE_PUBLIC_KEY- ? [new BatchSpanProcessor(new LangfuseExporter())]- : []),- ...(process.env.OTEL_EXPORTER_OTLP_ENDPOINT- ? [- new BatchSpanProcessor(- new OTLPTraceExporter({- url: process.env.OTEL_EXPORTER_OTLP_ENDPOINT,- }),- ),- ]- : []),- ...(process.env.HONEYCOMB_TEAM_ID- ? [- new BatchSpanProcessor(- new OTLPTraceExporter({- url: "https://api.honeycomb.io",- headers: {- "x-honeycomb-team": process.env.HONEYCOMB_TEAM_ID,- },- }),- ),- ]- : []),- ],- instrumentations: [getNodeAutoInstrumentations()],- })- : null;+const otelSdk = new NodeSDK({+ traceExporter: new OTLPTraceExporter(),+ instrumentations: [getNodeAutoInstrumentations()],+});-if (otelSdk) {- otelSdk.start();-}+otelSdk.start();const processExtractJobInternal = async (token: string,@@ -529,8 +488,6 @@ app.listen(workerPort, () => {}console.log("All jobs finished. Worker out!");- if (otelSdk) {- await otelSdk.shutdown();- }+ await otelSdk.shutdown();process.exit(0);})();diff --git a/apps/api/src/services/worker/scrape-worker.ts b/apps/api/src/services/worker/scrape-worker.tsindex 6ebd829d2..536a760c1 100644--- a/apps/api/src/services/worker/scrape-worker.ts+++ b/apps/api/src/services/worker/scrape-worker.ts@@ -50,7 +50,6 @@ import { calculateCreditsToBeBilled } from "../../lib/scrape-billing";import { getBillingQueue } from "../queue-service";import type { Logger } from "winston";import { finishCrawlIfNeeded } from "./crawl-logic";-import { LangfuseExporter } from "langfuse-vercel";import { getNodeAutoInstrumentations } from "@opentelemetry/auto-instrumentations-node";import { NodeSDK } from "@opentelemetry/sdk-node";import {@@ -60,10 +59,7 @@ import {UnknownError,} from "../../lib/error";import { serializeTransportableError } from "../../lib/error-serde";-import { BatchSpanProcessor } from "@opentelemetry/sdk-trace-node";-import { OTLPTraceExporter } from "@opentelemetry/exporter-trace-otlp-grpc";-import { resourceFromAttributes } from "@opentelemetry/resources";-import { ATTR_SERVICE_NAME } from "@opentelemetry/semantic-conventions";+import { OTLPTraceExporter } from "@opentelemetry/exporter-trace-otlp-http";import type { NuQJob } from "./nuq";import {ScrapeJobData,@@ -1144,56 +1140,18 @@ export const processJobInternal = async (job: NuQJob<ScrapeJobData>) => {}};-const shouldOtel =- process.env.LANGFUSE_PUBLIC_KEY ||- process.env.OTEL_EXPORTER_OTLP_ENDPOINT ||- process.env.HONEYCOMB_TEAM_ID;-const otelSdk = shouldOtel- ? new NodeSDK({- resource: resourceFromAttributes({- [ATTR_SERVICE_NAME]: "firecrawl-worker-scrape",- }),- spanProcessors: [- ...(process.env.LANGFUSE_PUBLIC_KEY- ? [new BatchSpanProcessor(new LangfuseExporter())]- : []),- ...(process.env.OTEL_EXPORTER_OTLP_ENDPOINT- ? [- new BatchSpanProcessor(- new OTLPTraceExporter({- url: process.env.OTEL_EXPORTER_OTLP_ENDPOINT,- }),- ),- ]- : []),- ...(process.env.HONEYCOMB_TEAM_ID- ? [- new BatchSpanProcessor(- new OTLPTraceExporter({- url: "https://api.honeycomb.io",- headers: {- "x-honeycomb-team": process.env.HONEYCOMB_TEAM_ID,- },- }),- ),- ]- : []),- ],- instrumentations: [getNodeAutoInstrumentations()],- })- : null;--if (otelSdk) {- otelSdk.start();-}+const otelSdk = new NodeSDK({+ traceExporter: new OTLPTraceExporter(),+ instrumentations: [getNodeAutoInstrumentations()],+});++otelSdk.start();const exitHandler = () => {- if (otelSdk) {- otelSdk.shutdown().finally(() => {- _logger.debug("OTEL shutdown");- process.exit(0);- });- }+ otelSdk.shutdown().finally(() => {+ _logger.debug("OTEL shutdown");+ process.exit(0);+ });};process.on("SIGINT", exitHandler);