Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(api): otel sidecar
Gergő Móricze6562c980bbf
diff --git a/apps/api/package.json b/apps/api/package.jsonindex c29baf861..69245b839 100644--- a/apps/api/package.json+++ b/apps/api/package.json@@ -6,12 +6,12 @@"scripts": {"start": "tsc && node dist/src/harness.js --start-built","dev": "tsx src/harness.ts --start",- "server": "tsc-watch --onSuccess \"node dist/src/index.js\"",- "server:production": "tsc && node dist/src/index.js",- "server:production:nobuild": "node dist/src/index.js",+ "server": "tsc-watch --onSuccess \"node --import ./dist/src/otel.js dist/src/index.js\"",+ "server:production": "tsc && node --import ./dist/src/otel.js dist/src/index.js",+ "server:production:nobuild": "node --import ./dist/src/otel.js dist/src/index.js","format": "prettier --write \"src/**/*.(js|ts)\"",- "flyio": "node dist/src/index.js",- "start:dev": "tsc-watch --onSuccess \"node dist/src/index.js\"",+ "flyio": "node --import ./dist/src/otel.js dist/src/index.js",+ "start:dev": "tsc-watch --onSuccess \"node --import ./dist/src/otel.js dist/src/index.js\"","build": "tsc","build:nosentry": "tsc","test": "jest --testPathIgnorePatterns=\"src/__tests__/e2e_noAuth/*\"",@@ -20,12 +20,12 @@"test:prod": "jest --testPathIgnorePatterns=\"(src/__tests__/e2e_noAuth|src/__tests__/e2e_full_withAuth|src/scraper/scrapeURL)\"","test:snips": "jest \"src/__tests__/snips/v[12]/.+\\.test\\.ts\"","harness": "tsx src/harness.ts",- "workers": "tsc-watch --onSuccess \"node dist/src/services/queue-worker.js\"",- "worker:production": "node dist/src/services/queue-worker.js",- "nuq-worker": "tsc-watch --onSuccess \"node dist/src/services/worker/nuq-worker.js\"",- "nuq-worker:production": "node dist/src/services/worker/nuq-worker.js",- "index-worker": "tsc-watch --onSuccess \"node dist/src/services/indexing/index-worker.js\"",- "index-worker:production": "node dist/src/services/indexing/index-worker.js",+ "workers": "tsc-watch --onSuccess \"node --import ./dist/src/otel.js dist/src/services/queue-worker.js\"",+ "worker:production": "node --import ./dist/src/otel.js dist/src/services/queue-worker.js",+ "nuq-worker": "tsc-watch --onSuccess \"node --import ./dist/src/otel.js dist/src/services/worker/nuq-worker.js\"",+ "nuq-worker:production": "node --import ./dist/src/otel.js dist/src/services/worker/nuq-worker.js",+ "index-worker": "tsc-watch --onSuccess \"node --import ./dist/src/otel.js dist/src/services/indexing/index-worker.js\"",+ "index-worker:production": "node --import ./dist/src/otel.js dist/src/services/indexing/index-worker.js","mongo-docker": "docker run -d -p 2717:27017 -v ./mongo-data:/data/db --name mongodb mongo:latest","mongo-docker-console": "docker exec -it mongodb mongosh","run-example": "npx ts-node src/example.ts",diff --git a/apps/api/src/harness.ts b/apps/api/src/harness.tsindex a740f444a..77e4d0b3b 100644--- a/apps/api/src/harness.ts+++ b/apps/api/src/harness.ts@@ -320,7 +320,7 @@ function startServices(command?: string[]): Services {const api = execForward("api",process.argv[2] === "--start-docker"- ? "node dist/src/index.js"+ ? "node --import ./dist/src/otel.js dist/src/index.js": "pnpm server:production:nobuild",{NUQ_REDUCE_NOISE: "true",@@ -330,7 +330,7 @@ function startServices(command?: string[]): Services {const worker = execForward("worker",process.argv[2] === "--start-docker"- ? "node dist/src/services/queue-worker.js"+ ? "node --import ./dist/src/otel.js dist/src/services/queue-worker.js": "pnpm worker:production",{NUQ_REDUCE_NOISE: "true",@@ -341,7 +341,7 @@ function startServices(command?: string[]): Services {execForward(`nuq-worker-${i}`,process.argv[2] === "--start-docker"- ? "node dist/src/services/worker/nuq-worker.js"+ ? "node --import ./dist/src/otel.js dist/src/services/worker/nuq-worker.js": "pnpm nuq-worker:production",{NUQ_WORKER_PORT: String(3006 + i),@@ -355,7 +355,7 @@ function startServices(command?: string[]): Services {? execForward("index-worker",process.argv[2] === "--start-docker"- ? "node dist/src/services/indexing/index-worker.js"+ ? "node --import ./dist/src/otel.js dist/src/services/indexing/index-worker.js": "pnpm index-worker:production",{NUQ_REDUCE_NOISE: "true",diff --git a/apps/api/src/index.ts b/apps/api/src/index.tsindex 598565536..e6b4854da 100644--- a/apps/api/src/index.ts+++ b/apps/api/src/index.ts@@ -1,4 +1,5 @@import "dotenv/config";+import { shutdownOtel } from "./otel";import "./services/sentry";import * as Sentry from "@sentry/node";import express, { NextFunction, Request, Response } from "express";@@ -30,9 +31,6 @@ import { attachWsProxy } from "./services/agentLivecastWS";import { cacheableLookup } from "./scraper/scrapeURL/lib/cacheableLookup";import { v2Router } from "./routes/v2";import domainFrequencyRouter from "./routes/domain-frequency";-import { NodeSDK } from "@opentelemetry/sdk-node";-import { getNodeAutoInstrumentations } from "@opentelemetry/auto-instrumentations-node";-import { OTLPTraceExporter } from "@opentelemetry/exporter-trace-otlp-http";import { nuqShutdown } from "./services/worker/nuq";import { getErrorContactMessage } from "./lib/deployment";import { initializeBlocklist } from "./scraper/WebScraper/utils/blocklist";@@ -53,13 +51,6 @@ logger.info("Network info dump", {cacheableLookup.install(http.globalAgent);cacheableLookup.install(https.globalAgent);-const otelSdk = new NodeSDK({- traceExporter: new OTLPTraceExporter(),- instrumentations: [getNodeAutoInstrumentations()],-});--otelSdk.start();-// Initialize Express with WebSocket supportconst expressApp = express();const ws = expressWs(expressApp);@@ -142,7 +133,7 @@ async function startServer(port = DEFAULT_PORT) {logger.info("Server closed.");nuqShutdown().finally(() => {logger.info("NUQ shutdown complete");- otelSdk.shutdown().finally(() => {+ shutdownOtel().finally(() => {logger.info("OTEL shutdown");process.exit(0);});diff --git a/apps/api/src/lib/bigquery-jobs.ts b/apps/api/src/lib/bigquery-jobs.tsindex 0fb7b7c7e..41517e30b 100644--- a/apps/api/src/lib/bigquery-jobs.ts+++ b/apps/api/src/lib/bigquery-jobs.ts@@ -345,26 +345,26 @@ export async function saveJobToBigQuery(/*** Queries jobs from BigQuery*/-export async function queryJobsFromBigQuery(- query: string,- params?: any[]-): Promise<any[]> {- if (!bigquery || !process.env.BIGQUERY_DATASET_ID) {- logger.warn("BigQuery not configured");- return [];- }-- try {- const options = {- query,- params,- location: process.env.BIGQUERY_LOCATION || "US",- };-- const [rows] = await bigquery.query(options);- return rows;- } catch (error) {- logger.error("Error querying jobs from BigQuery", { error, query });- throw error;- }-}+// export async function queryJobsFromBigQuery(+// query: string,+// params?: any[]+// ): Promise<any[]> {+// if (!bigquery || !process.env.BIGQUERY_DATASET_ID) {+// logger.warn("BigQuery not configured");+// return [];+// }++// try {+// const options = {+// query,+// params,+// location: process.env.BIGQUERY_LOCATION || "US",+// };++// const [rows] = await bigquery.query(options);+// return rows;+// } catch (error) {+// logger.error("Error querying jobs from BigQuery", { error, query });+// throw error;+// }+// }diff --git a/apps/api/src/lib/job-transform.ts b/apps/api/src/lib/job-transform.tsindex 38746ef32..fac5a8f02 100644--- a/apps/api/src/lib/job-transform.ts+++ b/apps/api/src/lib/job-transform.ts@@ -14,7 +14,7 @@ function cleanOfNull<T>(x: T): T {}}-export interface TransformOptions {+interface TransformOptions {/** Whether to include timestamp field (for BigQuery) */includeTimestamp?: boolean;/** Whether to serialize objects to JSON strings (for BigQuery) */@@ -28,7 +28,7 @@ export interface TransformOptions {*/export function transformJobForLogging(job: FirecrawlJob,- options: TransformOptions = {}+ options: TransformOptions = {},) {const {includeTimestamp = false,@@ -39,16 +39,22 @@ export function transformJobForLogging(const zeroDataRetention = job.zeroDataRetention ?? false;// Determine if docs should be included based on zero data retention and GCS usage- const shouldIncludeDocs = !zeroDataRetention &&- !((job.mode === "single_urls" || job.mode === "scrape") && process.env.GCS_BUCKET_NAME);+ const shouldIncludeDocs =+ !zeroDataRetention &&+ !(+ (job.mode === "single_urls" || job.mode === "scrape") &&+ process.env.GCS_BUCKET_NAME+ );const baseTransform = {job_id: job.job_id ? job.job_id : null,success: job.success,message: zeroDataRetention ? null : job.message,num_docs: job.num_docs,- docs: shouldIncludeDocs- ? (cleanNullValues ? cleanOfNull(job.docs) : job.docs)+ docs: shouldIncludeDocs+ ? cleanNullValues+ ? cleanOfNull(job.docs)+ : job.docs: null,time_taken: job.time_taken,team_id:@@ -56,9 +62,7 @@ export function transformJobForLogging(? null: job.team_id,mode: job.mode,- url: zeroDataRetention- ? "<redacted due to zero data retention>"- : job.url,+ url: zeroDataRetention ? "<redacted due to zero data retention>" : job.url,crawler_options: zeroDataRetention ? null : job.crawlerOptions,page_options: zeroDataRetention ? null : job.scrapeOptions,origin: zeroDataRetention ? null : job.origin,@@ -90,14 +94,14 @@ export function transformJobForLogging(return {...withTimestamp,docs: withTimestamp.docs ? JSON.stringify(withTimestamp.docs) : null,- crawler_options: withTimestamp.crawler_options- ? JSON.stringify(withTimestamp.crawler_options)+ crawler_options: withTimestamp.crawler_options+ ? JSON.stringify(withTimestamp.crawler_options): null,- page_options: withTimestamp.page_options- ? JSON.stringify(withTimestamp.page_options)+ page_options: withTimestamp.page_options+ ? JSON.stringify(withTimestamp.page_options): null,- cost_tracking: withTimestamp.cost_tracking- ? JSON.stringify(withTimestamp.cost_tracking)+ cost_tracking: withTimestamp.cost_tracking+ ? JSON.stringify(withTimestamp.cost_tracking): null,};}diff --git a/apps/api/src/otel.ts b/apps/api/src/otel.tsnew file mode 100644index 000000000..ed3e608af--- /dev/null+++ b/apps/api/src/otel.ts@@ -0,0 +1,14 @@+import { getNodeAutoInstrumentations } from "@opentelemetry/auto-instrumentations-node";+import { NodeSDK } from "@opentelemetry/sdk-node";+import { OTLPTraceExporter } from "@opentelemetry/exporter-trace-otlp-http";++const otelSdk = new NodeSDK({+ traceExporter: new OTLPTraceExporter(),+ instrumentations: [getNodeAutoInstrumentations()],+});++otelSdk.start();++export const shutdownOtel = async () => {+ await otelSdk.shutdown();+};diff --git a/apps/api/src/services/queue-worker.ts b/apps/api/src/services/queue-worker.tsindex e128d7ed1..7550f419b 100644--- a/apps/api/src/services/queue-worker.ts+++ b/apps/api/src/services/queue-worker.ts@@ -1,4 +1,5 @@import "dotenv/config";+import { shutdownOtel } from "../otel";import "./sentry";import * as Sentry from "@sentry/node";import {@@ -28,9 +29,6 @@ import http from "http";import https from "https";import { cacheableLookup } from "../scraper/scrapeURL/lib/cacheableLookup";import { robustFetch } from "../scraper/scrapeURL/lib/fetch";-import { NodeSDK } from "@opentelemetry/sdk-node";-import { getNodeAutoInstrumentations } from "@opentelemetry/auto-instrumentations-node";-import { OTLPTraceExporter } from "@opentelemetry/exporter-trace-otlp-http";import { BullMQOtel } from "bullmq-otel";import { getErrorContactMessage } from "../lib/deployment";import { initializeBlocklist } from "../scraper/WebScraper/utils/blocklist";@@ -56,13 +54,6 @@ const runningJobs: Set<string> = new Set();cacheableLookup.install(http.globalAgent);cacheableLookup.install(https.globalAgent);-const otelSdk = new NodeSDK({- traceExporter: new OTLPTraceExporter(),- instrumentations: [getNodeAutoInstrumentations()],-});--otelSdk.start();-const processExtractJobInternal = async (token: string,job: Job & { id: string },@@ -488,6 +479,6 @@ app.listen(workerPort, () => {}console.log("All jobs finished. Worker out!");- await otelSdk.shutdown();+ await shutdownOtel();process.exit(0);})();diff --git a/apps/api/src/services/worker/scrape-worker.ts b/apps/api/src/services/worker/scrape-worker.tsindex 536a760c1..17bcd4bea 100644--- a/apps/api/src/services/worker/scrape-worker.ts+++ b/apps/api/src/services/worker/scrape-worker.ts@@ -50,8 +50,6 @@ import { calculateCreditsToBeBilled } from "../../lib/scrape-billing";import { getBillingQueue } from "../queue-service";import type { Logger } from "winston";import { finishCrawlIfNeeded } from "./crawl-logic";-import { getNodeAutoInstrumentations } from "@opentelemetry/auto-instrumentations-node";-import { NodeSDK } from "@opentelemetry/sdk-node";import {RacedRedirectError,ScrapeJobTimeoutError,@@ -59,7 +57,6 @@ import {UnknownError,} from "../../lib/error";import { serializeTransportableError } from "../../lib/error-serde";-import { OTLPTraceExporter } from "@opentelemetry/exporter-trace-otlp-http";import type { NuQJob } from "./nuq";import {ScrapeJobData,@@ -68,7 +65,7 @@ import {ScrapeJobSingleUrls,} from "../../types";import { scrapeSitemap } from "../../scraper/crawler/sitemap";-import { filterUrl } from "@mendable/firecrawl-rs";+import { shutdownOtel } from "../../otel";const sleep = (ms: number) => new Promise(resolve => setTimeout(resolve, ms));@@ -1140,15 +1137,8 @@ export const processJobInternal = async (job: NuQJob<ScrapeJobData>) => {}};-const otelSdk = new NodeSDK({- traceExporter: new OTLPTraceExporter(),- instrumentations: [getNodeAutoInstrumentations()],-});--otelSdk.start();-const exitHandler = () => {- otelSdk.shutdown().finally(() => {+ shutdownOtel().finally(() => {_logger.debug("OTEL shutdown");process.exit(0);});