Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(scrape-worker): reintroduce OTEL for accurate LLM cost tracking (#1943)
Gergő Móriczae6a0524d2db
diff --git a/apps/api/src/services/worker/scrape-worker.ts b/apps/api/src/services/worker/scrape-worker.tsindex f678e16c2..84f80c9be 100644--- a/apps/api/src/services/worker/scrape-worker.ts+++ b/apps/api/src/services/worker/scrape-worker.ts@@ -793,15 +793,6 @@ export const processJobInternal = async (job: Job & { id: string }) => {crawlId: job.data?.crawl_id ?? undefined,zeroDataRetention: job.data?.zeroDataRetention ?? false,});-- // const langfuseOtel = process.env.LANGFUSE_PUBLIC_KEY ? new NodeSDK({- // traceExporter: new LangfuseExporter(),- // instrumentations: [getNodeAutoInstrumentations()],- // }) : null;-- // if (langfuseOtel) {- // langfuseOtel.start();- // }try {try {@@ -855,13 +846,29 @@ export const processJobInternal = async (job: Job & { id: string }) => {logger.debug("Job failed", { error });Sentry.captureException(error);throw error;- } finally {- // if (langfuseOtel) {- // langfuseOtel.shutdown().then(() => {- // logger.debug("Langfuse OTEL shutdown");- // });- // }}};-module.exports = processJobInternal;\ No newline at end of file+const langfuseOtel = process.env.LANGFUSE_PUBLIC_KEY ? new NodeSDK({+ traceExporter: new LangfuseExporter(),+ instrumentations: [getNodeAutoInstrumentations()],+}) : null;++if (langfuseOtel) {+ langfuseOtel.start();+}++module.exports = processJobInternal;++const exitHandler = () => {+ if (langfuseOtel) {+ langfuseOtel.shutdown().then(() => {+ _logger.debug("Langfuse OTEL shutdown");+ process.exit(0);+ });+ }+}++process.on("SIGINT", exitHandler);+process.on("SIGTERM", exitHandler);+process.on("exit", exitHandler);\ No newline at end of file