Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(api): replace Sentry with optional OpenTelemetry OTLP export (#4555)
Gergő Móriczbca56457b0d8
diff --git a/apps/api/.env.example b/apps/api/.env.exampleindex 35ef78bb1..5905d11f7 100644--- a/apps/api/.env.example+++ b/apps/api/.env.example@@ -153,3 +153,9 @@ CDP_API_KEY_SECRET=""## ENDPOINT PRICING TEST $0.01X402_ENDPOINT_PRICE_USD=0.01++# === OpenTelemetry (optional) ===+# Set an OTLP endpoint to export traces (http/protobuf, 100% sampled). Spans from+# zero-data-retention requests are never exported.+# OTEL_EXPORTER_OTLP_ENDPOINT=http://otel-collector:4318+# OTEL_SERVICE_NAME=firecrawl-apidiff --git a/apps/api/.gitignore b/apps/api/.gitignoreindex 1cc479f83..d953aacec 100644--- a/apps/api/.gitignore+++ b/apps/api/.gitignore@@ -8,7 +8,6 @@ dump.rdb/.next/.rdb-.sentryclirc.env.*diff --git a/apps/api/package.json b/apps/api/package.jsonindex a0240f4da..8a1a7e16a 100644--- a/apps/api/package.json+++ b/apps/api/package.json@@ -14,7 +14,6 @@"start:dev": "tsc-watch --compiler ./node_modules/typescript-7/bin/tsc --onSuccess \"node dist/src/index.js\"","build": "tsc","branding:print": "node src/scraper/scrapeURL/engines/fire-engine/branding-script/print-script.js",- "build:nosentry": "tsc","test": "vitest run --exclude='**/__tests__/e2e_noAuth/**'","test:local-no-auth": "vitest run --exclude='**/__tests__/e2e_withAuth/**'","test:full": "vitest run --exclude='**/__tests__/e2e_noAuth/**' --exclude='**/__tests__/e2e_withAuth/**'",@@ -41,14 +40,12 @@"zdr-worker:production": "node dist/src/services/zdr-worker.js","mongo-docker": "docker run -d -p 2717:27017 -v ./mongo-data:/data/db --name mongodb mongo:latest","mongo-docker-console": "docker exec -it mongodb mongosh",- "sentry:sourcemaps": "sentry-cli sourcemaps inject --org caleb-peffer --project firecrawl-scraper-js ./dist && sentry-cli sourcemaps upload --org caleb-peffer --project firecrawl-scraper-js ./dist","prepare": "cd ../.. && husky ./apps/api/.husky","knip": "knip"},"author": "","license": "ISC","devDependencies": {- "@sentry/core": "^10.67.0","@types/amqplib": "^0.10.7","@types/body-parser": "^1.19.2","@types/cors": "^2.8.13",@@ -91,8 +88,11 @@"@google-cloud/storage": "^7.21.0","@mendable/firecrawl-rs": "workspace:*","@openrouter/ai-sdk-provider": "^2.9.1",- "@sentry/cli": "^2.58.2",- "@sentry/node": "^10.67.0",+ "@opentelemetry/api": "^1.9.1",+ "@opentelemetry/exporter-trace-otlp-proto": "^0.222.0",+ "@opentelemetry/resources": "^2.11.0",+ "@opentelemetry/sdk-trace-node": "^2.11.0",+ "@opentelemetry/semantic-conventions": "^1.43.0","@types/ws": "^8.5.12","ai": "6.0.86","ajv": "^8.18.0",diff --git a/apps/api/pnpm-lock.yaml b/apps/api/pnpm-lock.yamlindex 4afe58716..4b37e5fbc 100644--- a/apps/api/pnpm-lock.yaml+++ b/apps/api/pnpm-lock.yaml@@ -105,12 +105,21 @@ importers:'@openrouter/ai-sdk-provider':specifier: ^2.9.1- '@sentry/cli':- specifier: ^2.58.2- version: 2.58.2- '@sentry/node':- specifier: ^10.67.0- version: 10.67.0(@opentelemetry/[email protected](@opentelemetry/[email protected]))+ '@opentelemetry/api':+ specifier: ^1.9.1+ version: 1.9.1+ '@opentelemetry/exporter-trace-otlp-proto':+ specifier: ^0.222.0+ version: 0.222.0(@opentelemetry/[email protected])+ '@opentelemetry/resources':+ specifier: ^2.11.0+ version: 2.11.0(@opentelemetry/[email protected])+ '@opentelemetry/sdk-trace-node':+ specifier: ^2.11.0+ version: 2.11.0(@opentelemetry/[email protected])+ '@opentelemetry/semantic-conventions':+ specifier: ^1.43.0+ version: 1.43.0'@types/ws':specifier: ^8.5.12version: 8.5.12@@ -206,7 +215,7 @@ importers:version: 2.9.0langsmith:specifier: ^0.6.0- version: 0.6.3(@opentelemetry/[email protected])(@opentelemetry/[email protected](@opentelemetry/[email protected]))([email protected])+ version: 0.6.3(@opentelemetry/[email protected])(@opentelemetry/[email protected](@opentelemetry/[email protected]))(@opentelemetry/[email protected](@opentelemetry/[email protected]))([email protected])lodash:specifier: ^4.18.0version: 4.18.1@@ -289,9 +298,6 @@ importers:specifier: 4.1.12version: 4.1.12devDependencies:- '@sentry/core':- specifier: ^10.67.0- version: 10.67.0'@types/amqplib':specifier: ^0.10.7version: 0.10.7@@ -514,17 +520,6 @@ packages:peerDependencies:zod: ^3.25.76 || ^4.1.8- '@apm-js-collab/[email protected]':- resolution: {integrity: sha512-Yidf5GOl60db80UxUtNdKK3pnY7obU/gs0xOfA0SCdnvVLMCvfYIer/egC3TqpPiT0Jg22eg3RlzcO+zKfPMcA==}- engines: {node: '>=18.0.0'}-- '@apm-js-collab/[email protected]':- resolution: {integrity: sha512-aN3Oq8r1J3gPJtCwErP664gM0+HhM1I1lujPr9TMTCcEl/joQQbpGpeMdts9B1+W2wHMsvioDMv5F4PvMWE6gw==}- hasBin: true-- '@apm-js-collab/[email protected]':- resolution: {integrity: sha512-mTvWz9rnQwx1U3h0XPTHaX7bgfkpipLLTQyjlC2cdhQpQEuoLT0AGzoydeoq2NxfEVv6fWOOETcSbb2nptleyw==}-'@asamuzakjp/[email protected]':resolution: {integrity: sha512-KVw6qIiCTUQhByfTd78h2yD1/00waTmm9uy/R7Ck/ctUyAPj+AEDLkQIdJW0T8+qGgj3j5bpNKK7Q3G+LedJWg==}engines: {node: ^20.19.0 || ^22.12.0 || >=24.0.0}@@ -1510,8 +1505,8 @@ packages:ai: ^6.0.0zod: ^3.25.0 || ^4.0.0- '@opentelemetry/[email protected]':- resolution: {integrity: sha512-CmVa4ImJ+ynfrPMNaAXHET6Bhb44SwzmfyVJFq9ni2jgXJR/l7C6gfVFddNmHP+ZOkP9cf4f9DBe68qVLTHc9w==}+ '@opentelemetry/[email protected]':+ resolution: {integrity: sha512-9mb1If+IF6u0ZVXkHQ6ogEae5HwA6ajIVUgpSDQyRASxft6BSXHvBvPooRle3yFN/fKnCdSOnuu0OC3PLcF6+g==}engines: {node: '>=8.0.0'}'@opentelemetry/[email protected]':@@ -1522,38 +1517,86 @@ packages:resolution: {integrity: sha512-gLyJlPHPZYdAk1JENA9LeHejZe1Ti77/pTeFm/nMXmQH/HFZlcS/O2XJB+L8fkbrNSqhdtlvjBVjxwUYanNH5Q==}engines: {node: '>=8.0.0'}+ '@opentelemetry/[email protected]':+ resolution: {integrity: sha512-Tr79DyWI8itsBdg+jH+opjfrwLzX+erk1/ExkIwhWoAVjVrJIn2y5+cGjTC0Vy8fyNIA/y8wuJPZwr1T3xCZeQ==}+ engines: {node: ^18.19.0 || >=20.6.0}+ peerDependencies:+ '@opentelemetry/api': '>=1.0.0 <1.10.0'++ '@opentelemetry/[email protected]':+ resolution: {integrity: sha512-7YP44XH0tV6+Mb54x2YGf84i7yi+31MBZlE8JwvozkxyTvXbSp10X7cI7YE49ChJ3shMJoBmCJF3+1QFBJctGA==}+ engines: {node: ^18.19.0 || >=20.6.0}+ peerDependencies:+ '@opentelemetry/api': '>=1.0.0 <1.10.0'+'@opentelemetry/[email protected]':resolution: {integrity: sha512-m2nckMT80NnmjTYSPjJQObBJ+8dgkoajEOUbznL8AHZ3T3yHRk2P7gI1PhEBc1+lOnrYE9UWrWHqJDsmqjmNbw==}engines: {node: ^18.19.0 || >=20.6.0}peerDependencies:'@opentelemetry/api': '>=1.0.0 <1.10.0'- '@opentelemetry/[email protected]':- resolution: {integrity: sha512-xQx3E2WxP1mDvKzxLxX+CTCtNLa560YJZ3087qYHerl2YmiKpv7AH+dAy7vmx+eVrZ5BwhfWUAVoKOoxCNHcpw==}+ '@opentelemetry/[email protected]':+ resolution: {integrity: sha512-imLrLzNhnABJt3tUyT63h6MqfZrG3Gdv3TE7+OUQTvwD088RfCUx3rF01lcgfu2KXBFk13vwi0yKZLvFEalkIA==}+ engines: {node: ^18.19.0 || >=20.6.0}+ peerDependencies:+ '@opentelemetry/api': ^1.3.0++ '@opentelemetry/[email protected]':+ resolution: {integrity: sha512-YbywG3veEm2Fb6TbdxRkuquWob6eVWXuA8/Ba1tXz9jHfUqpdE3keilOHEtPboC4CvS1bjeeVfNkWGOOrLj+lw==}+ engines: {node: ^18.19.0 || >=20.6.0}+ peerDependencies:+ '@opentelemetry/api': ^1.3.0++ '@opentelemetry/[email protected]':+ resolution: {integrity: sha512-/F3BZ89+CJQnZkMh2tCrtcdB+XT2Dxhj4FFE+WPQ//413hmFL0/RfEX6vgOIWGhiSzrkHWTK3+6SiT7K5/g/jQ==}engines: {node: ^18.19.0 || >=20.6.0}peerDependencies:'@opentelemetry/api': ^1.3.0+ '@opentelemetry/[email protected]':+ resolution: {integrity: sha512-Ie7+8q8MDF4FAEQCKVMTx3ReUvxiIAgIiiW3c9JdmP8+HMcDy20puT+AHjexnExgnbvBxjQ9fjkFDWrikJ2jQA==}+ engines: {node: ^18.19.0 || >=20.6.0}+ peerDependencies:+ '@opentelemetry/api': '>=1.3.0 <1.10.0'+'@opentelemetry/[email protected]':resolution: {integrity: sha512-jyA5MBLQ+Dkl3+JsZkUoUvL7yHvU64kLsvpXKarWm6347Sl1t1bXFTFykUePNpT5WH5pm9a2Qtt03iIYQhZ1Fg==}engines: {node: ^18.19.0 || >=20.6.0}peerDependencies:'@opentelemetry/api': '>=1.3.0 <1.10.0'+ '@opentelemetry/[email protected]':+ resolution: {integrity: sha512-+19YHODIjaUCArxleaJtuufFZVpz/xvvK+VllQqE+W8hHolxdoRwHfK/s667zezwh1hkx6FFF+oYzetYgqK+Bg==}+ engines: {node: ^18.19.0 || >=20.6.0}+ peerDependencies:+ '@opentelemetry/api': '>=1.4.0 <1.10.0'++ '@opentelemetry/[email protected]':+ resolution: {integrity: sha512-7GXXcObyHyDUUSG+L+kJoquty01bzm7ivE7+SSgXXJcHuPzGviptxwARmI2c+bnnxjexGQbJnyNlN8HxBP/Y7A==}+ engines: {node: ^18.19.0 || >=20.6.0}+ peerDependencies:+ '@opentelemetry/api': '>=1.9.0 <1.10.0'+'@opentelemetry/[email protected]':resolution: {integrity: sha512-Xx8RGS4H5XEBl01WuCreMIpiah9cCXMbSkeuIePPdD2cUpq/vUzYmj8E/MK1OsbOc93FuAD4jfn2WOacKwLn7Q==}engines: {node: ^18.19.0 || >=20.6.0}peerDependencies:'@opentelemetry/api': '>=1.9.0 <1.10.0'- '@opentelemetry/[email protected]':- resolution: {integrity: sha512-cp9zmTl62R8PJrpvFcmc8N2JQU/xfa0S+61q511Nji+QxCfZ8Ifvg7H27G8cANe4crg4RTrWsVvanHiXjSp6ag==}+ '@opentelemetry/[email protected]':+ resolution: {integrity: sha512-H19x/TX/LZdqiYOjM7fqtSxwlplC5pgelavqbQdHbhdq0q/AI/TGkM2dfGuuynTXmJPeF2HoZVoPDu+TGoW78A==}engines: {node: ^18.19.0 || >=20.6.0}peerDependencies:'@opentelemetry/api': '>=1.3.0 <1.10.0'- '@opentelemetry/[email protected]':- resolution: {integrity: sha512-sGA19HvtrrSKYsseHphluH6j3p6Xa3fqc7c7y8f/7mYWejc1lyDFcpSdD1kYa50HCLUeEo4zA5bW0pniaPszuw==}+ '@opentelemetry/[email protected]':+ resolution: {integrity: sha512-CuvCMJmZxswhNLlM2LfuLOW3h3fZujA4hsG4B+Sz4dX2zvaXO8Ng74cnDHWD64gLszTlhiG3c0iNUjj4g+0/sA==}+ engines: {node: ^18.19.0 || >=20.6.0}+ peerDependencies:+ '@opentelemetry/api': '>=1.0.0 <1.10.0'++ '@opentelemetry/[email protected]':+ resolution: {integrity: sha512-fFnTqGm8/G73GQVnxYi7LXa1ZVYEUvgL6XI1LpvV0bPC7WQ/ZGgKxCSl8FnlZBKto9JHHEFTO6s6CUpvvtwFrA==}engines: {node: ^18.19.0 || >=20.6.0}peerDependencies:'@opentelemetry/api': '>=1.3.0 <1.10.0'@@ -1858,103 +1901,6 @@ packages:'@selderee/[email protected]':resolution: {integrity: sha512-P33hHGdldxGabLFjPPpaTxVolMrzrcegejx+0GxjrIb9Zv48D8yAIA/QTDR2dFl7Uz7urX8aX6+5bCZslr+gWQ==}- '@sentry/[email protected]':- resolution: {integrity: sha512-MArsb3zLhA2/cbd4rTm09SmTpnEuZCoZOpuZYkrpDw1qzBVJmRFA1W1hGAQ9puzBIk/ubY3EUhhzuU3zN2uD6w==}- engines: {node: '>=10'}- os: [darwin]-- '@sentry/[email protected]':- resolution: {integrity: sha512-ay3OeObnbbPrt45cjeUyQjsx5ain1laj1tRszWj37NkKu55NZSp4QCg1gGBZ0gBGhckI9nInEsmKtix00alw2g==}- engines: {node: '>=10'}- cpu: [arm64]- os: [linux, freebsd, android]-- '@sentry/[email protected]':- resolution: {integrity: sha512-HU9lTCzcHqCz/7Mt5n+cv+nFuJdc1hGD2h35Uo92GgxX3/IujNvOUfF+nMX9j6BXH6hUt73R5c0Ycq9+a3Parg==}- engines: {node: '>=10'}- cpu: [arm]- os: [linux, freebsd, android]-- '@sentry/[email protected]':- resolution: {integrity: sha512-CN9p0nfDFsAT1tTGBbzOUGkIllwS3hygOUyTK7LIm9z+UHw5uNgNVqdM/3Vg+02ymjkjISNB3/+mqEM5osGXdA==}- engines: {node: '>=10'}- cpu: [x86, ia32]- os: [linux, freebsd, android]-- '@sentry/[email protected]':- resolution: {integrity: sha512-oX/LLfvWaJO50oBVOn4ZvG2SDWPq0MN8SV9eg5tt2nviq+Ryltfr7Rtoo+HfV+eyOlx1/ZXhq9Wm7OT3cQuz+A==}- engines: {node: '>=10'}- cpu: [x64]- os: [linux, freebsd, android]-- '@sentry/[email protected]':- resolution: {integrity: sha512-+cl3x2HPVMpoSVGVM1IDWlAEREZrrVQj4xBb0TRKII7g3hUxRsAIcsrr7+tSkie++0FuH4go/b5fGAv51OEF3w==}- engines: {node: '>=10'}- cpu: [arm64]- os: [win32]-- '@sentry/[email protected]':- resolution: {integrity: sha512-omFVr0FhzJ8oTJSg1Kf+gjLgzpYklY0XPfLxZ5iiMiYUKwF5uo1RJRdkUOiEAv0IqpUKnmKcmVCLaDxsWclB7Q==}- engines: {node: '>=10'}- cpu: [x86, ia32]- os: [win32]-- '@sentry/[email protected]':- resolution: {integrity: sha512-2NAFs9UxVbRztQbgJSP5i8TB9eJQ7xraciwj/93djrSMHSEbJ0vC47TME0iifgvhlHMs5vqETOKJtfbbpQAQFA==}- engines: {node: '>=10'}- cpu: [x64]- os: [win32]-- '@sentry/[email protected]':- resolution: {integrity: sha512-U4u62V4vaTWF+o40Mih8aOpQKqKUbZQt9A3LorIJwaE3tO3XFLRI70eWtW2se1Qmy0RZ74zB14nYcFNFl2t4Rw==}- engines: {node: '>= 10'}- hasBin: true-- '@sentry/[email protected]':- resolution: {integrity: sha512-fO9PLmHdVURcSPUpWCItWAtgKiMwGdJHbovoSEyLplX5sxs2ugvI4CBPTrkkgqhObnZOD0CnWBKDzSVQYBKEyQ==}- engines: {node: '>=14'}-- '@sentry/[email protected]':- resolution: {integrity: sha512-b6U3pJ8AUvN9aouq0vl+VZI8KT8RslBsfGMFuNwRr313zOmdmFJBZqTiUw9VGgJ2jGKxLO9alm9rlxBfX4hf+w==}- engines: {node: '>=18'}-- '@sentry/[email protected]':- resolution: {integrity: sha512-dBHHRwZyan1pOnFJ+sNBvR8TkXbZAfZU/jpxmALS3JZ2/8AGR7cQKL+b7SleKuJ7iUDZyklN3Nqi0i5JkcA+HA==}- engines: {node: '>=18'}- peerDependencies:- '@opentelemetry/api': ^1.9.0- '@opentelemetry/core': 2.9.0- '@opentelemetry/exporter-trace-otlp-http': '>=0.57.0 <1'- '@opentelemetry/instrumentation': '>=0.57.1 <1'- '@opentelemetry/sdk-trace-base': ^1.30.1 || ^2.1.0- peerDependenciesMeta:- '@opentelemetry/api':- optional: true- '@opentelemetry/core':- optional: true- '@opentelemetry/exporter-trace-otlp-http':- optional: true- '@opentelemetry/instrumentation':- optional: true- '@opentelemetry/sdk-trace-base':- optional: true-- '@sentry/[email protected]':- resolution: {integrity: sha512-SFKpZGqOCEFSmP93NdDP6ikZp4NS7A/JR8+2ofK3jF6Y9Vyox7pX0pxdOnPLpFcPtMybMahfWqSAWJvsFs4RmA==}- engines: {node: '>=18'}-- '@sentry/[email protected]':- resolution: {integrity: sha512-oLTOrAK1rOqmYRktOJZwz37B1seXPx1W2FTMVtzTVNjMFA/LZwGzePeZzhUOgzZgfLHixMd/ceWtGqoxAndcjQ==}- engines: {node: '>=18'}- peerDependencies:- '@opentelemetry/api': ^1.9.0- '@opentelemetry/core': 2.9.0- '@opentelemetry/sdk-trace-base': ^1.30.1 || ^2.1.0-- '@sentry/[email protected]':- resolution: {integrity: sha512-GQ9t+RSTx5s3b/aZrLFuL4nrwPLMah5NiZk5cjxJmgmOSgm3nMdO8gdqCedDch5B13F3YsxtaQYjSJnzLz8M5A==}- engines: {node: '>=18'}-'@standard-schema/[email protected]':resolution: {integrity: sha512-l2aFy5jALhniG5HgqrD6jXLi/rUWrKvqN/qJx6yoJsgKhblVd+iqqU4RCXavm/jPityDo5TCvKMnpjKnOriy0w==}@@ -2315,10 +2261,6 @@ packages:resolution: {integrity: sha512-Izi8RQcffqCeNVgFigKli1ssklIbpHnCYc6AknXGYoB6grJqyeby7jv12JUQgmTAnIDnbck1uxksT4dzN3PWBA==}engines: {node: '>=12'}- resolution: {integrity: sha512-LElXdjswlqjWrPpJFg1Fx4wpkOCxj1TDHlSV4PlaRxHGWko024xICaa97ZkMfs6DRKlCguiAI+rbXv5GWwXIkg==}- hasBin: true-resolution: {integrity: sha512-1A94B18jkJ3DYq284ohPxoXbfTA5HsQ7/Mf4DEhcyLx3Bz27Rh59iScbB6EPiP+B+joue6YCxcMXSbFC1tZKwA==}@@ -2455,9 +2397,6 @@ packages:resolution: {integrity: sha512-VqR8m68vM46BNnuZ5NtnGBKIE/DfN0cRIzg9n40EIq9NOv90ayxLBXA8fXC5gquFRGJSTRqBq25Jt2ECLR431Q==}engines: {node: '>= 6'}- resolution: {integrity: sha512-4bHTS2YuzUvtoLjdy+98ykbNB5jS0+07EvFNXerqZQJ89F7DI6ET7OQo/HJuW6K0aVsKA9hj9/RVb2kQVOrPDQ==}-resolution: {integrity: sha512-aCj4O5wKyszjMmDT4tZj93kxyydN/K5zPWSCe6/0AV/AA1pqe5ZBIw0a2ZfPQV7lL5/yb5HsUreJ6UFAF1tEQw==}engines: {node: '>=18'}@@ -2874,9 +2813,6 @@ packages:resolution: {integrity: sha512-n27zTYMjYu1aj4MjCWzSP7G9r75utsaoc8m61weK+W8JMBGGQybd43GstCXZ3WNmSFtGT9wi59qQTW6mhTR5LQ==}- resolution: {integrity: sha512-shc1dbU90Yl/xq1QrC7QRtfcwURZuVRfPhZbDoldJ1cn1gzDvBaBWlv0eFolj5+0znnPJz5TXLxsN77X/12KTA==}-resolution: {integrity: sha512-FGgH2h8zKNim9ljj7dankFPcICIK9Cp5bm+c2gQSYePhpaG5+esrLODihIorn+Pe6FGJzWhXQotPv73jTaldXA==}engines: {node: '>= 0.4'}@@ -2904,14 +2840,6 @@ packages:resolution: {integrity: sha512-TtpcNJ3XAzx3Gq8sWRzJaVajRs0uVxA2YAkdb1jm2YkPz4G6egUFAyA3n5vtEIZefPk5Wa4UXbKuS5fKkJWdgA==}engines: {node: '>=10'}- resolution: {integrity: sha512-Ap6G0WQwcU/LHsvLwON1fAQX9Zp0A2Y6Y/cJBl9r/JbW90Zyg4/zbG6zzKa2OTALELarYHmKu0GhpM5EO+7T0g==}- engines: {node: '>=0.10'}-- resolution: {integrity: sha512-MMdARuVEQziNTeJD8DgMqmhwR11BRQ/cBP+pLtYdSTnf3MIO8fFeiINEbX36ZdNlfU/7A9f3gUw49B3oQsvwBA==}- engines: {node: '>=4.0'}-resolution: {integrity: sha512-7RUKfXgSMMkzt6ZuXmqapOurLGPPfgj6l9uRZ7lRGolvk0y2yocc35LdcxKC5PQZdn2DMqioAQ2NoWcrTKmm6g==}@@ -3287,10 +3215,6 @@ packages:resolution: {integrity: sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==}- resolution: {integrity: sha512-jTd2FfOgOWOdgjkHuk/1Ms8VKFXkPs15ymYBETw1sAOrO/dY3XeGVRWir9qBbw7pXr0T2eTFwfCZ+N02HmiNGA==}- engines: {node: '>=18'}-resolution: {integrity: sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==}@@ -3660,10 +3584,6 @@ packages:resolution: {integrity: sha512-8q7VEgMJW4J8tcfVPy8g09NcQwZdbwFEqhe/WZkoIzjn/3TGDwtOCYtXGxA3O8tPzpczCCDgv+P2P5y00ZJOOg==}engines: {node: '>= 8'}- resolution: {integrity: sha512-Sz8FzjzI0kN13GK/6MVEsVzMZEPvOhnmmI1lU5+/1cGOiK3QUahntrNNtdVeihrO7t9JpoH75iMNXg6R6uWflQ==}- engines: {node: '>=18.0.0'}-resolution: {integrity: sha512-iclAHeNqNm68zFtnZ0e+1L2yUIdvzNoauKU4WBA3VvH/vPFieF7qfRlwUZU+DA9P9bPXIS90ulxoUoCH23sV2w==}engines: {node: '>= 0.6'}@@ -3713,9 +3633,6 @@ packages:resolution: {integrity: sha512-qOOzS1cBTWYF4BH8fVePDBOO9iptMnGUEZwNc/cMWnTV2nVLZ7VoNWEPHkYczZA0pdoA7dl6e7FL659nX9S2aw==}engines: {node: '>=16 || 14 >=14.17'}- resolution: {integrity: sha512-EGWKgxALGMgzvxYF1UyGTy0HXX/2vHLkw6+NvDKW2jypWbHpjQuj4UMcqQWXHERJhVGKikolT06G3bcKe4fi7w==}-resolution: {integrity: sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==}@@ -4027,10 +3944,6 @@ packages:resolution: {integrity: sha512-cdGef/drWFoydD1JsMzuFf8100nZl+GT+yacc2bEced5f9Rjk4z+WtFUTBu9PhOi9j/jfmBPu0mMEY4wIdAF8A==}engines: {node: '>= 0.6.0'}- resolution: {integrity: sha512-7PiHtLll5LdnKIMw100I+8xJXR5gW2QwWYkT6iJva0bXitZKa/XMrSbdmg3r2Xnaidz9Qumd0VPaMrZlF9V9sA==}- engines: {node: '>=0.4.0'}-resolution: {integrity: sha512-6ZiOBfCywsD4k1BN9IX0uZhF+tJkV8q8llP64G5Hajs4JOeVLPCwpPVcpXy3BwYiUGgyJzsJJQeOIv7+hDSq8g==}engines: {node: ^16 || ^18 || >=20}@@ -4051,9 +3964,6 @@ packages:resolution: {integrity: sha512-llQsMLSUDUPT44jdrU/O37qlnifitDP+ZwrmmZcoSKyLKvtZxpyV0n2/bD/N4tBAAZ/gJEdZU7KMraoK1+XYAg==}engines: {node: '>= 0.10'}- resolution: {integrity: sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg==}-resolution: {integrity: sha512-cJ+oHTW1VAEa8cJslgmUZrc+sjRKgAKl3Zyse6+PV38hZe/V6Z14TbCuXcan9F9ghlz4QrFr2c92TNF82UkYHA==}engines: {node: '>=10'}@@ -4152,10 +4062,6 @@ packages:resolution: {integrity: sha512-Xf0nWe6RseziFMu+Ap9biiUbmplq6S9/p+7w7YXP/JBHhrUDDUhwa+vANyubuqfZWTveU//DYVGsDG7RKL/vEw==}engines: {node: '>=0.10.0'}- resolution: {integrity: sha512-QT7FVMXfWOYFbeRBF6nu+I6tr2Tf3u0q8RIEjNob/heKY/nh7drD/k7eeMFmSQgnTtCzLDcCu/XEnpW2wk4xCQ==}- engines: {node: '>=9.3.0 || >=8.10.0 <9.0.0'}-resolution: {integrity: sha512-KigOCHcocU3XODJxsu8i/j8T9tzT4adHiecwORRQ0ZZFcp7ahwXuRU1m+yuO90C5ZUyGeGfocHDI14M3L3yDAQ==}@@ -4239,9 +4145,6 @@ packages:resolution: {integrity: sha512-5TF+l7p4+OsnP8BCCvSyZiSPc4x4//p5uPwK8TCnVPJYRmU2aYKMpOXvw8zM5a5JvuuCGN1jmsMwuU2W02ukfA==}- resolution: {integrity: sha512-xXR3KGeoxTNWPD4aBvL5NUpMTT7WMANr3EWnaS190QVkY52lqqcVRD7Q05UVbBhiWDGWMlJEUam9m7uFFGVScw==}-resolution: {integrity: sha512-FNAIBWCx9qcRhoHcgcJ0gvU7SN1lYU2ZXuSfl04bSC5OpvDHFyJCjdNHomPXxjQlCBU67YW64PzY7/VIEH7F2w==}engines: {node: '>=10'}@@ -4317,10 +4220,6 @@ packages:resolution: {integrity: sha512-UXWMKhLOwVKb728IUtQPXxfYU+usdybtUrK/8uGE8CQMvrhOpwvzDBwj0QhSL7MQc7vIsISBG8VQ8+IDQxpfQA==}engines: {node: '>=0.10.0'}- resolution: {integrity: sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==}- engines: {node: '>=0.10.0'}-resolution: {integrity: sha512-qxQJTx2ryR0Dw0ITYyekNQWpz6f8dGd7vffGNflQQ3Iqj9NJ6qiZ7ELpZsJ/QBhIVAiDfXdag3+Gp8RvWa62AA==}engines: {node: '>=12'}@@ -4976,30 +4875,6 @@ snapshots:'@ai-sdk/provider-utils': 4.0.23([email protected])zod: 4.1.12- '@apm-js-collab/[email protected]':- dependencies:- '@apm-js-collab/code-transformer': 0.18.0- es-module-lexer: 2.3.1- magic-string: 0.30.21- module-details-from-path: 1.0.4-- '@apm-js-collab/[email protected]':- dependencies:- '@types/estree': 1.0.9- astring: 1.9.0- esquery: 1.7.0- meriyah: 6.1.4- semifies: 1.0.0- source-map: 0.6.1-- '@apm-js-collab/[email protected]':- dependencies:- '@apm-js-collab/code-transformer': 0.18.0- debug: 4.4.3- module-details-from-path: 1.0.4- transitivePeerDependencies:- - supports-color-'@asamuzakjp/[email protected]':dependencies:'@asamuzakjp/generational-cache': 1.0.1@@ -5881,7 +5756,7 @@ snapshots:ai: 6.0.86([email protected])zod: 4.1.12- '@opentelemetry/[email protected]':+ '@opentelemetry/[email protected]':dependencies:'@opentelemetry/api': 1.9.1@@ -5889,19 +5764,48 @@ snapshots:'@opentelemetry/[email protected]': {}+ '@opentelemetry/[email protected](@opentelemetry/[email protected])':+ dependencies:+ '@opentelemetry/api': 1.9.1++ '@opentelemetry/[email protected](@opentelemetry/[email protected])':+ dependencies:+ '@opentelemetry/api': 1.9.1+ '@opentelemetry/semantic-conventions': 1.43.0+'@opentelemetry/[email protected](@opentelemetry/[email protected])':dependencies:'@opentelemetry/api': 1.9.1'@opentelemetry/semantic-conventions': 1.43.0- '@opentelemetry/[email protected](@opentelemetry/[email protected])':+ '@opentelemetry/[email protected](@opentelemetry/[email protected])':dependencies:'@opentelemetry/api': 1.9.1- '@opentelemetry/api-logs': 0.220.0- import-in-the-middle: 3.3.2- require-in-the-middle: 8.0.1- transitivePeerDependencies:- - supports-color+ '@opentelemetry/otlp-exporter-base': 0.222.0(@opentelemetry/[email protected])+ '@opentelemetry/otlp-transformer': 0.222.0(@opentelemetry/[email protected])+ '@opentelemetry/sdk-trace': 2.11.0(@opentelemetry/[email protected])++ '@opentelemetry/[email protected](@opentelemetry/[email protected])':+ dependencies:+ '@opentelemetry/api': 1.9.1+ '@opentelemetry/core': 2.11.0(@opentelemetry/[email protected])+ '@opentelemetry/otlp-transformer': 0.222.0(@opentelemetry/[email protected])++ '@opentelemetry/[email protected](@opentelemetry/[email protected])':+ dependencies:+ '@opentelemetry/api': 1.9.1+ '@opentelemetry/api-logs': 0.222.0+ '@opentelemetry/core': 2.11.0(@opentelemetry/[email protected])+ '@opentelemetry/resources': 2.11.0(@opentelemetry/[email protected])+ '@opentelemetry/sdk-logs': 0.222.0(@opentelemetry/[email protected])+ '@opentelemetry/sdk-metrics': 2.11.0(@opentelemetry/[email protected])+ '@opentelemetry/sdk-trace': 2.11.0(@opentelemetry/[email protected])++ '@opentelemetry/[email protected](@opentelemetry/[email protected])':+ dependencies:+ '@opentelemetry/api': 1.9.1+ '@opentelemetry/core': 2.11.0(@opentelemetry/[email protected])+ '@opentelemetry/semantic-conventions': 1.43.0'@opentelemetry/[email protected](@opentelemetry/[email protected])':dependencies:@@ -5909,25 +5813,46 @@ snapshots:'@opentelemetry/core': 2.9.0(@opentelemetry/[email protected])'@opentelemetry/semantic-conventions': 1.43.0+ '@opentelemetry/[email protected](@opentelemetry/[email protected])':+ dependencies:+ '@opentelemetry/api': 1.9.1+ '@opentelemetry/api-logs': 0.222.0+ '@opentelemetry/core': 2.11.0(@opentelemetry/[email protected])+ '@opentelemetry/resources': 2.11.0(@opentelemetry/[email protected])+ '@opentelemetry/semantic-conventions': 1.43.0++ '@opentelemetry/[email protected](@opentelemetry/[email protected])':+ dependencies:+ '@opentelemetry/api': 1.9.1+ '@opentelemetry/core': 2.11.0(@opentelemetry/[email protected])+ '@opentelemetry/resources': 2.11.0(@opentelemetry/[email protected])+'@opentelemetry/[email protected](@opentelemetry/[email protected])':dependencies:'@opentelemetry/api': 1.9.1'@opentelemetry/core': 2.9.0(@opentelemetry/[email protected])'@opentelemetry/resources': 2.9.0(@opentelemetry/[email protected])- '@opentelemetry/[email protected](@opentelemetry/[email protected])':+ '@opentelemetry/[email protected](@opentelemetry/[email protected])':dependencies:'@opentelemetry/api': 1.9.1- '@opentelemetry/core': 2.9.0(@opentelemetry/[email protected])- '@opentelemetry/resources': 2.9.0(@opentelemetry/[email protected])- '@opentelemetry/sdk-trace': 2.9.0(@opentelemetry/[email protected])+ '@opentelemetry/core': 2.11.0(@opentelemetry/[email protected])+ '@opentelemetry/resources': 2.11.0(@opentelemetry/[email protected])+ '@opentelemetry/sdk-trace': 2.11.0(@opentelemetry/[email protected])'@opentelemetry/semantic-conventions': 1.43.0- '@opentelemetry/[email protected](@opentelemetry/[email protected])':+ '@opentelemetry/[email protected](@opentelemetry/[email protected])':dependencies:'@opentelemetry/api': 1.9.1- '@opentelemetry/core': 2.9.0(@opentelemetry/[email protected])- '@opentelemetry/resources': 2.9.0(@opentelemetry/[email protected])+ '@opentelemetry/context-async-hooks': 2.11.0(@opentelemetry/[email protected])+ '@opentelemetry/core': 2.11.0(@opentelemetry/[email protected])+ '@opentelemetry/sdk-trace-base': 2.11.0(@opentelemetry/[email protected])++ '@opentelemetry/[email protected](@opentelemetry/[email protected])':+ dependencies:+ '@opentelemetry/api': 1.9.1+ '@opentelemetry/core': 2.11.0(@opentelemetry/[email protected])+ '@opentelemetry/resources': 2.11.0(@opentelemetry/[email protected])'@opentelemetry/semantic-conventions': 1.43.0'@opentelemetry/[email protected]': {}@@ -6112,101 +6037,6 @@ snapshots:domhandler: 5.0.3selderee: 0.11.0- '@sentry/[email protected]':- optional: true-- '@sentry/[email protected]':- optional: true-- '@sentry/[email protected]':- optional: true-- '@sentry/[email protected]':- optional: true-- '@sentry/[email protected]':- optional: true-- '@sentry/[email protected]':- optional: true-- '@sentry/[email protected]':- optional: true-- '@sentry/[email protected]':- optional: true-- '@sentry/[email protected]':- dependencies:- https-proxy-agent: 5.0.1- node-fetch: 2.7.0(patch_hash=cf2058381802eaf328e1cb26b185762ba41e2157a5545e7ed9bba56b7de0c49b)- progress: 2.0.3- proxy-from-env: 1.1.0- which: 2.0.2- optionalDependencies:- '@sentry/cli-darwin': 2.58.2- '@sentry/cli-linux-arm': 2.58.2- '@sentry/cli-linux-arm64': 2.58.2- '@sentry/cli-linux-i686': 2.58.2- '@sentry/cli-linux-x64': 2.58.2- '@sentry/cli-win32-arm64': 2.58.2- '@sentry/cli-win32-i686': 2.58.2- '@sentry/cli-win32-x64': 2.58.2- transitivePeerDependencies:- - encoding- - supports-color-- '@sentry/[email protected]': {}-- '@sentry/[email protected]':- dependencies:- '@sentry/conventions': 0.16.0-- '@sentry/[email protected](@opentelemetry/[email protected])(@opentelemetry/[email protected](@opentelemetry/[email protected]))(@opentelemetry/[email protected](@opentelemetry/[email protected]))(@opentelemetry/[email protected](@opentelemetry/[email protected]))':- dependencies:- '@sentry/conventions': 0.16.0- '@sentry/core': 10.67.0- '@sentry/opentelemetry': 10.67.0(@opentelemetry/[email protected])(@opentelemetry/[email protected](@opentelemetry/[email protected]))(@opentelemetry/[email protected](@opentelemetry/[email protected]))- import-in-the-middle: 3.3.2- optionalDependencies:- '@opentelemetry/api': 1.9.1- '@opentelemetry/core': 2.9.0(@opentelemetry/[email protected])- '@opentelemetry/instrumentation': 0.220.0(@opentelemetry/[email protected])- '@opentelemetry/sdk-trace-base': 2.9.0(@opentelemetry/[email protected])-- dependencies:- '@opentelemetry/api': 1.9.1- '@opentelemetry/instrumentation': 0.220.0(@opentelemetry/[email protected])- '@opentelemetry/sdk-trace-base': 2.9.0(@opentelemetry/[email protected])- '@sentry/conventions': 0.16.0- '@sentry/core': 10.67.0- '@sentry/node-core': 10.67.0(@opentelemetry/[email protected])(@opentelemetry/[email protected](@opentelemetry/[email protected]))(@opentelemetry/[email protected](@opentelemetry/[email protected]))(@opentelemetry/[email protected](@opentelemetry/[email protected]))- '@sentry/opentelemetry': 10.67.0(@opentelemetry/[email protected])(@opentelemetry/[email protected](@opentelemetry/[email protected]))(@opentelemetry/[email protected](@opentelemetry/[email protected]))- '@sentry/server-utils': 10.67.0- import-in-the-middle: 3.3.2- transitivePeerDependencies:- - '@opentelemetry/core'- - '@opentelemetry/exporter-trace-otlp-http'- - supports-color-- '@sentry/[email protected](@opentelemetry/[email protected])(@opentelemetry/[email protected](@opentelemetry/[email protected]))(@opentelemetry/[email protected](@opentelemetry/[email protected]))':- dependencies:- '@opentelemetry/api': 1.9.1- '@opentelemetry/core': 2.9.0(@opentelemetry/[email protected])- '@opentelemetry/sdk-trace-base': 2.9.0(@opentelemetry/[email protected])- '@sentry/conventions': 0.16.0- '@sentry/core': 10.67.0-- '@sentry/[email protected]':- dependencies:- '@apm-js-collab/code-transformer-bundler-plugins': 0.7.1- '@apm-js-collab/tracing-hooks': 0.13.0- '@sentry/conventions': 0.16.0- '@sentry/core': 10.67.0- transitivePeerDependencies:- - supports-color-'@standard-schema/[email protected]': {}'@taplo/[email protected]': {}@@ -6527,8 +6357,6 @@ snapshots:- [email protected]: {}-dependencies:tslib: 2.8.1@@ -6690,8 +6518,6 @@ snapshots:parse5: 7.1.2parse5-htmlparser2-tree-adapter: 7.0.0- [email protected]: {}-dependencies:restore-cursor: 5.1.0@@ -6949,8 +6775,6 @@ snapshots:- [email protected]: {}-dependencies:es-errors: 1.3.0@@ -6999,12 +6823,6 @@ snapshots:- dependencies:- estraverse: 5.3.0-- [email protected]: {}-dependencies:'@types/estree': 1.0.9@@ -7515,12 +7333,6 @@ snapshots:- dependencies:- cjs-module-lexer: 2.2.0- es-module-lexer: 2.3.1- module-details-from-path: 1.0.4-@@ -7690,12 +7502,13 @@ snapshots:- [email protected](@opentelemetry/[email protected])(@opentelemetry/[email protected](@opentelemetry/[email protected]))([email protected]):+ [email protected](@opentelemetry/[email protected])(@opentelemetry/[email protected](@opentelemetry/[email protected]))(@opentelemetry/[email protected](@opentelemetry/[email protected]))([email protected]):dependencies:p-queue: 6.6.2optionalDependencies:'@opentelemetry/api': 1.9.1- '@opentelemetry/sdk-trace-base': 2.9.0(@opentelemetry/[email protected])+ '@opentelemetry/exporter-trace-otlp-proto': 0.222.0(@opentelemetry/[email protected])+ '@opentelemetry/sdk-trace-base': 2.11.0(@opentelemetry/[email protected])ws: 8.21.0@@ -7857,8 +7670,6 @@ snapshots:- [email protected]: {}-@@ -7892,8 +7703,6 @@ snapshots:- [email protected]: {}-@@ -8187,11 +7996,9 @@ snapshots:- [email protected]: {}-dependencies:- '@opentelemetry/api': 1.9.0+ '@opentelemetry/api': 1.9.1tdigest: 0.1.2@@ -8219,8 +8026,6 @@ snapshots:forwarded: 0.2.0ipaddr.js: 1.9.1- [email protected]: {}-@@ -8328,13 +8133,6 @@ snapshots:- dependencies:- debug: 4.4.3- module-details-from-path: 1.0.4- transitivePeerDependencies:- - supports-color-@@ -8441,8 +8239,6 @@ snapshots:dependencies:parseley: 0.12.1- [email protected]: {}-@@ -8532,8 +8328,6 @@ snapshots:- [email protected]: {}-diff --git a/apps/api/pnpm-workspace.yaml b/apps/api/pnpm-workspace.yamlindex c5137401d..f6d8c7174 100644--- a/apps/api/pnpm-workspace.yaml+++ b/apps/api/pnpm-workspace.yaml@@ -60,8 +60,6 @@ patchedDependencies:"[email protected]": "patches/[email protected]"allowBuilds:"@mendable/firecrawl-rs": true- "@sentry-internal/node-cpu-profiler": true- "@sentry/cli": truebigint-buffer: true# Optional native accelerator for fast-crc32c (pulled in by# @google-cloud/bigtable); the pure-JS fallback is fine for our payloaddiff --git a/apps/api/src/__tests__/lib/branding/llm-failure-fallback.test.ts b/apps/api/src/__tests__/lib/branding/llm-failure-fallback.test.tsindex da2bec978..67118ccca 100644--- a/apps/api/src/__tests__/lib/branding/llm-failure-fallback.test.ts+++ b/apps/api/src/__tests__/lib/branding/llm-failure-fallback.test.ts@@ -6,9 +6,6 @@ vi.mock("ai", () => ({vi.mock("../../../lib/generic-ai", () => ({getModel: vi.fn().mockReturnValue({}),}));-vi.mock("../../../services/sentry", () => ({- captureExceptionWithZdrCheck: vi.fn(),-}));import { enhanceBrandingWithLLM } from "../../../lib/branding/llm";import { mergeBrandingResults } from "../../../lib/branding/merge";diff --git a/apps/api/src/__tests__/routes/not-found-metrics.routes.test.ts b/apps/api/src/__tests__/routes/not-found-metrics.routes.test.tsindex 1217c8ace..2626b9c98 100644--- a/apps/api/src/__tests__/routes/not-found-metrics.routes.test.ts+++ b/apps/api/src/__tests__/routes/not-found-metrics.routes.test.ts@@ -34,8 +34,12 @@ function appUnderTest() {v2.post("/scrape", (_req, res) => res.status(200).json({ success: true }));// Mirrors the real /v2/crawl/:jobId table (src/routes/v2.ts): only GET and// DELETE are registered, never PATCH.- v2.get("/crawl/:jobId", (_req, res) => res.status(200).json({ success: true }));- v2.delete("/crawl/:jobId", (_req, res) => res.status(200).json({ success: true }));+ v2.get("/crawl/:jobId", (_req, res) =>+ res.status(200).json({ success: true }),+ );+ v2.delete("/crawl/:jobId", (_req, res) =>+ res.status(200).json({ success: true }),+ );app.use("/v2", v2);app.use(notFoundHandler);diff --git a/apps/api/src/config.ts b/apps/api/src/config.tsindex 5ecce4076..fb1e655ec 100644--- a/apps/api/src/config.ts+++ b/apps/api/src/config.ts@@ -523,11 +523,15 @@ const configSchema = z.object({SYS_INFO_MAX_CACHE_DURATION: z.coerce.number().default(150),USE_GO_MARKDOWN_PARSER: z.stringbool().optional(),- // Sentry- SENTRY_DSN: z.string().optional(),- SENTRY_TRACE_SAMPLE_RATE: z.coerce.number().default(0.01),- SENTRY_ERROR_SAMPLE_RATE: z.coerce.number().default(0.05),SENTRY_ENVIRONMENT: z.string().default("production"),++ // OpenTelemetry. Tracing is off unless an OTLP endpoint is set; spans are then+ // exported over http/protobuf at 100% sampling, and the SDK honors the+ // standard OTEL_EXPORTER_OTLP_* / OTEL_BSP_* / OTEL_RESOURCE_ATTRIBUTES+ // variables. Zero-data-retention spans are never exported (see otel-tracer).+ OTEL_EXPORTER_OTLP_ENDPOINT: emptyStringAsUndefined(z.string().url()),+ OTEL_EXPORTER_OTLP_TRACES_ENDPOINT: emptyStringAsUndefined(z.string().url()),+ OTEL_SERVICE_NAME: emptyStringAsUndefined(z.string()),NUQ_POD_NAME: z.string().default("main"),// Billingdiff --git a/apps/api/src/controllers/v0/admin/check-fire-engine.ts b/apps/api/src/controllers/v0/admin/check-fire-engine.tsindex ae78abd9e..bc038633c 100644--- a/apps/api/src/controllers/v0/admin/check-fire-engine.ts+++ b/apps/api/src/controllers/v0/admin/check-fire-engine.ts@@ -1,6 +1,5 @@import { logger } from "../../../lib/logger";import { config } from "../../../config";-import * as Sentry from "@sentry/node";import { Request, Response } from "express";export async function checkFireEngine(req: Request, res: Response) {@@ -63,14 +62,12 @@ export async function checkFireEngine(req: Request, res: Response) {method: "checkFireEngine",error: lastError,});- Sentry.captureException(lastError);return res.status(500).json({success: false,error: "Internal server error - all retry attempts failed",});} catch (error) {logger.error(error);- Sentry.captureException(error);return res.status(500).json({success: false,error: "Internal server error",diff --git a/apps/api/src/controllers/v0/crawl-cancel.ts b/apps/api/src/controllers/v0/crawl-cancel.tsindex fa552014f..c86836a38 100644--- a/apps/api/src/controllers/v0/crawl-cancel.ts+++ b/apps/api/src/controllers/v0/crawl-cancel.ts@@ -3,7 +3,6 @@ import { authenticateUser } from "../auth";import { RateLimiterMode } from "../../../src/types";import { logger } from "../../../src/lib/logger";import { getCrawl, saveCrawl } from "../../../src/lib/crawl-redis";-import * as Sentry from "@sentry/node";import { configDotenv } from "dotenv";import { redisEvictConnection } from "../../../src/services/redis";import { crawlGroup } from "../../services/worker/nuq-router";@@ -83,7 +82,6 @@ export async function crawlCancelController(req: Request, res: Response) {status: "cancelled",});} catch (error) {- Sentry.captureException(error);logger.error(error);return res.status(500).json({ error: error.message });}diff --git a/apps/api/src/controllers/v0/crawl-status.ts b/apps/api/src/controllers/v0/crawl-status.tsindex ee23113c8..8726214e3 100644--- a/apps/api/src/controllers/v0/crawl-status.ts+++ b/apps/api/src/controllers/v0/crawl-status.ts@@ -10,7 +10,6 @@ import {type StoredCrawl,} from "../../../src/lib/crawl-redis";import { supabaseGetScrapesByRequestId } from "../../../src/lib/supabase-jobs";-import * as Sentry from "@sentry/node";import { configDotenv } from "dotenv";import { toLegacyDocument } from "../v1/types";import type { DBScrape, PseudoJob } from "../v1/crawl-status";@@ -297,7 +296,6 @@ export async function crawlStatusController(req: Request, res: Response) {.map(x => toLegacyDocument(x, sc.internalOptions)),});} catch (error) {- Sentry.captureException(error);logger.error(error);return res.status(500).json({ error: error.message });}diff --git a/apps/api/src/controllers/v0/crawl.ts b/apps/api/src/controllers/v0/crawl.tsindex fe1bccb57..2e497a9b5 100644--- a/apps/api/src/controllers/v0/crawl.ts+++ b/apps/api/src/controllers/v0/crawl.ts@@ -26,7 +26,6 @@ import {} from "../../../src/lib/crawl-redis";import { redisEvictConnection } from "../../../src/services/redis";import { checkAndUpdateURL } from "../../../src/lib/validateUrl";-import * as Sentry from "@sentry/node";import { getJobPriority } from "../../lib/job-priority";import { url as urlSchema } from "../v1/types";import { ZodError } from "zod";@@ -312,7 +311,6 @@ export async function crawlController(req: Request, res: Response) {logger,);for (const job of jobs) {- // add with sentry instrumentationawait addScrapeJob(job.data, job.jobId, job.priority);}});@@ -347,7 +345,6 @@ export async function crawlController(req: Request, res: Response) {res.json({ jobId: id });} catch (error) {- Sentry.captureException(error);logger.error(error);return res.status(500).json({error: error instanceof ZodError ? "Invalid URL" : error.message,diff --git a/apps/api/src/controllers/v0/scrape.ts b/apps/api/src/controllers/v0/scrape.tsindex 9d71790c1..30f3787e6 100644--- a/apps/api/src/controllers/v0/scrape.ts+++ b/apps/api/src/controllers/v0/scrape.ts@@ -15,7 +15,6 @@ import { addScrapeJob, waitForJob } from "../../services/queue-jobs";import { redisEvictConnection } from "../../../src/services/redis";import { v7 as uuidv7 } from "uuid";import { logger } from "../../lib/logger";-import * as Sentry from "@sentry/node";import { getJobPriority } from "../../lib/job-priority";import { ZodError } from "zod";import { Document as V0Document } from "./../../lib/entities";@@ -323,7 +322,6 @@ export async function scrapeController(req: Request, res: Response) {return res.status(result.returnCode).json(result);} catch (error) {- Sentry.captureException(error);logger.error("Scrape error occcurred", { error });return res.status(500).json({error:diff --git a/apps/api/src/controllers/v0/search.ts b/apps/api/src/controllers/v0/search.tsindex 432799157..de30f90c8 100644--- a/apps/api/src/controllers/v0/search.ts+++ b/apps/api/src/controllers/v0/search.ts@@ -12,7 +12,6 @@ import { v7 as uuidv7 } from "uuid";import { logger } from "../../lib/logger";import { redisEvictConnection } from "../../../src/services/redis";import { addScrapeJob, waitForJob } from "../../services/queue-jobs";-import * as Sentry from "@sentry/node";import { getJobPriority } from "../../lib/job-priority";import {fromLegacyScrapeOptions,@@ -261,7 +260,6 @@ export async function searchController(req: Request, res: Response) {return res.status(402).json({ error: "Insufficient credits" });}} catch (error) {- Sentry.captureException(error);logger.error(error);return res.status(500).json({ error: "Internal server error" });}@@ -301,7 +299,6 @@ export async function searchController(req: Request, res: Response) {return res.status(408).json({ error: error.message });}- Sentry.captureException(error);logger.error("Unhandled error occurred in search", { error });return res.status(500).json({ error: error.message });}diff --git a/apps/api/src/controllers/v1/crawl-cancel.ts b/apps/api/src/controllers/v1/crawl-cancel.tsindex 2e1d80c82..7fc58c1cb 100644--- a/apps/api/src/controllers/v1/crawl-cancel.ts+++ b/apps/api/src/controllers/v1/crawl-cancel.ts@@ -1,7 +1,6 @@import { Response } from "express";import { logger } from "../../lib/logger";import { getCrawl, getCrawlJobs, saveCrawl } from "../../lib/crawl-redis";-import * as Sentry from "@sentry/node";import { configDotenv } from "dotenv";import { RequestWithAuth } from "./types";import { crawlGroup } from "../../services/worker/nuq-router";@@ -49,7 +48,6 @@ export async function crawlCancelController(status: "cancelled",});} catch (error) {- Sentry.captureException(error);logger.error(error);return res.status(500).json({ error: error.message });}diff --git a/apps/api/src/controllers/v1/crawl-status-ws.ts b/apps/api/src/controllers/v1/crawl-status-ws.tsindex 762c25213..ea01e508b 100644--- a/apps/api/src/controllers/v1/crawl-status-ws.ts+++ b/apps/api/src/controllers/v1/crawl-status-ws.ts@@ -17,7 +17,6 @@ import {getDoneJobsOrdered,} from "../../lib/crawl-redis";import { getJobs, PseudoJob } from "./crawl-status";-import * as Sentry from "@sentry/node";import { getConcurrencyLimitedJobs } from "../../lib/concurrency-limit";import { scrapeQueue, NuQJobStatus } from "../../services/worker/nuq-router";import { getErrorContactMessage } from "../../lib/deployment";@@ -188,8 +187,6 @@ export async function crawlStatusWSController(await crawlStatusWS(ws, req);} catch (err) {- Sentry.captureException(err);-const id = uuidv7();let verbose = JSON.stringify(err);if (verbose === "{}") {diff --git a/apps/api/src/controllers/v1/deep-research.ts b/apps/api/src/controllers/v1/deep-research.tsindex 919126963..b6faff9a6 100644--- a/apps/api/src/controllers/v1/deep-research.ts+++ b/apps/api/src/controllers/v1/deep-research.ts@@ -2,7 +2,6 @@ import { v7 as uuidv7 } from "uuid";import { Request, Response } from "express";import { ErrorResponse, extractOptions, RequestWithAuth } from "./types";import { getDeepResearchQueue } from "../../services/queue-service";-import * as Sentry from "@sentry/node";import { saveDeepResearch } from "../../lib/deep-research/deep-research-redis";import { z } from "zod";import { logRequest } from "../../services/logging/log_job";diff --git a/apps/api/src/controllers/v1/generate-llmstxt.ts b/apps/api/src/controllers/v1/generate-llmstxt.tsindex ced86715d..3ad13dda1 100644--- a/apps/api/src/controllers/v1/generate-llmstxt.ts+++ b/apps/api/src/controllers/v1/generate-llmstxt.ts@@ -7,7 +7,6 @@ import {RequestWithAuth,} from "./types";import { getGenerateLlmsTxtQueue } from "../../services/queue-service";-import * as Sentry from "@sentry/node";import { saveGeneratedLlmsTxt } from "../../lib/generate-llmstxt/generate-llmstxt-redis";import { logRequest } from "../../services/logging/log_job";import { externalRequestId } from "../../lib/external-request-id";diff --git a/apps/api/src/controllers/v1/scrape.ts b/apps/api/src/controllers/v1/scrape.tsindex 6269822f0..279ac1dd3 100644--- a/apps/api/src/controllers/v1/scrape.ts+++ b/apps/api/src/controllers/v1/scrape.ts@@ -27,8 +27,14 @@ import { AbortManagerThrownError } from "../../scraper/scrapeURL/lib/abortManageimport { logRequest } from "../../services/logging/log_job";import { externalRequestId } from "../../lib/external-request-id";import { getErrorContactMessage } from "../../lib/deployment";-import { captureExceptionWithZdrCheck } from "../../services/sentry";import { getScrapeZDR } from "../../lib/zdr-helpers";+import {+ withSpan,+ setSpanAttributes,+ recordSpanException,+ SpanKind,+ type Span,+} from "../../lib/otel-tracer";import {adjustKeylessCredits,keylessLimitBody,@@ -43,6 +49,31 @@ import { getEffectiveConcurrencyLimit } from "../../lib/concurrency-limit";export async function scrapeController(req: RequestWithAuth<{}, ScrapeResponse, ScrapeRequest>,res: Response<ScrapeResponse>,+) {+ // Resolved before the root span starts so the whole request trace stays+ // unrecorded for zero-data-retention requests (see otel-tracer).+ const zeroDataRetentionTrace =+ getScrapeZDR(req.acuc?.flags) === "forced" ||+ req.body?.zeroDataRetention === true;++ return withSpan(+ "api.scrape.request",+ span => scrapeControllerInner(req, res, span),+ {+ kind: SpanKind.SERVER,+ attributes: {+ "api.version": "v1",+ "scrape.team_id": req.auth.team_id,+ },+ zeroDataRetention: zeroDataRetentionTrace,+ },+ );+}++async function scrapeControllerInner(+ req: RequestWithAuth<{}, ScrapeResponse, ScrapeRequest>,+ res: Response<ScrapeResponse>,+ span: Span,) {// Get timing data from middleware (includes all middleware processing time)const middlewareStartTime =@@ -342,17 +373,10 @@ export async function scrapeController(path: req.path,teamId: req.auth.team_id,});- captureExceptionWithZdrCheck(e, {- tags: {- errorId: id,- version: "v1",- teamId: req.auth.team_id,- },- extra: {- path: req.path,- url: req.body.url,- },- zeroDataRetention,+ recordSpanException(span, e);+ setSpanAttributes(span, {+ "scrape.status_code": 500,+ "scrape.error_id": id,});return res.status(500).json({success: false,diff --git a/apps/api/src/controllers/v1/search.ts b/apps/api/src/controllers/v1/search.tsindex e5e641473..9429d3338 100644--- a/apps/api/src/controllers/v1/search.ts+++ b/apps/api/src/controllers/v1/search.ts@@ -20,7 +20,6 @@ import {} from "../../lib/key-restriction";import type { Logger } from "winston";import { ScrapeJobTimeoutError } from "../../lib/error";-import { captureExceptionWithZdrCheck } from "../../services/sentry";import { z } from "zod";import { executeSearch } from "../../search/execute";import { resolveThreatProtection } from "../../lib/threat-protection/request";@@ -374,9 +373,6 @@ export async function searchController(});}- captureExceptionWithZdrCheck(error, {- extra: { zeroDataRetention },- });logger.error("Unhandled error occurred in search", {version: "v1",error,diff --git a/apps/api/src/controllers/v1/types.ts b/apps/api/src/controllers/v1/types.tsindex 291a02729..2381a32ec 100644--- a/apps/api/src/controllers/v1/types.ts+++ b/apps/api/src/controllers/v1/types.ts@@ -1,4 +1,4 @@-import { Request, Response } from "express";+import { Request } from "express";import { config } from "../../config";import { z } from "zod";import { protocolIncluded, checkUrl } from "../../lib/validateUrl";@@ -1410,11 +1410,6 @@ export interface RequestWithAuth<account?: Account;}-export interface ResponseWithSentry<ResBody = undefined>- extends Response<ResBody> {- sentry?: string;-}-export function toLegacyCrawlerOptions(x: CrawlerOptions) {return {includes: x.includePaths,diff --git a/apps/api/src/controllers/v2/__tests__/search-developer-ledger.test.ts b/apps/api/src/controllers/v2/__tests__/search-developer-ledger.test.tsindex 6b2f93c4d..fb4abcb1a 100644--- a/apps/api/src/controllers/v2/__tests__/search-developer-ledger.test.ts+++ b/apps/api/src/controllers/v2/__tests__/search-developer-ledger.test.ts@@ -51,11 +51,6 @@ vi.mock("../../../lib/key-restriction", () => ({checkKeyEndpointRestriction: vi.fn().mockResolvedValue({ allowed: true }),}));-vi.mock("../../../services/sentry", () => ({- applyZdrScope: vi.fn(),- captureExceptionWithZdrCheck: vi.fn(),-}));-vi.mock("../../../lib/logger", () => ({logger: {info: vi.fn(),diff --git a/apps/api/src/controllers/v2/crawl-cancel.ts b/apps/api/src/controllers/v2/crawl-cancel.tsindex a304860ac..17484b891 100644--- a/apps/api/src/controllers/v2/crawl-cancel.ts+++ b/apps/api/src/controllers/v2/crawl-cancel.ts@@ -6,7 +6,6 @@ import {saveCrawl,StoredCrawl,} from "../../lib/crawl-redis";-import * as Sentry from "@sentry/node";import { configDotenv } from "dotenv";import { RequestWithAuth, scrapeOptions } from "./types";import { crawlGroup } from "../../services/worker/nuq-router";@@ -63,7 +62,6 @@ export async function crawlCancelController(status: "cancelled",});} catch (error) {- Sentry.captureException(error);logger.error(error);return res.status(500).json({ error: error.message });}diff --git a/apps/api/src/controllers/v2/crawl-status-ws.ts b/apps/api/src/controllers/v2/crawl-status-ws.tsindex b44137223..d1ec7fa2f 100644--- a/apps/api/src/controllers/v2/crawl-status-ws.ts+++ b/apps/api/src/controllers/v2/crawl-status-ws.ts@@ -18,7 +18,6 @@ import {getDoneJobsOrdered,} from "../../lib/crawl-redis";import { getJobs, PseudoJob } from "./crawl-status";-import * as Sentry from "@sentry/node";import { getConcurrencyLimitedJobs } from "../../lib/concurrency-limit";import { scrapeQueue, NuQJobStatus } from "../../services/worker/nuq-router";import { getErrorContactMessage } from "../../lib/deployment";@@ -217,8 +216,6 @@ export async function crawlStatusWSController(await crawlStatusWS(ws, req);} catch (err) {- Sentry.captureException(err);-const id = uuidv7();let verbose = JSON.stringify(err);if (verbose === "{}") {diff --git a/apps/api/src/controllers/v2/feedback/record.ts b/apps/api/src/controllers/v2/feedback/record.tsindex 0eb116d83..a43b86e22 100644--- a/apps/api/src/controllers/v2/feedback/record.ts+++ b/apps/api/src/controllers/v2/feedback/record.ts@@ -1,11 +1,17 @@import { v7 as uuidv7 } from "uuid";import { config } from "../../../config";import { logger as _logger } from "../../../lib/logger";+import {+ withSpan,+ setSpanAttributes,+ recordSpanException,+ type Span,+} from "../../../lib/otel-tracer";+import { getScrapeZDR, getSearchZDR } from "../../../lib/zdr-helpers";import {autumnService,featureIdForBillingEndpoint,} from "../../../services/autumn/autumn.service";-import { captureExceptionWithZdrCheck } from "../../../services/sentry";import {EndpointFeedbackErrorCode,RequestWithAuth,@@ -238,9 +244,49 @@ async function refundCredits(params: {}}+/**+ * Team-level zero data retention for the endpoint the feedback refers to. Unlike+ * `shouldSkipPersistenceForForcedZdr` this ignores the persistence override:+ * the trace must stay unrecorded whenever the team is forced into ZDR.+ */+function isForcedZdrTeam(+ req: RequestWithAuth<any, any, any>,+ options: FeedbackRecordOptions,+): boolean {+ if (options.endpoint === "search") {+ const searchZDR = getSearchZDR(req.acuc?.flags);+ return searchZDR === "forced-zdr" || searchZDR === "forced-anon";+ }++ if (options.endpoint === "scrape" || options.endpoint === "parse") {+ return getScrapeZDR(req.acuc?.flags) === "forced";+ }++ return false;+}+export async function recordEndpointFeedback(req: RequestWithAuth<any, any, any>,options: FeedbackRecordOptions,+): Promise<FeedbackRecordResult> {+ return withSpan(+ "api.feedback.record",+ span => recordEndpointFeedbackInner(req, options, span),+ {+ attributes: {+ "feedback.endpoint": options.endpoint,+ "feedback.job_id": options.jobId,+ "feedback.team_id": req.auth.team_id,+ },+ zeroDataRetention: isForcedZdrTeam(req, options),+ },+ );+}++async function recordEndpointFeedbackInner(+ req: RequestWithAuth<any, any, any>,+ options: FeedbackRecordOptions,+ span: Span,): Promise<FeedbackRecordResult> {const logger = _logger.child({module: "api/v2",@@ -252,6 +298,7 @@ export async function recordEndpointFeedback(if (shouldSkipPersistenceForForcedZdr(req, options)) {logger.info("Skipping feedback persistence for forced ZDR team");+ setSpanAttributes(span, { "feedback.zero_data_retention": true });return zdrFeedbackSuccess(options);}@@ -265,7 +312,9 @@ export async function recordEndpointFeedback(if ("status" in jobOrFailure) return jobOrFailure;if (shouldSkipPersistenceForJobZdr(jobOrFailure, options)) {+ // Learned late: the attribute drops this span and flags the trace.logger.info("Skipping feedback persistence for ZDR job");+ setSpanAttributes(span, { "feedback.zero_data_retention": true });return zdrFeedbackSuccess(options);}@@ -365,10 +414,10 @@ export async function recordEndpointFeedback(},};} catch (error) {- captureExceptionWithZdrCheck(error);logger.error("Unhandled error while recording endpoint feedback", {error,});+ recordSpanException(span, error);return feedbackFailure(500,"INTERNAL",diff --git a/apps/api/src/controllers/v2/parse-upload.ts b/apps/api/src/controllers/v2/parse-upload.tsindex d3f204ada..f971ee99e 100644--- a/apps/api/src/controllers/v2/parse-upload.ts+++ b/apps/api/src/controllers/v2/parse-upload.ts@@ -11,6 +11,7 @@ import { getRedisConnection } from "../../services/queue-service";import { RequestWithAuth, UploadedParseFile } from "./types";import { detectUploadedFileKind, getSupportedParseFileTypes } from "./parse";import { isImageOcrEnabled } from "../../lib/image-ocr-gate";+import { getScrapeZDR } from "../../lib/zdr-helpers";import { reportPipelineError } from "../../lib/redis-pipeline";const PARSE_UPLOAD_MAX_BYTES = 50 * 1024 * 1024;@@ -278,140 +279,148 @@ export async function parseUploadUrlController(req: RequestWithAuth<{}, any, any>,res: Response,) {- return withSpan("api.parse.upload_url", async span => {- const parsed = uploadInitSchema.safeParse(req.body ?? {});- if (!parsed.success) {- return res.status(400).json({- success: false,- code: "BAD_REQUEST",- error:- parsed.error.issues[0]?.message ?? "Invalid upload init request.",- });- }-- const { filename, contentType, declaredSizeBytes } = parsed.data;- const imageOcrEnabled = isImageOcrEnabled(req.acuc?.flags);- if (!isSupportedParseUpload(filename, contentType, imageOcrEnabled)) {- return res.status(400).json({- success: false,- code: "UNSUPPORTED_FILE_TYPE",- error: `Unsupported upload type. Supported file extensions include ${getSupportedParseFileTypes(imageOcrEnabled)}, or matching supported MIME types.`,- });- }-- cleanupExpiredLocalUploads();- const driver = getParseUploadDriver();- if (!getEffectiveRefSecret()) {- return res.status(503).json({- success: false,- code: "PARSE_UPLOAD_REF_SECRET_MISSING",- error: "Parse upload references are not configured.",- });- }-- if (driver === "local" && !isLocalUploadAdapterAllowed()) {- return res.status(503).json({- success: false,- code: "PARSE_UPLOAD_STORAGE_DISABLED",- error:- "Local parse upload storage is disabled outside development/test.",- });- }+ return withSpan(+ "api.parse.upload_url",+ async span => {+ const parsed = uploadInitSchema.safeParse(req.body ?? {});+ if (!parsed.success) {+ return res.status(400).json({+ success: false,+ code: "BAD_REQUEST",+ error:+ parsed.error.issues[0]?.message ?? "Invalid upload init request.",+ });+ }- const uploadId = uuidv7();- const expiresAt = Date.now() + PARSE_UPLOAD_TTL_MS;- const objectPath = makeObjectPath(req.auth.team_id, uploadId, filename);- const refPayload: ParseUploadRefPayload = {- v: 1,- driver,- uploadId,- teamId: req.auth.team_id,- objectPath,- filename: sanitizeFilename(filename),- contentType,- expiresAt,- maxBytes: PARSE_UPLOAD_MAX_BYTES,- };- const uploadRef = signUploadRef(refPayload);+ const { filename, contentType, declaredSizeBytes } = parsed.data;+ const imageOcrEnabled = isImageOcrEnabled(req.acuc?.flags);+ if (!isSupportedParseUpload(filename, contentType, imageOcrEnabled)) {+ return res.status(400).json({+ success: false,+ code: "UNSUPPORTED_FILE_TYPE",+ error: `Unsupported upload type. Supported file extensions include ${getSupportedParseFileTypes(imageOcrEnabled)}, or matching supported MIME types.`,+ });+ }- setSpanAttributes(span, {- "parse_upload.driver": driver,- "parse_upload.team_id": req.auth.team_id,- "parse_upload.declared_size": declaredSizeBytes,- });+ cleanupExpiredLocalUploads();+ const driver = getParseUploadDriver();+ if (!getEffectiveRefSecret()) {+ return res.status(503).json({+ success: false,+ code: "PARSE_UPLOAD_REF_SECRET_MISSING",+ error: "Parse upload references are not configured.",+ });+ }- if (driver === "gcs") {- if (!config.GCS_PARSE_UPLOAD_BUCKET_NAME) {+ if (driver === "local" && !isLocalUploadAdapterAllowed()) {return res.status(503).json({success: false,code: "PARSE_UPLOAD_STORAGE_DISABLED",- error: "Parse upload storage is not configured.",+ error:+ "Local parse upload storage is disabled outside development/test.",});}- const bucket = getStorageClient().bucket(- config.GCS_PARSE_UPLOAD_BUCKET_NAME,- );- const file = bucket.file(objectPath);- const [policy] = await file.generateSignedPostPolicyV4({- expires: expiresAt,- fields: {- "Content-Type": contentType || "application/octet-stream",- },- conditions: [- ["content-length-range", 1, PARSE_UPLOAD_MAX_BYTES],- ["eq", "$Content-Type", contentType || "application/octet-stream"],- ],+ const uploadId = uuidv7();+ const expiresAt = Date.now() + PARSE_UPLOAD_TTL_MS;+ const objectPath = makeObjectPath(req.auth.team_id, uploadId, filename);+ const refPayload: ParseUploadRefPayload = {+ v: 1,+ driver,+ uploadId,+ teamId: req.auth.team_id,+ objectPath,+ filename: sanitizeFilename(filename),+ contentType,+ expiresAt,+ maxBytes: PARSE_UPLOAD_MAX_BYTES,+ };+ const uploadRef = signUploadRef(refPayload);++ setSpanAttributes(span, {+ "parse_upload.driver": driver,+ "parse_upload.team_id": req.auth.team_id,+ "parse_upload.declared_size": declaredSizeBytes,});+ if (driver === "gcs") {+ if (!config.GCS_PARSE_UPLOAD_BUCKET_NAME) {+ return res.status(503).json({+ success: false,+ code: "PARSE_UPLOAD_STORAGE_DISABLED",+ error: "Parse upload storage is not configured.",+ });+ }++ const bucket = getStorageClient().bucket(+ config.GCS_PARSE_UPLOAD_BUCKET_NAME,+ );+ const file = bucket.file(objectPath);+ const [policy] = await file.generateSignedPostPolicyV4({+ expires: expiresAt,+ fields: {+ "Content-Type": contentType || "application/octet-stream",+ },+ conditions: [+ ["content-length-range", 1, PARSE_UPLOAD_MAX_BYTES],+ ["eq", "$Content-Type", contentType || "application/octet-stream"],+ ],+ });++ if (+ await reserveUnparsedUploadRefOrRespond(+ res,+ req.auth.team_id,+ uploadId,+ )+ ) {+ return;+ }++ return res.status(200).json({+ success: true,+ data: {+ uploadUrl: policy.url,+ uploadRef,+ method: "POST",+ headers: {},+ fields: policy.fields,+ expiresAt: new Date(expiresAt).toISOString(),+ maxSizeBytes: PARSE_UPLOAD_MAX_BYTES,+ },+ });+ }+if (await reserveUnparsedUploadRefOrRespond(res, req.auth.team_id, uploadId)) {return;}+ localUploads.set(uploadId, {+ filename: sanitizeFilename(filename),+ contentType,+ teamId: req.auth.team_id,+ expiresAt,+ maxBytes: PARSE_UPLOAD_MAX_BYTES,+ });+return res.status(200).json({success: true,data: {- uploadUrl: policy.url,+ uploadUrl: `${buildPublicBaseUrl(req)}/v2/parse/upload/${uploadId}?uploadRef=${encodeURIComponent(uploadRef)}`,uploadRef,- method: "POST",- headers: {},- fields: policy.fields,+ method: "PUT",+ headers: {+ "Content-Type": contentType || "application/octet-stream",+ },expiresAt: new Date(expiresAt).toISOString(),maxSizeBytes: PARSE_UPLOAD_MAX_BYTES,},});- }-- if (- await reserveUnparsedUploadRefOrRespond(res, req.auth.team_id, uploadId)- ) {- return;- }-- localUploads.set(uploadId, {- filename: sanitizeFilename(filename),- contentType,- teamId: req.auth.team_id,- expiresAt,- maxBytes: PARSE_UPLOAD_MAX_BYTES,- });-- return res.status(200).json({- success: true,- data: {- uploadUrl: `${buildPublicBaseUrl(req)}/v2/parse/upload/${uploadId}?uploadRef=${encodeURIComponent(uploadRef)}`,- uploadRef,- method: "PUT",- headers: {- "Content-Type": contentType || "application/octet-stream",- },- expiresAt: new Date(expiresAt).toISOString(),- maxSizeBytes: PARSE_UPLOAD_MAX_BYTES,- },- });- });+ },+ { zeroDataRetention: getScrapeZDR(req.acuc?.flags) === "forced" },+ );}export async function parseLocalUploadController(req: Request, res: Response) {diff --git a/apps/api/src/controllers/v2/parse.ts b/apps/api/src/controllers/v2/parse.tsindex fef769b70..4b689d4e7 100644--- a/apps/api/src/controllers/v2/parse.ts+++ b/apps/api/src/controllers/v2/parse.ts@@ -24,7 +24,6 @@ import { getJobPriority } from "../../lib/job-priority";import { logRequest } from "../../services/logging/log_job";import { externalRequestId } from "../../lib/external-request-id";import { getErrorContactMessage } from "../../lib/deployment";-import { captureExceptionWithZdrCheck } from "../../services/sentry";import type { BillingMetadata } from "../../services/billing/types";import { getScrapeZDR } from "../../lib/zdr-helpers";import {@@ -295,6 +294,12 @@ export async function parseController(req: RequestWithAuth<{}, ScrapeResponse, ParseRequest>,res: Response<ScrapeResponse>,) {+ // Resolved before the root span starts so the whole request trace stays+ // unrecorded for zero-data-retention requests (see otel-tracer).+ const zeroDataRetentionTrace =+ getScrapeZDR(req.acuc?.flags) === "forced" ||+ req.body?.zeroDataRetention === true;+return withSpan("api.parse.request",async span => {@@ -658,18 +663,6 @@ export async function parseController(path: req.path,teamId: req.auth.team_id,});- captureExceptionWithZdrCheck(e, {- tags: {- errorId: id,- version: "v2",- teamId: req.auth.team_id,- },- extra: {- path: req.path,- fileName: req.body.file.filename,- },- zeroDataRetention,- });setSpanAttributes(span, {"parse.status_code": 500,"parse.error_id": id,@@ -771,6 +764,7 @@ export async function parseController("http.route": "/v2/parse",},kind: SpanKind.SERVER,+ zeroDataRetention: zeroDataRetentionTrace,},);}diff --git a/apps/api/src/controllers/v2/scrape.ts b/apps/api/src/controllers/v2/scrape.tsindex 61197cda9..9615dba9f 100644--- a/apps/api/src/controllers/v2/scrape.ts+++ b/apps/api/src/controllers/v2/scrape.ts@@ -30,7 +30,6 @@ import { getJobPriority } from "../../lib/job-priority";import { logRequest } from "../../services/logging/log_job";import { externalRequestId } from "../../lib/external-request-id";import { getErrorContactMessage } from "../../lib/deployment";-import { captureExceptionWithZdrCheck } from "../../services/sentry";import type { BillingMetadata } from "../../services/billing/types";import { getScrapeZDR } from "../../lib/zdr-helpers";import {@@ -51,6 +50,13 @@ export async function scrapeController(req: RequestWithAuth<{}, ScrapeResponse, ScrapeRequest>,res: Response<ScrapeResponse>,) {+ // Resolved before the root span starts so the whole request trace stays+ // unrecorded for zero-data-retention requests (see otel-tracer).+ const zeroDataRetentionTrace =+ getScrapeZDR(req.acuc?.flags) === "forced" ||+ req.body?.zeroDataRetention === true ||+ req.body?.lockdown === true;+return withSpan("api.scrape.request",async span => {@@ -525,18 +531,6 @@ export async function scrapeController(path: req.path,teamId: req.auth.team_id,});- captureExceptionWithZdrCheck(e, {- tags: {- errorId: id,- version: "v2",- teamId: req.auth.team_id,- },- extra: {- path: req.path,- url: req.body.url,- },- zeroDataRetention,- });setSpanAttributes(span, {"scrape.status_code": 500,"scrape.error_id": id,@@ -642,6 +636,7 @@ export async function scrapeController("http.route": "/v2/scrape",},kind: SpanKind.SERVER,+ zeroDataRetention: zeroDataRetentionTrace,},);}diff --git a/apps/api/src/controllers/v2/search.ts b/apps/api/src/controllers/v2/search.tsindex 4d773290a..b115cae2a 100644--- a/apps/api/src/controllers/v2/search.ts+++ b/apps/api/src/controllers/v2/search.ts@@ -24,13 +24,16 @@ import { logger as _logger } from "../../lib/logger";import { ScrapeJobTimeoutError } from "../../lib/error";import { z } from "zod";import { CategoryOption } from "../../lib/search-query-builder";-import {- applyZdrScope,- captureExceptionWithZdrCheck,-} from "../../services/sentry";import { executeSearch } from "../../search/execute";import type { BillingMetadata } from "../../services/billing/types";import { getSearchForcedKind, getSearchZDR } from "../../lib/zdr-helpers";+import {+ withSpan,+ setSpanAttributes,+ recordSpanException,+ SpanKind,+ type Span,+} from "../../lib/otel-tracer";import { projectSearchTotalCredits } from "../../lib/keyless-credit-projection";import { applyAgentAuthDiscoveryHeader } from "../../lib/agent-auth-discovery";import { resolveThreatProtection } from "../../lib/threat-protection/request";@@ -47,6 +50,35 @@ import { isAgentInteropSecretValid } from "../../lib/agent-interop";export async function searchController(req: RequestWithAuth<{}, SearchResponse, SearchRequest>,res: Response<SearchResponse>,+) {+ // Resolved before any span starts so the whole request stays unrecorded for+ // zero-data-retention and anonymous searches (see otel-tracer).+ const enterprise: unknown[] = Array.isArray(req.body?.enterprise)+ ? req.body.enterprise+ : [];+ const zeroDataRetentionTrace =+ Boolean(getSearchForcedKind(req.acuc?.flags)) ||+ enterprise.includes("zdr") ||+ enterprise.includes("anon");++ return withSpan(+ "api.search.request",+ span => searchControllerInner(req, res, span),+ {+ kind: SpanKind.SERVER,+ attributes: {+ "api.version": "v2",+ "search.team_id": req.auth.team_id,+ },+ zeroDataRetention: zeroDataRetentionTrace,+ },+ );+}++async function searchControllerInner(+ req: RequestWithAuth<{}, SearchResponse, SearchRequest>,+ res: Response<SearchResponse>,+ span: Span,) {const middlewareStartTime =(req as any).requestTiming?.startTime || new Date().getTime();@@ -169,7 +201,6 @@ export async function searchController(const isZDROrAnon = isZDR || isAnon;zeroDataRetention = isZDROrAnon ?? false;logger = logger.child({ zeroDataRetention });- applyZdrScope(zeroDataRetention);// Verify the team has searchZDR enabled before allowing enterprise ZDR/anonif (isZDROrAnon && !teamForcedKind) {@@ -406,13 +437,12 @@ export async function searchController(});}- captureExceptionWithZdrCheck(error, {- extra: { zeroDataRetention },- });logger.error("Unhandled error occurred in search", {version: "v2",error,});+ recordSpanException(span, error);+ setSpanAttributes(span, { "search.status_code": 500 });return res.status(500).json({success: false,error: error.message,diff --git a/apps/api/src/index.ts b/apps/api/src/index.tsindex 8beb321a2..8e1b29bf1 100644--- a/apps/api/src/index.ts+++ b/apps/api/src/index.ts@@ -1,8 +1,6 @@import "dotenv/config";import { config } from "./config";-import "./services/sentry";-import { setSentryServiceTag } from "./services/sentry";-import * as Sentry from "@sentry/node";+import { shutdownTracing } from "./otel";import express, { NextFunction, Request, Response } from "express";import bodyParser from "body-parser";import cors from "cors";@@ -20,11 +18,7 @@ import http from "node:http";import https from "node:https";import { v1Router } from "./routes/v1";import expressWs from "express-ws";-import {- ErrorResponse,- RequestWithMaybeACUC,- ResponseWithSentry,-} from "./controllers/v1/types";+import { ErrorResponse, RequestWithMaybeACUC } from "./controllers/v1/types";import { ZodError } from "zod";import { QueueFullError } from "./lib/queue-full-error";import { v7 as uuidv7 } from "uuid";@@ -75,8 +69,6 @@ const app = ws.app;global.isProduction = config.IS_PRODUCTION;-setSentryServiceTag("api");-// Capture the exact request bytes so integrations that sign the raw payload// (e.g. Slack's X-Slack-Signature) can verify it after body parsing. Typed with// the node http types body-parser's `verify` hook expects.@@ -187,6 +179,7 @@ async function startServer(port = DEFAULT_PORT) {shutdownWebhookQueue().finally(() => {shutdownIndexerQueue().finally(async () => {await shutdownPubSubLogging();+ await shutdownTracing();logger.info("NUQ shutdown complete");process.exit(0);});@@ -265,13 +258,11 @@ app.use(},);-Sentry.setupExpressErrorHandler(app);-app.use((err: unknown,req: RequestWithMaybeACUC<{}, ErrorResponse, undefined>,- res: ResponseWithSentry<ErrorResponse>,+ res: Response<ErrorResponse>,next: NextFunction,) => {if (@@ -298,7 +289,7 @@ app.use(});}- const id = res.sentry ?? uuidv7();+ const id = uuidv7();logger.error("Error occurred in request! (" + req.path + ") -- ID " + id + " -- ",diff --git a/apps/api/src/lib/branding/llm.ts b/apps/api/src/lib/branding/llm.tsindex b877615fd..bdd96c58b 100644--- a/apps/api/src/lib/branding/llm.ts+++ b/apps/api/src/lib/branding/llm.ts@@ -5,7 +5,6 @@ import { BrandingEnhancement, getBrandingEnhancementSchema } from "./schema";import { buildBrandingPrompt } from "./prompt";import { BrandingLLMInput } from "./types";import { getModel } from "../generic-ai";-import { captureExceptionWithZdrCheck } from "../../services/sentry";function isDebugBrandingEnabled(input: BrandingLLMInput): boolean {return (@@ -121,6 +120,9 @@ export async function enhanceBrandingWithLLM(temperature: 0.1,experimental_telemetry: {isEnabled: true,+ // The input carries the page screenshot / raw page content; too large+ // for span attributes. Outputs stay recorded.+ recordInputs: false,functionId: "enhanceBrandingWithLLM",metadata: {teamId: input.teamId || "unknown",@@ -193,22 +195,6 @@ export async function enhanceBrandingWithLLM(},);} else {- captureExceptionWithZdrCheck(error, {- tags: {- feature: "branding-llm",- model: modelName,- ...(input.scrapeId ? { scrape_id: input.scrapeId } : {}),- ...(input.teamId ? { team_id: input.teamId } : {}),- },- extra: {- url: input.url,- buttonsCount: input.buttons?.length || 0,- logoCandidatesCount: input.logoCandidates?.length || 0,- promptLength: prompt.length,- hasScreenshot: !!input.screenshot,- },- zeroDataRetention: input.zeroDataRetention,- });logger.error("LLM branding enhancement failed", {error,buttonsCount: input.buttons?.length || 0,diff --git a/apps/api/src/lib/crawl-redis.ts b/apps/api/src/lib/crawl-redis.tsindex f9a2b1ee1..52a2633bb 100644--- a/apps/api/src/lib/crawl-redis.ts+++ b/apps/api/src/lib/crawl-redis.ts@@ -39,35 +39,39 @@ export type StoredCrawl = {};export async function saveCrawl(id: string, crawl: StoredCrawl) {- return await withSpan("firecrawl-redis-save-crawl", async span => {- setSpanAttributes(span, {- "crawl.id": id,- "crawl.team_id": crawl.team_id,- "crawl.zero_data_retention": crawl.zeroDataRetention || false,- operation: "save_crawl",- });+ return await withSpan(+ "firecrawl-redis-save-crawl",+ async span => {+ setSpanAttributes(span, {+ "crawl.id": id,+ "crawl.team_id": crawl.team_id,+ "crawl.zero_data_retention": crawl.zeroDataRetention || false,+ operation: "save_crawl",+ });- _logger.debug("Saving crawl " + id + " to Redis...", {- crawl,- module: "crawl-redis",- method: "saveCrawl",- crawlId: id,- teamId: crawl.team_id,- zeroDataRetention: crawl.zeroDataRetention,- });+ _logger.debug("Saving crawl " + id + " to Redis...", {+ crawl,+ module: "crawl-redis",+ method: "saveCrawl",+ crawlId: id,+ teamId: crawl.team_id,+ zeroDataRetention: crawl.zeroDataRetention,+ });- await redisEvictConnection.set(- "crawl:" + id,- JSON.stringify(crawl),- "EX",- 24 * 60 * 60,- );- await redisEvictConnection.sadd("crawls_by_team_id:" + crawl.team_id, id);- await redisEvictConnection.expire(- "crawls_by_team_id:" + crawl.team_id,- 24 * 60 * 60,- );- });+ await redisEvictConnection.set(+ "crawl:" + id,+ JSON.stringify(crawl),+ "EX",+ 24 * 60 * 60,+ );+ await redisEvictConnection.sadd("crawls_by_team_id:" + crawl.team_id, id);+ await redisEvictConnection.expire(+ "crawls_by_team_id:" + crawl.team_id,+ 24 * 60 * 60,+ );+ },+ { zeroDataRetention: crawl.zeroDataRetention },+ );}export async function recordRobotsBlocked(crawlId: string, url: string) {@@ -134,6 +138,7 @@ export async function getCrawl(id: string): Promise<StoredCrawl | null> {setSpanAttributes(span, {"crawl.found": true,"crawl.team_id": crawl.team_id,+ "crawl.zero_data_retention": crawl.zeroDataRetention === true,});return crawl;@@ -669,103 +674,58 @@ export async function lockURL(url: string,__logger: Logger = _logger,): Promise<boolean> {- return await withSpan("firecrawl-redis-lock-url", async span => {- const normalizedUrl = normalizeURL(url, sc);- setSpanAttributes(span, {- "crawl.id": id,- "crawl.url": normalizedUrl,- "crawl.team_id": sc.team_id,- operation: "lock_url",- });-- if (typeof sc.crawlerOptions?.limit === "number") {- if (- (await redisEvictConnection.scard("crawl:" + id + ":visited_unique")) >=- sc.crawlerOptions.limit- ) {- setSpanAttributes(span, { "crawl.limit_reached": true });- return false;- }- }-- const lockTarget = !sc.crawlerOptions?.deduplicateSimilarURLs- ? normalizedUrl- : generateURLPermutations(normalizedUrl)[0].href;-- // The commands are idempotent, so retry the pipeline on failure: the- // SADD may land before a failing command (e.g. a failed EXPIRE), and- // giving up then would leave the URL marked visited without being- // queued — a caller retry would read SADD=0 and skip it. Track whether- // any attempt reported the SADD as newly added, since a retry after a- // partial success reads 0.- let sawNewLock = false;- let locked = false;- let lastError: unknown = null;- for (let attempt = 1; attempt <= 3; attempt++) {- let attemptError: unknown = null;- try {- const pipeline = redisEvictConnection.pipeline();- pipeline.sadd("crawl:" + id + ":visited", lockTarget);- pipeline.expire("crawl:" + id + ":visited", 24 * 60 * 60);- const attemptResults = await pipeline.exec();- if (attemptResults?.[0]?.[1] === 1) {- sawNewLock = true;- }- attemptError = firstPipelineError(attemptResults);- } catch (error) {- attemptError = error;- }-- if (attemptError === null) {- locked = true;- lastError = null;- break;- }- lastError = attemptError;-- __logger.error("Redis pipeline command failed", {- module: "crawl-redis",- method: "lockURL",- crawlId: id,- url: normalizedUrl,- attempt,- error: attemptError,+ return await withSpan(+ "firecrawl-redis-lock-url",+ async span => {+ const normalizedUrl = normalizeURL(url, sc);+ setSpanAttributes(span, {+ "crawl.id": id,+ "crawl.url": normalizedUrl,+ "crawl.team_id": sc.team_id,+ operation: "lock_url",});- if (attempt < 3) {- await new Promise(resolve => setTimeout(resolve, 50 * 2 ** attempt));+ if (typeof sc.crawlerOptions?.limit === "number") {+ if (+ (await redisEvictConnection.scard(+ "crawl:" + id + ":visited_unique",+ )) >= sc.crawlerOptions.limit+ ) {+ setSpanAttributes(span, { "crawl.limit_reached": true });+ return false;+ }}- }-- if (!locked) {- throw new Error("URL lock pipeline failed after retries", {- cause: lastError,- });- }-- const res = sawNewLock;- if (res) {- // The visited SADD has already landed, so a caller retry would skip- // this URL as already-visited and it would be lost from the crawl.- // The visited_unique bookkeeping is idempotent — retry it before- // exposing the failure.+ const lockTarget = !sc.crawlerOptions?.deduplicateSimilarURLs+ ? normalizedUrl+ : generateURLPermutations(normalizedUrl)[0].href;++ // The commands are idempotent, so retry the pipeline on failure: the+ // SADD may land before a failing command (e.g. a failed EXPIRE), and+ // giving up then would leave the URL marked visited without being+ // queued — a caller retry would read SADD=0 and skip it. Track whether+ // any attempt reported the SADD as newly added, since a retry after a+ // partial success reads 0.+ let sawNewLock = false;+ let locked = false;let lastError: unknown = null;for (let attempt = 1; attempt <= 3; attempt++) {let attemptError: unknown = null;try {- const uniquePipeline = redisEvictConnection.pipeline();- uniquePipeline.sadd("crawl:" + id + ":visited_unique", normalizedUrl);- uniquePipeline.expire(- "crawl:" + id + ":visited_unique",- 24 * 60 * 60,- );- attemptError = firstPipelineError(await uniquePipeline.exec());+ const pipeline = redisEvictConnection.pipeline();+ pipeline.sadd("crawl:" + id + ":visited", lockTarget);+ pipeline.expire("crawl:" + id + ":visited", 24 * 60 * 60);+ const attemptResults = await pipeline.exec();+ if (attemptResults?.[0]?.[1] === 1) {+ sawNewLock = true;+ }+ attemptError = firstPipelineError(attemptResults);} catch (error) {attemptError = error;}if (attemptError === null) {+ locked = true;lastError = null;break;}@@ -785,29 +745,42 @@ export async function lockURL(}}- if (lastError !== null) {- // Roll back the locks we just acquired, so a caller retry can- // re-lock and queue this URL instead of skipping it as- // already-visited. visited_unique may have landed during a retried- // attempt whose EXPIRE failed, so remove it too — otherwise the- // stale member consumes crawl-limit budget for a URL that was- // never queued. The rollback itself is retried; if it still fails,- // Redis is having a sustained outage and the canonical logs are- // the signal.+ if (!locked) {+ throw new Error("URL lock pipeline failed after retries", {+ cause: lastError,+ });+ }++ const res = sawNewLock;++ if (res) {+ // The visited SADD has already landed, so a caller retry would skip+ // this URL as already-visited and it would be lost from the crawl.+ // The visited_unique bookkeeping is idempotent — retry it before+ // exposing the failure.+ let lastError: unknown = null;for (let attempt = 1; attempt <= 3; attempt++) {- let rollbackError: unknown = null;+ let attemptError: unknown = null;try {- const rollback = redisEvictConnection.pipeline();- rollback.srem("crawl:" + id + ":visited", lockTarget);- rollback.srem("crawl:" + id + ":visited_unique", normalizedUrl);- rollbackError = firstPipelineError(await rollback.exec());+ const uniquePipeline = redisEvictConnection.pipeline();+ uniquePipeline.sadd(+ "crawl:" + id + ":visited_unique",+ normalizedUrl,+ );+ uniquePipeline.expire(+ "crawl:" + id + ":visited_unique",+ 24 * 60 * 60,+ );+ attemptError = firstPipelineError(await uniquePipeline.exec());} catch (error) {- rollbackError = error;+ attemptError = error;}- if (rollbackError === null) {+ if (attemptError === null) {+ lastError = null;break;}+ lastError = attemptError;__logger.error("Redis pipeline command failed", {module: "crawl-redis",@@ -815,7 +788,7 @@ export async function lockURL(crawlId: id,url: normalizedUrl,attempt,- error: rollbackError,+ error: attemptError,});if (attempt < 3) {@@ -824,15 +797,57 @@ export async function lockURL();}}- throw new Error("Unique URL lock pipeline failed after retries", {- cause: lastError,- });++ if (lastError !== null) {+ // Roll back the locks we just acquired, so a caller retry can+ // re-lock and queue this URL instead of skipping it as+ // already-visited. visited_unique may have landed during a retried+ // attempt whose EXPIRE failed, so remove it too — otherwise the+ // stale member consumes crawl-limit budget for a URL that was+ // never queued. The rollback itself is retried; if it still fails,+ // Redis is having a sustained outage and the canonical logs are+ // the signal.+ for (let attempt = 1; attempt <= 3; attempt++) {+ let rollbackError: unknown = null;+ try {+ const rollback = redisEvictConnection.pipeline();+ rollback.srem("crawl:" + id + ":visited", lockTarget);+ rollback.srem("crawl:" + id + ":visited_unique", normalizedUrl);+ rollbackError = firstPipelineError(await rollback.exec());+ } catch (error) {+ rollbackError = error;+ }++ if (rollbackError === null) {+ break;+ }++ __logger.error("Redis pipeline command failed", {+ module: "crawl-redis",+ method: "lockURL",+ crawlId: id,+ url: normalizedUrl,+ attempt,+ error: rollbackError,+ });++ if (attempt < 3) {+ await new Promise(resolve =>+ setTimeout(resolve, 50 * 2 ** attempt),+ );+ }+ }+ throw new Error("Unique URL lock pipeline failed after retries", {+ cause: lastError,+ });+ }}- }- setSpanAttributes(span, { "crawl.url_locked": res });- return res;- });+ setSpanAttributes(span, { "crawl.url_locked": res });+ return res;+ },+ { zeroDataRetention: sc.zeroDataRetention },+ );}/// NOTE: does not check limit. only use if limit is checked beforehand e.g. with sitemapdiff --git a/apps/api/src/lib/error.ts b/apps/api/src/lib/error.tsindex 67ed0f5a9..39cd45096 100644--- a/apps/api/src/lib/error.ts+++ b/apps/api/src/lib/error.ts@@ -389,7 +389,7 @@ export class ActionsNotSupportedError extends TransportableError {/*** Error thrown when a job is cancelled (expected flow control, not a real error)- * This should not be sent to Sentry as it's expected behavior when a crawl/batch is cancelled+ * This is expected behavior when a crawl/batch is cancelled*/export class JobCancelledError extends Error {constructor() {diff --git a/apps/api/src/lib/gcs-jobs.ts b/apps/api/src/lib/gcs-jobs.tsindex 3203da93a..92d86f179 100644--- a/apps/api/src/lib/gcs-jobs.ts+++ b/apps/api/src/lib/gcs-jobs.ts@@ -90,74 +90,78 @@ async function saveJobToGCS(params: {const attempts: GCSOperationAttempt[] = [];- return await withSpan("firecrawl-gcs-save-job", async span => {- setSpanAttributes(span, {- "gcs.operation": "save_job",- "job.id": params.id,- "job.request_id": params.request_id,- "job.team_id": params.team_id,- "job.mode": params.mode,- "job.success": params.is_successful,- "job.num_docs": params.num_docs,- });-- if (!config.GCS_BUCKET_NAME) {- setSpanAttributes(span, { "gcs.bucket_configured": false });- return;- }+ return await withSpan(+ "firecrawl-gcs-save-job",+ async span => {+ setSpanAttributes(span, {+ "gcs.operation": "save_job",+ "job.id": params.id,+ "job.request_id": params.request_id,+ "job.team_id": params.team_id,+ "job.mode": params.mode,+ "job.success": params.is_successful,+ "job.num_docs": params.num_docs,+ });- const bucket = storageManualRetries.bucket(config.GCS_BUCKET_NAME);- const blob = bucket.file(filename);+ if (!config.GCS_BUCKET_NAME) {+ setSpanAttributes(span, { "gcs.bucket_configured": false });+ return;+ }- let backoffUsed = BACKOFF_PARAMS;+ const bucket = storageManualRetries.bucket(config.GCS_BUCKET_NAME);+ const blob = bucket.file(filename);- const data = JSON.stringify(params.data);+ let backoffUsed = BACKOFF_PARAMS;- // Save job docs with retry- // Due to retries and resumable uploads, this is:- // if data is smaller than or exactly 3MB: best case 1 request, worst case 3 requests- // if data is larger than 3MB: best case 2 requests, worst case 6 requests- for (let i = 0; i < backoffUsed.length; i++) {- const backoffMs = backoffUsed[i];- if (backoffMs > 0) {- await new Promise(resolve => setTimeout(resolve, backoffMs));- }+ const data = JSON.stringify(params.data);- const saveStart = Date.now();- try {- await blob.save(data, {- metadata: {- contentType: "application/json",- metadata: params.metadata,- },- resumable: data.length > 3 * 1024 * 1024, // 3MB, 5MB official limit- });- attempts.push({- error: null,- timeMs: Date.now() - saveStart,- backoffMs,- });- break;- } catch (error) {- if (- error instanceof ApiError &&- (error.code === 429 || error.code === 503)- ) {- // switch to slower backoff parameters for rate limiting or server overloaded errors- backoffUsed = BACKOFF_SLOWDOWN_PARAMS;+ // Save job docs with retry+ // Due to retries and resumable uploads, this is:+ // if data is smaller than or exactly 3MB: best case 1 request, worst case 3 requests+ // if data is larger than 3MB: best case 2 requests, worst case 6 requests+ for (let i = 0; i < backoffUsed.length; i++) {+ const backoffMs = backoffUsed[i];+ if (backoffMs > 0) {+ await new Promise(resolve => setTimeout(resolve, backoffMs));}- attempts.push({ error, timeMs: Date.now() - saveStart, backoffMs });-- if (i === BACKOFF_PARAMS.length - 1) {- setSpanAttributes(span, { "gcs.save_successful": false });- throw error;+ const saveStart = Date.now();+ try {+ await blob.save(data, {+ metadata: {+ contentType: "application/json",+ metadata: params.metadata,+ },+ resumable: data.length > 3 * 1024 * 1024, // 3MB, 5MB official limit+ });+ attempts.push({+ error: null,+ timeMs: Date.now() - saveStart,+ backoffMs,+ });+ break;+ } catch (error) {+ if (+ error instanceof ApiError &&+ (error.code === 429 || error.code === 503)+ ) {+ // switch to slower backoff parameters for rate limiting or server overloaded errors+ backoffUsed = BACKOFF_SLOWDOWN_PARAMS;+ }++ attempts.push({ error, timeMs: Date.now() - saveStart, backoffMs });++ if (i === BACKOFF_PARAMS.length - 1) {+ setSpanAttributes(span, { "gcs.save_successful": false });+ throw error;+ }}}- }- setSpanAttributes(span, { "gcs.save_successful": true });- })+ setSpanAttributes(span, { "gcs.save_successful": true });+ },+ { zeroDataRetention: params.zeroDataRetention },+ ).then(x => {if (attempts.length === 0) {return x;diff --git a/apps/api/src/lib/html-to-markdown-client.ts b/apps/api/src/lib/html-to-markdown-client.tsindex bdaf12cd0..20d7ced40 100644--- a/apps/api/src/lib/html-to-markdown-client.ts+++ b/apps/api/src/lib/html-to-markdown-client.ts@@ -9,7 +9,6 @@ import axios, { AxiosInstance, AxiosError } from "axios";import { config } from "../config";import { logger } from "./logger";import type { Logger } from "winston";-import * as Sentry from "@sentry/node";interface ConvertRequest {html: string;@@ -112,22 +111,6 @@ export async function convertHTMLToMarkdownWithHttpService(serviceUrl: url,});- // Capture in Sentry with additional context (omit identifying fields- // and content sizes when ZDR so nothing is retained in Sentry either).- Sentry.captureException(error, {- tags: {- service: "html-to-markdown",- status_code: statusCode,- ...(requestId && !zeroDataRetention ? { request_id: requestId } : {}),- },- extra: {- serviceUrl: url,- errorMessage,- errorDetails,- ...(zeroDataRetention ? {} : { inputSize: html.length }),- },- });-// Include details in error message if availableconst fullErrorMessage = errorDetails? `HTML to Markdown conversion failed: ${errorMessage} - ${errorDetails}`@@ -141,11 +124,6 @@ export async function convertHTMLToMarkdownWithHttpService(error,},);- Sentry.captureException(error, {- tags: {- ...(requestId && !zeroDataRetention ? { request_id: requestId } : {}),- },- });throw error;}}diff --git a/apps/api/src/lib/html-to-markdown.ts b/apps/api/src/lib/html-to-markdown.tsindex 1b217f3b5..eeed6d66a 100644--- a/apps/api/src/lib/html-to-markdown.ts+++ b/apps/api/src/lib/html-to-markdown.ts@@ -1,7 +1,5 @@import koffi from "koffi";import { config } from "../config";-import "../services/sentry";-import * as Sentry from "@sentry/node";import { logger } from "./logger";import type { Logger } from "winston";import { stat } from "fs/promises";@@ -82,12 +80,6 @@ export async function parseMarkdown("Error converting HTML to Markdown with HTTP service, falling back to original parser",{ error },);- Sentry.captureException(error, {- tags: {- fallback: "original_parser",- ...(requestId && !zeroDataRetention ? { request_id: requestId } : {}),- },- });}}@@ -103,11 +95,6 @@ export async function parseMarkdown(!(error instanceof Error) ||error.message !== "Go shared library not found") {- Sentry.captureException(error, {- tags: {- ...(requestId && !zeroDataRetention ? { request_id: requestId } : {}),- },- });contextLogger.error(`Error converting HTML to Markdown with Go parser: ${error}`,);diff --git a/apps/api/src/lib/otel-tracer.test.ts b/apps/api/src/lib/otel-tracer.test.tsnew file mode 100644index 000000000..134b6e0a1--- /dev/null+++ b/apps/api/src/lib/otel-tracer.test.ts@@ -0,0 +1,244 @@+import {+ context,+ propagation,+ SpanStatusCode,+ trace,+} from "@opentelemetry/api";+import {+ InMemorySpanExporter,+ type NodeTracerProvider,+} from "@opentelemetry/sdk-trace-node";+import {+ createTracerProvider,+ serializeTraceContext,+ setSpanAttributes,+ SpanKind,+ withSpan,+ withTraceContextAsync,+ withZeroDataRetention,+ type SerializedTraceContext,+} from "./otel-tracer";++describe("otel-tracer", () => {+ const exporter = new InMemorySpanExporter();+ let provider: NodeTracerProvider;+ // The provider's env detector reads these; keep the suite hermetic.+ const savedEnv = {+ OTEL_SERVICE_NAME: process.env.OTEL_SERVICE_NAME,+ OTEL_RESOURCE_ATTRIBUTES: process.env.OTEL_RESOURCE_ATTRIBUTES,+ };++ beforeAll(() => {+ delete process.env.OTEL_SERVICE_NAME;+ delete process.env.OTEL_RESOURCE_ATTRIBUTES;+ provider = createTracerProvider({+ exporter,+ serviceName: "test-service",+ serviceInstanceId: "pod-1",+ });+ provider.register();+ });++ afterAll(async () => {+ await provider.shutdown();+ trace.disable();+ context.disable();+ propagation.disable();+ for (const [key, value] of Object.entries(savedEnv)) {+ if (value === undefined) delete process.env[key];+ else process.env[key] = value;+ }+ });++ beforeEach(() => {+ exporter.reset();+ });++ async function exportedSpans() {+ await provider.forceFlush();+ return exporter.getFinishedSpans();+ }++ it("exports spans with kind, attributes, status, and resource", async () => {+ const result = await withSpan(+ "test.root",+ async span => {+ setSpanAttributes(span, {+ "a.string": "x",+ "a.number": 1,+ "a.undefined": undefined,+ "a.null": null,+ });+ return 42;+ },+ { kind: SpanKind.SERVER, attributes: { initial: true } },+ );++ expect(result).toBe(42);+ const spans = await exportedSpans();+ expect(spans).toHaveLength(1);+ expect(spans[0].name).toBe("test.root");+ expect(spans[0].kind).toBe(SpanKind.SERVER);+ expect(spans[0].attributes).toEqual({+ initial: true,+ "a.string": "x",+ "a.number": 1,+ });+ expect(spans[0].status.code).toBe(SpanStatusCode.OK);+ expect(spans[0].resource.attributes["service.name"]).toBe("test-service");+ expect(spans[0].resource.attributes["service.instance.id"]).toBe("pod-1");+ });++ it("records the exception, marks the span as errored, and rethrows", async () => {+ await expect(+ withSpan("test.fail", async () => {+ throw new Error("boom");+ }),+ ).rejects.toThrow("boom");++ const [span] = await exportedSpans();+ expect(span.status).toEqual({+ code: SpanStatusCode.ERROR,+ message: "boom",+ });+ expect(span.events.map(event => event.name)).toContain("exception");+ });++ it("parents nested and library spans under the active span", async () => {+ await withSpan("test.parent", async () => {+ await withSpan("test.child", async () => {});+ // The AI SDK reports through the global tracer, like this.+ trace.getTracer("ai").startSpan("ai.generateText").end();+ });++ const spans = await exportedSpans();+ const parent = spans.find(span => span.name === "test.parent")!;+ expect(spans.map(span => span.name).sort()).toEqual([+ "ai.generateText",+ "test.child",+ "test.parent",+ ]);+ for (const span of spans.filter(span => span !== parent)) {+ expect(span.parentSpanContext?.spanId).toBe(parent.spanContext().spanId);+ }+ });++ it("records nothing started under zero data retention", async () => {+ await withSpan(+ "zdr.root",+ async span => {+ expect(span.isRecording()).toBe(false);+ await withSpan("zdr.child", async () => {});+ trace.getTracer("ai").startSpan("ai.generateText").end();+ },+ { zeroDataRetention: true },+ );+ await withZeroDataRetention(true, () =>+ withSpan("zdr.nested", async () => {}),+ );+ await withSpan("kept.root", async () => {}, { zeroDataRetention: false });++ expect((await exportedSpans()).map(span => span.name)).toEqual([+ "kept.root",+ ]);+ });++ it("drops spans that end with a zero_data_retention attribute", async () => {+ await withSpan("attr.dropped", async span => {+ setSpanAttributes(span, { "nuq.zero_data_retention": true });+ });+ await withSpan("attr.kept", async span => {+ setSpanAttributes(span, { "nuq.zero_data_retention": false });+ });++ expect((await exportedSpans()).map(span => span.name)).toEqual([+ "attr.kept",+ ]);+ });++ it("suppresses descendants once a live span is flagged zero data retention", async () => {+ await withSpan("late.root", async span => {+ setSpanAttributes(span, { "nuq.zero_data_retention": true });+ await withSpan("late.child", async () => {});+ trace.getTracer("ai").startSpan("ai.generateText").end();+ });++ expect(await exportedSpans()).toHaveLength(0);+ });++ it("drops in-flight spans of a trace flagged after they started", async () => {+ await withSpan("late.root", async root => {+ await withSpan("late.child", async () => {+ setSpanAttributes(root, { "crawl.zero_data_retention": true });+ });+ });+ await withSpan("other.root", async () => {});++ expect((await exportedSpans()).map(span => span.name)).toEqual([+ "other.root",+ ]);+ });++ it("continues a serialized trace and inherits the sampled decision", async () => {+ let serialized: SerializedTraceContext = {};+ let apiSpanId = "";+ let traceId = "";+ await withSpan("api.request", async span => {+ serialized = serializeTraceContext();+ apiSpanId = span.spanContext().spanId;+ traceId = span.spanContext().traceId;+ });+ expect(serialized.traceparent).toBe(`00-${traceId}-${apiSpanId}-01`);++ await withTraceContextAsync(serialized, () =>+ withSpan("worker.process", async () => {}),+ );++ const worker = (await exportedSpans()).find(+ span => span.name === "worker.process",+ )!;+ expect(worker.spanContext().traceId).toBe(traceId);+ expect(worker.parentSpanContext?.spanId).toBe(apiSpanId);+ });++ it("keeps a continued zero-data-retention trace unrecorded", async () => {+ let serialized: SerializedTraceContext = {};+ await withSpan(+ "api.zdr",+ async () => {+ serialized = serializeTraceContext();+ },+ { zeroDataRetention: true },+ );+ expect(serialized.traceparent).toMatch(/-00$/);++ await withTraceContextAsync(serialized, () =>+ withSpan("worker.process", async () => {}),+ );++ expect(await exportedSpans()).toHaveLength(0);+ });++ it("starts a fresh trace for missing or legacy serialized contexts", async () => {+ await withTraceContextAsync(undefined, () =>+ withSpan("worker.none", async () => {}),+ );+ await withTraceContextAsync(+ { sentryTrace: "legacy" } as unknown as SerializedTraceContext,+ () => withSpan("worker.legacy", async () => {}),+ );++ const spans = await exportedSpans();+ expect(spans.map(span => span.name).sort()).toEqual([+ "worker.legacy",+ "worker.none",+ ]);+ for (const span of spans) {+ expect(span.parentSpanContext).toBeUndefined();+ }+ });++ it("serializes an empty context outside of a span", () => {+ expect(serializeTraceContext()).toEqual({});+ });+});diff --git a/apps/api/src/lib/otel-tracer.ts b/apps/api/src/lib/otel-tracer.tsindex bca592fe1..aaeb53c2f 100644--- a/apps/api/src/lib/otel-tracer.ts+++ b/apps/api/src/lib/otel-tracer.ts@@ -1,71 +1,286 @@-import * as Sentry from "@sentry/node";-import type { Span } from "@sentry/node";-import type { SpanAttributeValue, SpanAttributes } from "@sentry/core";+import {+ context,+ createContextKey,+ isSpanContextValid,+ propagation,+ SpanKind,+ SpanStatusCode,+ trace,+ type Attributes,+ type AttributeValue,+ type Context,+ type Link,+ type Span,+} from "@opentelemetry/api";+import {+ AlwaysOnSampler,+ BatchSpanProcessor,+ NodeTracerProvider,+ ParentBasedSampler,+ SamplingDecision,+ type ReadableSpan,+ type Sampler,+ type SamplingResult,+ type SpanExporter,+ type SpanProcessor,+} from "@opentelemetry/sdk-trace-node";+import {+ defaultResource,+ detectResources,+ envDetector,+ resourceFromAttributes,+} from "@opentelemetry/resources";+import { ATTR_SERVICE_NAME } from "@opentelemetry/semantic-conventions";-export enum SpanKind {- INTERNAL = 0,- SERVER = 1,- CLIENT = 2,- PRODUCER = 3,- CONSUMER = 4,+export { SpanKind } from "@opentelemetry/api";+export type { Span } from "@opentelemetry/api";++const TRACER_NAME = "firecrawl-api";++// Zero data retention (ZDR) spans must never leave the process. Two layers+// enforce that:+// 1. `withZeroDataRetention` flags the active context; the sampler turns every+// span started under it (our own and library spans such as the AI SDK's)+// into a non-recording span, so nothing is even buffered.+// 2. Any span that ends carrying a `zero_data_retention` /+// `*.zero_data_retention` attribute set to `true` is dropped before export,+// covering spans that only learn about ZDR after they started.+const ZERO_DATA_RETENTION_CONTEXT_KEY = createContextKey(+ "firecrawl.zero_data_retention",+);++export function withZeroDataRetention<T>(+ zeroDataRetention: boolean | undefined,+ fn: () => T,+): T {+ if (!zeroDataRetention) {+ return fn();+ }++ return context.with(+ context.active().setValue(ZERO_DATA_RETENTION_CONTEXT_KEY, true),+ fn,+ );+}++function isZeroDataRetentionContext(ctx: Context): boolean {+ return ctx.getValue(ZERO_DATA_RETENTION_CONTEXT_KEY) === true;+}++// Traces flagged ZDR after their root span started (a `*.zero_data_retention`+// attribute set from inside a *recording* span). Descendants started afterwards+// are not recorded and spans of these traces still in flight are dropped when+// they end. Only recording spans register here: a non-recording span already+// suppresses its descendants through parent-based sampling, so the registry+// only sees the rare late-discovery case and stays far below its cap. The cap+// is a memory guard, not an expected steady state.+const ZERO_DATA_RETENTION_TRACE_LIMIT = 100_000;+const zeroDataRetentionTraces = new Set<string>();++function markTraceZeroDataRetention(traceId: string): void {+ if (zeroDataRetentionTraces.has(traceId)) {+ return;+ }++ zeroDataRetentionTraces.add(traceId);+ if (zeroDataRetentionTraces.size > ZERO_DATA_RETENTION_TRACE_LIMIT) {+ const oldest = zeroDataRetentionTraces.values().next().value;+ if (oldest !== undefined) {+ zeroDataRetentionTraces.delete(oldest);+ }+ }+}++function hasZeroDataRetentionAttribute(attributes: Attributes): boolean {+ for (const [key, value] of Object.entries(attributes)) {+ if (+ value === true &&+ (key === "zero_data_retention" || key.endsWith(".zero_data_retention"))+ ) {+ return true;+ }+ }++ return false;+}++class ZeroDataRetentionSampler implements Sampler {+ constructor(private readonly delegate: Sampler) {}++ shouldSample(+ ctx: Context,+ traceId: string,+ spanName: string,+ spanKind: SpanKind,+ attributes: Attributes,+ links: Link[],+ ): SamplingResult {+ if (hasZeroDataRetentionAttribute(attributes)) {+ markTraceZeroDataRetention(traceId);+ return { decision: SamplingDecision.NOT_RECORD };+ }++ if (+ isZeroDataRetentionContext(ctx) ||+ zeroDataRetentionTraces.has(traceId)+ ) {+ return { decision: SamplingDecision.NOT_RECORD };+ }++ return this.delegate.shouldSample(+ ctx,+ traceId,+ spanName,+ spanKind,+ attributes,+ links,+ );+ }++ toString(): string {+ return `ZeroDataRetentionSampler(${this.delegate.toString()})`;+ }+}++class ZeroDataRetentionSpanProcessor implements SpanProcessor {+ constructor(private readonly delegate: SpanProcessor) {}++ onStart(+ span: Parameters<SpanProcessor["onStart"]>[0],+ parentContext: Context,+ ): void {+ this.delegate.onStart(span, parentContext);+ }++ onEnd(span: ReadableSpan): void {+ const traceId = span.spanContext().traceId;+ if (hasZeroDataRetentionAttribute(span.attributes)) {+ markTraceZeroDataRetention(traceId);+ return;+ }++ if (zeroDataRetentionTraces.has(traceId)) {+ return;+ }++ this.delegate.onEnd(span);+ }++ forceFlush(): Promise<void> {+ return this.delegate.forceFlush();+ }++ shutdown(): Promise<void> {+ return this.delegate.shutdown();+ }+}++interface TracerProviderOptions {+ exporter: SpanExporter;+ serviceName: string;+ serviceInstanceId?: string;}-const SPAN_STATUS_OK = 1;-const SPAN_STATUS_ERROR = 2;+/**+ * Builds the tracer provider used by every Firecrawl process: 100% sampling+ * (respecting the sampled flag of a remote parent), ZDR filtering, and batched+ * export. Callers decide whether to `register()` it globally.+ */+export function createTracerProvider(+ options: TracerProviderOptions,+): NodeTracerProvider {+ const resource = defaultResource()+ .merge(detectResources({ detectors: [envDetector] }))+ .merge(+ resourceFromAttributes({+ [ATTR_SERVICE_NAME]: options.serviceName,+ ...(options.serviceInstanceId+ ? { "service.instance.id": options.serviceInstanceId }+ : {}),+ }),+ );+ return new NodeTracerProvider({+ resource,+ sampler: new ZeroDataRetentionSampler(+ new ParentBasedSampler({ root: new AlwaysOnSampler() }),+ ),+ spanProcessors: [+ new ZeroDataRetentionSpanProcessor(+ new BatchSpanProcessor(options.exporter),+ ),+ ],+ });+}++/**+ * W3C trace context carrier, stored on queue jobs so a worker can continue the+ * trace the API request started.+ */export interface SerializedTraceContext {- sentryTrace?: string;+ traceparent?: string;+ tracestate?: string;baggage?: string;}export function serializeTraceContext(): SerializedTraceContext {- const traceData = Sentry.getTraceData();+ const carrier: Record<string, string> = {};+ propagation.inject(context.active(), carrier);- return {- sentryTrace: traceData["sentry-trace"],- baggage: traceData["baggage"],- };+ const serialized: SerializedTraceContext = {};+ if (carrier.traceparent) serialized.traceparent = carrier.traceparent;+ if (carrier.tracestate) serialized.tracestate = carrier.tracestate;+ if (carrier.baggage) serialized.baggage = carrier.baggage;+ return serialized;}export async function withTraceContextAsync<T>(- serializedContext: SerializedTraceContext,+ serializedContext: SerializedTraceContext | undefined,fn: () => Promise<T>,): Promise<T> {- if (!serializedContext.sentryTrace) {+ if (!serializedContext?.traceparent) {return fn();}- return Sentry.continueTrace(- {- sentryTrace: serializedContext.sentryTrace,- baggage: serializedContext.baggage,- },- fn,- );+ const carrier: Record<string, string> = {+ traceparent: serializedContext.traceparent,+ };+ if (serializedContext.tracestate) {+ carrier.tracestate = serializedContext.tracestate;+ }+ if (serializedContext.baggage) {+ carrier.baggage = serializedContext.baggage;+ }++ return context.with(propagation.extract(context.active(), carrier), fn);}-type Attributes = SpanAttributes;+type SpanAttributeInput = Record<string, AttributeValue | null | undefined>;-interface SpanOptions {- attributes?: Attributes;- kind?: SpanKind;- op?: string;-}+function cleanAttributes(+ attributes: SpanAttributeInput | undefined,+): Attributes | undefined {+ if (!attributes) {+ return undefined;+ }-function spanKindToOp(kind?: SpanKind): string | undefined {- switch (kind) {- case SpanKind.SERVER:- return "http.server";- case SpanKind.CLIENT:- return "http.client";- case SpanKind.PRODUCER:- return "queue.publish";- case SpanKind.CONSUMER:- return "queue.process";- case SpanKind.INTERNAL:- default:- return undefined;+ const cleaned: Attributes = {};+ for (const [key, value] of Object.entries(attributes)) {+ if (value !== undefined && value !== null) {+ cleaned[key] = value;+ }}+ return cleaned;+}++interface SpanOptions {+ attributes?: SpanAttributeInput;+ kind?: SpanKind;+ /**+ * Marks the span and everything started inside it as zero data retention:+ * nothing is recorded or exported.+ */+ zeroDataRetention?: boolean;}export async function withSpan<T>(@@ -73,35 +288,61 @@ export async function withSpan<T>(fn: (span: Span) => Promise<T>,options?: SpanOptions,): Promise<T> {- const op = options?.op || spanKindToOp(options?.kind);+ const tracer = trace.getTracer(TRACER_NAME);- return Sentry.startSpan(- {+ return withZeroDataRetention(options?.zeroDataRetention, () =>+ tracer.startActiveSpan(name,- op,- attributes: options?.attributes,- },- async span => {- try {- const result = await fn(span);- span.setStatus({ code: SPAN_STATUS_OK });- return result;- } catch (error) {- span.setStatus({- code: SPAN_STATUS_ERROR,- message: error instanceof Error ? error.message : String(error),- });-- throw error;- }- },+ { kind: options?.kind, attributes: cleanAttributes(options?.attributes) },+ async span => {+ try {+ const result = await fn(span);+ span.setStatus({ code: SpanStatusCode.OK });+ return result;+ } catch (error) {+ recordSpanException(span, error);+ throw error;+ } finally {+ span.end();+ }+ },+ ),);}-export function setSpanAttributes(span: Span, attributes: Attributes): void {- Object.entries(attributes).forEach(([key, value]) => {- if (value !== undefined && value !== null) {- span.setAttribute(key, value as SpanAttributeValue);- }+/**+ * Marks a span as failed and attaches the error as an exception event, for+ * failures that are handled (returned) rather than thrown out of `withSpan`.+ */+export function recordSpanException(span: Span, error: unknown): void {+ span.recordException(error instanceof Error ? error : String(error));+ span.setStatus({+ code: SpanStatusCode.ERROR,+ message: error instanceof Error ? error.message : String(error),});}++export function setSpanAttributes(+ span: Span,+ attributes: SpanAttributeInput,+): void {+ const cleaned = cleanAttributes(attributes);+ if (!cleaned) {+ return;+ }++ span.setAttributes(cleaned);++ // Flagging a live, recording span as ZDR also covers everything started under+ // it from now on, plus spans of the trace that are still in flight. A+ // non-recording span needs no registry entry: its descendants are already+ // dropped by parent-based sampling.+ const spanContext = span.spanContext();+ if (+ span.isRecording() &&+ hasZeroDataRetentionAttribute(cleaned) &&+ isSpanContextValid(spanContext)+ ) {+ markTraceZeroDataRetention(spanContext.traceId);+ }+}diff --git a/apps/api/src/lib/queue-full-error.ts b/apps/api/src/lib/queue-full-error.tsindex 5e9e44245..3cea8e5ca 100644--- a/apps/api/src/lib/queue-full-error.ts+++ b/apps/api/src/lib/queue-full-error.ts@@ -1,5 +1,4 @@const QUEUE_FULL_ERROR_NAME = "QueueFullError";-const QUEUE_FULL_ERROR_MESSAGE_PREFIX = "Queue limit reached:";export class QueueFullError extends Error {statusCode = 429;@@ -11,41 +10,3 @@ export class QueueFullError extends Error {this.name = QUEUE_FULL_ERROR_NAME;}}--function getStringField(- error: Record<string, unknown>,- fields: string[],-): string {- for (const field of fields) {- if (field in error && error[field] !== undefined && error[field] !== null) {- return String(error[field]);- }- }-- return "";-}--export function isQueueFullError(error: unknown): boolean {- const objectError =- error && typeof error === "object"- ? (error as Record<string, unknown>)- : null;-- const errorName = objectError- ? getStringField(objectError, ["name", "type"])- : "";- if (errorName === QUEUE_FULL_ERROR_NAME) {- return true;- }-- const errorMessage =- error instanceof Error- ? error.message- : typeof error === "string"- ? error- : objectError- ? getStringField(objectError, ["message", "value"])- : "";-- return errorMessage.startsWith(QUEUE_FULL_ERROR_MESSAGE_PREFIX);-}diff --git a/apps/api/src/lib/retry-utils.ts b/apps/api/src/lib/retry-utils.tsindex e32269949..c794bbbdb 100644--- a/apps/api/src/lib/retry-utils.ts+++ b/apps/api/src/lib/retry-utils.ts@@ -1,4 +1,3 @@-import * as Sentry from "@sentry/node";import { logger } from "./logger";const RETRY_DELAYS = [500, 1500, 3000] as const;@@ -47,7 +46,6 @@ export async function attemptRequest<T>(}} catch (error) {logger.error("Fire Engine API request failed:", error);- Sentry.captureException(error);}return null;}@@ -107,7 +105,6 @@ export async function executeWithRetry<T>(}logger.error(`Attempt ${attempt + 1} failed:`, error);- Sentry.captureException(error);}// Wait before retry (except on last attempt)diff --git a/apps/api/src/lib/supabase-jobs.ts b/apps/api/src/lib/supabase-jobs.tsindex 0c9d8db59..46e6672eb 100644--- a/apps/api/src/lib/supabase-jobs.ts+++ b/apps/api/src/lib/supabase-jobs.ts@@ -3,7 +3,6 @@ import { eq, inArray, and } from "drizzle-orm";import { db, dbRr } from "../db/connection";import * as schema from "../db/schema";import { logger } from "./logger";-import * as Sentry from "@sentry/node";/*** Get a single scrape by ID from the scrapes table@@ -38,7 +37,6 @@ export const supabaseGetScrapesById = async (.where(inArray(schema.scrapes.id, scrapeIds));} catch (error) {logger.error(`Error in supabaseGetScrapesById: ${error}`);- Sentry.captureException(error);return [];}};@@ -58,7 +56,6 @@ export const supabaseGetScrapesByRequestId = async (.where(eq(schema.scrapes.request_id, requestId));} catch (error) {logger.error(`Error in supabaseGetScrapesByRequestId: ${error}`);- Sentry.captureException(error);return [];}};diff --git a/apps/api/src/lib/withAuth.ts b/apps/api/src/lib/withAuth.tsindex a9d602cbd..305322aaa 100644--- a/apps/api/src/lib/withAuth.ts+++ b/apps/api/src/lib/withAuth.ts@@ -1,6 +1,5 @@import { AuthResponse } from "../../src/types";import { logger } from "./logger";-import * as Sentry from "@sentry/node";import { configDotenv } from "dotenv";import { config } from "../config";configDotenv();diff --git a/apps/api/src/otel.ts b/apps/api/src/otel.tsnew file mode 100644index 000000000..feecb4709--- /dev/null+++ b/apps/api/src/otel.ts@@ -0,0 +1,57 @@+import { diag, DiagLogLevel, type DiagLogger } from "@opentelemetry/api";+import { OTLPTraceExporter } from "@opentelemetry/exporter-trace-otlp-proto";+import { config } from "./config";+import { logger } from "./lib/logger";+import { createTracerProvider } from "./lib/otel-tracer";+import type { NodeTracerProvider } from "@opentelemetry/sdk-trace-node";++// Imported for its side effects by every entry point, before anything that+// starts spans. Tracing stays off unless an OTLP endpoint is configured, in+// which case spans go to `OTEL_EXPORTER_OTLP_TRACES_ENDPOINT` or+// `OTEL_EXPORTER_OTLP_ENDPOINT` + `/v1/traces` over http/protobuf. The exporter+// reads the standard `OTEL_EXPORTER_OTLP_*` variables (headers, timeout,+// compression) itself.+const endpoint =+ config.OTEL_EXPORTER_OTLP_TRACES_ENDPOINT ||+ config.OTEL_EXPORTER_OTLP_ENDPOINT;++let provider: NodeTracerProvider | undefined;++if (endpoint) {+ const otelLogger = logger.child({ module: "otel" });+ const diagLogger: DiagLogger = {+ error: (message, ...args) => otelLogger.error(message, { args }),+ warn: (message, ...args) => otelLogger.warn(message, { args }),+ info: (message, ...args) => otelLogger.info(message, { args }),+ debug: (message, ...args) => otelLogger.debug(message, { args }),+ verbose: (message, ...args) => otelLogger.debug(message, { args }),+ };+ diag.setLogger(diagLogger, DiagLogLevel.WARN);++ const serviceName = config.OTEL_SERVICE_NAME ?? "firecrawl-api";+ provider = createTracerProvider({+ exporter: new OTLPTraceExporter(),+ serviceName,+ serviceInstanceId: config.NUQ_POD_NAME,+ });+ provider.register();++ otelLogger.info("OpenTelemetry tracing enabled", { endpoint, serviceName });+}++/**+ * Flushes buffered spans and stops the exporter. Bounded so a stuck collector+ * cannot hold up process shutdown.+ */+export async function shutdownTracing(timeoutMs = 5000): Promise<void> {+ if (!provider) {+ return;+ }++ const active = provider;+ provider = undefined;+ await Promise.race([+ active.shutdown().catch(() => {}),+ new Promise<void>(resolve => setTimeout(resolve, timeoutMs).unref()),+ ]);+}diff --git a/apps/api/src/scraper/scrapeURL/engines/fire-engine/checkStatus.ts b/apps/api/src/scraper/scrapeURL/engines/fire-engine/checkStatus.tsindex e847dae1f..d200a13c3 100644--- a/apps/api/src/scraper/scrapeURL/engines/fire-engine/checkStatus.ts+++ b/apps/api/src/scraper/scrapeURL/engines/fire-engine/checkStatus.ts@@ -1,5 +1,4 @@import { Logger } from "winston";-import * as Sentry from "@sentry/node";import { z } from "zod";import { robustFetch } from "../../lib/fetch";diff --git a/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts b/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.tsindex 3c4f35aeb..67b1082ed 100644--- a/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts+++ b/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts@@ -25,7 +25,6 @@ import {FEPageLoadFailed,ProxySelectionError,} from "../../error";-import * as Sentry from "@sentry/node";import { gunzipSync } from "node:zlib";import { specialtyScrapeCheck } from "../utils/specialtyHandler";import {@@ -202,7 +201,6 @@ async function performFireEngineScrape<`An unexpeceted error occurred while calling checkStatus. Error counter is now at ${errors.length}.`,{ error, jobId: (scrape as any).jobId },);- Sentry.captureException(error);}}diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/index.tsindex 955489d5e..f3875d2c5 100644--- a/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts@@ -58,7 +58,6 @@ import { runFirePdfByReferenceAttempt } from "./fire-pdf/by-reference-flow";import { decideFirePdfAsyncRoute } from "./fire-pdf/routing";import { scrapePDFWithParsePDF } from "./pdfParse";import { toPublicBlocks } from "./blocks";-import { captureExceptionWithZdrCheck } from "../../../../services/sentry";import { isPdfBuffer, PDF_SNIFF_WINDOW } from "./pdfUtils";import { comparePdfOutputs } from "./shadowComparison";import { withPdfExtractionPermit } from "./semaphore";@@ -370,14 +369,6 @@ export async function scrapePDF(meta: Meta): Promise<EngineScrapeResult> {error,url: meta.rewrittenUrl ?? meta.url,});- captureExceptionWithZdrCheck(error, {- extra: {- zeroDataRetention: meta.internalOptions.zeroDataRetention ?? false,- scrapeId: meta.id,- teamId: meta.internalOptions.teamId,- url: meta.rewrittenUrl ?? meta.url,- },- });}} else {// Rust extraction enabled (fast / auto modes).@@ -483,14 +474,6 @@ export async function scrapePDF(meta: Meta): Promise<EngineScrapeResult> {error,url: meta.rewrittenUrl ?? meta.url,});- captureExceptionWithZdrCheck(error, {- extra: {- zeroDataRetention: meta.internalOptions.zeroDataRetention ?? false,- scrapeId: meta.id,- teamId: meta.internalOptions.teamId,- url: meta.rewrittenUrl ?? meta.url,- },- });// effectivePageCount stays 0 — skip time budget check}}@@ -882,15 +865,6 @@ export async function scrapePDF(meta: Meta): Promise<EngineScrapeResult> {"RunPod MU failed to parse PDF (could be due to timeout) -- falling back to parse-pdf",{ error },);- captureExceptionWithZdrCheck(error, {- extra: {- zeroDataRetention:- meta.internalOptions.zeroDataRetention ?? false,- scrapeId: meta.id,- teamId: meta.internalOptions.teamId,- url: meta.rewrittenUrl ?? meta.url,- },- });const muV1DurationMs = Date.now() - muV1StartedAt;meta.logger.child({ method: "scrapePDF/MUv1Experiment" })diff --git a/apps/api/src/scraper/scrapeURL/index.ts b/apps/api/src/scraper/scrapeURL/index.tsindex dfe3bae08..3f797fcd2 100644--- a/apps/api/src/scraper/scrapeURL/index.ts+++ b/apps/api/src/scraper/scrapeURL/index.ts@@ -1,8 +1,10 @@import { Logger } from "winston";import { config } from "../../config";-import * as Sentry from "@sentry/node";-import { withSpan, setSpanAttributes } from "../../lib/otel-tracer";-import { captureExceptionWithZdrCheck } from "../../services/sentry";+import {+ withSpan,+ setSpanAttributes,+ recordSpanException,+} from "../../lib/otel-tracer";import {applyScrapeOptionsDefaults,@@ -1227,597 +1229,621 @@ export async function scrapeURL(internalOptions: InternalOptions,costTracking: CostTracking,): Promise<ScrapeUrlResponse> {- return withSpan("scrape.pipeline", async span => {- const meta = await buildMetaObject(- id,- url,- options,- internalOptions,- costTracking,- );-- const startTime = Date.now();+ return withSpan(+ "scrape.pipeline",+ async span => {+ const meta = await buildMetaObject(+ id,+ url,+ options,+ internalOptions,+ costTracking,+ );- // Set initial span attributes- setSpanAttributes(span, {- "scrape.id": id,- "scrape.url": url,- "scrape.team_id": internalOptions.teamId,- "scrape.crawl_id": internalOptions.crawlId,- "scrape.zero_data_retention": internalOptions.zeroDataRetention,- "scrape.force_engine": Array.isArray(internalOptions.forceEngine)- ? internalOptions.forceEngine.join(",")- : internalOptions.forceEngine,- "scrape.features": Array.from(meta.featureFlags).join(","),- });+ const startTime = Date.now();- meta.logger.info("scrapeURL entered");-- // Threat protection: check the target URL BEFORE any engine selection- // or outbound fetch. The policy is resolved at the controller layer and- // threaded through internalOptions; absent policy = zero overhead.- // The dedup map is scoped to this one scrape: the initial check and any- // redirect re-check on the same URL share a single scan (one fee); a- // redirect to a different URL is a second scan and bills a second fee- // (see calculateThreatScanCredits).- const threatPolicy = internalOptions.threatProtection;- const threatDedup: ThreatCheckDedup = new Map();- if (threatPolicy && threatPolicy.mode !== "off") {- const initialUrl = meta.rewrittenUrl ?? meta.url;- const decision = await checkUrl(initialUrl, threatPolicy, {- teamId: internalOptions.teamId,- dedup: threatDedup,+ // Set initial span attributes+ setSpanAttributes(span, {+ "scrape.id": id,+ "scrape.url": url,+ "scrape.team_id": internalOptions.teamId,+ "scrape.crawl_id": internalOptions.crawlId,+ "scrape.zero_data_retention": internalOptions.zeroDataRetention,+ "scrape.force_engine": Array.isArray(internalOptions.forceEngine)+ ? internalOptions.forceEngine.join(",")+ : internalOptions.forceEngine,+ "scrape.features": Array.from(meta.featureFlags).join(","),});- meta.threatDecisions.push(decision);- if (!decision.allowed) {- meta.logger.info("URL blocked by threat protection policy", {- url: initialUrl,- domain: decision.domain,- rule: decision.rule,++ meta.logger.info("scrapeURL entered");++ // Threat protection: check the target URL BEFORE any engine selection+ // or outbound fetch. The policy is resolved at the controller layer and+ // threaded through internalOptions; absent policy = zero overhead.+ // The dedup map is scoped to this one scrape: the initial check and any+ // redirect re-check on the same URL share a single scan (one fee); a+ // redirect to a different URL is a second scan and bills a second fee+ // (see calculateThreatScanCredits).+ const threatPolicy = internalOptions.threatProtection;+ const threatDedup: ThreatCheckDedup = new Map();+ if (threatPolicy && threatPolicy.mode !== "off") {+ const initialUrl = meta.rewrittenUrl ?? meta.url;+ const decision = await checkUrl(initialUrl, threatPolicy, {+ teamId: internalOptions.teamId,+ dedup: threatDedup,});+ meta.threatDecisions.push(decision);+ if (!decision.allowed) {+ meta.logger.info("URL blocked by threat protection policy", {+ url: initialUrl,+ domain: decision.domain,+ rule: decision.rule,+ });+ setSpanAttributes(span, {+ "scrape.blocked_by_threat_protection": true,+ });+ return {+ success: false,+ error: new UnsafeDomainBlockedError(initialUrl, decision),+ threatDecisions: meta.threatDecisions,+ };+ }+ }++ if (meta.rewrittenUrl) {+ meta.logger.info("Rewriting URL");setSpanAttributes(span, {- "scrape.blocked_by_threat_protection": true,+ "scrape.rewritten_url": meta.rewrittenUrl,});- return {- success: false,- error: new UnsafeDomainBlockedError(initialUrl, decision),- threatDecisions: meta.threatDecisions,- };}- }- if (meta.rewrittenUrl) {- meta.logger.info("Rewriting URL");- setSpanAttributes(span, {- "scrape.rewritten_url": meta.rewrittenUrl,- });- }-- if (internalOptions.isPreCrawl === true) {- setSpanAttributes(span, {- "scrape.is_precrawl": true,- });- }-- if (shouldCheckRobots(options, internalOptions)) {- const result = await withSpan("scrape.robots_check", async robotsSpan => {- const urlToCheck = meta.rewrittenUrl || meta.url;- meta.logger.info("Checking robots.txt", { url: urlToCheck });-- const urlObj = new URL(urlToCheck);- const isRobotsTxtPath = urlObj.pathname === "/robots.txt";-- setSpanAttributes(robotsSpan, {- "robots.url": urlToCheck,- "robots.is_robots_txt_path": isRobotsTxtPath,+ if (internalOptions.isPreCrawl === true) {+ setSpanAttributes(span, {+ "scrape.is_precrawl": true,});+ }- if (!isRobotsTxtPath) {- try {- let robotsTxt: string | undefined;- if (internalOptions.crawlId) {- const crawl = await getCrawl(internalOptions.crawlId);- robotsTxt = crawl?.robots;- }-- if (!robotsTxt) {- const { content } = await fetchRobotsTxt(- {- url: urlToCheck,- zeroDataRetention: internalOptions.zeroDataRetention || false,- location: options.location,- },- id,- meta.logger,- meta.abort.asSignal(),- );- robotsTxt = content;- }+ if (shouldCheckRobots(options, internalOptions)) {+ const result = await withSpan(+ "scrape.robots_check",+ async robotsSpan => {+ const urlToCheck = meta.rewrittenUrl || meta.url;+ meta.logger.info("Checking robots.txt", { url: urlToCheck });- const checker = createRobotsChecker(urlToCheck, robotsTxt);- const isAllowed = isUrlAllowedByRobots(urlToCheck, checker.robots);+ const urlObj = new URL(urlToCheck);+ const isRobotsTxtPath = urlObj.pathname === "/robots.txt";setSpanAttributes(robotsSpan, {- "robots.allowed": isAllowed,+ "robots.url": urlToCheck,+ "robots.is_robots_txt_path": isRobotsTxtPath,});- if (!isAllowed) {- meta.logger.info("URL blocked by robots.txt", {- url: urlToCheck,- });- setSpanAttributes(span, {- "scrape.blocked_by_robots": true,- });- throw new CrawlDenialError("URL blocked by robots.txt");- }- } catch (error) {- if (error instanceof CrawlDenialError) {- throw error;+ if (!isRobotsTxtPath) {+ try {+ let robotsTxt: string | undefined;+ if (internalOptions.crawlId) {+ const crawl = await getCrawl(internalOptions.crawlId);+ robotsTxt = crawl?.robots;+ }++ if (!robotsTxt) {+ const { content } = await fetchRobotsTxt(+ {+ url: urlToCheck,+ zeroDataRetention:+ internalOptions.zeroDataRetention || false,+ location: options.location,+ },+ id,+ meta.logger,+ meta.abort.asSignal(),+ );+ robotsTxt = content;+ }++ const checker = createRobotsChecker(urlToCheck, robotsTxt);+ const isAllowed = isUrlAllowedByRobots(+ urlToCheck,+ checker.robots,+ );++ setSpanAttributes(robotsSpan, {+ "robots.allowed": isAllowed,+ });++ if (!isAllowed) {+ meta.logger.info("URL blocked by robots.txt", {+ url: urlToCheck,+ });+ setSpanAttributes(span, {+ "scrape.blocked_by_robots": true,+ });+ throw new CrawlDenialError("URL blocked by robots.txt");+ }+ } catch (error) {+ if (error instanceof CrawlDenialError) {+ throw error;+ }+ meta.logger.debug(+ "Failed to fetch robots.txt, allowing scrape",+ {+ error,+ url: urlToCheck,+ },+ );+ setSpanAttributes(robotsSpan, {+ "robots.fetch_failed": true,+ });+ }}- meta.logger.debug("Failed to fetch robots.txt, allowing scrape", {+ },+ ).catch(error => {+ if (error instanceof CrawlDenialError) {+ return {+ success: false as const,error,- url: urlToCheck,- });- setSpanAttributes(robotsSpan, {- "robots.fetch_failed": true,- });+ };}- }- }).catch(error => {- if (error instanceof CrawlDenialError) {+ throw error;+ });+ if (result) {return {- success: false as const,- error,+ ...result,+ threatDecisions: meta.threatDecisions,};}- throw error;- });- if (result) {- return {- ...result,- threatDecisions: meta.threatDecisions,- };}- }- // Initialize retry tracker with configured limits- const retryTracker = new ScrapeRetryTracker(- {- maxAttempts: config.SCRAPE_MAX_ATTEMPTS,- maxFeatureToggles: config.SCRAPE_MAX_FEATURE_TOGGLES,- maxFeatureRemovals: config.SCRAPE_MAX_FEATURE_REMOVALS,- maxPdfPrefetches: config.SCRAPE_MAX_PDF_PREFETCHES,- maxDocumentPrefetches: config.SCRAPE_MAX_DOCUMENT_PREFETCHES,- },- meta.logger,- );+ // Initialize retry tracker with configured limits+ const retryTracker = new ScrapeRetryTracker(+ {+ maxAttempts: config.SCRAPE_MAX_ATTEMPTS,+ maxFeatureToggles: config.SCRAPE_MAX_FEATURE_TOGGLES,+ maxFeatureRemovals: config.SCRAPE_MAX_FEATURE_REMOVALS,+ maxPdfPrefetches: config.SCRAPE_MAX_PDF_PREFETCHES,+ maxDocumentPrefetches: config.SCRAPE_MAX_DOCUMENT_PREFETCHES,+ },+ meta.logger,+ );- try {- let result: ScrapeUrlResponse;- while (true) {- try {- result = await scrapeURLLoop(meta);- break;- } catch (error) {- if (- error instanceof AddFeatureError &&- (meta.internalOptions.forceEngine === undefined ||- Array.isArray(meta.internalOptions.forceEngine))- ) {- retryTracker.record("feature_toggle", error);- // A file handoff names the one parser that can open the file;- // the file flag the URL extension implied earlier gives way to it.- const nextFeatureFlags = applyHandoffFeatureFlags(- meta.featureFlags,- error.featureFlags,- );- meta.logger.debug(- "More feature flags requested by scraper: adding " +- error.featureFlags.join(", "),- {- error,- existingFlags: meta.featureFlags,- droppedFlags: [...meta.featureFlags].filter(- flag => !nextFeatureFlags.has(flag),- ),- },- );- meta.featureFlags = nextFeatureFlags;- if (error.pdfPrefetch) {- meta.pdfPrefetch = error.pdfPrefetch;- } else if (error.pdfPrefetch === null) {- // Browser round trip ran but delivered no file. Preserve the- // null sentinel: the antibot branches below still retry (the- // empty handoff may be transient), but the proxy-failure- // branches fail fast instead of re-running the browser.- meta.pdfPrefetch = null;- }- if (error.documentPrefetch) {- meta.documentPrefetch = error.documentPrefetch;- } else if (error.documentPrefetch === null) {- meta.documentPrefetch = null;- }- if (error.imagePrefetch) {- meta.imagePrefetch = error.imagePrefetch;- } else if (error.imagePrefetch === null) {- meta.imagePrefetch = null;- }- } else if (- error instanceof RemoveFeatureError &&- (meta.internalOptions.forceEngine === undefined ||- Array.isArray(meta.internalOptions.forceEngine))- ) {- retryTracker.record("feature_removal", error);- meta.logger.debug(- "Incorrect feature flags reported by scraper: removing " +- error.featureFlags.join(","),- { error, existingFlags: meta.featureFlags },- );- meta.featureFlags = new Set(- [...meta.featureFlags].filter(- x => !error.featureFlags.includes(x),- ),- );- } else if (- error instanceof PDFAntibotError &&- meta.internalOptions.forceEngine === undefined- ) {- // null = browser ran but delivered no file (possibly transient) —- // still worth one more browser round trip, so only a real- // prefetch object fails here.- if (meta.pdfPrefetch != null) {- meta.logger.error(- "PDF was prefetched and still blocked by antibot, failing",+ try {+ let result: ScrapeUrlResponse;+ while (true) {+ try {+ result = await scrapeURLLoop(meta);+ break;+ } catch (error) {+ if (+ error instanceof AddFeatureError &&+ (meta.internalOptions.forceEngine === undefined ||+ Array.isArray(meta.internalOptions.forceEngine))+ ) {+ retryTracker.record("feature_toggle", error);+ // A file handoff names the one parser that can open the file;+ // the file flag the URL extension implied earlier gives way to it.+ const nextFeatureFlags = applyHandoffFeatureFlags(+ meta.featureFlags,+ error.featureFlags,);- throw error;- } else {- retryTracker.record("pdf_antibot", error);meta.logger.debug(- "PDF was blocked by anti-bot, prefetching with chrome-cdp",- );- meta.featureFlags = new Set(- [...meta.featureFlags].filter(x => x !== "pdf"),- );- }- } else if (- error instanceof PDFFetchProxyError &&- meta.internalOptions.forceEngine === undefined- ) {- // meta.pdfPrefetch distinguishes "browser never attempted"- // (undefined — clear the pdf flag so the browser engine fetches- // the file through fire-engine's proxies) from "browser attempted,- // came back empty" (null — fail fast: another round trip would- // only burn the shared antibot+proxy prefetch budget).- if (meta.pdfPrefetch !== undefined) {- meta.logger.error(- "PDF was prefetched and the direct fetch still failed at the proxy, failing",+ "More feature flags requested by scraper: adding " ++ error.featureFlags.join(", "),+ {+ error,+ existingFlags: meta.featureFlags,+ droppedFlags: [...meta.featureFlags].filter(+ flag => !nextFeatureFlags.has(flag),+ ),+ },);- throw error;- } else {- retryTracker.record("pdf_fetch_proxy", error);+ meta.featureFlags = nextFeatureFlags;+ if (error.pdfPrefetch) {+ meta.pdfPrefetch = error.pdfPrefetch;+ } else if (error.pdfPrefetch === null) {+ // Browser round trip ran but delivered no file. Preserve the+ // null sentinel: the antibot branches below still retry (the+ // empty handoff may be transient), but the proxy-failure+ // branches fail fast instead of re-running the browser.+ meta.pdfPrefetch = null;+ }+ if (error.documentPrefetch) {+ meta.documentPrefetch = error.documentPrefetch;+ } else if (error.documentPrefetch === null) {+ meta.documentPrefetch = null;+ }+ if (error.imagePrefetch) {+ meta.imagePrefetch = error.imagePrefetch;+ } else if (error.imagePrefetch === null) {+ meta.imagePrefetch = null;+ }+ } else if (+ error instanceof RemoveFeatureError &&+ (meta.internalOptions.forceEngine === undefined ||+ Array.isArray(meta.internalOptions.forceEngine))+ ) {+ retryTracker.record("feature_removal", error);meta.logger.debug(- "PDF direct download failed at the proxy, prefetching with chrome-cdp",+ "Incorrect feature flags reported by scraper: removing " ++ error.featureFlags.join(","),+ { error, existingFlags: meta.featureFlags },);meta.featureFlags = new Set(- [...meta.featureFlags].filter(x => x !== "pdf"),- );- }- } else if (- error instanceof DocumentAntibotError &&- meta.internalOptions.forceEngine === undefined- ) {- // null = browser ran but delivered no file (possibly transient) —- // still worth one more browser round trip, so only a real- // prefetch object fails here.- if (meta.documentPrefetch != null) {- meta.logger.error(- "Document was prefetched and still blocked by antibot, failing",+ [...meta.featureFlags].filter(+ x => !error.featureFlags.includes(x),+ ),);- throw error;+ } else if (+ error instanceof PDFAntibotError &&+ meta.internalOptions.forceEngine === undefined+ ) {+ // null = browser ran but delivered no file (possibly transient) —+ // still worth one more browser round trip, so only a real+ // prefetch object fails here.+ if (meta.pdfPrefetch != null) {+ meta.logger.error(+ "PDF was prefetched and still blocked by antibot, failing",+ );+ throw error;+ } else {+ retryTracker.record("pdf_antibot", error);+ meta.logger.debug(+ "PDF was blocked by anti-bot, prefetching with chrome-cdp",+ );+ meta.featureFlags = new Set(+ [...meta.featureFlags].filter(x => x !== "pdf"),+ );+ }+ } else if (+ error instanceof PDFFetchProxyError &&+ meta.internalOptions.forceEngine === undefined+ ) {+ // meta.pdfPrefetch distinguishes "browser never attempted"+ // (undefined — clear the pdf flag so the browser engine fetches+ // the file through fire-engine's proxies) from "browser attempted,+ // came back empty" (null — fail fast: another round trip would+ // only burn the shared antibot+proxy prefetch budget).+ if (meta.pdfPrefetch !== undefined) {+ meta.logger.error(+ "PDF was prefetched and the direct fetch still failed at the proxy, failing",+ );+ throw error;+ } else {+ retryTracker.record("pdf_fetch_proxy", error);+ meta.logger.debug(+ "PDF direct download failed at the proxy, prefetching with chrome-cdp",+ );+ meta.featureFlags = new Set(+ [...meta.featureFlags].filter(x => x !== "pdf"),+ );+ }+ } else if (+ error instanceof DocumentAntibotError &&+ meta.internalOptions.forceEngine === undefined+ ) {+ // null = browser ran but delivered no file (possibly transient) —+ // still worth one more browser round trip, so only a real+ // prefetch object fails here.+ if (meta.documentPrefetch != null) {+ meta.logger.error(+ "Document was prefetched and still blocked by antibot, failing",+ );+ throw error;+ } else {+ retryTracker.record("document_antibot", error);+ meta.logger.debug(+ "Document was blocked by anti-bot, prefetching with chrome-cdp",+ );+ meta.featureFlags = new Set(+ [...meta.featureFlags].filter(x => x !== "document"),+ );+ }+ } else if (+ error instanceof DocumentFetchProxyError &&+ meta.internalOptions.forceEngine === undefined+ ) {+ // Same undefined-vs-null distinction as the PDF branch above.+ if (meta.documentPrefetch !== undefined) {+ meta.logger.error(+ "Document was prefetched and the direct fetch still failed at the proxy, failing",+ );+ throw error;+ } else {+ retryTracker.record("document_fetch_proxy", error);+ meta.logger.debug(+ "Document direct download failed at the proxy, prefetching with chrome-cdp",+ );+ meta.featureFlags = new Set(+ [...meta.featureFlags].filter(x => x !== "document"),+ );+ }} else {- retryTracker.record("document_antibot", error);- meta.logger.debug(- "Document was blocked by anti-bot, prefetching with chrome-cdp",- );- meta.featureFlags = new Set(- [...meta.featureFlags].filter(x => x !== "document"),- );- }- } else if (- error instanceof DocumentFetchProxyError &&- meta.internalOptions.forceEngine === undefined- ) {- // Same undefined-vs-null distinction as the PDF branch above.- if (meta.documentPrefetch !== undefined) {- meta.logger.error(- "Document was prefetched and the direct fetch still failed at the proxy, failing",- );throw error;- } else {- retryTracker.record("document_fetch_proxy", error);- meta.logger.debug(- "Document direct download failed at the proxy, prefetching with chrome-cdp",- );- meta.featureFlags = new Set(- [...meta.featureFlags].filter(x => x !== "document"),- );}- } else {- throw error;}}- }- // Threat protection: if the scrape ended up on a different URL than- // requested (redirect), re-check the destination URL. This closes the- // "clean URL redirects to a blocked URL" bypass vector — including- // same-domain redirects onto a flagged path.- if (threatPolicy && threatPolicy.mode !== "off" && result.success) {- const initialUrl = meta.rewrittenUrl ?? meta.url;- const finalUrl = result.document.metadata.url;- if (- finalUrl &&- canonicalizeUrl(finalUrl) !== canonicalizeUrl(initialUrl)- ) {- const decision = await checkUrl(finalUrl, threatPolicy, {- teamId: internalOptions.teamId,- dedup: threatDedup,- });- meta.threatDecisions.push(decision);- if (!decision.allowed) {- meta.logger.info(- "Redirect destination blocked by threat protection policy",- {- url: finalUrl,- domain: decision.domain,- initialUrl,- rule: decision.rule,- },- );- setSpanAttributes(span, {- "scrape.blocked_by_threat_protection": true,+ // Threat protection: if the scrape ended up on a different URL than+ // requested (redirect), re-check the destination URL. This closes the+ // "clean URL redirects to a blocked URL" bypass vector — including+ // same-domain redirects onto a flagged path.+ if (threatPolicy && threatPolicy.mode !== "off" && result.success) {+ const initialUrl = meta.rewrittenUrl ?? meta.url;+ const finalUrl = result.document.metadata.url;+ if (+ finalUrl &&+ canonicalizeUrl(finalUrl) !== canonicalizeUrl(initialUrl)+ ) {+ const decision = await checkUrl(finalUrl, threatPolicy, {+ teamId: internalOptions.teamId,+ dedup: threatDedup,});- throw new UnsafeDomainBlockedError(finalUrl, decision);+ meta.threatDecisions.push(decision);+ if (!decision.allowed) {+ meta.logger.info(+ "Redirect destination blocked by threat protection policy",+ {+ url: finalUrl,+ domain: decision.domain,+ initialUrl,+ rule: decision.rule,+ },+ );+ setSpanAttributes(span, {+ "scrape.blocked_by_threat_protection": true,+ });+ throw new UnsafeDomainBlockedError(finalUrl, decision);+ }}}- }-- meta.logger.debug("scrapeURL metrics", {- module: "scrapeURL/metrics",- timeTaken: Date.now() - startTime,- maxAgeValid: (meta.options.maxAge ?? 0) > 0,- shouldUseIndex: shouldUseIndex(meta),- success: result.success,- indexHit:- result.success && result.document.metadata.cacheState === "hit",- });- if (useIndex) {- meta.logger.debug("scrapeURL index metrics", {- module: "scrapeURL/index-metrics",+ meta.logger.debug("scrapeURL metrics", {+ module: "scrapeURL/metrics",timeTaken: Date.now() - startTime,- changeTrackingEnabled: !!hasFormatOfType(- meta.options.formats,- "changeTracking",- ),- summaryEnabled: !!hasFormatOfType(meta.options.formats, "summary"),- jsonEnabled: !!hasFormatOfType(meta.options.formats, "json"),- screenshotEnabled: !!hasFormatOfType(- meta.options.formats,- "screenshot",- ),- imagesEnabled: !!hasFormatOfType(meta.options.formats, "images"),- brandingEnabled: !!hasFormatOfType(meta.options.formats, "branding"),- pdfMaxPages: getPDFMaxPages(meta.options.parsers),- maxAge: meta.options.maxAge,- headers: meta.options.headers- ? Object.keys(meta.options.headers).length- : 0,- actions: meta.options.actions?.length ?? 0,- proxy: meta.options.proxy,+ maxAgeValid: (meta.options.maxAge ?? 0) > 0,+ shouldUseIndex: shouldUseIndex(meta),success: result.success,indexHit:result.success && result.document.metadata.cacheState === "hit",});- }- setSpanAttributes(span, {- "scrape.success": true,- "scrape.duration_ms": Date.now() - startTime,- "scrape.index_hit":- result.success && result.document.metadata.cacheState === "hit",- });+ if (useIndex) {+ meta.logger.debug("scrapeURL index metrics", {+ module: "scrapeURL/index-metrics",+ timeTaken: Date.now() - startTime,+ changeTrackingEnabled: !!hasFormatOfType(+ meta.options.formats,+ "changeTracking",+ ),+ summaryEnabled: !!hasFormatOfType(meta.options.formats, "summary"),+ jsonEnabled: !!hasFormatOfType(meta.options.formats, "json"),+ screenshotEnabled: !!hasFormatOfType(+ meta.options.formats,+ "screenshot",+ ),+ imagesEnabled: !!hasFormatOfType(meta.options.formats, "images"),+ brandingEnabled: !!hasFormatOfType(+ meta.options.formats,+ "branding",+ ),+ pdfMaxPages: getPDFMaxPages(meta.options.parsers),+ maxAge: meta.options.maxAge,+ headers: meta.options.headers+ ? Object.keys(meta.options.headers).length+ : 0,+ actions: meta.options.actions?.length ?? 0,+ proxy: meta.options.proxy,+ success: result.success,+ indexHit:+ result.success && result.document.metadata.cacheState === "hit",+ });+ }- return meta.threatDecisions.length > 0- ? { ...result, threatDecisions: meta.threatDecisions }- : result;- } catch (error) {- // if (Object.values(meta.results).length > 0 && Object.values(meta.results).every(x => x.state === "error" && x.error instanceof FEPageLoadFailed)) {- // throw new FEPageLoadFailed();- // } else- // A timed-out large-PDF scrape leaves its fire-pdf job running by- // design (fire-pdf/async.ts cancel policy); upgrade the timeout- // error IN PLACE so the caller learns processing continues and- // when a retry of the same URL picks the result up. Covers both- // the engine race's own timer and the abort manager's inner- // timeout, which exits through the early rethrow below.- const timeoutCandidate =- error instanceof AbortManagerThrownError ? error.inner : error;- if (- meta.largePdfProcessing?.current &&- timeoutCandidate instanceof ScrapeJobTimeoutError &&- timeoutCandidate.processing === undefined- ) {- const composed = composeTimeoutProcessing({- ...meta.largePdfProcessing.current,- nowMs: Date.now(),+ setSpanAttributes(span, {+ "scrape.success": true,+ "scrape.duration_ms": Date.now() - startTime,+ "scrape.index_hit":+ result.success && result.document.metadata.cacheState === "hit",});- timeoutCandidate.processing = composed.details;- timeoutCandidate.message = composed.message;- }- meta.logger.debug("scrapeURL metrics", {- module: "scrapeURL/metrics",- timeTaken: Date.now() - startTime,- maxAgeValid: (meta.options.maxAge ?? 0) > 0,- shouldUseIndex: shouldUseIndex(meta),- success: false,- indexHit: false,- });+ return meta.threatDecisions.length > 0+ ? { ...result, threatDecisions: meta.threatDecisions }+ : result;+ } catch (error) {+ // if (Object.values(meta.results).length > 0 && Object.values(meta.results).every(x => x.state === "error" && x.error instanceof FEPageLoadFailed)) {+ // throw new FEPageLoadFailed();+ // } else+ // A timed-out large-PDF scrape leaves its fire-pdf job running by+ // design (fire-pdf/async.ts cancel policy); upgrade the timeout+ // error IN PLACE so the caller learns processing continues and+ // when a retry of the same URL picks the result up. Covers both+ // the engine race's own timer and the abort manager's inner+ // timeout, which exits through the early rethrow below.+ const timeoutCandidate =+ error instanceof AbortManagerThrownError ? error.inner : error;+ if (+ meta.largePdfProcessing?.current &&+ timeoutCandidate instanceof ScrapeJobTimeoutError &&+ timeoutCandidate.processing === undefined+ ) {+ const composed = composeTimeoutProcessing({+ ...meta.largePdfProcessing.current,+ nowMs: Date.now(),+ });+ timeoutCandidate.processing = composed.details;+ timeoutCandidate.message = composed.message;+ }- if (useIndex) {- meta.logger.debug("scrapeURL index metrics", {- module: "scrapeURL/index-metrics",+ meta.logger.debug("scrapeURL metrics", {+ module: "scrapeURL/metrics",timeTaken: Date.now() - startTime,- changeTrackingEnabled: !!hasFormatOfType(- meta.options.formats,- "changeTracking",- ),- summaryEnabled: !!hasFormatOfType(meta.options.formats, "summary"),- jsonEnabled: !!hasFormatOfType(meta.options.formats, "json"),- screenshotEnabled: !!hasFormatOfType(- meta.options.formats,- "screenshot",- ),- imagesEnabled: !!hasFormatOfType(meta.options.formats, "images"),- brandingEnabled: !!hasFormatOfType(meta.options.formats, "branding"),- pdfMaxPages: getPDFMaxPages(meta.options.parsers),- maxAge: meta.options.maxAge,- headers: meta.options.headers- ? Object.keys(meta.options.headers).length- : 0,- actions: meta.options.actions?.length ?? 0,- proxy: meta.options.proxy,+ maxAgeValid: (meta.options.maxAge ?? 0) > 0,+ shouldUseIndex: shouldUseIndex(meta),success: false,indexHit: false,});- }- // Set error attributes on span- let errorType = "unknown";- if (error instanceof NoEnginesLeftError) {- errorType = "NoEnginesLeftError";- meta.logger.warn("scrapeURL: All scraping engines failed!", { error });- } else if (error instanceof LLMRefusalError) {- errorType = "LLMRefusalError";- meta.logger.warn("scrapeURL: LLM refused to extract content", {- error,- });- } else if (- error instanceof Error &&- error.message.includes("Invalid schema for response_format")- ) {- errorType = "LLMSchemaError";- // TODO: separate into custom error- meta.logger.warn("scrapeURL: LLM schema error", { error });- // TODO: results?- } else if (error instanceof SiteError) {- errorType = "SiteError";- meta.logger.warn("scrapeURL: Site failed to load in browser", {- error,- });- } else if (error instanceof SSLError) {- errorType = "SSLError";- meta.logger.warn("scrapeURL: SSL error", { error });- } else if (error instanceof ActionError) {- errorType = "ActionError";- meta.logger.warn("scrapeURL: Action(s) failed to complete", { error });- } else if (error instanceof UnsupportedFileError) {- errorType = "UnsupportedFileError";- meta.logger.warn("scrapeURL: Tried to scrape unsupported file", {- error,- });- } else if (error instanceof PDFInsufficientTimeError) {- errorType = "PDFInsufficientTimeError";- meta.logger.warn("scrapeURL: Insufficient time to process PDF", {- error,- });- } else if (error instanceof PDFOCRRequiredError) {- errorType = "PDFOCRRequiredError";- meta.logger.warn(- "scrapeURL: PDF requires OCR but fast mode was requested",- {+ if (useIndex) {+ meta.logger.debug("scrapeURL index metrics", {+ module: "scrapeURL/index-metrics",+ timeTaken: Date.now() - startTime,+ changeTrackingEnabled: !!hasFormatOfType(+ meta.options.formats,+ "changeTracking",+ ),+ summaryEnabled: !!hasFormatOfType(meta.options.formats, "summary"),+ jsonEnabled: !!hasFormatOfType(meta.options.formats, "json"),+ screenshotEnabled: !!hasFormatOfType(+ meta.options.formats,+ "screenshot",+ ),+ imagesEnabled: !!hasFormatOfType(meta.options.formats, "images"),+ brandingEnabled: !!hasFormatOfType(+ meta.options.formats,+ "branding",+ ),+ pdfMaxPages: getPDFMaxPages(meta.options.parsers),+ maxAge: meta.options.maxAge,+ headers: meta.options.headers+ ? Object.keys(meta.options.headers).length+ : 0,+ actions: meta.options.actions?.length ?? 0,+ proxy: meta.options.proxy,+ success: false,+ indexHit: false,+ });+ }++ // Set error attributes on span+ let errorType = "unknown";+ if (error instanceof NoEnginesLeftError) {+ errorType = "NoEnginesLeftError";+ meta.logger.warn("scrapeURL: All scraping engines failed!", {error,- },- );- } else if (error instanceof PDFPrefetchFailed) {- errorType = "PDFPrefetchFailed";- meta.logger.warn(- "scrapeURL: Failed to prefetch PDF that is protected by anti-bot",- { error },- );- } else if (error instanceof DocumentPrefetchFailed) {- errorType = "DocumentPrefetchFailed";- meta.logger.warn(- "scrapeURL: Failed to prefetch document that is protected by anti-bot",- { error },- );- } else if (error instanceof PDFFetchProxyError) {- errorType = "PDFFetchProxyError";- meta.logger.warn(- "scrapeURL: PDF download failed at the proxy and could not be recovered via browser prefetch",- { error },- );- } else if (error instanceof DocumentFetchProxyError) {- errorType = "DocumentFetchProxyError";- meta.logger.warn(- "scrapeURL: Document download failed at the proxy and could not be recovered via browser prefetch",- { error },- );- } else if (error instanceof BrandingNotSupportedError) {- errorType = "BrandingNotSupportedError";- meta.logger.warn("scrapeURL: Branding not supported for this content", {- error,- });- } else if (error instanceof ProxySelectionError) {- errorType = "ProxySelectionError";- meta.logger.warn("scrapeURL: Proxy selection error", { error });- } else if (error instanceof DNSResolutionError) {- errorType = "DNSResolutionError";- meta.logger.warn("scrapeURL: DNS resolution error", { error });- } else if (error instanceof ScrapeRetryLimitError) {- errorType = "ScrapeRetryLimitError";- meta.logger.warn("scrapeURL: Retry limit reached", {- error,- retryStats: error.stats,- });- } else if (error instanceof UnsafeDomainBlockedError) {- errorType = "UnsafeDomainBlockedError";- meta.logger.warn(- "scrapeURL: Domain blocked by threat protection policy",- {+ });+ } else if (error instanceof LLMRefusalError) {+ errorType = "LLMRefusalError";+ meta.logger.warn("scrapeURL: LLM refused to extract content", {error,- domain: error.domain,- rule: error.decision.rule,- },- );- } else if (error instanceof AbortManagerThrownError) {- errorType = "AbortManagerThrownError";- throw error.inner;- } else {- captureExceptionWithZdrCheck(error, {- extra: {- zeroDataRetention: internalOptions.zeroDataRetention ?? false,- },- });- meta.logger.error("scrapeURL: Unexpected error happened", { error });- // TODO: results?- }+ });+ } else if (+ error instanceof Error &&+ error.message.includes("Invalid schema for response_format")+ ) {+ errorType = "LLMSchemaError";+ // TODO: separate into custom error+ meta.logger.warn("scrapeURL: LLM schema error", { error });+ // TODO: results?+ } else if (error instanceof SiteError) {+ errorType = "SiteError";+ meta.logger.warn("scrapeURL: Site failed to load in browser", {+ error,+ });+ } else if (error instanceof SSLError) {+ errorType = "SSLError";+ meta.logger.warn("scrapeURL: SSL error", { error });+ } else if (error instanceof ActionError) {+ errorType = "ActionError";+ meta.logger.warn("scrapeURL: Action(s) failed to complete", {+ error,+ });+ } else if (error instanceof UnsupportedFileError) {+ errorType = "UnsupportedFileError";+ meta.logger.warn("scrapeURL: Tried to scrape unsupported file", {+ error,+ });+ } else if (error instanceof PDFInsufficientTimeError) {+ errorType = "PDFInsufficientTimeError";+ meta.logger.warn("scrapeURL: Insufficient time to process PDF", {+ error,+ });+ } else if (error instanceof PDFOCRRequiredError) {+ errorType = "PDFOCRRequiredError";+ meta.logger.warn(+ "scrapeURL: PDF requires OCR but fast mode was requested",+ {+ error,+ },+ );+ } else if (error instanceof PDFPrefetchFailed) {+ errorType = "PDFPrefetchFailed";+ meta.logger.warn(+ "scrapeURL: Failed to prefetch PDF that is protected by anti-bot",+ { error },+ );+ } else if (error instanceof DocumentPrefetchFailed) {+ errorType = "DocumentPrefetchFailed";+ meta.logger.warn(+ "scrapeURL: Failed to prefetch document that is protected by anti-bot",+ { error },+ );+ } else if (error instanceof PDFFetchProxyError) {+ errorType = "PDFFetchProxyError";+ meta.logger.warn(+ "scrapeURL: PDF download failed at the proxy and could not be recovered via browser prefetch",+ { error },+ );+ } else if (error instanceof DocumentFetchProxyError) {+ errorType = "DocumentFetchProxyError";+ meta.logger.warn(+ "scrapeURL: Document download failed at the proxy and could not be recovered via browser prefetch",+ { error },+ );+ } else if (error instanceof BrandingNotSupportedError) {+ errorType = "BrandingNotSupportedError";+ meta.logger.warn(+ "scrapeURL: Branding not supported for this content",+ {+ error,+ },+ );+ } else if (error instanceof ProxySelectionError) {+ errorType = "ProxySelectionError";+ meta.logger.warn("scrapeURL: Proxy selection error", { error });+ } else if (error instanceof DNSResolutionError) {+ errorType = "DNSResolutionError";+ meta.logger.warn("scrapeURL: DNS resolution error", { error });+ } else if (error instanceof ScrapeRetryLimitError) {+ errorType = "ScrapeRetryLimitError";+ meta.logger.warn("scrapeURL: Retry limit reached", {+ error,+ retryStats: error.stats,+ });+ } else if (error instanceof UnsafeDomainBlockedError) {+ errorType = "UnsafeDomainBlockedError";+ meta.logger.warn(+ "scrapeURL: Domain blocked by threat protection policy",+ {+ error,+ domain: error.domain,+ rule: error.decision.rule,+ },+ );+ } else if (error instanceof AbortManagerThrownError) {+ errorType = "AbortManagerThrownError";+ throw error.inner;+ } else {+ meta.logger.error("scrapeURL: Unexpected error happened", { error });+ recordSpanException(span, error);+ // TODO: results?+ }- setSpanAttributes(span, {- "scrape.success": false,- "scrape.error": error instanceof Error ? error.message : String(error),- "scrape.error_type": errorType,- "scrape.duration_ms": Date.now() - startTime,- });+ setSpanAttributes(span, {+ "scrape.success": false,+ "scrape.error":+ error instanceof Error ? error.message : String(error),+ "scrape.error_type": errorType,+ "scrape.duration_ms": Date.now() - startTime,+ });- return {- success: false,- error,- ...(meta.threatDecisions.length > 0- ? { threatDecisions: meta.threatDecisions }- : {}),- };- }- });+ return {+ success: false,+ error,+ ...(meta.threatDecisions.length > 0+ ? { threatDecisions: meta.threatDecisions }+ : {}),+ };+ }+ },+ { zeroDataRetention: internalOptions.zeroDataRetention },+ );}diff --git a/apps/api/src/scraper/scrapeURL/lib/fetch.ts b/apps/api/src/scraper/scrapeURL/lib/fetch.tsindex 59c26f45f..498a9ed1a 100644--- a/apps/api/src/scraper/scrapeURL/lib/fetch.ts+++ b/apps/api/src/scraper/scrapeURL/lib/fetch.ts@@ -1,6 +1,5 @@import { Logger } from "winston";import { z, ZodError } from "zod";-import * as Sentry from "@sentry/node";import { MockState, saveMock } from "./mock";import { fireEngineURL } from "../engines/fire-engine/scrape";import { fetch, Response, FormData, Agent } from "undici";@@ -137,7 +136,6 @@ export async function robustFetch<if (error instanceof AbortManagerThrownError) {throw error;} else if (!ignoreFailure) {- Sentry.captureException(error);if (tryCount > 1) {logger.debug("Request failed, trying " + (tryCount - 1) + " more times",diff --git a/apps/api/src/scraper/scrapeURL/lib/promptInjectionGuard.ts b/apps/api/src/scraper/scrapeURL/lib/promptInjectionGuard.tsindex cdc80fa85..ed7dabc4a 100644--- a/apps/api/src/scraper/scrapeURL/lib/promptInjectionGuard.ts+++ b/apps/api/src/scraper/scrapeURL/lib/promptInjectionGuard.ts@@ -17,7 +17,6 @@ import {} from "../../../lib/cost-tracking";import { calculateCost } from "../transformers/llmExtract";import { PromptInjectionDetectedError } from "../error";-import { captureExceptionWithZdrCheck } from "../../../services/sentry";const GUARD_MODEL = "gpt-4o-mini";const GUARD_MAX_CHUNK_CHARS = 32000;@@ -109,6 +108,9 @@ async function classifyChunk(},experimental_telemetry: {isEnabled: !zeroDataRetention,+ // The input carries the page screenshot / raw page content; too large+ // for span attributes. Outputs stay recorded.+ recordInputs: false,functionId: metadata.functionId? metadata.functionId + "/promptInjectionGuard": "promptInjectionGuard",@@ -160,13 +162,6 @@ async function classifyChunk();}- if (!NoObjectGeneratedError.isInstance(error)) {- captureExceptionWithZdrCheck(error, {- tags: { feature: "prompt-injection-guard" },- extra: { teamId: metadata.teamId },- });- }-// Deliberately fail-open so a guard outage doesn't take down the whole scrape.logger.warn("Prompt injection guard call failed; proceeding without a guard verdict (fail-open)",diff --git a/apps/api/src/search/fireEngine.ts b/apps/api/src/search/fireEngine.tsindex 0529e499e..6070285f5 100644--- a/apps/api/src/search/fireEngine.ts+++ b/apps/api/src/search/fireEngine.ts@@ -1,6 +1,5 @@import { config } from "../config";import { SearchResult } from "../../src/lib/entities";-import * as Sentry from "@sentry/node";import { logger } from "../lib/logger";import { executeWithRetry, attemptRequest } from "../lib/retry-utils";import { useFireEngine } from "../scraper/scrapeURL/engines/fire-engine/available";diff --git a/apps/api/src/search/v2/fireEngine-v2.ts b/apps/api/src/search/v2/fireEngine-v2.tsindex 88bea6a14..fd27262a9 100644--- a/apps/api/src/search/v2/fireEngine-v2.ts+++ b/apps/api/src/search/v2/fireEngine-v2.ts@@ -4,7 +4,6 @@ import {SearchV2Response,SearchResultType,} from "../../lib/entities";-import * as Sentry from "@sentry/node";import { logger } from "../../lib/logger";import { executeWithRetry, attemptRequest } from "../../lib/retry-utils";import { useFireEngine } from "../../scraper/scrapeURL/engines/fire-engine/available";diff --git a/apps/api/src/services/billing/__tests__/batch_billing.test.ts b/apps/api/src/services/billing/__tests__/batch_billing.test.tsindex d207476f3..2bef47022 100644--- a/apps/api/src/services/billing/__tests__/batch_billing.test.ts+++ b/apps/api/src/services/billing/__tests__/batch_billing.test.ts@@ -4,33 +4,23 @@ import { vi } from "vitest";// reads at build time must be created in vi.hoisted(). (Jest left jest.mock// un-hoisted here because `jest` was imported from @jest/globals.) The `redis`// stub below stays module-level: its factory only captures it lazily.-const {- captureException,- logger,- withAuth,- trackCredits,- refundCredits,- billTeam7,-} = vi.hoisted(() => {- const logger: any = {- info: vi.fn(),- warn: vi.fn(),- error: vi.fn(),- child: vi.fn(() => logger),- };- return {- captureException: vi.fn(),- logger,- withAuth: vi.fn((fn: any) => fn),- trackCredits: vi.fn<(args: any) => Promise<boolean>>(),- refundCredits: vi.fn<(args: any) => Promise<void>>(),- billTeam7: vi.fn<(params: any) => Promise<{ api_key: string }[]>>(),- };-});--vi.mock("@sentry/node", () => ({- captureException,-}));+const { logger, withAuth, trackCredits, refundCredits, billTeam7 } = vi.hoisted(+ () => {+ const logger: any = {+ info: vi.fn(),+ warn: vi.fn(),+ error: vi.fn(),+ child: vi.fn(() => logger),+ };+ return {+ logger,+ withAuth: vi.fn((fn: any) => fn),+ trackCredits: vi.fn<(args: any) => Promise<boolean>>(),+ refundCredits: vi.fn<(args: any) => Promise<void>>(),+ billTeam7: vi.fn<(params: any) => Promise<{ api_key: string }[]>>(),+ };+ },+);vi.mock("../../../lib/logger", () => ({logger,@@ -151,7 +141,6 @@ describe("processBillingBatch", () => {expect(billTeam7).toHaveBeenCalled();expect(trackCredits).not.toHaveBeenCalled();- expect(captureException).not.toHaveBeenCalled();});it("does not re-track even when the op was already tracked at request time", async () => {@@ -179,10 +168,9 @@ describe("processBillingBatch", () => {},featureId: "CREDITS",});- expect(captureException).toHaveBeenCalled();});- it("captures exceptions and refunds when billing throws", async () => {+ it("refunds when billing throws", async () => {queue = [makeOp({ autumnTrackInRequest: true })];billTeam7.mockRejectedValueOnce(new Error("rpc exploded"));@@ -198,7 +186,6 @@ describe("processBillingBatch", () => {},featureId: "CREDITS",});- expect(captureException).toHaveBeenCalled();});it("continues processing later groups when an Autumn refund fails", async () => {@@ -232,6 +219,5 @@ describe("processBillingBatch", () => {expect(billTeam7).toHaveBeenCalledTimes(2);// The batch never tracks usage to Autumn, regardless of the request-time flag.expect(trackCredits).not.toHaveBeenCalled();- expect(captureException).toHaveBeenCalled();});});diff --git a/apps/api/src/services/billing/batch_billing.ts b/apps/api/src/services/billing/batch_billing.tsindex f09c80175..56bf07bac 100644--- a/apps/api/src/services/billing/batch_billing.ts+++ b/apps/api/src/services/billing/batch_billing.ts@@ -1,7 +1,6 @@import { logger } from "../../lib/logger";import { getRedisConnection } from "../queue-service";import { billTeam7 } from "../../db/rpc";-import * as Sentry from "@sentry/node";import { withAuth } from "../../lib/withAuth";import {autumnService,@@ -53,10 +52,7 @@ async function confirmExchangeOutcomes(// Exchange queues at most EXCHANGE_CONFIRM_CONCURRENCY waiters per// invocation on the shared budget instead of one per operation.let nextIndex = 0;- const workerCount = Math.min(- EXCHANGE_CONFIRM_CONCURRENCY,- operations.length,- );+ const workerCount = Math.min(EXCHANGE_CONFIRM_CONCURRENCY, operations.length);await Promise.all(Array.from({ length: workerCount }, async () => {while (true) {@@ -157,13 +153,6 @@ async function refundRequestTrackedCredits(group: GroupedBillingOperation) {credits: requestTrackedCredits,billing: group.billing,});- Sentry.captureException(error, {- data: {- operation: "batch_billing_refund",- team_id: group.team_id,- credits: requestTrackedCredits,- },- });}}@@ -299,24 +288,12 @@ export async function processBillingBatch() {error,group,});- Sentry.captureException(error, {- data: {- operation: "batch_billing",- team_id: group.team_id,- credits: group.total_credits,- },- });}}logger.info("✅ Billing batch processing completed successfully");} catch (error) {logger.error("Error processing billing batch", { error });- Sentry.captureException(error, {- data: {- operation: "batch_billing_process",- },- });} finally {await releaseLock();}@@ -412,13 +389,6 @@ export async function queueBillingOperation(return { success: true };} catch (error) {logger.error("Error queueing billing operation", { error, team_id });- Sentry.captureException(error, {- data: {- operation: "queue_billing",- team_id,- credits,- },- });return { success: false, error };}}@@ -455,7 +425,6 @@ async function supaBillTeam(is_extract,});} catch (error) {- Sentry.captureException(error);_logger.error("Failed to bill team.", { error });return { success: false, error };}diff --git a/apps/api/src/services/cclog-worker.ts b/apps/api/src/services/cclog-worker.tsindex 1637dfaf7..84cf10e9c 100644--- a/apps/api/src/services/cclog-worker.ts+++ b/apps/api/src/services/cclog-worker.ts@@ -1,7 +1,6 @@import "dotenv/config";import { config } from "../config";-import "./sentry";-import { setSentryServiceTag } from "./sentry";+import { shutdownTracing } from "../otel";import Express from "express";import { logger as _logger } from "../lib/logger";import {@@ -34,8 +33,6 @@ async function sendHeartbeat() {}(async () => {- setSentryServiceTag("cclog-worker");-let isShuttingDown = false;let tickInFlight = false;@@ -68,7 +65,8 @@ async function sendHeartbeat() {await sleep(1000);}- server.close(() => {+ server.close(async () => {+ await shutdownTracing();_logger.info("cclog worker shut down");process.exit(0);});diff --git a/apps/api/src/services/extract-worker.ts b/apps/api/src/services/extract-worker.tsindex 15e07efb4..043652bae 100644--- a/apps/api/src/services/extract-worker.ts+++ b/apps/api/src/services/extract-worker.ts@@ -1,8 +1,6 @@import "dotenv/config";import { config } from "../config";-import "./sentry";-import { setSentryServiceTag } from "./sentry";-import * as Sentry from "@sentry/node";+import { shutdownTracing } from "../otel";import { logger as _logger } from "../lib/logger";import { configDotenv } from "dotenv";import { ExtractResult } from "../lib/extract/types";@@ -11,7 +9,6 @@ import { performExtraction_F0 } from "../lib/extract/fire-0/extraction-service-fimport { createWebhookSender, WebhookEvent } from "./webhook";import Express from "express";import { getErrorContactMessage } from "../lib/deployment";-import { TransportableError } from "../lib/error";import { initializeBlocklist } from "../scraper/WebScraper/utils/blocklist";import { initializeEngineForcing } from "../scraper/WebScraper/utils/engine-forcing";import {@@ -104,15 +101,6 @@ const processExtractJob = async (error,});- // Filter out TransportableErrors (flow control)- if (!(error instanceof TransportableError)) {- Sentry.captureException(error, {- data: {- extractId: data.extractId,- },- });- }-await updateExtract(data.extractId, {status: "failed",error:@@ -216,6 +204,7 @@ async function shutdown() {_logger.info("Shutting down extract worker...");await shutdownExtractQueue();await shutdownPubSubLogging();+ await shutdownTracing();_logger.info("Extract worker shut down");process.exit(0);}@@ -233,8 +222,6 @@ if (require.main === module) {}(async () => {- setSentryServiceTag("extract-worker");-await initializeBlocklist().catch(e => {_logger.error("Failed to initialize blocklist", { error: e });process.exit(1);diff --git a/apps/api/src/services/indexing/index-worker.ts b/apps/api/src/services/indexing/index-worker.tsindex 2de767688..22258ecae 100644--- a/apps/api/src/services/indexing/index-worker.ts+++ b/apps/api/src/services/indexing/index-worker.ts@@ -1,8 +1,6 @@import "dotenv/config";import { config } from "../../config";-import "../sentry";-import { setSentryServiceTag } from "../sentry";-import * as Sentry from "@sentry/node";+import { shutdownTracing } from "../../otel";import { Job, Queue, Worker } from "bullmq";import { logger as _logger, logger } from "../../lib/logger";import {@@ -125,7 +123,6 @@ const processBillingJobInternal = async (token: string, job: Job) => {await job.moveToCompleted({ success: true }, token, false);} catch (error) {logger.error("Error processing billing job", { error });- Sentry.captureException(error);err = error;await job.moveToFailed(error, token, false);} finally {@@ -555,7 +552,6 @@ const processPrecrawlJob = async (token: string, job: Job) => {submittedCrawls++;} catch (e) {logger.error("Error adding precrawl job to queue", { error: e });- Sentry.captureException(e);}}@@ -580,7 +576,6 @@ const processPrecrawlJob = async (token: string, job: Job) => {});} catch (e) {logger.error("Error processing precrawl job", { error: e });- Sentry.captureException(e);await job.moveToFailed(e, token, false);} finally {clearInterval(extendLockInterval);@@ -687,6 +682,7 @@ const workerFun = async (}logger.info("All jobs finished. Worker exiting!");await shutdownPubSubLogging();+ await shutdownTracing();process.exit(0);};@@ -699,8 +695,6 @@ const BROWSER_ACTIVITY_INSERT_INTERVAL = 10000;// Start the workers(async () => {- setSentryServiceTag("index-worker");-// Start billing worker and batch processingstartBillingBatchProcessing();const billingWorkerPromise = workerFun(diff --git a/apps/api/src/services/logging/log_job.test.ts b/apps/api/src/services/logging/log_job.test.tsindex d4f39b59b..ce6eda1f8 100644--- a/apps/api/src/services/logging/log_job.test.ts+++ b/apps/api/src/services/logging/log_job.test.ts@@ -4,7 +4,6 @@ import { vi } from "vitest";// vi.hoisted() (also hoisted). Under Jest these worked because importing `jest`// from @jest/globals disables jest.mock hoisting.const {- captureException,logger,values,insert,@@ -38,7 +37,6 @@ const {};});return {- captureException: vi.fn(),logger,values,insert,@@ -58,10 +56,6 @@ vi.mock("@google-cloud/pubsub", () => ({},}));-vi.mock("@sentry/node", () => ({- captureException,-}));-vi.mock("../../config", () => ({config: {GCS_BUCKET_NAME: undefined,@@ -177,28 +171,6 @@ describe("logSearch", () => {expect(search.options.query).toBe("nested\u0000query");});- it("uses sanitized data in Sentry insert failure context", async () => {- values.mockRejectedValueOnce(- Object.assign(new Error("unsupported Unicode escape sequence"), {- code: "22P05",- }),- );-- await logSearch(- makeSearch({- query: "bad\u0000query",- options: { query: "bad\u0000query" },- }),- );-- expect(captureException).toHaveBeenCalled();- const context = captureException.mock.calls[0][1] as {- extra: { data: string };- };- expect(context.extra.data).not.toContain("\\u0000");- expect(context.extra.data).toContain("badquery");- });-it("reports serialization failures without losing the PostgreSQL attempt", async () => {const search = makeSearch({ options: { unsupported: 1n } });@@ -286,7 +258,6 @@ describe("logRequest", () => {"Failed to publish log to Pub/Sub",expect.objectContaining({ error: expect.any(Error) }),);- expect(captureException).toHaveBeenCalled();});it("does not hold the caller on a slow publish", async () => {diff --git a/apps/api/src/services/logging/log_job.ts b/apps/api/src/services/logging/log_job.tsindex 7e3ada468..9ed445b8a 100644--- a/apps/api/src/services/logging/log_job.ts+++ b/apps/api/src/services/logging/log_job.ts@@ -6,7 +6,6 @@ import "dotenv/config";import { logger as _logger } from "../../lib/logger";import { EXTERNAL_REQUEST_ID_MAX_BYTES } from "../../lib/external-request-id";import { configDotenv } from "dotenv";-import * as Sentry from "@sentry/node";import type { PgTable } from "drizzle-orm/pg-core";import {saveDeepResearchToGCS,@@ -122,9 +121,6 @@ function getPubSubClient(logger: Logger): PubSub | null {} catch (error) {pubSubClient = null;logger.error("Failed to initialize Pub/Sub log publisher", { error });- Sentry.captureException(error, {- tags: { operation: "initializePubSubLogPublisher" },- });return null;}}@@ -197,10 +193,6 @@ async function publishLog(table: string, data: any, logger: Logger) {logId: data.id,durationMs: Date.now() - startedAt,});- Sentry.captureException(error, {- tags: { table, operation: "publishPubSubLog" },- extra: { logId: data.id },- });}}@@ -259,10 +251,6 @@ async function shutdownPubSubLoggingOnce(): Promise<void> {if (errors.length > 0) {logger.error("Failed to drain Pub/Sub log publisher", { errors });- Sentry.captureException(errors[0], {- tags: { operation: "flushPubSubLogPublisher" },- extra: { failures: errors.length },- });} else {logger.info("Pub/Sub log publisher drained", {durationMs: Date.now() - startedAt,@@ -283,9 +271,6 @@ async function shutdownPubSubLoggingOnce(): Promise<void> {]);} catch (error) {logger.error("Failed to close Pub/Sub log publisher", { error });- Sentry.captureException(error, {- tags: { operation: "closePubSubLogPublisher" },- });} finally {clearTimeout(closeDeadline);}@@ -355,25 +340,6 @@ async function robustInsert(});} else {logger.error("Failed to insert into database", { attempts });- // Report to Sentry with context- Sentry.captureException(- attempts[attempts.length - 1]?.error ||- new Error("Database insert failed after 10 attempts"),- {- tags: {- table,- operation: "robustInsert",- },- extra: {- table,- data: JSON.stringify(data).substring(0, 500), // Limit size- attempts: 10,- lastError: attempts[attempts.length - 1]?.error- ? JSON.stringify(attempts[attempts.length - 1].error)- : null,- },- },- );}} else {const start = Date.now();@@ -384,18 +350,6 @@ async function robustInsert(} catch (error) {attempts.push({ error, timeMs: Date.now() - start, backoffMs: 0 });logger.error("Failed to insert into database", { attempts });- // Report to Sentry- Sentry.captureException(error, {- tags: {- table,- operation: "robustInsert",- force: "false",- },- extra: {- table,- data: JSON.stringify(data).substring(0, 500), // Limit size- },- });}}}diff --git a/apps/api/src/services/queue-worker.ts b/apps/api/src/services/queue-worker.tsindex ffe3085c0..3f98da518 100644--- a/apps/api/src/services/queue-worker.ts+++ b/apps/api/src/services/queue-worker.ts@@ -1,8 +1,6 @@import "dotenv/config";import { config } from "../config";-import "./sentry";-import { setSentryServiceTag } from "./sentry";-import * as Sentry from "@sentry/node";+import { shutdownTracing } from "../otel";import {getDeepResearchQueue,getGenerateLlmsTxtQueue,@@ -24,7 +22,6 @@ import { initializeEngineForcing } from "../scraper/WebScraper/utils/engine-forcimport { crawlFinishedQueue, NuQJob, scrapeQueue } from "./worker/nuq";import { finishCrawlSuper } from "./worker/crawl-logic";import { getCrawl } from "../lib/crawl-redis";-import { TransportableError } from "../lib/error";import {processMonitorCheckJob,reconcileRunningMonitorChecks,@@ -102,15 +99,6 @@ const processDeepResearchJobInternal = async (} catch (error) {logger.error(`🚫 Job errored ${job.id} - ${error}`, { error });- // Skip TransportableErrors: they're flow control, not failures.- if (!(error instanceof TransportableError)) {- Sentry.captureException(error, {- data: {- job: job.id,- },- });- }-try {await job.moveToFailed(error, token, false);} catch (e) {@@ -178,15 +166,6 @@ const processGenerateLlmsTxtJobInternal = async (} catch (error) {logger.error(`🚫 Job errored ${job.id} - ${error}`, { error });- // Skip TransportableErrors: they're flow control, not failures.- if (!(error instanceof TransportableError)) {- Sentry.captureException(error, {- data: {- job: job.id,- },- });- }-try {await job.moveToFailed(error, token, false);} catch (e) {@@ -456,8 +435,6 @@ app.listen(workerPort, (error?: Error) => {});(async () => {- setSentryServiceTag("queue-worker");-await initializeBlocklist().catch(e => {_logger.error("Failed to initialize blocklist", { error: e });process.exit(1);@@ -514,5 +491,6 @@ app.listen(workerPort, (error?: Error) => {_logger.info("All jobs finished. Shutting down...");await shutdownPubSubLogging();+ await shutdownTracing();process.exit(0);})();diff --git a/apps/api/src/services/sentry.test.ts b/apps/api/src/services/sentry.test.tsdeleted file mode 100644index f23917b5b..000000000--- a/apps/api/src/services/sentry.test.ts+++ /dev/null@@ -1,133 +0,0 @@-import { vi } from "vitest";--// vi.mock is hoisted to the top of the file, so anything its factories reference-// must be created in vi.hoisted() (which also hoists). Under Jest these worked-// because importing `jest` from @jest/globals disables jest.mock hoisting.-const { captureException, init, setTag, scopeSetExtra, scopeSetTag } = vi.hoisted(- () => ({- captureException: vi.fn(),- init: vi.fn(),- setTag: vi.fn(),- scopeSetExtra: vi.fn(),- scopeSetTag: vi.fn(),- }),-);--vi.mock("@sentry/node", () => ({- captureException,- getCurrentScope: () => ({- setExtra: scopeSetExtra,- setTag: scopeSetTag,- }),- init,- setTag,- vercelAIIntegration: vi.fn(() => ({})),-}));--vi.mock("../config", () => ({- config: {- NUQ_POD_NAME: "test-pod",- SENTRY_DSN: undefined,- SENTRY_ENVIRONMENT: "test",- SENTRY_ERROR_SAMPLE_RATE: 1,- SENTRY_TRACE_SAMPLE_RATE: 0,- },-}));--vi.mock("../lib/logger", () => ({- logger: {- info: vi.fn(),- },-}));--vi.mock("../scraper/scrapeURL/error", () => ({- AddFeatureError: class AddFeatureError extends Error {},- RemoveFeatureError: class RemoveFeatureError extends Error {},- EngineError: class EngineError extends Error {},-}));--vi.mock("../scraper/scrapeURL/lib/abortManager", () => ({- AbortManagerThrownError: class AbortManagerThrownError extends Error {},-}));--vi.mock("../lib/error", () => ({- JobCancelledError: class JobCancelledError extends Error {},-}));--import { QueueFullError } from "../lib/queue-full-error";-import {- captureExceptionWithZdrCheck,- shouldIgnoreSentryException,-} from "./sentry";--describe("sentry filtering", () => {- beforeEach(() => {- vi.clearAllMocks();- });-- it("does not capture queue full errors", () => {- captureExceptionWithZdrCheck(new QueueFullError(50000, 50000));-- expect(captureException).not.toHaveBeenCalled();- });-- it("recognizes Sentry exception payloads for queue full errors", () => {- expect(- shouldIgnoreSentryException({- type: "QueueFullError",- value:- "Queue limit reached: your team has 50000 jobs queued (limit: 50000).",- }),- ).toBe(true);- });-- it("recognizes serialized transportable errors in Sentry exception values", () => {- expect(- shouldIgnoreSentryException({- type: "Error",- value: 'SCRAPE_TIMEOUT|{"message":"timeout"}',- }),- ).toBe(true);- });-- it("recognizes actions-not-supported Sentry exception values", () => {- expect(- shouldIgnoreSentryException({- type: "Error",- value:- 'SCRAPE_ACTIONS_NOT_SUPPORTED|{"message":"Actions are not supported by any available engines. Actions require Fire Engine (fire-engine) to be enabled."}',- }),- ).toBe(true);- });-- it("recognizes cancellation errors in Sentry exception values", () => {- expect(- shouldIgnoreSentryException({- type: "Error",- value: "Parent crawl/batch scrape was cancelled",- }),- ).toBe(true);- });-- it("does not ignore unexpected Sentry exception values", () => {- expect(- shouldIgnoreSentryException({- type: "Error",- value: "boom",- }),- ).toBe(false);- });-- it("still captures unexpected errors", () => {- const error = new Error("boom");-- captureExceptionWithZdrCheck(error, {- tags: { module: "test" },- zeroDataRetention: false,- });-- expect(captureException).toHaveBeenCalledWith(error, {- tags: { module: "test" },- });- });-});diff --git a/apps/api/src/services/sentry.ts b/apps/api/src/services/sentry.tsdeleted file mode 100644index f02ae61b9..000000000--- a/apps/api/src/services/sentry.ts+++ /dev/null@@ -1,214 +0,0 @@-import * as Sentry from "@sentry/node";-import { logger } from "../lib/logger";-import { config } from "../config";-import {- AddFeatureError,- RemoveFeatureError,- EngineError,-} from "../scraper/scrapeURL/error";-import { AbortManagerThrownError } from "../scraper/scrapeURL/lib/abortManager";-import { JobCancelledError } from "../lib/error";-import { isQueueFullError } from "../lib/queue-full-error";--type CaptureContext = {- tags?: Record<string, string>;- extra?: Record<string, any>;- level?: Sentry.SeverityLevel;- fingerprint?: string[];- contexts?: Record<string, any>;- user?: {- id?: string;- username?: string;- email?: string;- ip_address?: string;- [key: string]: any;- };- [key: string]: any;-};--const transportableErrorCodes = [- "SCRAPE_ALL_ENGINES_FAILED",- "SCRAPE_DNS_RESOLUTION_ERROR",- "SCRAPE_SITE_ERROR",- "SCRAPE_SSL_ERROR",- "SCRAPE_PROXY_SELECTION_ERROR",- "SCRAPE_ZDR_VIOLATION_ERROR",- "SCRAPE_UNSUPPORTED_FILE_ERROR",- "SCRAPE_PDF_ANTIBOT_ERROR",- "SCRAPE_PDF_FETCH_PROXY_ERROR",- "SCRAPE_ACTION_ERROR",- "SCRAPE_PDF_INSUFFICIENT_TIME_ERROR",- "SCRAPE_PDF_PREFETCH_FAILED",- "SCRAPE_DOCUMENT_ANTIBOT_ERROR",- "SCRAPE_DOCUMENT_FETCH_PROXY_ERROR",- "SCRAPE_DOCUMENT_PREFETCH_FAILED",- "SCRAPE_ACTIONS_NOT_SUPPORTED",- "SCRAPE_TIMEOUT",- "MAP_TIMEOUT",- "SCRAPE_UNKNOWN_ERROR",- "SCRAPE_RACED_REDIRECT_ERROR",- "SCRAPE_SITEMAP_ERROR",- "CRAWL_DENIAL",-];--function getStringField(- error: Record<string, unknown>,- fields: string[],-): string {- for (const field of fields) {- if (error[field] !== undefined && error[field] !== null) {- return String(error[field]);- }- }-- return "";-}--function getErrorMessage(error: unknown): string {- if (error instanceof Error) {- return String(error.message || "");- }-- if (typeof error === "string") {- return error;- }-- if (error && typeof error === "object") {- return getStringField(error as Record<string, unknown>, [- "message",- "value",- ]);- }-- return "";-}--export function shouldIgnoreSentryException(error: unknown): boolean {- if (isQueueFullError(error)) {- return true;- }-- if (!error || typeof error !== "object") {- return false;- }-- if (- error instanceof AddFeatureError ||- error instanceof RemoveFeatureError ||- error instanceof AbortManagerThrownError ||- error instanceof EngineError ||- error instanceof JobCancelledError- ) {- return true;- }-- // We sometimes rethrow TransportableErrors across process boundaries as a- // plain Error with message like `${code}|${json}` (see error-serde.ts).- // In that case, `error.code` is missing, so extract it from the message.- const errorCodeFromField = "code" in error ? String(error.code) : "";- const errorMessage = getErrorMessage(error);-- // Ignore cancellation errors (fallback for serialized errors)- if (- errorMessage === "Parent crawl/batch scrape was cancelled" ||- errorMessage.includes("Parent crawl/batch scrape was cancelled")- ) {- return true;- }-- const errorCodeFromMessage =- errorCodeFromField ||- (errorMessage.includes("|") ? errorMessage.split("|", 1)[0] : "");- const errorCode = errorCodeFromMessage;-- return transportableErrorCodes.includes(errorCode);-}--if (config.SENTRY_DSN) {- logger.info("Setting up Sentry...");-- Sentry.init({- dsn: config.SENTRY_DSN,- integrations: integrations => [- ...integrations,- Sentry.vercelAIIntegration({- recordInputs: false,- recordOutputs: false,- }),- ],- tracesSampler: samplingContext => {- // trace all AI spans, sample 1% of all others- return samplingContext.name?.startsWith("ai.")- ? 1.0- : config.SENTRY_TRACE_SAMPLE_RATE;- },- sampleRate: config.SENTRY_ERROR_SAMPLE_RATE,- serverName: config.NUQ_POD_NAME,- environment: config.SENTRY_ENVIRONMENT,- beforeSend(event, hint) {- const zeroDataRetention =- event.tags?.zeroDataRetention === "true" ||- event.tags?.zero_data_retention === "true" ||- event.extra?.zeroDataRetention === true ||- event.extra?.zero_data_retention === true;-- if (zeroDataRetention) {- return null;- }-- const error = hint?.originalException;- const sentryException = event.exception?.values?.[0];-- if (- shouldIgnoreSentryException(error) ||- shouldIgnoreSentryException(sentryException)- ) {- return null;- }-- return event;- },- });-}--export function captureExceptionWithZdrCheck(- error: any,- context?: (CaptureContext & { zeroDataRetention?: boolean }) | null,-) {- const zeroDataRetention =- context?.zeroDataRetention ??- (context?.extra as any)?.zeroDataRetention ??- (context?.data as any)?.zeroDataRetention;-- if (zeroDataRetention) {- return;- }-- if (shouldIgnoreSentryException(error)) {- return;- }-- const { zeroDataRetention: _zdr, ...sentryContext } = context || {};-- return Sentry.captureException(error, sentryContext);-}--export function applyZdrScope(zeroDataRetention?: boolean) {- if (!zeroDataRetention) {- return;- }-- const scope = Sentry.getCurrentScope();- scope.setTag("zeroDataRetention", "true");- scope.setExtra("zeroDataRetention", true);-}--/**- * Set the service type tag for this Sentry instance- * This helps distinguish between API server and worker errors in Sentry- */-export function setSentryServiceTag(serviceType: string) {- if (config.SENTRY_DSN) {- Sentry.setTag("service_type", serviceType);- }-}diff --git a/apps/api/src/services/worker/nuq-fdb-worker.ts b/apps/api/src/services/worker/nuq-fdb-worker.tsindex f51104bc2..b971018c9 100644--- a/apps/api/src/services/worker/nuq-fdb-worker.ts+++ b/apps/api/src/services/worker/nuq-fdb-worker.ts@@ -1,6 +1,5 @@import "dotenv/config";-import "../sentry";-import { setSentryServiceTag } from "../sentry";+import "../../otel";import { config } from "../../config";import { logger as _logger } from "../../lib/logger";import { getCrawl } from "../../lib/crawl-redis";@@ -133,8 +132,6 @@ function startCrawlFinishedLoop() {}(async () => {- setSentryServiceTag("nuq-fdb-worker");-let crawlFinishedLoop: ReturnType<typeof startCrawlFinishedLoop> | null =null;diff --git a/apps/api/src/services/worker/nuq-prefetch-worker.ts b/apps/api/src/services/worker/nuq-prefetch-worker.tsindex 3456e88f3..5b4d8b69a 100644--- a/apps/api/src/services/worker/nuq-prefetch-worker.ts+++ b/apps/api/src/services/worker/nuq-prefetch-worker.ts@@ -1,7 +1,6 @@import "dotenv/config";import { config } from "../../config";-import "../sentry";-import { setSentryServiceTag } from "../sentry";+import { shutdownTracing } from "../../otel";import {scrapeQueue,nuqGetLocalMetrics,@@ -13,8 +12,6 @@ import Express from "express";import { logger } from "../../lib/logger";(async () => {- setSentryServiceTag("nuq-prefetch-worker");-const app = Express();app.get("/metrics", (_, res) =>@@ -46,6 +43,7 @@ import { logger } from "../../lib/logger";async function shutdown() {server.close();await nuqShutdown();+ await shutdownTracing();process.exit(0);}diff --git a/apps/api/src/services/worker/nuq-reconciler-worker.ts b/apps/api/src/services/worker/nuq-reconciler-worker.tsindex 865da6549..16e6f9322 100644--- a/apps/api/src/services/worker/nuq-reconciler-worker.ts+++ b/apps/api/src/services/worker/nuq-reconciler-worker.ts@@ -1,7 +1,6 @@import "dotenv/config";import { config } from "../../config";-import "../sentry";-import { setSentryServiceTag } from "../sentry";+import { shutdownTracing } from "../../otel";import { logger as _logger } from "../../lib/logger";import { reconcileConcurrencyQueue } from "../../lib/concurrency-queue-reconciler";import { repairCrawlJobDoneMarkers } from "../../lib/crawl-redis";@@ -26,8 +25,6 @@ const reconcilerJobsRecoveredTotal = new Counter({});(async () => {- setSentryServiceTag("nuq-reconciler-worker");-let isShuttingDown = false;let reconcilerInFlight = false;@@ -72,7 +69,8 @@ const reconcilerJobsRecoveredTotal = new Counter({await new Promise(resolve => setTimeout(resolve, 1000));}- server.close(() => {+ server.close(async () => {+ await shutdownTracing();_logger.info("NuQ reconciler worker shut down");process.exit(0);});diff --git a/apps/api/src/services/worker/nuq-worker-runner.ts b/apps/api/src/services/worker/nuq-worker-runner.tsindex 3af1a4145..259a332e1 100644--- a/apps/api/src/services/worker/nuq-worker-runner.ts+++ b/apps/api/src/services/worker/nuq-worker-runner.ts@@ -1,4 +1,6 @@import { config } from "../../config";+import { withZeroDataRetention } from "../../lib/otel-tracer";+import { shutdownTracing } from "../../otel";import { logger as _logger } from "../../lib/logger";import { jobDurationSeconds } from "../../lib/job-metrics";import { processJobInternal } from "./scrape-worker";@@ -175,35 +177,43 @@ export async function runNuqWorker(options: {clearInterval(lockRenewInterval);- if (processResult.ok) {- endJobTimer({ status: "success" });- if (- !(await options.queue.jobFinish(- job.id,- job.lock!,- processResult.data,- logger,- ))- ) {- logger.warn("Could not update job status");- }- } else {- endJobTimer({ status: "failed" });- if (- !(await options.queue.jobFail(- job.id,- job.lock!,- processResult.error instanceof Error- ? processResult.error.message- : typeof processResult.error === "string"- ? processResult.error- : JSON.stringify(processResult.error),- logger,- ))- ) {- logger.warn("Could not update job status");+ // The job span has ended by now; keep the queue bookkeeping (which records+ // the failure reason) unrecorded for zero-data-retention jobs too.+ const zeroDataRetention =+ (job.data as { zeroDataRetention?: boolean } | undefined)+ ?.zeroDataRetention === true;++ await withZeroDataRetention(zeroDataRetention, async () => {+ if (processResult.ok) {+ endJobTimer({ status: "success" });+ if (+ !(await options.queue.jobFinish(+ job.id,+ job.lock!,+ processResult.data,+ logger,+ ))+ ) {+ logger.warn("Could not update job status");+ }+ } else {+ endJobTimer({ status: "failed" });+ if (+ !(await options.queue.jobFail(+ job.id,+ job.lock!,+ processResult.error instanceof Error+ ? processResult.error.message+ : typeof processResult.error === "string"+ ? processResult.error+ : JSON.stringify(processResult.error),+ logger,+ ))+ ) {+ logger.warn("Could not update job status");+ }}- }+ });}_logger.info("NuQ worker shutting down", {@@ -215,6 +225,7 @@ export async function runNuqWorker(options: {await options.beforeShutdown?.();await options.shutdown?.();await shutdownPubSubLogging();+ await shutdownTracing();_logger.info("NuQ worker shut down", { module: options.serviceName });process.exit(0);});diff --git a/apps/api/src/services/worker/nuq-worker.ts b/apps/api/src/services/worker/nuq-worker.tsindex dab1132f8..c4e23b7de 100644--- a/apps/api/src/services/worker/nuq-worker.ts+++ b/apps/api/src/services/worker/nuq-worker.ts@@ -1,12 +1,9 @@import "dotenv/config";-import "../sentry";-import { setSentryServiceTag } from "../sentry";+import "../../otel";import { nuqGetLocalMetrics, nuqHealthCheck, scrapeQueue } from "./nuq";import { runNuqWorker } from "./nuq-worker-runner";(async () => {- setSentryServiceTag("nuq-worker");-await runNuqWorker({serviceName: "nuq-worker",queue: scrapeQueue,diff --git a/apps/api/src/services/worker/nuq.ts b/apps/api/src/services/worker/nuq.tsindex fb5b1ca64..c65d1785c 100644--- a/apps/api/src/services/worker/nuq.ts+++ b/apps/api/src/services/worker/nuq.ts@@ -810,60 +810,64 @@ class NuQ<JobData = any, JobReturnValue = any> {data: JobData,options: NuQJobOptions,): Promise<NuQJob<JobData, JobReturnValue>> {- return withSpan("nuq.addJob", async span => {- setSpanAttributes(span, {- "nuq.queue_name": this.queueName,- "nuq.job_id": id,- "nuq.priority": options.priority ?? 0,- "nuq.zero_data_retention": (data as any)?.zeroDataRetention ?? false,- "nuq.listenable": options.listenable ?? false,- });-- const start = Date.now();- try {- const result = this.rowToJob(- (- await nuqPool.query(- `INSERT INTO ${this.queueName}${options.backlogged ? "_backlog" : ""} (id, data, priority, listen_channel_id, owner_id, group_id${options.backlogged ? ", times_out_at" : ""}) VALUES ($1, $2, $3, $4, $5, $6${options.backlogged ? ", $7" : ""}) RETURNING ${(options.backlogged ? this.jobBacklogReturning : this.jobReturning).join(", ")};`,- [- id,- data,- options.priority ?? 0,- options.listenable ? this.listenChannelId : null,- normalizeOwnerId(options.ownerId),- options.groupId ?? null,- ...(options.backlogged- ? [- options.backloggedTimesOutAt- ? options.backloggedTimesOutAt.toISOString()- : null,- ]- : []),- ],- )- ).rows[0],- options.backlogged,- )!;-+ return withSpan(+ "nuq.addJob",+ async span => {setSpanAttributes(span, {- "nuq.job_created": true,+ "nuq.queue_name": this.queueName,+ "nuq.job_id": id,+ "nuq.priority": options.priority ?? 0,+ "nuq.zero_data_retention": (data as any)?.zeroDataRetention ?? false,+ "nuq.listenable": options.listenable ?? false,});- return result;- } finally {- const duration = Date.now() - start;- setSpanAttributes(span, {- "nuq.duration_ms": duration,- });- logger.info("nuqAddJob metrics", {- module: "nuq/metrics",- method: "nuqAddJob",- duration,- scrapeId: id,- zeroDataRetention: (data as any)?.zeroDataRetention ?? false,- });- }- });+ const start = Date.now();+ try {+ const result = this.rowToJob(+ (+ await nuqPool.query(+ `INSERT INTO ${this.queueName}${options.backlogged ? "_backlog" : ""} (id, data, priority, listen_channel_id, owner_id, group_id${options.backlogged ? ", times_out_at" : ""}) VALUES ($1, $2, $3, $4, $5, $6${options.backlogged ? ", $7" : ""}) RETURNING ${(options.backlogged ? this.jobBacklogReturning : this.jobReturning).join(", ")};`,+ [+ id,+ data,+ options.priority ?? 0,+ options.listenable ? this.listenChannelId : null,+ normalizeOwnerId(options.ownerId),+ options.groupId ?? null,+ ...(options.backlogged+ ? [+ options.backloggedTimesOutAt+ ? options.backloggedTimesOutAt.toISOString()+ : null,+ ]+ : []),+ ],+ )+ ).rows[0],+ options.backlogged,+ )!;++ setSpanAttributes(span, {+ "nuq.job_created": true,+ });++ return result;+ } finally {+ const duration = Date.now() - start;+ setSpanAttributes(span, {+ "nuq.duration_ms": duration,+ });+ logger.info("nuqAddJob metrics", {+ module: "nuq/metrics",+ method: "nuqAddJob",+ duration,+ scrapeId: id,+ zeroDataRetention: (data as any)?.zeroDataRetention ?? false,+ });+ }+ },+ { zeroDataRetention: (data as any)?.zeroDataRetention === true },+ );}public async addJobIfNotExists(@@ -871,60 +875,64 @@ class NuQ<JobData = any, JobReturnValue = any> {data: JobData,options: NuQJobOptions,): Promise<NuQJob<JobData, JobReturnValue> | null> {- return withSpan("nuq.addJob", async span => {- setSpanAttributes(span, {- "nuq.queue_name": this.queueName,- "nuq.job_id": id,- "nuq.priority": options.priority ?? 0,- "nuq.zero_data_retention": (data as any)?.zeroDataRetention ?? false,- "nuq.listenable": options.listenable ?? false,- });-- const start = Date.now();- try {- const result = this.rowToJob(- (- await nuqPool.query(- `INSERT INTO ${this.queueName}${options.backlogged ? "_backlog" : ""} (id, data, priority, listen_channel_id, owner_id, group_id${options.backlogged ? ", times_out_at" : ""}) VALUES ($1, $2, $3, $4, $5, $6${options.backlogged ? ", $7" : ""}) ON CONFLICT (id) DO NOTHING RETURNING ${(options.backlogged ? this.jobBacklogReturning : this.jobReturning).join(", ")};`,- [- id,- data,- options.priority ?? 0,- options.listenable ? this.listenChannelId : null,- normalizeOwnerId(options.ownerId),- options.groupId ?? null,- ...(options.backlogged- ? [- options.backloggedTimesOutAt- ? options.backloggedTimesOutAt.toISOString()- : null,- ]- : []),- ],- )- ).rows[0],- options.backlogged,- );-+ return withSpan(+ "nuq.addJob",+ async span => {setSpanAttributes(span, {- "nuq.job_created": result !== null,+ "nuq.queue_name": this.queueName,+ "nuq.job_id": id,+ "nuq.priority": options.priority ?? 0,+ "nuq.zero_data_retention": (data as any)?.zeroDataRetention ?? false,+ "nuq.listenable": options.listenable ?? false,});- return result;- } finally {- const duration = Date.now() - start;- setSpanAttributes(span, {- "nuq.duration_ms": duration,- });- logger.info("nuqAddJob metrics", {- module: "nuq/metrics",- method: "nuqAddJob",- duration,- scrapeId: id,- zeroDataRetention: (data as any)?.zeroDataRetention ?? false,- });- }- });+ const start = Date.now();+ try {+ const result = this.rowToJob(+ (+ await nuqPool.query(+ `INSERT INTO ${this.queueName}${options.backlogged ? "_backlog" : ""} (id, data, priority, listen_channel_id, owner_id, group_id${options.backlogged ? ", times_out_at" : ""}) VALUES ($1, $2, $3, $4, $5, $6${options.backlogged ? ", $7" : ""}) ON CONFLICT (id) DO NOTHING RETURNING ${(options.backlogged ? this.jobBacklogReturning : this.jobReturning).join(", ")};`,+ [+ id,+ data,+ options.priority ?? 0,+ options.listenable ? this.listenChannelId : null,+ normalizeOwnerId(options.ownerId),+ options.groupId ?? null,+ ...(options.backlogged+ ? [+ options.backloggedTimesOutAt+ ? options.backloggedTimesOutAt.toISOString()+ : null,+ ]+ : []),+ ],+ )+ ).rows[0],+ options.backlogged,+ );++ setSpanAttributes(span, {+ "nuq.job_created": result !== null,+ });++ return result;+ } finally {+ const duration = Date.now() - start;+ setSpanAttributes(span, {+ "nuq.duration_ms": duration,+ });+ logger.info("nuqAddJob metrics", {+ module: "nuq/metrics",+ method: "nuqAddJob",+ duration,+ scrapeId: id,+ zeroDataRetention: (data as any)?.zeroDataRetention ?? false,+ });+ }+ },+ { zeroDataRetention: (data as any)?.zeroDataRetention === true },+ );}public async addJobs(@@ -934,142 +942,150 @@ class NuQ<JobData = any, JobReturnValue = any> {options: NuQJobOptions;}>,): Promise<NuQJob<JobData, JobReturnValue>[]> {- return withSpan("nuq.addJobs", async span => {- setSpanAttributes(span, {- "nuq.queue_name": this.queueName,- "nuq.jobs_count": jobs.length,- });+ return withSpan(+ "nuq.addJobs",+ async span => {+ setSpanAttributes(span, {+ "nuq.queue_name": this.queueName,+ "nuq.jobs_count": jobs.length,+ });- if (jobs.length === 0) {- return [];- }+ if (jobs.length === 0) {+ return [];+ }- const start = Date.now();- try {- // Separate jobs into backlogged and non-backlogged groups- const regularJobs: typeof jobs = [];- const backloggedJobs: typeof jobs = [];+ const start = Date.now();+ try {+ // Separate jobs into backlogged and non-backlogged groups+ const regularJobs: typeof jobs = [];+ const backloggedJobs: typeof jobs = [];- for (const job of jobs) {- if (job.options.backlogged) {- backloggedJobs.push(job);- } else {- regularJobs.push(job);+ for (const job of jobs) {+ if (job.options.backlogged) {+ backloggedJobs.push(job);+ } else {+ regularJobs.push(job);+ }}- }- const results: NuQJob<JobData, JobReturnValue>[] = [];-- // Batch size: 6 params per job, stay well under PG's 65535 param limit- // 1000 jobs = 6000 params, leaving plenty of headroom- const BATCH_SIZE = 1000;-- // Helper function to build and execute bulk insert with batching- const bulkInsert = async (- jobsToInsert: typeof jobs,- tableSuffix: string,- ) => {- if (jobsToInsert.length === 0) return;-- // Process in batches- for (- let offset = 0;- offset < jobsToInsert.length;- offset += BATCH_SIZE- ) {- const batch = jobsToInsert.slice(offset, offset + BATCH_SIZE);-- // Build the VALUES clause and parameters array- const valuesPlaceholders: string[] = [];- const params: any[] = [];-- const columns = [- "id",- "data",- "priority",- "listen_channel_id",- "owner_id",- "group_id",- ...(tableSuffix === "_backlog" ? ["times_out_at"] : []),- ];-- for (let i = 0; i < batch.length; i++) {- const job = batch[i];- const baseIdx = i * columns.length + 1;-- valuesPlaceholders.push(- `(${new Array(columns.length)- .fill(0)- .map((_, i) => "$" + (baseIdx + i))- .join(", ")})`,- );+ const results: NuQJob<JobData, JobReturnValue>[] = [];++ // Batch size: 6 params per job, stay well under PG's 65535 param limit+ // 1000 jobs = 6000 params, leaving plenty of headroom+ const BATCH_SIZE = 1000;++ // Helper function to build and execute bulk insert with batching+ const bulkInsert = async (+ jobsToInsert: typeof jobs,+ tableSuffix: string,+ ) => {+ if (jobsToInsert.length === 0) return;++ // Process in batches+ for (+ let offset = 0;+ offset < jobsToInsert.length;+ offset += BATCH_SIZE+ ) {+ const batch = jobsToInsert.slice(offset, offset + BATCH_SIZE);++ // Build the VALUES clause and parameters array+ const valuesPlaceholders: string[] = [];+ const params: any[] = [];++ const columns = [+ "id",+ "data",+ "priority",+ "listen_channel_id",+ "owner_id",+ "group_id",+ ...(tableSuffix === "_backlog" ? ["times_out_at"] : []),+ ];++ for (let i = 0; i < batch.length; i++) {+ const job = batch[i];+ const baseIdx = i * columns.length + 1;++ valuesPlaceholders.push(+ `(${new Array(columns.length)+ .fill(0)+ .map((_, i) => "$" + (baseIdx + i))+ .join(", ")})`,+ );- params.push(- ...[- job.id,- job.data,- job.options.priority ?? 0,- job.options.listenable ? this.listenChannelId : null,- normalizeOwnerId(job.options.ownerId),- job.options.groupId ?? null,- ...(tableSuffix === "_backlog"- ? [- job.options.backloggedTimesOutAt- ? job.options.backloggedTimesOutAt.toISOString()- : null,- ]- : []),- ],- );- }+ params.push(+ ...[+ job.id,+ job.data,+ job.options.priority ?? 0,+ job.options.listenable ? this.listenChannelId : null,+ normalizeOwnerId(job.options.ownerId),+ job.options.groupId ?? null,+ ...(tableSuffix === "_backlog"+ ? [+ job.options.backloggedTimesOutAt+ ? job.options.backloggedTimesOutAt.toISOString()+ : null,+ ]+ : []),+ ],+ );+ }- const query = `INSERT INTO ${this.queueName}${tableSuffix} (${columns.join(", ")}) VALUES ${valuesPlaceholders.join(", ")} RETURNING ${(tableSuffix === "_backlog" ? this.jobBacklogReturning : this.jobReturning).join(", ")};`;+ const query = `INSERT INTO ${this.queueName}${tableSuffix} (${columns.join(", ")}) VALUES ${valuesPlaceholders.join(", ")} RETURNING ${(tableSuffix === "_backlog" ? this.jobBacklogReturning : this.jobReturning).join(", ")};`;- const result = await nuqPool.query(query, params);+ const result = await nuqPool.query(query, params);- // Convert rows to jobs and maintain order- const jobMap = new Map(- result.rows.map(row => [- row.id,- this.rowToJob(row, tableSuffix === "_backlog")!,- ]),- );+ // Convert rows to jobs and maintain order+ const jobMap = new Map(+ result.rows.map(row => [+ row.id,+ this.rowToJob(row, tableSuffix === "_backlog")!,+ ]),+ );- for (const job of batch) {- const insertedJob = jobMap.get(job.id);- if (insertedJob) {- results.push(insertedJob);+ for (const job of batch) {+ const insertedJob = jobMap.get(job.id);+ if (insertedJob) {+ results.push(insertedJob);+ }}}- }- };+ };- // Insert regular jobs- await bulkInsert(regularJobs, "");+ // Insert regular jobs+ await bulkInsert(regularJobs, "");- // Insert backlogged jobs- await bulkInsert(backloggedJobs, "_backlog");+ // Insert backlogged jobs+ await bulkInsert(backloggedJobs, "_backlog");- setSpanAttributes(span, {- "nuq.jobs_created": results.length,- "nuq.regular_jobs_count": regularJobs.length,- "nuq.backlogged_jobs_count": backloggedJobs.length,- });+ setSpanAttributes(span, {+ "nuq.jobs_created": results.length,+ "nuq.regular_jobs_count": regularJobs.length,+ "nuq.backlogged_jobs_count": backloggedJobs.length,+ });- return results;- } finally {- const duration = Date.now() - start;- setSpanAttributes(span, {- "nuq.duration_ms": duration,- });- logger.info("nuqAddJobs metrics", {- module: "nuq/metrics",- method: "nuqAddJobs",- duration,- jobsCount: jobs.length,- });- }- });+ return results;+ } finally {+ const duration = Date.now() - start;+ setSpanAttributes(span, {+ "nuq.duration_ms": duration,+ });+ logger.info("nuqAddJobs metrics", {+ module: "nuq/metrics",+ method: "nuqAddJobs",+ duration,+ jobsCount: jobs.length,+ });+ }+ },+ {+ zeroDataRetention: jobs.some(+ job => (job.data as any)?.zeroDataRetention === true,+ ),+ },+ );}public async promoteJobFromBacklogOrAdd(@@ -1077,21 +1093,23 @@ class NuQ<JobData = any, JobReturnValue = any> {data: JobData,options: NuQJobOptions,): Promise<NuQJob<JobData, JobReturnValue> | null> {- return withSpan("nuq.promoteJobFromBacklogOrAdd", async span => {- setSpanAttributes(span, {- "nuq.queue_name": this.queueName,- "nuq.job_id": id,- "nuq.priority": options.priority ?? 0,- "nuq.zero_data_retention": (data as any)?.zeroDataRetention ?? false,- "nuq.listenable": options.listenable ?? false,- });+ return withSpan(+ "nuq.promoteJobFromBacklogOrAdd",+ async span => {+ setSpanAttributes(span, {+ "nuq.queue_name": this.queueName,+ "nuq.job_id": id,+ "nuq.priority": options.priority ?? 0,+ "nuq.zero_data_retention": (data as any)?.zeroDataRetention ?? false,+ "nuq.listenable": options.listenable ?? false,+ });- const start = Date.now();- try {- const result = this.rowToJob(- (- await nuqPool.query(- `+ const start = Date.now();+ try {+ const result = this.rowToJob(+ (+ await nuqPool.query(+ `WITH ins AS (INSERT INTO ${this.queueName} (id, data, created_at, priority, listen_channel_id, owner_id, group_id)SELECT b.id, b.data, b.created_at, b.priority, b.listen_channel_id, b.owner_id, b.group_id@@ -1106,31 +1124,33 @@ class NuQ<JobData = any, JobReturnValue = any> {)SELECT * FROM ins`,- [id],- )- ).rows[0],- );+ [id],+ )+ ).rows[0],+ );++ if (!result) {+ return await this.addJobIfNotExists(id, data, {+ ...options,+ backlogged: false,+ });+ }- if (!result) {- return await this.addJobIfNotExists(id, data, {- ...options,- backlogged: false,+ return result;+ } finally {+ const duration = Date.now() - start;+ setSpanAttributes(span, {+ "nuq.duration_ms": duration,+ });+ logger.info("nuqPromoteJobFromBacklogOrAdd metrics", {+ module: "nuq/metrics",+ method: "nuqPromoteJobFromBacklogOrAdd",+ duration,});}-- return result;- } finally {- const duration = Date.now() - start;- setSpanAttributes(span, {- "nuq.duration_ms": duration,- });- logger.info("nuqPromoteJobFromBacklogOrAdd metrics", {- module: "nuq/metrics",- method: "nuqPromoteJobFromBacklogOrAdd",- duration,- });- }- });+ },+ { zeroDataRetention: (data as any)?.zeroDataRetention === true },+ );}private readonly nuqWaitMode =diff --git a/apps/api/src/services/worker/scrape-worker.ts b/apps/api/src/services/worker/scrape-worker.tsindex 87b838f69..e8b3737c1 100644--- a/apps/api/src/services/worker/scrape-worker.ts+++ b/apps/api/src/services/worker/scrape-worker.ts@@ -1,7 +1,5 @@import { configDotenv } from "dotenv";import { config } from "../../config";-import * as Sentry from "@sentry/node";-import { applyZdrScope, captureExceptionWithZdrCheck } from "../sentry";import http from "http";import https from "https";@@ -93,6 +91,7 @@ import { scrapeSitemap } from "../../scraper/crawler/sitemap";import {withTraceContextAsync,withSpan,+ withZeroDataRetention,setSpanAttributes,} from "../../lib/otel-tracer";import { ScrapeUrlResponse } from "../../scraper/scrapeURL";@@ -303,9 +302,6 @@ async function billScrapeJob(status: "void",});}- captureExceptionWithZdrCheck(error, {- extra: { zeroDataRetention: job.data.zeroDataRetention ?? false },- });return creditsToBeBilled;}}@@ -363,7 +359,6 @@ async function processJob(job: NuQJob<ScrapeJobSingleUrls>) {teamId: job.data?.team_id ?? undefined,zeroDataRetention: job.data?.zeroDataRetention ?? false,});- applyZdrScope(job.data?.zeroDataRetention);logger.info(`🐂 Worker taking job ${job.id}`, { url: job.data.url });const start = job.data.startTime ?? Date.now();const remainingTime = job.data.scrapeOptions.timeout@@ -995,16 +990,6 @@ async function processJob(job: NuQJob<ScrapeJobSingleUrls>) {} else {logger.error(`🐂 Job errored ${job.id} - ${error}`, { error });- // Filter out TransportableErrors (flow control)- if (!(error instanceof TransportableError)) {- captureExceptionWithZdrCheck(error, {- data: {- job: job.id,- },- extra: { zeroDataRetention: job.data.zeroDataRetention ?? false },- });- }-if (error instanceof CustomError) {// Here we handle the error, then save the failed joblogger.error(error.message); // or any other error handling@@ -1662,8 +1647,13 @@ export const processJobInternal = async (job: NuQJob<ScrapeJobData>) => {zeroDataRetention: job.data?.zeroDataRetention ?? false,});- // Restore trace context if available and execute within span- if (job.data.traceContext) {+ // Restore trace context if available and execute within span. The ZDR+ // context is applied either way so nothing below records for ZDR jobs.+ return withZeroDataRetention(job.data.zeroDataRetention === true, () => {+ if (!job.data.traceContext) {+ return processJobWithTracing(job, logger);+ }+return withTraceContextAsync(job.data.traceContext, () =>withSpan("worker.scrape.process", async span => {setSpanAttributes(span, {@@ -1677,9 +1667,7 @@ export const processJobInternal = async (job: NuQJob<ScrapeJobData>) => {return processJobWithTracing(job, logger);}),);- } else {- return processJobWithTracing(job, logger);- }+ });};async function processJobWithTracing(job: NuQJob<ScrapeJobData>, logger: any) {@@ -1767,20 +1755,6 @@ async function processJobWithTracing(job: NuQJob<ScrapeJobData>, logger: any) {} catch (error) {logger.warn("Job failed", { error });- // Filter out expected errors (flow control, not real errors)- if (- error instanceof TransportableError ||- error instanceof JobCancelledError ||- error instanceof RacedRedirectError ||- error instanceof ScrapeJobTimeoutError- ) {- // These are expected flow control errors, don't send to Sentry- } else {- captureExceptionWithZdrCheck(error, {- extra: { zeroDataRetention: job.data.zeroDataRetention ?? false },- });- }-if (job.data.skipNuq) {throw error;} else {diff --git a/apps/api/src/services/zdr-worker.ts b/apps/api/src/services/zdr-worker.tsindex 5dde4ddec..1ed439d7d 100644--- a/apps/api/src/services/zdr-worker.ts+++ b/apps/api/src/services/zdr-worker.ts@@ -1,6 +1,5 @@import "dotenv/config";-import "./sentry";-import { setSentryServiceTag } from "./sentry";+import { shutdownTracing } from "../otel";import { logger } from "../lib/logger";import { zdrcleaner } from "../lib/zdrcleaner";@@ -17,12 +16,11 @@ process.on("SIGTERM", () => {});(async () => {- setSentryServiceTag("zdr-worker");-while (!isShuttingDown) {await zdrcleaner();}+ await shutdownTracing();logger.info("zdr-worker exiting");process.exit(0);})();diff --git a/apps/api/src/types.ts b/apps/api/src/types.tsindex ef87771b0..31fe6dbc2 100644--- a/apps/api/src/types.ts+++ b/apps/api/src/types.ts@@ -11,7 +11,7 @@ import { InternalOptions } from "./scraper/scrapeURL";import type { CostTracking } from "./lib/cost-tracking";import type { BillingMetadata } from "./services/billing/types";import { webhookSchema } from "./services/webhook/schema";-import { SerializedTraceContext } from "./lib/otel-tracer";+import type { SerializedTraceContext } from "./lib/otel-tracer";type ScrapeJobCommon = {concurrencyLimited?: boolean;@@ -60,7 +60,6 @@ type ScrapeJobSingleUrlsUnique = {from_extract?: boolean;startTime?: number;- sentry?: any;is_extract?: boolean;apiKeyId: number | null;